1 /* 2 * Copyright (c) 2004-2009 Voltaire, Inc. All rights reserved. 3 * Copyright (c) 2002-2005 Mellanox Technologies LTD. All rights reserved. 4 * Copyright (c) 1996-2003 Intel Corporation. All rights reserved. 5 * 6 * This software is available to you under a choice of one of two 7 * licenses. You may choose to be licensed under the terms of the GNU 8 * General Public License (GPL) Version 2, available from the file 9 * COPYING in the main directory of this source tree, or the 10 * OpenIB.org BSD license below: 11 * 12 * Redistribution and use in source and binary forms, with or 13 * without modification, are permitted provided that the following 14 * conditions are met: 15 * 16 * - Redistributions of source code must retain the above 17 * copyright notice, this list of conditions and the following 18 * disclaimer. 19 * 20 * - Redistributions in binary form must reproduce the above 21 * copyright notice, this list of conditions and the following 22 * disclaimer in the documentation and/or other materials 23 * provided with the distribution. 24 * 25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 32 * SOFTWARE. 33 * 34 */ 35 36 /* 37 * Abstract: 38 * Implementation of osm_req_t. 39 * This object represents the generic attribute requester. 40 * This object is part of the opensm family of objects. 41 * 42 */ 43 44 /* 45 Next available error code: 0x300 46 */ 47 48 #if HAVE_CONFIG_H 49 # include <config.h> 50 #endif /* HAVE_CONFIG_H */ 51 52 #ifdef OSM_VENDOR_INTF_AL 53 54 #include <stdlib.h> 55 #include <string.h> 56 #include <complib/cl_qlist.h> 57 #include <complib/cl_thread.h> 58 #include <complib/cl_math.h> 59 #include <complib/cl_debug.h> 60 #include <iba/ib_types.h> 61 #include <opensm/osm_madw.h> 62 #include <opensm/osm_log.h> 63 #include <opensm/osm_mad_pool.h> 64 #include <vendor/osm_vendor_api.h> 65 66 /****s* OpenSM: Vendor AL/osm_al_bind_info_t 67 * NAME 68 * osm_al_bind_info_t 69 * 70 * DESCRIPTION 71 * Structure containing bind information. 72 * 73 * SYNOPSIS 74 */ 75 typedef struct _osm_al_bind_info { 76 osm_vendor_t *p_vend; 77 void *client_context; 78 ib_qp_handle_t h_qp; 79 ib_mad_svc_handle_t h_svc; 80 uint8_t port_num; 81 ib_pool_key_t pool_key; 82 osm_vend_mad_recv_callback_t rcv_callback; 83 osm_vend_mad_send_err_callback_t send_err_callback; 84 osm_mad_pool_t *p_osm_pool; 85 ib_av_handle_t h_dr_av; 86 87 } osm_al_bind_info_t; 88 /* 89 * FIELDS 90 * p_vend 91 * Pointer to the vendor object. 92 * 93 * client_context 94 * User's context passed during osm_bind 95 * 96 * h_qp 97 * Handle the QP for this bind. 98 * 99 * h_qp_svc 100 * Handle the QP mad service for this bind. 101 * 102 * port_num 103 * Port number (within the HCA) of the bound port. 104 * 105 * pool_key 106 * Pool key returned by all for this QP. 107 * 108 * h_dr_av 109 * Address vector handle used for all directed route SMPs. 110 * 111 * SEE ALSO 112 *********/ 113 114 inline static ib_api_status_t 115 __osm_al_convert_wcs(IN ib_wc_status_t const wc_status) 116 { 117 switch (wc_status) { 118 case IB_WCS_SUCCESS: 119 return (IB_SUCCESS); 120 121 case IB_WCS_TIMEOUT_RETRY_ERR: 122 return (IB_TIMEOUT); 123 124 default: 125 return (IB_ERROR); 126 } 127 } 128 129 static void __osm_al_ca_err_callback(IN ib_async_event_rec_t * p_async_rec) 130 { 131 osm_vendor_t *p_vend = (osm_vendor_t *) p_async_rec->context; 132 OSM_LOG_ENTER(p_vend->p_log); 133 134 osm_log(p_vend->p_log, OSM_LOG_ERROR, 135 "__osm_al_ca_err_callback: ERR 3B01: " 136 "Event on channel adapter (%s).\n", 137 ib_get_async_event_str(p_async_rec->code)); 138 139 OSM_LOG_EXIT(p_vend->p_log); 140 } 141 142 static void __osm_al_ca_destroy_callback(IN void *context) 143 { 144 osm_al_bind_info_t *p_bind = (osm_al_bind_info_t *) context; 145 osm_vendor_t *p_vend = p_bind->p_vend; 146 OSM_LOG_ENTER(p_vend->p_log); 147 148 osm_log(p_vend->p_log, OSM_LOG_INFO, 149 "__osm_al_ca_destroy_callback: " 150 "Closing local channel adapter.\n"); 151 152 OSM_LOG_EXIT(p_vend->p_log); 153 } 154 155 static void __osm_al_err_callback(IN ib_async_event_rec_t * p_async_rec) 156 { 157 osm_al_bind_info_t *p_bind = 158 (osm_al_bind_info_t *) p_async_rec->context; 159 osm_vendor_t *p_vend = p_bind->p_vend; 160 OSM_LOG_ENTER(p_vend->p_log); 161 162 osm_log(p_vend->p_log, OSM_LOG_ERROR, 163 "__osm_al_err_callback: ERR 3B02: " 164 "Error on QP (%s).\n", 165 ib_get_async_event_str(p_async_rec->code)); 166 167 OSM_LOG_EXIT(p_vend->p_log); 168 } 169 170 static void 171 __osm_al_send_callback(IN void *mad_svc_context, IN ib_mad_element_t * p_elem) 172 { 173 osm_al_bind_info_t *const p_bind = 174 (osm_al_bind_info_t *) mad_svc_context; 175 osm_vendor_t *const p_vend = p_bind->p_vend; 176 osm_madw_t *const p_madw = (osm_madw_t *) p_elem->context1; 177 osm_vend_wrap_t *const p_vw = osm_madw_get_vend_ptr(p_madw); 178 ib_mad_t *p_mad; 179 180 OSM_LOG_ENTER(p_vend->p_log); 181 182 CL_ASSERT(p_vw); 183 CL_ASSERT(p_vw->h_av); 184 185 /* 186 Destroy the address vector as necessary. 187 */ 188 if (p_vw->h_av != p_bind->h_dr_av) { 189 if (osm_log_is_active(p_vend->p_log, OSM_LOG_DEBUG)) { 190 osm_log(p_vend->p_log, OSM_LOG_DEBUG, 191 "__osm_al_send_callback: " 192 "Destroying av handle %p.\n", p_vw->h_av); 193 } 194 195 ib_destroy_av(p_vw->h_av); 196 } 197 198 p_mad = ib_get_mad_buf(p_elem); 199 200 if (p_elem->resp_expected) { 201 /* 202 If the send was unsuccessful, notify the user 203 for MADs that were expecting a response. 204 A NULL mad wrapper parameter is the user's clue 205 that the transaction turned sour. 206 207 Otherwise, do nothing for successful sends when a 208 reponse is expected. The mad will be returned to the 209 pool later. 210 */ 211 p_madw->status = __osm_al_convert_wcs(p_elem->status); 212 if (p_elem->status != IB_WCS_SUCCESS) { 213 osm_log(p_vend->p_log, OSM_LOG_DEBUG, 214 "__osm_al_send_callback: " 215 "MAD completed with work queue error: %s.\n", 216 ib_get_wc_status_str(p_elem->status)); 217 /* 218 Return any wrappers to the pool that may have been 219 pre-emptively allocated to handle a receive. 220 */ 221 if (p_vw->p_resp_madw) { 222 osm_mad_pool_put(p_bind->p_osm_pool, 223 p_vw->p_resp_madw); 224 p_vw->p_resp_madw = NULL; 225 } 226 227 p_bind->send_err_callback(p_bind->client_context, 228 p_madw); 229 } 230 } else { 231 osm_log(p_vend->p_log, OSM_LOG_DEBUG, 232 "__osm_al_send_callback: " 233 "Returning MAD to pool, TID = 0x%" PRIx64 ".\n", 234 cl_ntoh64(p_mad->trans_id)); 235 osm_mad_pool_put(p_bind->p_osm_pool, p_madw); 236 goto Exit; 237 } 238 239 Exit: 240 OSM_LOG_EXIT(p_vend->p_log); 241 } 242 243 static void 244 __osm_al_rcv_callback(IN void *mad_svc_context, IN ib_mad_element_t * p_elem) 245 { 246 osm_al_bind_info_t *const p_bind = 247 (osm_al_bind_info_t *) mad_svc_context; 248 osm_vendor_t *const p_vend = p_bind->p_vend; 249 osm_madw_t *p_old_madw; 250 osm_madw_t *p_new_madw; 251 osm_vend_wrap_t *p_old_vw; 252 osm_vend_wrap_t *p_new_vw; 253 ib_mad_t *p_new_mad; 254 osm_mad_addr_t mad_addr; 255 256 OSM_LOG_ENTER(p_vend->p_log); 257 258 CL_ASSERT(p_elem->context1 == NULL); 259 CL_ASSERT(p_elem->context2 == NULL); 260 261 p_new_mad = ib_get_mad_buf(p_elem); 262 263 /* 264 In preperation for initializing the new mad wrapper, 265 Initialize the mad_addr structure for the received wire MAD. 266 */ 267 mad_addr.dest_lid = p_elem->remote_lid; 268 mad_addr.path_bits = p_elem->path_bits; 269 270 /* TO DO - figure out which #define to use for the 2.5 Gb rate... */ 271 mad_addr.static_rate = 0; 272 273 if (p_new_mad->mgmt_class == IB_MCLASS_SUBN_LID || 274 p_new_mad->mgmt_class == IB_MCLASS_SUBN_DIR) { 275 mad_addr.addr_type.smi.source_lid = p_elem->remote_lid; 276 } else { 277 mad_addr.addr_type.gsi.remote_qp = p_elem->remote_qp; 278 mad_addr.addr_type.gsi.remote_qkey = p_elem->remote_qkey; 279 mad_addr.addr_type.gsi.pkey_ix = p_elem->pkey_index; 280 mad_addr.addr_type.gsi.service_level = p_elem->remote_sl; 281 mad_addr.addr_type.gsi.global_route = FALSE; 282 } 283 284 /* 285 If this MAD is a response to a previous request, 286 then grab our pre-allocated MAD wrapper. 287 Otherwise, allocate a new MAD wrapper. 288 */ 289 if (ib_mad_is_response(p_new_mad)) { 290 CL_ASSERT(p_elem->send_context1 != NULL); 291 CL_ASSERT(p_elem->send_context2 == NULL); 292 293 p_old_madw = (osm_madw_t *) p_elem->send_context1; 294 p_old_vw = osm_madw_get_vend_ptr(p_old_madw); 295 p_new_madw = p_old_vw->p_resp_madw; 296 297 CL_ASSERT(p_new_madw); 298 299 osm_madw_init(p_new_madw, p_bind, p_elem->size, &mad_addr); 300 osm_madw_set_mad(p_new_madw, p_new_mad); 301 } else { 302 CL_ASSERT(p_elem->send_context1 == NULL); 303 CL_ASSERT(p_elem->send_context2 == NULL); 304 305 p_new_madw = osm_mad_pool_get_wrapper(p_bind->p_osm_pool, 306 p_bind, p_elem->size, 307 p_new_mad, &mad_addr); 308 } 309 310 CL_ASSERT(p_new_madw); 311 p_new_vw = osm_madw_get_vend_ptr(p_new_madw); 312 313 p_new_vw->h_bind = p_bind; 314 p_new_vw->size = p_elem->size; 315 p_new_vw->p_elem = p_elem; 316 p_new_vw->h_av = 0; 317 p_new_vw->p_resp_madw = NULL; 318 319 osm_log(p_vend->p_log, OSM_LOG_DEBUG, 320 "__osm_al_rcv_callback: " 321 "Calling receive callback function %p.\n", 322 p_bind->rcv_callback); 323 324 p_bind->rcv_callback(p_new_madw, p_bind->client_context, 325 p_elem->send_context1); 326 327 OSM_LOG_EXIT(p_vend->p_log); 328 } 329 330 ib_api_status_t 331 osm_vendor_init(IN osm_vendor_t * const p_vend, 332 IN osm_log_t * const p_log, IN const uint32_t timeout) 333 { 334 ib_api_status_t status; 335 OSM_LOG_ENTER(p_log); 336 337 p_vend->p_log = p_log; 338 339 /* 340 Open our instance of AL. 341 */ 342 status = ib_open_al(&p_vend->h_al); 343 if (status != IB_SUCCESS) { 344 osm_log(p_vend->p_log, OSM_LOG_ERROR, 345 "osm_vendor_init: ERR 3B03: " 346 "Error opening AL (%s).\n", ib_get_err_str(status)); 347 348 goto Exit; 349 } 350 351 p_vend->timeout = timeout; 352 353 Exit: 354 OSM_LOG_EXIT(p_log); 355 return (status); 356 } 357 358 osm_vendor_t *osm_vendor_new(IN osm_log_t * const p_log, 359 IN const uint32_t timeout) 360 { 361 ib_api_status_t status; 362 osm_vendor_t *p_vend; 363 364 OSM_LOG_ENTER(p_log); 365 366 p_vend = malloc(sizeof(*p_vend)); 367 if (p_vend == NULL) { 368 osm_log(p_vend->p_log, OSM_LOG_ERROR, 369 "osm_vendor_new: ERR 3B04: " 370 "Unable to allocate vendor object.\n"); 371 goto Exit; 372 } 373 374 memset(p_vend, 0, sizeof(*p_vend)); 375 376 status = osm_vendor_init(p_vend, p_log, timeout); 377 if (status != IB_SUCCESS) { 378 free(p_vend); 379 p_vend = NULL; 380 } 381 382 Exit: 383 OSM_LOG_EXIT(p_log); 384 return (p_vend); 385 } 386 387 void osm_vendor_delete(IN osm_vendor_t ** const pp_vend) 388 { 389 /* TO DO - fill this in */ 390 ib_close_al((*pp_vend)->h_al); 391 free(*pp_vend); 392 *pp_vend = NULL; 393 } 394 395 static ib_api_status_t 396 __osm_ca_info_init(IN osm_vendor_t * const p_vend, 397 IN osm_ca_info_t * const p_ca_info, 398 IN const ib_net64_t ca_guid) 399 { 400 ib_api_status_t status; 401 402 OSM_LOG_ENTER(p_vend->p_log); 403 404 p_ca_info->guid = ca_guid; 405 406 if (osm_log_is_active(p_vend->p_log, OSM_LOG_VERBOSE)) { 407 osm_log(p_vend->p_log, OSM_LOG_VERBOSE, 408 "__osm_ca_info_init: " 409 "Querying CA 0x%" PRIx64 ".\n", cl_ntoh64(ca_guid)); 410 } 411 412 status = ib_query_ca_by_guid(p_vend->h_al, ca_guid, NULL, 413 &p_ca_info->attr_size); 414 if ((status != IB_INSUFFICIENT_MEMORY) && (status != IB_SUCCESS)) { 415 osm_log(p_vend->p_log, OSM_LOG_ERROR, 416 "__osm_ca_info_init: ERR 3B05: " 417 "Unexpected status getting CA attributes (%s).\n", 418 ib_get_err_str(status)); 419 goto Exit; 420 } 421 422 CL_ASSERT(p_ca_info->attr_size); 423 424 p_ca_info->p_attr = malloc(p_ca_info->attr_size); 425 if (p_ca_info->p_attr == NULL) { 426 osm_log(p_vend->p_log, OSM_LOG_ERROR, 427 "__osm_ca_info_init: ERR 3B06: " 428 "Unable to allocate attribute storage.\n"); 429 goto Exit; 430 } 431 432 status = ib_query_ca_by_guid(p_vend->h_al, ca_guid, p_ca_info->p_attr, 433 &p_ca_info->attr_size); 434 if (status != IB_SUCCESS) { 435 osm_log(p_vend->p_log, OSM_LOG_ERROR, 436 "__osm_ca_info_init: ERR 3B07: " 437 "Unexpected status getting CA attributes (%s).\n", 438 ib_get_err_str(status)); 439 goto Exit; 440 } 441 442 Exit: 443 OSM_LOG_EXIT(p_vend->p_log); 444 return (status); 445 } 446 447 void 448 osm_ca_info_destroy(IN osm_vendor_t * const p_vend, 449 IN osm_ca_info_t * const p_ca_info) 450 { 451 OSM_LOG_ENTER(p_vend->p_log); 452 453 if (p_ca_info->p_attr) 454 free(p_ca_info->p_attr); 455 456 free(p_ca_info); 457 458 OSM_LOG_EXIT(p_vend->p_log); 459 } 460 461 osm_ca_info_t *osm_ca_info_new(IN osm_vendor_t * const p_vend, 462 IN const ib_net64_t ca_guid) 463 { 464 ib_api_status_t status; 465 osm_ca_info_t *p_ca_info; 466 467 OSM_LOG_ENTER(p_vend->p_log); 468 469 CL_ASSERT(ca_guid); 470 471 p_ca_info = malloc(sizeof(*p_ca_info)); 472 if (p_ca_info == NULL) 473 goto Exit; 474 475 memset(p_ca_info, 0, sizeof(*p_ca_info)); 476 477 status = __osm_ca_info_init(p_vend, p_ca_info, ca_guid); 478 if (status != IB_SUCCESS) { 479 osm_ca_info_destroy(p_vend, p_ca_info); 480 p_ca_info = NULL; 481 goto Exit; 482 } 483 484 Exit: 485 OSM_LOG_EXIT(p_vend->p_log); 486 return (p_ca_info); 487 } 488 489 static ib_api_status_t 490 __osm_vendor_get_ca_guids(IN osm_vendor_t * const p_vend, 491 IN ib_net64_t ** const p_guids, 492 IN unsigned * const p_num_guids) 493 { 494 ib_api_status_t status; 495 496 OSM_LOG_ENTER(p_vend->p_log); 497 498 CL_ASSERT(p_guids); 499 CL_ASSERT(p_num_guids); 500 501 status = ib_get_ca_guids(p_vend->h_al, NULL, p_num_guids); 502 if ((status != IB_INSUFFICIENT_MEMORY) && (status != IB_SUCCESS)) { 503 osm_log(p_vend->p_log, OSM_LOG_ERROR, 504 "__osm_vendor_get_ca_guids: ERR 3B08: " 505 "Unexpected status getting CA GUID array (%s).\n", 506 ib_get_err_str(status)); 507 goto Exit; 508 } 509 510 if (*p_num_guids == 0) { 511 osm_log(p_vend->p_log, OSM_LOG_ERROR, 512 "__osm_vendor_get_ca_guids: ERR 3B09: " 513 "No available channel adapters.\n"); 514 status = IB_INSUFFICIENT_RESOURCES; 515 goto Exit; 516 } 517 518 *p_guids = malloc(*p_num_guids * sizeof(**p_guids)); 519 if (*p_guids == NULL) { 520 osm_log(p_vend->p_log, OSM_LOG_ERROR, 521 "__osm_vendor_get_ca_guids: ERR 3B10: " 522 "Unable to allocate CA GUID array.\n"); 523 goto Exit; 524 } 525 526 status = ib_get_ca_guids(p_vend->h_al, *p_guids, p_num_guids); 527 CL_ASSERT(*p_num_guids); 528 529 if (osm_log_is_active(p_vend->p_log, OSM_LOG_VERBOSE)) { 530 osm_log(p_vend->p_log, OSM_LOG_VERBOSE, 531 "__osm_vendor_get_ca_guids: " 532 "Detected %u local channel adapters.\n", *p_num_guids); 533 } 534 535 Exit: 536 OSM_LOG_EXIT(p_vend->p_log); 537 return (status); 538 } 539 540 /****f* OpenSM: CA Info/osm_ca_info_get_pi_ptr 541 * NAME 542 * osm_ca_info_get_pi_ptr 543 * 544 * DESCRIPTION 545 * Returns a pointer to the port attribute of the specified port 546 * owned by this CA. 547 * 548 * SYNOPSIS 549 */ 550 static ib_port_attr_t *__osm_ca_info_get_port_attr_ptr(IN const osm_ca_info_t * 551 const p_ca_info, 552 IN const uint8_t index) 553 { 554 return (&p_ca_info->p_attr->p_port_attr[index]); 555 } 556 557 /* 558 * PARAMETERS 559 * p_ca_info 560 * [in] Pointer to a CA Info object. 561 * 562 * index 563 * [in] Port "index" for which to retrieve the port attribute. 564 * The index is the offset into the ca's internal array 565 * of port attributes. 566 * 567 * RETURN VALUE 568 * Returns a pointer to the port attribute of the specified port 569 * owned by this CA. 570 * 571 * NOTES 572 * 573 * SEE ALSO 574 *********/ 575 576 ib_api_status_t 577 osm_vendor_get_all_port_attr(IN osm_vendor_t * const p_vend, 578 IN ib_port_attr_t * const p_attr_array, 579 IN uint32_t * const p_num_ports) 580 { 581 ib_api_status_t status; 582 583 uint32_t ca; 584 unsigned ca_count; 585 uint32_t port_count = 0; 586 uint8_t port_num; 587 uint32_t total_ports = 0; 588 ib_net64_t *p_ca_guid = NULL; 589 osm_ca_info_t *p_ca_info; 590 591 OSM_LOG_ENTER(p_vend->p_log); 592 593 CL_ASSERT(p_vend); 594 CL_ASSERT(p_vend->p_ca_info == NULL); 595 596 /* 597 1) Determine the number of CA's 598 2) Allocate an array big enough to hold the ca info objects. 599 3) Call again to retrieve the guids. 600 */ 601 status = __osm_vendor_get_ca_guids(p_vend, &p_ca_guid, &ca_count); 602 603 p_vend->p_ca_info = malloc(ca_count * sizeof(*p_vend->p_ca_info)); 604 if (p_vend->p_ca_info == NULL) { 605 osm_log(p_vend->p_log, OSM_LOG_ERROR, 606 "osm_vendor_get_all_port_attr: ERR 3B11: " 607 "Unable to allocate CA information array.\n"); 608 goto Exit; 609 } 610 611 memset(p_vend->p_ca_info, 0, ca_count * sizeof(*p_vend->p_ca_info)); 612 p_vend->ca_count = ca_count; 613 614 /* 615 For each CA, retrieve the port info attributes 616 */ 617 for (ca = 0; ca < ca_count; ca++) { 618 p_ca_info = &p_vend->p_ca_info[ca]; 619 620 status = __osm_ca_info_init(p_vend, p_ca_info, p_ca_guid[ca]); 621 622 if (status != IB_SUCCESS) { 623 osm_log(p_vend->p_log, OSM_LOG_ERROR, 624 "osm_vendor_get_all_port_attr: ERR 3B12: " 625 "Unable to initialize CA Info object (%s).\n", 626 ib_get_err_str(status)); 627 } 628 629 total_ports += osm_ca_info_get_num_ports(p_ca_info); 630 } 631 632 /* 633 If the user supplied enough storage, return the port guids, 634 otherwise, return the appropriate error. 635 */ 636 if (*p_num_ports >= total_ports) { 637 for (ca = 0; ca < ca_count; ca++) { 638 uint32_t num_ports; 639 640 p_ca_info = &p_vend->p_ca_info[ca]; 641 642 num_ports = osm_ca_info_get_num_ports(p_ca_info); 643 644 for (port_num = 0; port_num < num_ports; port_num++) { 645 p_attr_array[port_count] = 646 *__osm_ca_info_get_port_attr_ptr(p_ca_info, 647 port_num); 648 port_count++; 649 } 650 } 651 } else { 652 status = IB_INSUFFICIENT_MEMORY; 653 } 654 655 *p_num_ports = total_ports; 656 657 Exit: 658 if (p_ca_guid) 659 free(p_ca_guid); 660 661 OSM_LOG_EXIT(p_vend->p_log); 662 return (status); 663 } 664 665 ib_net64_t 666 osm_vendor_get_ca_guid(IN osm_vendor_t * const p_vend, 667 IN const ib_net64_t port_guid) 668 { 669 uint8_t index; 670 uint8_t num_ports; 671 uint32_t num_guids = 0; 672 osm_ca_info_t *p_ca_info; 673 uint32_t ca; 674 675 OSM_LOG_ENTER(p_vend->p_log); 676 677 CL_ASSERT(port_guid); 678 /* 679 First, locate the HCA that owns this port. 680 */ 681 if (p_vend->p_ca_info == NULL) { 682 /* 683 Initialize the osm_ca_info_t array which allows 684 us to match port GUID to CA. 685 */ 686 osm_vendor_get_all_port_attr(p_vend, NULL, &num_guids); 687 } 688 689 CL_ASSERT(p_vend->p_ca_info); 690 CL_ASSERT(p_vend->ca_count); 691 692 for (ca = 0; ca < p_vend->ca_count; ca++) { 693 p_ca_info = &p_vend->p_ca_info[ca]; 694 695 num_ports = osm_ca_info_get_num_ports(p_ca_info); 696 CL_ASSERT(num_ports); 697 698 for (index = 0; index < num_ports; index++) { 699 if (port_guid == 700 osm_ca_info_get_port_guid(p_ca_info, index)) { 701 OSM_LOG_EXIT(p_vend->p_log); 702 return (osm_ca_info_get_ca_guid(p_ca_info)); 703 } 704 } 705 } 706 707 /* 708 No local CA owns this guid! 709 */ 710 osm_log(p_vend->p_log, OSM_LOG_ERROR, 711 "osm_vendor_get_ca_guid: ERR 3B13: " 712 "Unable to determine CA guid.\n"); 713 714 OSM_LOG_EXIT(p_vend->p_log); 715 return (0); 716 } 717 718 uint8_t 719 osm_vendor_get_port_num(IN osm_vendor_t * const p_vend, 720 IN const ib_net64_t port_guid) 721 { 722 uint8_t index; 723 uint8_t num_ports; 724 uint32_t num_guids = 0; 725 osm_ca_info_t *p_ca_info; 726 uint32_t ca; 727 728 OSM_LOG_ENTER(p_vend->p_log); 729 730 CL_ASSERT(port_guid); 731 /* 732 First, locate the HCA that owns this port. 733 */ 734 if (p_vend->p_ca_info == NULL) { 735 /* 736 Initialize the osm_ca_info_t array which allows 737 us to match port GUID to CA. 738 */ 739 osm_vendor_get_all_port_attr(p_vend, NULL, &num_guids); 740 } 741 742 CL_ASSERT(p_vend->p_ca_info); 743 CL_ASSERT(p_vend->ca_count); 744 745 for (ca = 0; ca < p_vend->ca_count; ca++) { 746 p_ca_info = &p_vend->p_ca_info[ca]; 747 748 num_ports = osm_ca_info_get_num_ports(p_ca_info); 749 CL_ASSERT(num_ports); 750 751 for (index = 0; index < num_ports; index++) { 752 if (port_guid == 753 osm_ca_info_get_port_guid(p_ca_info, index)) { 754 OSM_LOG_EXIT(p_vend->p_log); 755 return (osm_ca_info_get_port_num 756 (p_ca_info, index)); 757 } 758 } 759 } 760 761 /* 762 No local CA owns this guid! 763 */ 764 osm_log(p_vend->p_log, OSM_LOG_ERROR, 765 "osm_vendor_get_port_num: ERR 3B30: " 766 "Unable to determine CA guid.\n"); 767 768 OSM_LOG_EXIT(p_vend->p_log); 769 return (0); 770 } 771 772 static ib_api_status_t 773 __osm_vendor_open_ca(IN osm_vendor_t * const p_vend, 774 IN const ib_net64_t port_guid) 775 { 776 ib_net64_t ca_guid; 777 ib_api_status_t status; 778 779 OSM_LOG_ENTER(p_vend->p_log); 780 781 ca_guid = osm_vendor_get_ca_guid(p_vend, port_guid); 782 if (ca_guid == 0) { 783 osm_log(p_vend->p_log, OSM_LOG_ERROR, 784 "__osm_vendor_open_ca: ERR 3B31: " 785 "Bad port GUID value 0x%" PRIx64 ".\n", 786 cl_ntoh64(port_guid)); 787 status = IB_ERROR; 788 goto Exit; 789 } 790 791 osm_log(p_vend->p_log, OSM_LOG_VERBOSE, 792 "__osm_vendor_open_ca: " 793 "Opening HCA 0x%" PRIx64 ".\n", cl_ntoh64(ca_guid)); 794 795 status = ib_open_ca(p_vend->h_al, 796 ca_guid, 797 __osm_al_ca_err_callback, p_vend, &p_vend->h_ca); 798 799 if (status != IB_SUCCESS) { 800 osm_log(p_vend->p_log, OSM_LOG_ERROR, 801 "__osm_vendor_open_ca: ERR 3B15: " 802 "Unable to open CA (%s).\n", ib_get_err_str(status)); 803 goto Exit; 804 } 805 806 CL_ASSERT(p_vend->h_ca); 807 808 status = ib_alloc_pd(p_vend->h_ca, IB_PDT_ALIAS, p_vend, &p_vend->h_pd); 809 810 if (status != IB_SUCCESS) { 811 ib_close_ca(p_vend->h_ca, __osm_al_ca_destroy_callback); 812 osm_log(p_vend->p_log, OSM_LOG_ERROR, 813 "__osm_vendor_open_ca: ERR 3B16: " 814 "Unable to allocate protection domain (%s).\n", 815 ib_get_err_str(status)); 816 goto Exit; 817 } 818 819 CL_ASSERT(p_vend->h_pd); 820 821 Exit: 822 OSM_LOG_EXIT(p_vend->p_log); 823 return (status); 824 } 825 826 static void 827 __osm_vendor_init_av(IN const osm_al_bind_info_t * p_bind, 828 IN ib_av_attr_t * p_av) 829 { 830 memset(p_av, 0, sizeof(*p_av)); 831 p_av->port_num = p_bind->port_num; 832 p_av->dlid = IB_LID_PERMISSIVE; 833 } 834 835 osm_bind_handle_t 836 osm_vendor_bind(IN osm_vendor_t * const p_vend, 837 IN osm_bind_info_t * const p_user_bind, 838 IN osm_mad_pool_t * const p_mad_pool, 839 IN osm_vend_mad_recv_callback_t mad_recv_callback, 840 IN osm_vend_mad_send_err_callback_t send_err_callback, 841 IN void *context) 842 { 843 ib_net64_t port_guid; 844 osm_al_bind_info_t *p_bind = 0; 845 ib_api_status_t status; 846 ib_qp_create_t qp_create; 847 ib_mad_svc_t mad_svc; 848 ib_av_attr_t av; 849 850 OSM_LOG_ENTER(p_vend->p_log); 851 852 CL_ASSERT(p_user_bind); 853 CL_ASSERT(p_mad_pool); 854 CL_ASSERT(mad_recv_callback); 855 CL_ASSERT(send_err_callback); 856 857 port_guid = p_user_bind->port_guid; 858 859 osm_log(p_vend->p_log, OSM_LOG_INFO, 860 "osm_vendor_bind: " 861 "Binding to port 0x%" PRIx64 ".\n", cl_ntoh64(port_guid)); 862 863 if (p_vend->h_ca == 0) { 864 osm_log(p_vend->p_log, OSM_LOG_DEBUG, 865 "osm_vendor_bind: " 866 "Opening CA that owns port 0x%" PRIx64 ".\n", 867 port_guid); 868 869 status = __osm_vendor_open_ca(p_vend, port_guid); 870 if (status != IB_SUCCESS) { 871 osm_log(p_vend->p_log, OSM_LOG_ERROR, 872 "osm_vendor_bind: ERR 3B17: " 873 "Unable to Open CA (%s).\n", 874 ib_get_err_str(status)); 875 goto Exit; 876 } 877 } 878 879 p_bind = malloc(sizeof(*p_bind)); 880 if (p_bind == NULL) { 881 osm_log(p_vend->p_log, OSM_LOG_ERROR, 882 "osm_vendor_bind: ERR 3B18: " 883 "Unable to allocate internal bind object.\n"); 884 goto Exit; 885 } 886 887 memset(p_bind, 0, sizeof(*p_bind)); 888 p_bind->p_vend = p_vend; 889 p_bind->client_context = context; 890 p_bind->port_num = osm_vendor_get_port_num(p_vend, port_guid); 891 p_bind->rcv_callback = mad_recv_callback; 892 p_bind->send_err_callback = send_err_callback; 893 p_bind->p_osm_pool = p_mad_pool; 894 895 CL_ASSERT(p_bind->port_num); 896 897 /* 898 Get the proper QP. 899 */ 900 memset(&qp_create, 0, sizeof(qp_create)); 901 902 switch (p_user_bind->mad_class) { 903 case IB_MCLASS_SUBN_LID: 904 case IB_MCLASS_SUBN_DIR: 905 qp_create.qp_type = IB_QPT_QP0_ALIAS; 906 break; 907 908 case IB_MCLASS_SUBN_ADM: 909 default: 910 qp_create.qp_type = IB_QPT_QP1_ALIAS; 911 break; 912 } 913 914 qp_create.sq_depth = p_user_bind->send_q_size; 915 qp_create.rq_depth = p_user_bind->recv_q_size; 916 qp_create.sq_sge = OSM_AL_SQ_SGE; 917 qp_create.rq_sge = OSM_AL_RQ_SGE; 918 919 status = ib_get_spl_qp(p_vend->h_pd, 920 port_guid, 921 &qp_create, 922 p_bind, 923 __osm_al_err_callback, 924 &p_bind->pool_key, &p_bind->h_qp); 925 926 if (status != IB_SUCCESS) { 927 free(p_bind); 928 osm_log(p_vend->p_log, OSM_LOG_ERROR, 929 "osm_vendor_bind: ERR 3B19: " 930 "Unable to get QP handle (%s).\n", 931 ib_get_err_str(status)); 932 goto Exit; 933 } 934 935 CL_ASSERT(p_bind->h_qp); 936 CL_ASSERT(p_bind->pool_key); 937 938 memset(&mad_svc, 0, sizeof(mad_svc)); 939 940 mad_svc.mad_svc_context = p_bind; 941 mad_svc.pfn_mad_send_cb = __osm_al_send_callback; 942 mad_svc.pfn_mad_recv_cb = __osm_al_rcv_callback; 943 mad_svc.mgmt_class = p_user_bind->mad_class; 944 mad_svc.mgmt_version = p_user_bind->class_version; 945 mad_svc.support_unsol = p_user_bind->is_responder; 946 mad_svc.method_array[IB_MAD_METHOD_GET] = TRUE; 947 mad_svc.method_array[IB_MAD_METHOD_SET] = TRUE; 948 mad_svc.method_array[IB_MAD_METHOD_DELETE] = TRUE; 949 mad_svc.method_array[IB_MAD_METHOD_TRAP] = TRUE; 950 mad_svc.method_array[IB_MAD_METHOD_GETTABLE] = TRUE; 951 952 status = ib_reg_mad_svc(p_bind->h_qp, &mad_svc, &p_bind->h_svc); 953 954 if (status != IB_SUCCESS) { 955 free(p_bind); 956 osm_log(p_vend->p_log, OSM_LOG_ERROR, 957 "osm_vendor_bind: ERR 3B21: " 958 "Unable to register QP0 MAD service (%s).\n", 959 ib_get_err_str(status)); 960 goto Exit; 961 } 962 963 __osm_vendor_init_av(p_bind, &av); 964 965 status = ib_create_av(p_vend->h_pd, &av, &p_bind->h_dr_av); 966 if (status != IB_SUCCESS) { 967 osm_log(p_vend->p_log, OSM_LOG_ERROR, 968 "osm_vendor_bind: ERR 3B22: " 969 "Unable to create address vector (%s).\n", 970 ib_get_err_str(status)); 971 972 goto Exit; 973 } 974 975 if (osm_log_is_active(p_vend->p_log, OSM_LOG_DEBUG)) { 976 osm_log(p_vend->p_log, OSM_LOG_DEBUG, 977 "osm_vendor_bind: " 978 "Allocating av handle %p.\n", p_bind->h_dr_av); 979 } 980 981 Exit: 982 OSM_LOG_EXIT(p_vend->p_log); 983 return ((osm_bind_handle_t) p_bind); 984 } 985 986 ib_mad_t *osm_vendor_get(IN osm_bind_handle_t h_bind, 987 IN const uint32_t mad_size, 988 IN osm_vend_wrap_t * const p_vw) 989 { 990 ib_mad_t *p_mad; 991 osm_al_bind_info_t *p_bind = (osm_al_bind_info_t *) h_bind; 992 osm_vendor_t *p_vend = p_bind->p_vend; 993 ib_api_status_t status; 994 995 OSM_LOG_ENTER(p_vend->p_log); 996 997 CL_ASSERT(p_vw); 998 999 p_vw->size = mad_size; 1000 p_vw->h_bind = h_bind; 1001 1002 /* 1003 Retrieve a MAD element from the pool and give the user direct 1004 access to its buffer. 1005 */ 1006 status = ib_get_mad(p_bind->pool_key, mad_size, &p_vw->p_elem); 1007 if (status != IB_SUCCESS) { 1008 osm_log(p_vend->p_log, OSM_LOG_ERROR, 1009 "osm_vendor_get: ERR 3B25: " 1010 "Unable to acquire MAD (%s).\n", 1011 ib_get_err_str(status)); 1012 1013 p_mad = NULL; 1014 goto Exit; 1015 } 1016 1017 CL_ASSERT(p_vw->p_elem); 1018 p_mad = ib_get_mad_buf(p_vw->p_elem); 1019 1020 if (osm_log_get_level(p_vend->p_log) >= OSM_LOG_DEBUG) { 1021 osm_log(p_vend->p_log, OSM_LOG_DEBUG, 1022 "osm_vendor_get: " 1023 "Acquired MAD %p, size = %u.\n", p_mad, mad_size); 1024 } 1025 1026 Exit: 1027 OSM_LOG_EXIT(p_vend->p_log); 1028 return (p_mad); 1029 } 1030 1031 void 1032 osm_vendor_put(IN osm_bind_handle_t h_bind, IN osm_vend_wrap_t * const p_vw) 1033 { 1034 osm_al_bind_info_t *p_bind = (osm_al_bind_info_t *) h_bind; 1035 osm_vendor_t *p_vend = p_bind->p_vend; 1036 ib_api_status_t status; 1037 1038 OSM_LOG_ENTER(p_vend->p_log); 1039 1040 CL_ASSERT(p_vw); 1041 CL_ASSERT(p_vw->p_elem); 1042 CL_ASSERT(p_vw->h_bind == h_bind); 1043 1044 if (osm_log_get_level(p_vend->p_log) >= OSM_LOG_DEBUG) { 1045 osm_log(p_vend->p_log, OSM_LOG_DEBUG, 1046 "osm_vendor_put: " 1047 "Retiring MAD %p.\n", ib_get_mad_buf(p_vw->p_elem)); 1048 } 1049 1050 status = ib_put_mad(p_vw->p_elem); 1051 if (status != IB_SUCCESS) { 1052 osm_log(p_vend->p_log, OSM_LOG_ERROR, 1053 "osm_vendor_put: ERR 3B26: " 1054 "Unable to retire MAD (%s).\n", ib_get_err_str(status)); 1055 } 1056 1057 OSM_LOG_EXIT(p_vend->p_log); 1058 } 1059 1060 ib_api_status_t 1061 osm_vendor_send(IN osm_bind_handle_t h_bind, 1062 IN osm_madw_t * const p_madw, IN boolean_t const resp_expected) 1063 { 1064 osm_al_bind_info_t *const p_bind = h_bind; 1065 osm_vendor_t *const p_vend = p_bind->p_vend; 1066 osm_vend_wrap_t *const p_vw = osm_madw_get_vend_ptr(p_madw); 1067 osm_mad_addr_t *const p_mad_addr = osm_madw_get_mad_addr_ptr(p_madw); 1068 ib_mad_t *const p_mad = osm_madw_get_mad_ptr(p_madw); 1069 ib_api_status_t status; 1070 ib_mad_element_t *p_elem; 1071 ib_av_attr_t av; 1072 1073 OSM_LOG_ENTER(p_vend->p_log); 1074 1075 CL_ASSERT(p_vw->h_bind == h_bind); 1076 CL_ASSERT(p_vw->p_elem); 1077 1078 p_elem = p_vw->p_elem; 1079 1080 /* 1081 If a response is expected to this MAD, then preallocate 1082 a mad wrapper to contain the wire MAD received in the 1083 response. Allocating a wrapper here allows for easier 1084 failure paths than after we already received the wire mad. 1085 */ 1086 if (resp_expected) { 1087 p_vw->p_resp_madw = 1088 osm_mad_pool_get_wrapper_raw(p_bind->p_osm_pool); 1089 if (p_vw->p_resp_madw == NULL) { 1090 osm_log(p_vend->p_log, OSM_LOG_ERROR, 1091 "osm_vendor_send: ERR 3B27: " 1092 "Unable to allocate MAD wrapper.\n"); 1093 status = IB_INSUFFICIENT_RESOURCES; 1094 goto Exit; 1095 } 1096 } else 1097 p_vw->p_resp_madw = NULL; 1098 1099 /* 1100 For all sends other than directed route SM MADs, 1101 acquire an address vector for the destination. 1102 */ 1103 if (p_mad->mgmt_class != IB_MCLASS_SUBN_DIR) { 1104 memset(&av, 0, sizeof(av)); 1105 av.port_num = p_bind->port_num; 1106 av.dlid = p_mad_addr->dest_lid; 1107 av.static_rate = p_mad_addr->static_rate; 1108 av.path_bits = p_mad_addr->path_bits; 1109 1110 if ((p_mad->mgmt_class != IB_MCLASS_SUBN_LID) && 1111 (p_mad->mgmt_class != IB_MCLASS_SUBN_DIR)) { 1112 av.sl = p_mad_addr->addr_type.gsi.service_level; 1113 1114 if (p_mad_addr->addr_type.gsi.global_route) { 1115 av.grh_valid = TRUE; 1116 /* ANIL */ 1117 /* av.grh = p_mad_addr->addr_type.gsi.grh_info; */ 1118 } 1119 } 1120 1121 if (osm_log_is_active(p_vend->p_log, OSM_LOG_DEBUG)) { 1122 osm_log(p_vend->p_log, OSM_LOG_DEBUG, 1123 "osm_vendor_send: " 1124 "av.port_num 0x%X, " 1125 "av.dlid 0x%X, " 1126 "av.static_rate %d, " 1127 "av.path_bits %d.\n", 1128 av.port_num, cl_ntoh16(av.dlid), 1129 av.static_rate, av.path_bits); 1130 } 1131 1132 status = ib_create_av(p_vend->h_pd, &av, &p_vw->h_av); 1133 if (status != IB_SUCCESS) { 1134 osm_log(p_vend->p_log, OSM_LOG_ERROR, 1135 "osm_vendor_send: ERR 3B28: " 1136 "Unable to create address vector (%s).\n", 1137 ib_get_err_str(status)); 1138 1139 if (p_vw->p_resp_madw) 1140 osm_mad_pool_put(p_bind->p_osm_pool, 1141 p_vw->p_resp_madw); 1142 goto Exit; 1143 } 1144 1145 if (osm_log_is_active(p_vend->p_log, OSM_LOG_DEBUG)) { 1146 osm_log(p_vend->p_log, OSM_LOG_DEBUG, 1147 "osm_vendor_send: " 1148 "Allocating av handle %p.\n", p_vw->h_av); 1149 } 1150 } else { 1151 p_vw->h_av = p_bind->h_dr_av; 1152 } 1153 1154 p_elem->h_av = p_vw->h_av; 1155 1156 p_elem->context1 = p_madw; 1157 p_elem->context2 = NULL; 1158 1159 p_elem->immediate_data = 0; 1160 p_elem->p_grh = NULL; 1161 p_elem->resp_expected = resp_expected; 1162 p_elem->retry_cnt = OSM_DEFAULT_RETRY_COUNT; 1163 1164 p_elem->send_opt = IB_SEND_OPT_SIGNALED; 1165 p_elem->timeout_ms = p_vend->timeout; 1166 1167 /* Completion information. */ 1168 p_elem->status = 0; /* Not trusting AL */ 1169 1170 if ((p_mad->mgmt_class == IB_MCLASS_SUBN_LID) || 1171 (p_mad->mgmt_class == IB_MCLASS_SUBN_DIR)) { 1172 p_elem->remote_qp = 0; 1173 p_elem->remote_qkey = 0; 1174 } else { 1175 p_elem->remote_qp = p_mad_addr->addr_type.gsi.remote_qp; 1176 p_elem->remote_qkey = p_mad_addr->addr_type.gsi.remote_qkey; 1177 osm_log(p_vend->p_log, OSM_LOG_DEBUG, 1178 "osm_vendor_send: " 1179 "remote qp = 0x%X, remote qkey = 0x%X.\n", 1180 cl_ntoh32(p_elem->remote_qp), 1181 cl_ntoh32(p_elem->remote_qkey)); 1182 } 1183 1184 status = ib_send_mad(p_bind->h_svc, p_elem, NULL); 1185 if (status != IB_SUCCESS) { 1186 osm_log(p_vend->p_log, OSM_LOG_ERROR, 1187 "osm_vendor_send: ERR 3B29: " 1188 "Send failed (%s).\n", ib_get_err_str(status)); 1189 if (p_vw->p_resp_madw) 1190 osm_mad_pool_put(p_bind->p_osm_pool, p_vw->p_resp_madw); 1191 goto Exit; 1192 } 1193 1194 Exit: 1195 OSM_LOG_EXIT(p_vend->p_log); 1196 return (status); 1197 } 1198 1199 ib_api_status_t osm_vendor_local_lid_change(IN osm_bind_handle_t h_bind) 1200 { 1201 osm_al_bind_info_t *p_bind = (osm_al_bind_info_t *) h_bind; 1202 osm_vendor_t *p_vend = p_bind->p_vend; 1203 ib_av_attr_t av; 1204 ib_api_status_t status; 1205 1206 OSM_LOG_ENTER(p_vend->p_log); 1207 1208 /* 1209 The only thing we need to do is refresh the directed 1210 route address vector. 1211 */ 1212 __osm_vendor_init_av(p_bind, &av); 1213 1214 status = ib_destroy_av(p_bind->h_dr_av); 1215 if (status != IB_SUCCESS) { 1216 osm_log(p_vend->p_log, OSM_LOG_ERROR, 1217 "osm_vendor_local_lid_change: ERR 3B32: " 1218 "Unable to destroy address vector (%s).\n", 1219 ib_get_err_str(status)); 1220 1221 goto Exit; 1222 } 1223 1224 status = ib_create_av(p_vend->h_pd, &av, &p_bind->h_dr_av); 1225 if (status != IB_SUCCESS) { 1226 osm_log(p_vend->p_log, OSM_LOG_ERROR, 1227 "osm_vendor_local_lid_change: ERR 3B33: " 1228 "Unable to create address vector (%s).\n", 1229 ib_get_err_str(status)); 1230 1231 goto Exit; 1232 } 1233 1234 Exit: 1235 OSM_LOG_EXIT(p_vend->p_log); 1236 return (status); 1237 } 1238 1239 void osm_vendor_set_sm(IN osm_bind_handle_t h_bind, IN boolean_t is_sm_val) 1240 { 1241 osm_al_bind_info_t *p_bind = (osm_al_bind_info_t *) h_bind; 1242 osm_vendor_t *p_vend = p_bind->p_vend; 1243 ib_api_status_t status; 1244 ib_port_attr_mod_t attr_mod; 1245 1246 OSM_LOG_ENTER(p_vend->p_log); 1247 1248 memset(&attr_mod, 0, sizeof(attr_mod)); 1249 1250 attr_mod.cap.sm = is_sm_val; 1251 1252 status = ib_modify_ca(p_vend->h_ca, p_bind->port_num, 1253 IB_CA_MOD_IS_SM, &attr_mod); 1254 1255 if (status != IB_SUCCESS) { 1256 osm_log(p_vend->p_log, OSM_LOG_ERROR, 1257 "osm_vendor_set_sm: ERR 3B34: " 1258 "Unable set 'IS_SM' bit to:%u in port attributes (%s).\n", 1259 is_sm_val, ib_get_err_str(status)); 1260 } 1261 1262 OSM_LOG_EXIT(p_vend->p_log); 1263 } 1264 1265 void osm_vendor_set_debug(IN osm_vendor_t * const p_vend, IN int32_t level) 1266 { 1267 1268 } 1269 1270 #endif /* OSM_VENDOR_INTF_AL */ 1271