1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 /* 27 * ibtl_impl.c 28 * 29 * This file contains the IBTF module's initialization and 30 * IBTF Clients/Modules registration routines. 31 */ 32 33 #include <sys/modctl.h> 34 #include <sys/sunndi.h> 35 #include <sys/ib/ibtl/impl/ibtl.h> 36 #include <sys/ib/ibtl/impl/ibtl_ibnex.h> 37 38 /* 39 * Globals. 40 */ 41 static char ibtf[] = "ibtl_impl"; 42 43 extern ibtl_ibnex_callback_t ibtl_ibnex_callback_routine; 44 45 /* 46 * ibtl_clnt_list: 47 * 48 * Head of the list of IBT Client Instances. The IBT Client List 49 * is modified by IBTF on an IBT client's ibt_attach/ibt_detach call. 50 * 51 * ibtl_hca_list: 52 * 53 * Head of the list of HCA devices. The HCA List is modified by IBTF on 54 * a CI's ibc_attach/ibc_detach call. 55 * The datap of the list elements points to an ibtl_hca_devinfo_s 56 * structure. 57 * 58 * (ibc_attach) 59 * ibtl_hca_list -> ibtl_hca_devinfo_t--> ... -->ibtl_hca_devinfo_t 60 * [per-hca_dev] | ^ {nth HCA Dev} 61 * | | 62 * | ibtl_hca_t (ibt_open_hca) 63 * | ^ | 64 * | | | 65 * v | V 66 * ibtl_clnt_list -> ibtl_clnt_t--> ...--> {n'th Module} 67 * [per-client_instance] (ibt_attach) 68 * 69 */ 70 71 /* Global List of IBT Client Instances, and associated mutex. */ 72 struct ibtl_clnt_s *ibtl_clnt_list = NULL; 73 kmutex_t ibtl_clnt_list_mutex; 74 75 /* Lock for the race between the client and CM to free QPs. */ 76 kmutex_t ibtl_free_qp_mutex; 77 78 /* Lock for the race between the client closing the HCA and QPN being freed. */ 79 kcondvar_t ibtl_close_hca_cv; 80 81 /* Global List of HCA Devices, and associated mutex. */ 82 struct ibtl_hca_devinfo_s *ibtl_hca_list = NULL; 83 84 /* Well-known async handlers and associated client private. */ 85 ibt_async_handler_t ibtl_cm_async_handler; 86 ibt_async_handler_t ibtl_dm_async_handler; 87 ibt_async_handler_t ibtl_ibma_async_handler; 88 void *ibtl_cm_clnt_private; 89 void *ibtl_dm_clnt_private; 90 void *ibtl_ibma_clnt_private; 91 92 extern int ib_hw_status; 93 _NOTE(SCHEME_PROTECTS_DATA("Scheme protects data", ib_hw_status)) 94 95 /* 96 * Misc Module Declarations. 97 */ 98 extern struct mod_ops mod_miscops; 99 static struct modlmisc modlmisc = { 100 &mod_miscops, /* Type of module - misc. */ 101 "IB Transport Layer" /* Name of the Module. */ 102 }; 103 104 static struct modlinkage modlinkage = { 105 MODREV_1, (void *)&modlmisc, NULL 106 }; 107 108 109 /* 110 * IBTF Loadable Module Routines. 111 */ 112 113 int 114 _init(void) 115 { 116 int rval; 117 118 if ((rval = mod_install(&modlinkage)) != 0) 119 return (rval); 120 121 /* 122 * initialize IBTL ib2usec table 123 */ 124 ibtl_ib2usec_init(); 125 126 /* 127 * Initialize Logging 128 */ 129 ibtl_logging_initialization(); 130 131 /* 132 * Initialize the Alloc QP States. 133 */ 134 ibtl_init_cep_states(); 135 136 /* 137 * Initialize all Global Link Lists. 138 */ 139 mutex_init(&ibtl_clnt_list_mutex, NULL, MUTEX_DEFAULT, NULL); 140 mutex_init(&ibtl_free_qp_mutex, NULL, MUTEX_DEFAULT, NULL); 141 cv_init(&ibtl_close_hca_cv, NULL, CV_DEFAULT, NULL); 142 143 mutex_init(&ibtl_qp_mutex, NULL, MUTEX_DEFAULT, NULL); 144 cv_init(&ibtl_qp_cv, NULL, CV_DEFAULT, NULL); 145 146 ibtl_thread_init(); 147 148 return (rval); 149 } 150 151 152 /* 153 * The IBTF Module is never unloaded. Actually there is no need of this 154 * routine, but provided just in case. 155 */ 156 int 157 _fini(void) 158 { 159 int rval; 160 161 if ((rval = mod_remove(&modlinkage)) != 0) { 162 return (rval); 163 } 164 165 ibtl_thread_fini(); 166 167 mutex_destroy(&ibtl_clnt_list_mutex); 168 mutex_destroy(&ibtl_free_qp_mutex); 169 cv_destroy(&ibtl_close_hca_cv); 170 mutex_destroy(&ibtl_qp_mutex); 171 cv_destroy(&ibtl_qp_cv); 172 173 /* 174 * Stop Logging 175 */ 176 ibtl_logging_destroy(); 177 178 return (rval); 179 } 180 181 182 int 183 _info(struct modinfo *modinfop) 184 { 185 /* Return the Module Information. */ 186 return (mod_info(&modlinkage, modinfop)); 187 } 188 189 190 /* 191 * IBTF Client Registration Routines. 192 */ 193 194 /* 195 * Function: 196 * ibt_attach 197 * Input: 198 * modinfop - Client Module info structure. 199 * arg - usually client's dip 200 * clnt_private - client's private data pointer. 201 * Output: 202 * ibt_hdl_p - pointer to client's specific IBT handle, 203 * which is opaque to clients. 204 * Returns: 205 * IBT_SUCCESS 206 * IBT_INVALID_PARAM 207 * Called by: 208 * IBTF Client module during its attach() to register its instance 209 * to IBTF. 210 * Description: 211 * Registers the IBTF client module instance and returns an opaque 212 * handler to the client to be used for future calls to IBTF. 213 * Adds this client module instance to ibtl_clnt_list list. 214 * Records well-known async handlers. 215 */ 216 ibt_status_t 217 ibt_attach(ibt_clnt_modinfo_t *mod_infop, dev_info_t *arg, void *clnt_private, 218 ibt_clnt_hdl_t *ibt_hdl_p) 219 { 220 dev_info_t *pdip; 221 ibtl_clnt_t *clntp; 222 223 IBTF_DPRINTF_L3(ibtf, "ibt_attach(%p, %p, %p)", 224 mod_infop, arg, clnt_private); 225 226 if (mod_infop->mi_clnt_name == NULL) { 227 IBTF_DPRINTF_L1(ibtf, "ibt_attach: " 228 "IB client needs to specify its name"); 229 return (IBT_INVALID_PARAM); 230 } 231 232 /* 233 * Validate the Transport API version. 234 */ 235 if (mod_infop->mi_ibt_version != IBTI_V_CURR) { 236 IBTF_DPRINTF_L1(ibtf, "ibt_attach: IB client '%s' has an " 237 "invalid IB TI Version '%d'", mod_infop->mi_clnt_name, 238 mod_infop->mi_ibt_version); 239 return (IBT_NOT_SUPPORTED); 240 } 241 242 if (mod_infop->mi_async_handler == NULL) { 243 IBTF_DPRINTF_L2(ibtf, "ibt_attach: Client '%s' has not\n" 244 " provided an Asynchronous Event Handler.\n" 245 " This will be required soon.", 246 mod_infop->mi_clnt_name); 247 } 248 249 /* 250 * Check out Client's Class information. If it is not of mgmt class, 251 * we expect 'arg' to be Not NULL and point to client driver's 252 * device info struct. 253 */ 254 if ((!IBT_CLNT_MGMT_CLASS(mod_infop->mi_clnt_class)) && 255 (arg == NULL)) { 256 IBTF_DPRINTF_L1(ibtf, "ibt_attach: " 257 "arg not set with driver's dip."); 258 return (IBT_INVALID_PARAM); 259 } 260 261 if (!IBT_CLNT_MGMT_CLASS(mod_infop->mi_clnt_class)) { 262 pdip = ddi_get_parent(arg); 263 if (pdip == NULL || 264 ibtl_ibnex_valid_hca_parent(pdip) != IBT_SUCCESS) { 265 IBTF_DPRINTF_L2(ibtf, "ibt_attach: " 266 "client %s is not a child of IB nexus driver.", 267 ddi_driver_name(arg)); 268 return (IBT_INVALID_PARAM); 269 } 270 } 271 272 mutex_enter(&ibtl_clnt_list_mutex); 273 if (mod_infop->mi_clnt_class == IBT_CM) { 274 if (ibtl_cm_async_handler != NULL) { 275 IBTF_DPRINTF_L1(ibtf, "ibt_attach: " 276 "CM is already attached."); 277 mutex_exit(&ibtl_clnt_list_mutex); 278 return (IBT_INVALID_PARAM); 279 } 280 ibtl_cm_async_handler = mod_infop->mi_async_handler; 281 ibtl_cm_clnt_private = clnt_private; 282 } else if (mod_infop->mi_clnt_class == IBT_DM) { 283 if (ibtl_dm_async_handler != NULL) { 284 IBTF_DPRINTF_L1(ibtf, "ibt_attach: " 285 "DM is already attached."); 286 mutex_exit(&ibtl_clnt_list_mutex); 287 return (IBT_INVALID_PARAM); 288 } 289 ibtl_dm_async_handler = mod_infop->mi_async_handler; 290 ibtl_dm_clnt_private = clnt_private; 291 } else if (mod_infop->mi_clnt_class == IBT_IBMA) { 292 if (ibtl_ibma_async_handler != NULL) { 293 IBTF_DPRINTF_L1(ibtf, "ibt_attach: " 294 "IBMF is already attached."); 295 mutex_exit(&ibtl_clnt_list_mutex); 296 return (IBT_INVALID_PARAM); 297 } 298 ibtl_ibma_async_handler = mod_infop->mi_async_handler; 299 ibtl_ibma_clnt_private = clnt_private; 300 } 301 302 /* Allocate the memory for per-client-device info structure */ 303 clntp = kmem_zalloc(sizeof (ibtl_clnt_t), KM_SLEEP); 304 305 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(clntp->clnt_modinfop, 306 clntp->clnt_dip, clntp->clnt_name, clntp->clnt_async_cnt, 307 clntp->clnt_private)) 308 /* Update the Client info structure */ 309 clntp->clnt_modinfop = mod_infop; /* IBT Client's Mod Info */ 310 clntp->clnt_private = clnt_private; /* IBT Client's private */ 311 clntp->clnt_dip = arg; /* IBT Client's dip */ 312 clntp->clnt_async_cnt = 0; 313 /* using a count of 7 below guarantees it is NULL terminated */ 314 (void) strncpy(clntp->clnt_name, mod_infop->mi_clnt_name, 7); 315 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(clntp->clnt_modinfop, 316 clntp->clnt_dip, clntp->clnt_name, clntp->clnt_async_cnt, 317 clntp->clnt_private)) 318 319 /* 320 * Update Client Device Instance List. 321 */ 322 clntp->clnt_list_link = ibtl_clnt_list; 323 ibtl_clnt_list = clntp; 324 mutex_exit(&ibtl_clnt_list_mutex); 325 326 /* 327 * The ibt_hdl_p is a opaque handle which is the address of 328 * ibt_clnt_t structure passed back to the clients. 329 * The client will pass on this handle in its future calls to IBTF. 330 */ 331 *ibt_hdl_p = clntp; 332 333 return (IBT_SUCCESS); 334 } 335 336 337 /* 338 * Function: 339 * ibt_detach 340 * Input: 341 * ibt_hdl - IBT Handle as returned during ibt_attach call. 342 * Output: 343 * none 344 * Returns: 345 * IBT_SUCCESS 346 * IBT_INVALID_PARAM. 347 * Called by: 348 * IBTF Client module during its detach() to de-register its instance 349 * from IBTF. 350 * Description: 351 * Deregisters the IBTF client module instance from the IBTF. 352 * All resources and any reference to this ibt_hdl will be removed. 353 */ 354 ibt_status_t 355 ibt_detach(ibt_clnt_hdl_t ibt_hdl) 356 { 357 ibtl_clnt_t **clntpp; 358 359 IBTF_DPRINTF_L3(ibtf, "ibt_detach(%p)", ibt_hdl); 360 361 mutex_enter(&ibtl_clnt_list_mutex); 362 clntpp = &ibtl_clnt_list; 363 for (; *clntpp != NULL; clntpp = &(*clntpp)->clnt_list_link) 364 if (*clntpp == ibt_hdl) 365 break; 366 if (*clntpp == NULL) { 367 IBTF_DPRINTF_L1(ibtf, "ibt_detach: Client @ %p Not Found", 368 ibt_hdl); 369 mutex_exit(&ibtl_clnt_list_mutex); 370 return (IBT_INVALID_PARAM); 371 } 372 373 /* 374 * Check out whether the client has freed all its resources. 375 * If not done, then fail the detach. 376 * 377 * viz. A client has to close all the HCA they have opened, 378 * i.e. the HCA List maintained for clients has to be empty. 379 * If this list is not empty, then the client has not performed 380 * complete clean-up, so fail the detach. 381 */ 382 if (ibt_hdl->clnt_hca_list != NULL) { 383 mutex_exit(&ibtl_clnt_list_mutex); 384 385 IBTF_DPRINTF_L2(ibtf, "ibt_detach: " 386 "ERROR: Client '%s' has not closed all of its HCAs", 387 ibt_hdl->clnt_modinfop->mi_clnt_name); 388 return (IBT_HCA_RESOURCES_NOT_FREED); 389 } 390 391 if (ibt_hdl->clnt_srv_cnt != 0) { 392 mutex_exit(&ibtl_clnt_list_mutex); 393 IBTF_DPRINTF_L2(ibtf, "ibt_detach: client '%s' still has " 394 "services or subnet_notices registered", 395 ibt_hdl->clnt_modinfop->mi_clnt_name); 396 return (IBT_HCA_RESOURCES_NOT_FREED); 397 } 398 399 /* 400 * Delete the entry of this module from the ibtl_clnt_list List. 401 */ 402 *clntpp = ibt_hdl->clnt_list_link; /* remove us */ 403 404 /* make sure asyncs complete before freeing */ 405 ibtl_free_clnt_async_check(ibt_hdl); 406 407 if (ibt_hdl->clnt_modinfop->mi_clnt_class == IBT_CM) { 408 ibtl_cm_async_handler = NULL; 409 ibtl_cm_clnt_private = NULL; 410 } else if (ibt_hdl->clnt_modinfop->mi_clnt_class == IBT_DM) { 411 ibtl_dm_async_handler = NULL; 412 ibtl_dm_clnt_private = NULL; 413 } else if (ibt_hdl->clnt_modinfop->mi_clnt_class == IBT_IBMA) { 414 ibtl_ibma_async_handler = NULL; 415 ibtl_ibma_clnt_private = NULL; 416 } 417 mutex_exit(&ibtl_clnt_list_mutex); 418 419 /* Free up the memory of per-client info struct. */ 420 kmem_free(ibt_hdl, sizeof (ibtl_clnt_t)); 421 422 return (IBT_SUCCESS); 423 } 424 425 static void 426 ibtl_set_ibhw_status() 427 { 428 ib_hw_status++; 429 } 430 431 static void 432 ibtl_clear_ibhw_status() 433 { 434 ib_hw_status--; 435 } 436 437 /* 438 * Function: 439 * ibc_init 440 * Input: 441 * modlp - Pointer to IBC client module linkage structure 442 * Output: 443 * None 444 * Returns: 445 * 0 always for now 446 * Called by: 447 * CI client calls IBTF during its _init() to register HCA with 448 * Solaris I/O framework. 449 * Description: 450 * Initializes the CI clients module linkage structure with 451 * default bus_ops structure 452 */ 453 int 454 ibc_init(struct modlinkage *modlp) 455 { 456 ibtl_ibnex_cb_args_t cb_args; 457 458 mutex_enter(&ibtl_clnt_list_mutex); 459 cb_args.cb_flag = IBTL_IBNEX_IBC_INIT; 460 cb_args.cb_modlp = modlp; 461 if (ibtl_ibnex_callback_routine) { 462 (void) ((*ibtl_ibnex_callback_routine)(&cb_args)); 463 } 464 mutex_exit(&ibtl_clnt_list_mutex); 465 return (0); 466 } 467 468 469 /* 470 * Function: 471 * ibc_fini 472 * Input: 473 * modlp - Pointer to IBC client module linkage structure 474 * Output: 475 * None 476 * Returns: 477 * None 478 * Called by: 479 * CI client calls IBTF during its _fini() to remove HCA with 480 * Solaris I/O framework. 481 * Description: 482 * Undo what is done during ibc_init 483 */ 484 void 485 ibc_fini(struct modlinkage *modlp) 486 { 487 ibtl_ibnex_cb_args_t cb_args; 488 489 mutex_enter(&ibtl_clnt_list_mutex); 490 cb_args.cb_flag = IBTL_IBNEX_IBC_FINI; 491 cb_args.cb_modlp = modlp; 492 if (ibtl_ibnex_callback_routine) { 493 (void) ((*ibtl_ibnex_callback_routine)(&cb_args)); 494 } 495 mutex_exit(&ibtl_clnt_list_mutex); 496 } 497 498 /* 499 * Function: 500 * ibc_attach 501 * Input: 502 * info_p - IBC HCA Info. 503 * Output: 504 * ibc_hdl_p - IBC Client's HCA Handle. 505 * Returns: 506 * IBC_SUCCESS 507 * IBC_FAILURE 508 * Called by: 509 * CI calls IBTF during its attach() to register HCA Device with IBTF. 510 * Description: 511 * Registers the presence of HCA device by providing the HCA device info 512 * structure and provides an opaque HCA handler for future calls to this 513 * HCA device. 514 */ 515 ibc_status_t 516 ibc_attach(ibc_clnt_hdl_t *ibc_hdl_p, ibc_hca_info_t *info_p) 517 { 518 ibtl_hca_devinfo_t *hca_devp; 519 uint_t nports; 520 ibt_status_t status; 521 522 IBTF_DPRINTF_L2(ibtf, "ibc_attach(%p, %p)", ibc_hdl_p, info_p); 523 524 /* Validate the Transport API version */ 525 if (info_p->hca_ci_vers != IBCI_V3) { 526 IBTF_DPRINTF_L1(ibtf, "ibc_attach: Invalid IB CI Version '%d'", 527 info_p->hca_ci_vers); 528 return (IBC_FAILURE); 529 } 530 531 if (info_p->hca_attr == NULL) { 532 IBTF_DPRINTF_L1(ibtf, "ibc_attach: " 533 "HCA Attributes must be specified."); 534 return (IBC_FAILURE); 535 } 536 537 nports = info_p->hca_attr->hca_nports; 538 if (nports == 0) { 539 IBTF_DPRINTF_L1(ibtf, "ibc_attach: " 540 "Number of ports must be valid"); 541 return (IBC_FAILURE); 542 } 543 544 if (info_p->hca_attr->hca_max_port_pkey_tbl_sz == 0) { 545 IBTF_DPRINTF_L1(ibtf, "ibc_attach: " 546 "Number of Partitions must be at least 1"); 547 return (IBC_FAILURE); 548 } 549 550 if ((info_p->hca_attr->hca_flags & IBT_HCA_CURRENT_QP_STATE) == 0) { 551 IBTF_DPRINTF_L1(ibtf, "ibc_attach: " 552 "HCA driver must support QP current state checking"); 553 return (IBC_FAILURE); 554 } 555 556 if ((info_p->hca_attr->hca_flags & IBT_HCA_PORT_UP) == 0) { 557 IBTF_DPRINTF_L1(ibtf, "ibc_attach: " 558 "HCA driver must support PORT_UP async events"); 559 return (IBC_FAILURE); 560 } 561 562 /* 563 * Install IB nexus driver (if not installed already) 564 */ 565 ibtl_set_ibhw_status(); 566 if (ndi_devi_config_vhci("ib", 0) == NULL) { 567 IBTF_DPRINTF_L2(ibtf, "ibc_attach: IB nexus attach failed"); 568 ibtl_clear_ibhw_status(); 569 return (IBC_FAILURE); 570 } 571 572 ibtl_thread_init2(); 573 574 /* Allocate the memory for per-client info structure */ 575 hca_devp = kmem_zalloc(sizeof (ibtl_hca_devinfo_t) + 576 (nports - 1) * sizeof (ibtl_async_port_status_t), KM_SLEEP); 577 578 mutex_enter(&ibtl_clnt_list_mutex); 579 580 /* Update HCA dev info structure */ 581 hca_devp->hd_ibc_hca_hdl = info_p->hca_handle; 582 hca_devp->hd_ibc_ops = info_p->hca_ops; 583 hca_devp->hd_hca_attr = info_p->hca_attr; 584 hca_devp->hd_hca_dip = info_p->hca_dip; 585 586 status = ibtl_init_hca_portinfo(hca_devp); 587 if (status != IBT_SUCCESS) { 588 mutex_exit(&ibtl_clnt_list_mutex); 589 IBTF_DPRINTF_L1(ibtf, "ibc_attach: call to ibc_query_hca_ports " 590 "failed: status = %d", status); 591 kmem_free(hca_devp, sizeof (ibtl_hca_devinfo_t) + 592 (nports - 1) * sizeof (ibtl_async_port_status_t)); 593 return (IBC_FAILURE); 594 } 595 596 /* Register the with MPxIO as PHCI */ 597 if (ibtl_ibnex_phci_register(info_p->hca_dip) != IBT_SUCCESS) { 598 mutex_exit(&ibtl_clnt_list_mutex); 599 IBTF_DPRINTF_L1(ibtf, "ibc_attach: MPxIO register failed"); 600 kmem_free(hca_devp, sizeof (ibtl_hca_devinfo_t) + 601 (nports - 1) * sizeof (ibtl_async_port_status_t)); 602 return (IBC_FAILURE); 603 } 604 605 /* Initialize the Client List for this HCA. */ 606 hca_devp->hd_state = IBTL_HCA_DEV_ATTACHED; 607 608 /* lock out asyncs until after we announce the new HCA */ 609 hca_devp->hd_async_busy = 1; 610 611 cv_init(&hca_devp->hd_async_task_cv, NULL, CV_DEFAULT, NULL); 612 cv_init(&hca_devp->hd_async_busy_cv, NULL, CV_DEFAULT, NULL); 613 614 /* init portinfo locking variables */ 615 hca_devp->hd_portinfo_locked_port = 0; 616 cv_init(&hca_devp->hd_portinfo_cv, NULL, CV_DEFAULT, NULL); 617 618 mutex_exit(&ibtl_clnt_list_mutex); 619 620 /* 621 * The ibc_hdl_p points to an opaque handle which is the address 622 * of ibt_hca_devinfo_t structure passed back to the CI. 623 * The CI will pass on this handle in its future upcalls to IBTF. 624 */ 625 *ibc_hdl_p = hca_devp; 626 627 return (IBC_SUCCESS); 628 } 629 630 631 /* 632 * Function: 633 * ibc_post_attach 634 * Input: 635 * ibc_hdl - IBC Client's HCA Handle. 636 * Returns: 637 * none 638 * Called by: 639 * CI calls IBTF during its attach() after a successful ibc_attach(). 640 * Description: 641 * Announces to all known clients the existence of this HCA (by GUID). 642 */ 643 void 644 ibc_post_attach(ibc_clnt_hdl_t ibc_hdl) 645 { 646 IBTF_DPRINTF_L2(ibtf, "ibc_post_attach(%p)", ibc_hdl); 647 648 /* 649 * Update the HCA Device List. 650 */ 651 mutex_enter(&ibtl_clnt_list_mutex); 652 ibc_hdl->hd_hca_dev_link = ibtl_hca_list; 653 ibtl_hca_list = ibc_hdl; 654 mutex_exit(&ibtl_clnt_list_mutex); 655 656 /* notify all IBT Client Device Instances of the new HCA Device */ 657 ibtl_announce_new_hca(ibc_hdl); 658 } 659 660 661 /* 662 * Function: 663 * ibc_pre_detach 664 * Input: 665 * ibc_clnt_hdl - IBC HCA Handle as returned during ibc_attach call. 666 * cmd - DDI_DETACH/DDI_SUSPEND command. 667 * Output: 668 * none 669 * Returns: 670 * IBC_SUCCESS 671 * IBC_FAILURE. 672 * Called by: 673 * CI to try to get all IBTF clients to close the HCA device. 674 * Description: 675 * Attempts to deregister the HCA device entry from the IBTF. 676 * If all resources are freed by the IBTF clients and this HCA 677 * is closed, then IBC_SUCCESS is returned. 678 */ 679 ibc_status_t 680 ibc_pre_detach(ibc_clnt_hdl_t hca_devp, ddi_detach_cmd_t cmd) 681 { 682 ibtl_hca_devinfo_t **hcapp, *hcap; 683 684 IBTF_DPRINTF_L2(ibtf, "ibc_pre_detach(%p, 0x%x)", hca_devp, cmd); 685 686 /* 687 * Return failure, if command is not DDI_DETACH 688 */ 689 switch (cmd) { 690 case DDI_DETACH: 691 break; 692 default: 693 return (IBC_FAILURE); /* TBD: DDI_FAILURE */ 694 } 695 696 /* Make sure this HCA is on the HCA Device List. */ 697 mutex_enter(&ibtl_clnt_list_mutex); 698 hcap = ibtl_hca_list; 699 while (hcap != NULL) { 700 if (hcap == hca_devp) 701 break; 702 hcap = hcap->hd_hca_dev_link; 703 } 704 if (hcap == NULL) { 705 mutex_exit(&ibtl_clnt_list_mutex); 706 return (IBC_FAILURE); 707 } 708 709 /* 710 * Initially set the state to "Detaching". 711 */ 712 hca_devp->hd_state = IBTL_HCA_DEV_DETACHING; 713 714 /* 715 * Try to detach all IBTI clients, and continue only if all 716 * of the detaches succeed. 717 */ 718 if (ibtl_detach_all_clients(hca_devp)) { 719 hca_devp->hd_state = IBTL_HCA_DEV_ATTACHED; /* fix hd_state */ 720 mutex_exit(&ibtl_clnt_list_mutex); 721 722 return (IBC_FAILURE); 723 } 724 725 /* 726 * Check to see if all clients closed this HCA, or not. 727 * We only succeed if all clients cooperated. 728 */ 729 if (hca_devp->hd_clnt_list != NULL) { 730 hca_devp->hd_state = IBTL_HCA_DEV_ATTACHED; 731 mutex_exit(&ibtl_clnt_list_mutex); 732 IBTF_DPRINTF_L2(ibtf, "ibc_pre_detach: HCA still has attached " 733 "clients"); 734 return (IBC_FAILURE); 735 } 736 737 /* 738 * mark this device as detached 739 */ 740 hca_devp->hd_state = IBTL_HCA_DEV_DETACHED; 741 742 /* Delete the entry for this hca_devp from hca_head_list */ 743 hcapp = &ibtl_hca_list; 744 while (*hcapp != NULL) { 745 if (*hcapp == hca_devp) 746 break; 747 hcapp = &(*hcapp)->hd_hca_dev_link; 748 } 749 750 if (ibtl_ibnex_phci_unregister(hca_devp->hd_hca_dip) != IBT_SUCCESS) { 751 hca_devp->hd_state = IBTL_HCA_DEV_ATTACHED; /* fix hd_state */ 752 mutex_exit(&ibtl_clnt_list_mutex); 753 IBTF_DPRINTF_L1(ibtf, "ibc_pre_detach: PHCI unregister failed"); 754 return (IBC_FAILURE); 755 } 756 757 if (*hcapp == NULL) { 758 hca_devp->hd_state = IBTL_HCA_DEV_ATTACHED; /* fix hd_state */ 759 mutex_exit(&ibtl_clnt_list_mutex); 760 IBTF_DPRINTF_L1(ibtf, "ibc_pre_detach: HCA not attached"); 761 return (IBC_FAILURE); 762 } 763 *hcapp = hca_devp->hd_hca_dev_link; 764 ibtl_fast_gid_cache_valid = B_FALSE; /* invalidate fast_gid_cache */ 765 mutex_exit(&ibtl_clnt_list_mutex); 766 767 return (IBC_SUCCESS); 768 } 769 770 /* 771 * Function: 772 * ibc_detach 773 * Input: 774 * ibc_clnt_hdl - IBC HCA Handle as returned during ibc_attach call. 775 * Output: 776 * none 777 * Returns: 778 * None 779 * Called by: 780 * CI to detach the HCA device from IBTF. 781 * Description: 782 * Do the second step of detaching the HCA, which is required 783 * after a successful ibc_pre_detach. 784 */ 785 void 786 ibc_detach(ibc_clnt_hdl_t hca_devp) 787 { 788 IBTF_DPRINTF_L2(ibtf, "ibc_detach(%p)", hca_devp); 789 790 mutex_enter(&ibtl_clnt_list_mutex); 791 if (hca_devp->hd_state != IBTL_HCA_DEV_DETACHED) { 792 mutex_exit(&ibtl_clnt_list_mutex); 793 IBTF_DPRINTF_L0(ibtf, "ibc_detach: HCA has not successfully " 794 "pre-detached"); 795 return; 796 } 797 798 cv_destroy(&hca_devp->hd_async_task_cv); 799 cv_destroy(&hca_devp->hd_async_busy_cv); 800 cv_destroy(&hca_devp->hd_portinfo_cv); 801 802 kmem_free(hca_devp->hd_portinfop, hca_devp->hd_portinfo_len); 803 mutex_exit(&ibtl_clnt_list_mutex); 804 805 /* Free up the memory of per-client info struct */ 806 kmem_free(hca_devp, sizeof (ibtl_hca_devinfo_t) + 807 (hca_devp->hd_hca_attr->hca_nports - 1) * 808 sizeof (ibtl_async_port_status_t)); 809 ibtl_clear_ibhw_status(); 810 } 811 812 /* 813 * Function: 814 * ibt_ci_data_in() 815 * 816 * Input: 817 * hca_hdl HCA Handle. 818 * flags IBT_COMPLETE_ALLOC - Finish a deferred alloc. 819 * object Identifies the type object pointed to by 820 * ibt_object_handle. 821 * 822 * ibt_object_handle The handle of the object to be associated with 823 * the data in/out 824 * 825 * data_p Pointer data passed in to the CI. The buffer 826 * should be allocated by the caller. 827 * 828 * data_sz The size of the buffer pointed to by 829 * data_p. 830 * Output: 831 * 832 * Returns: 833 * IBT_SUCCESS 834 * IBT_NOT_SUPPORTED Feature not supported. 835 * IBT_INVALID_PARAM Invalid object type specified. 836 * IBT_HCA_HDL_INVALID 837 * IBT_AH_HDL_INVALID/IBT_UD_DEST_HDL_INVALID 838 * IBT_CHAN_HDL_INVALID/IBT_QP_HDL_INVALID 839 * IBT_CQ_HDL_INVALID 840 * IBT_EEC_HDL_INVALID 841 * IBT_RDD_HDL_INVALID 842 * IBT_MW_HDL_INVALID 843 * IBT_PD_HDL_INVALID 844 * IBT_SRQ_HDL_INVALID 845 * 846 * Description: 847 * Exchange CI private data for the specified CI object. 848 */ 849 ibt_status_t 850 ibt_ci_data_in(ibt_hca_hdl_t hca, ibt_ci_data_flags_t flags, 851 ibt_object_type_t object, void *ibt_object_handle, void *data_p, 852 size_t data_sz) 853 { 854 ibt_status_t retval; 855 void *ci_obj_hdl; 856 857 IBTF_DPRINTF_L3(ibtf, "ibt_ci_data_in(%p, %x, %d, %p, %p, %d)", 858 hca, flags, object, ibt_object_handle, data_p, data_sz); 859 860 switch (object) { 861 case IBT_HDL_HCA: 862 ci_obj_hdl = (void *) 863 (IBTL_HCA2CIHCA(((ibt_hca_hdl_t)ibt_object_handle))); 864 break; 865 866 case IBT_HDL_CHANNEL: 867 ci_obj_hdl = (void *) 868 (IBTL_CHAN2CIQP(((ibt_channel_hdl_t)ibt_object_handle))); 869 break; 870 871 case IBT_HDL_CQ: 872 ci_obj_hdl = (void *) 873 (((ibt_cq_hdl_t)(ibt_object_handle))->cq_ibc_cq_hdl); 874 break; 875 876 case IBT_HDL_EEC: 877 ci_obj_hdl = (void *) 878 (((ibt_eec_hdl_t)(ibt_object_handle))->eec_ibc_eec_hdl); 879 break; 880 881 case IBT_HDL_UD_DEST: 882 ci_obj_hdl = (void *) 883 (((ibt_ud_dest_hdl_t)(ibt_object_handle))->ud_ah); 884 break; 885 886 case IBT_HDL_SRQ: 887 ci_obj_hdl = (void *) 888 (((ibt_srq_hdl_t)(ibt_object_handle))->srq_ibc_srq_hdl); 889 break; 890 891 default: 892 ci_obj_hdl = ibt_object_handle; 893 break; 894 } 895 896 retval = (IBTL_HCA2CIHCAOPS_P(hca)->ibc_ci_data_in)(IBTL_HCA2CIHCA(hca), 897 flags, object, ci_obj_hdl, data_p, data_sz); 898 899 if (retval != IBT_SUCCESS) { 900 IBTF_DPRINTF_L2(ibtf, "ibt_ci_data_in: Failed : %d", retval); 901 } 902 return (retval); 903 } 904 905 /* 906 * Function: 907 * ibt_ci_data_out() 908 * 909 * Input: 910 * hca_hdl HCA Handle. 911 * flags IBT_COMPLETE_ALLOC - Finish a deferred alloc. 912 * object Identifies the type object pointed to by 913 * ibt_object_handle. 914 * 915 * ibt_object_handle The handle of the object to be associated with 916 * the data in/out 917 * 918 * data_p Pointer to a buffer in which to return the CI 919 * private data. The buffer should be allocated 920 * by the caller. 921 * 922 * data_sz The size of the buffer pointed to by 923 * data_p. 924 * Output: 925 * 926 * Returns: 927 * IBT_SUCCESS 928 * IBT_NOT_SUPPORTED Feature not supported. 929 * IBT_INSUFF_RESOURCE The buffer pointed to by data_p was too 930 * small to hold the data. 931 * IBT_INVALID_PARAM Invalid object type specified. 932 * IBT_HCA_HDL_INVALID 933 * IBT_AH_HDL_INVALID/IBT_UD_DEST_HDL_INVALID 934 * IBT_CHAN_HDL_INVALID/IBT_QP_HDL_INVALID 935 * IBT_CQ_HDL_INVALID 936 * IBT_EEC_HDL_INVALID 937 * IBT_RDD_HDL_INVALID 938 * IBT_MW_HDL_INVALID 939 * IBT_PD_HDL_INVALID 940 * IBT_SRQ_HDL_INVALID 941 * 942 * Description: 943 * Exchange CI private data for the specified CI object. 944 */ 945 ibt_status_t 946 ibt_ci_data_out(ibt_hca_hdl_t hca, ibt_ci_data_flags_t flags, 947 ibt_object_type_t object, void *ibt_object_handle, void *data_p, 948 size_t data_sz) 949 { 950 ibt_status_t retval; 951 void *ci_obj_hdl; 952 953 IBTF_DPRINTF_L3(ibtf, "ibt_ci_data_out(%p, %x, %d, %p, %p, %d)", 954 hca, flags, object, ibt_object_handle, data_p, data_sz); 955 956 switch (object) { 957 case IBT_HDL_HCA: 958 ci_obj_hdl = (void *) 959 (IBTL_HCA2CIHCA(((ibt_hca_hdl_t)ibt_object_handle))); 960 break; 961 962 case IBT_HDL_CHANNEL: 963 ci_obj_hdl = (void *) 964 (IBTL_CHAN2CIQP(((ibt_channel_hdl_t)ibt_object_handle))); 965 break; 966 967 case IBT_HDL_CQ: 968 ci_obj_hdl = (void *) 969 (((ibt_cq_hdl_t)(ibt_object_handle))->cq_ibc_cq_hdl); 970 break; 971 972 case IBT_HDL_EEC: 973 ci_obj_hdl = (void *) 974 (((ibt_eec_hdl_t)(ibt_object_handle))->eec_ibc_eec_hdl); 975 break; 976 977 case IBT_HDL_UD_DEST: 978 ci_obj_hdl = (void *) 979 (((ibt_ud_dest_hdl_t)(ibt_object_handle))->ud_ah); 980 break; 981 982 case IBT_HDL_SRQ: 983 ci_obj_hdl = (void *) 984 (((ibt_srq_hdl_t)(ibt_object_handle))->srq_ibc_srq_hdl); 985 break; 986 987 default: 988 ci_obj_hdl = ibt_object_handle; 989 break; 990 } 991 992 retval = (IBTL_HCA2CIHCAOPS_P(hca)->ibc_ci_data_out) 993 (IBTL_HCA2CIHCA(hca), flags, object, ci_obj_hdl, data_p, data_sz); 994 995 if (retval != IBT_SUCCESS) { 996 IBTF_DPRINTF_L2(ibtf, "ibt_ci_data_out: Failed : %d", retval); 997 } 998 return (retval); 999 } 1000 1001 1002 /* 1003 * FMA Support functions. 1004 */ 1005 1006 #define IBTL_ENA_MASK 0xC0000000 1007 #define IBTL_ENA_POSSIBLE 0x80000000 1008 #define IBTL_TYPE_SHIFT 27 1009 1010 /* 1011 * Function: 1012 * ibt_get_module_failure() 1013 * 1014 * Input: 1015 * type Identifies the failing IB module. 1016 * ena '0' or the data for Fault Management 1017 * Architecture (ENA). 1018 * 1019 * Returns: 1020 * status Special IB failure status. 1021 * 1022 * Description: 1023 * XXX Just stubbed out to return failures with no data for Fault 1024 * Management Architecture (ENAs) at the moment XXX 1025 */ 1026 ibt_status_t 1027 ibt_get_module_failure(ibt_failure_type_t type, uint64_t ena) 1028 { 1029 ibt_status_t ret; 1030 1031 IBTF_DPRINTF_L3(ibtf, "ibt_get_module_failure(%d, 0x%llX)", type, ena); 1032 1033 switch (type) { 1034 case IBT_FAILURE_CI: 1035 case IBT_FAILURE_IBMF: 1036 case IBT_FAILURE_IBCM: 1037 case IBT_FAILURE_IBDM: 1038 case IBT_FAILURE_IBTL: 1039 case IBT_FAILURE_IBSM: 1040 ret = IBTL_ENA_POSSIBLE | (type << IBTL_TYPE_SHIFT); 1041 break; 1042 default: 1043 ret = IBT_FAILURE; 1044 } 1045 IBTF_DPRINTF_L3(ibtf, "ibt_get_module_failure: ret = 0x%lX", ret); 1046 return (ret); 1047 } 1048 1049 1050 /* 1051 * Function: 1052 * ibc_get_ci_failure() 1053 * 1054 * Input: 1055 * ena '0' or the data for Fault Management 1056 * Architecture (ENA). 1057 * 1058 * Returns: 1059 * status Special CI failure status. 1060 * 1061 * Description: 1062 * Just use the function above to do the job. 1063 */ 1064 ibt_status_t 1065 ibc_get_ci_failure(uint64_t ena) 1066 { 1067 return (ibt_get_module_failure(IBT_FAILURE_CI, ena)); 1068 } 1069 1070 1071 /* 1072 * ibt_check_failure() 1073 * Function to test for special case failures. 1074 * 1075 * status An ibt_status_t returned from an IBTF function call. 1076 * 1077 * reserved_p NULL, or a pointer to where we store the data for 1078 * Fault Management Architecture (ENA). 1079 * 1080 * Description: 1081 * XXX Still need to determine the data for Fault Management Architecture 1082 * (ENA), using 0 for now XXX 1083 */ 1084 ibt_failure_type_t 1085 ibt_check_failure(ibt_status_t status, uint64_t *reserved_p) 1086 { 1087 ibt_failure_type_t type; 1088 1089 IBTF_DPRINTF_L3(ibtf, "ibt_check_failure(%X)", status); 1090 1091 if ((status & IBTL_ENA_MASK) == IBTL_ENA_POSSIBLE) { 1092 type = status & ~IBTL_ENA_POSSIBLE >> IBTL_TYPE_SHIFT; 1093 1094 /* XXX Need more work here... */ 1095 if (reserved_p != NULL) 1096 *reserved_p = 0; 1097 } else { 1098 type = IBT_FAILURE_STANDARD; 1099 if (reserved_p != NULL) 1100 *reserved_p = 0; /* No FMA Data Available. */ 1101 } 1102 IBTF_DPRINTF_L3(ibtf, "ibt_check_failure: type = 0x%X", type); 1103 return (type); 1104 } 1105