1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved. 23 */ 24 25 /* 26 * ibtl_impl.c 27 * 28 * This file contains the IBTF module's initialization and 29 * IBTF Clients/Modules registration routines. 30 */ 31 32 #include <sys/modctl.h> 33 #include <sys/sunndi.h> 34 #include <sys/ib/ibtl/impl/ibtl.h> 35 #include <sys/ib/ibtl/impl/ibtl_ibnex.h> 36 37 /* 38 * Globals. 39 */ 40 static char ibtf[] = "ibtl_impl"; 41 42 extern ibtl_ibnex_callback_t ibtl_ibnex_callback_routine; 43 44 /* 45 * ibtl_clnt_list: 46 * 47 * Head of the list of IBT Client Instances. The IBT Client List 48 * is modified by IBTF on an IBT client's ibt_attach/ibt_detach call. 49 * 50 * ibtl_hca_list: 51 * 52 * Head of the list of HCA devices. The HCA List is modified by IBTF on 53 * a CI's ibc_attach/ibc_detach call. 54 * The datap of the list elements points to an ibtl_hca_devinfo_s 55 * structure. 56 * 57 * (ibc_attach) 58 * ibtl_hca_list -> ibtl_hca_devinfo_t--> ... -->ibtl_hca_devinfo_t 59 * [per-hca_dev] | ^ {nth HCA Dev} 60 * | | 61 * | ibtl_hca_t (ibt_open_hca) 62 * | ^ | 63 * | | | 64 * v | V 65 * ibtl_clnt_list -> ibtl_clnt_t--> ...--> {n'th Module} 66 * [per-client_instance] (ibt_attach) 67 * 68 */ 69 70 /* Global List of IBT Client Instances, and associated mutex. */ 71 struct ibtl_clnt_s *ibtl_clnt_list = NULL; 72 kmutex_t ibtl_clnt_list_mutex; 73 74 /* Lock for the race between the client and CM to free QPs. */ 75 kmutex_t ibtl_free_qp_mutex; 76 77 /* Lock for the race between the client closing the HCA and QPN being freed. */ 78 kcondvar_t ibtl_close_hca_cv; 79 80 /* Global List of HCA Devices, and associated mutex. */ 81 struct ibtl_hca_devinfo_s *ibtl_hca_list = NULL; 82 83 /* Well-known async handlers and associated client private. */ 84 ibt_async_handler_t ibtl_cm_async_handler; 85 ibt_async_handler_t ibtl_dm_async_handler; 86 ibt_async_handler_t ibtl_ibma_async_handler; 87 void *ibtl_cm_clnt_private; 88 void *ibtl_dm_clnt_private; 89 void *ibtl_ibma_clnt_private; 90 91 extern int ib_hw_status; 92 _NOTE(SCHEME_PROTECTS_DATA("Scheme protects data", ib_hw_status)) 93 94 /* 95 * Misc Module Declarations. 96 */ 97 extern struct mod_ops mod_miscops; 98 static struct modlmisc modlmisc = { 99 &mod_miscops, /* Type of module - misc. */ 100 "IB Transport Layer" /* Name of the Module. */ 101 }; 102 103 static struct modlinkage modlinkage = { 104 MODREV_1, (void *)&modlmisc, NULL 105 }; 106 107 static void ibtl_kstat_init(ibtl_hca_devinfo_t *); 108 static void ibtl_kstat_fini(ibtl_hca_devinfo_t *); 109 static void ibtl_kstat_stats_create(ibtl_hca_devinfo_t *, uint_t); 110 static void ibtl_kstat_pkeys_create(ibtl_hca_devinfo_t *, uint_t); 111 112 extern kmutex_t ibtl_part_attr_mutex; 113 114 /* 115 * IBTF Loadable Module Routines. 116 */ 117 118 int 119 _init(void) 120 { 121 int rval; 122 123 if ((rval = mod_install(&modlinkage)) != 0) 124 return (rval); 125 126 /* 127 * initialize IBTL ib2usec table 128 */ 129 ibtl_ib2usec_init(); 130 131 /* 132 * Initialize Logging 133 */ 134 ibtl_logging_initialization(); 135 136 /* 137 * Initialize the Alloc QP States. 138 */ 139 ibtl_init_cep_states(); 140 141 /* 142 * Initialize all Global Link Lists. 143 */ 144 mutex_init(&ibtl_clnt_list_mutex, NULL, MUTEX_DEFAULT, NULL); 145 mutex_init(&ibtl_free_qp_mutex, NULL, MUTEX_DEFAULT, NULL); 146 cv_init(&ibtl_close_hca_cv, NULL, CV_DEFAULT, NULL); 147 148 mutex_init(&ibtl_qp_mutex, NULL, MUTEX_DEFAULT, NULL); 149 cv_init(&ibtl_qp_cv, NULL, CV_DEFAULT, NULL); 150 151 mutex_init(&ibtl_part_attr_mutex, NULL, MUTEX_DEFAULT, NULL); 152 153 ibtl_thread_init(); 154 155 return (rval); 156 } 157 158 159 /* 160 * The IBTF Module is never unloaded. Actually there is no need of this 161 * routine, but provided just in case. 162 */ 163 int 164 _fini(void) 165 { 166 int rval; 167 168 if ((rval = mod_remove(&modlinkage)) != 0) { 169 return (rval); 170 } 171 172 ibtl_thread_fini(); 173 174 mutex_destroy(&ibtl_clnt_list_mutex); 175 mutex_destroy(&ibtl_free_qp_mutex); 176 cv_destroy(&ibtl_close_hca_cv); 177 mutex_destroy(&ibtl_qp_mutex); 178 cv_destroy(&ibtl_qp_cv); 179 mutex_destroy(&ibtl_part_attr_mutex); 180 181 /* 182 * Stop Logging 183 */ 184 ibtl_logging_destroy(); 185 186 return (rval); 187 } 188 189 190 int 191 _info(struct modinfo *modinfop) 192 { 193 /* Return the Module Information. */ 194 return (mod_info(&modlinkage, modinfop)); 195 } 196 197 198 /* 199 * IBTF Client Registration Routines. 200 */ 201 202 /* 203 * Function: 204 * ibt_attach 205 * Input: 206 * modinfop - Client Module info structure. 207 * arg - usually client's dip 208 * clnt_private - client's private data pointer. 209 * Output: 210 * ibt_hdl_p - pointer to client's specific IBT handle, 211 * which is opaque to clients. 212 * Returns: 213 * IBT_SUCCESS 214 * IBT_INVALID_PARAM 215 * Called by: 216 * IBTF Client module during its attach() to register its instance 217 * to IBTF. 218 * Description: 219 * Registers the IBTF client module instance and returns an opaque 220 * handler to the client to be used for future calls to IBTF. 221 * Adds this client module instance to ibtl_clnt_list list. 222 * Records well-known async handlers. 223 */ 224 ibt_status_t 225 ibt_attach(ibt_clnt_modinfo_t *mod_infop, dev_info_t *arg, void *clnt_private, 226 ibt_clnt_hdl_t *ibt_hdl_p) 227 { 228 dev_info_t *pdip; 229 ibtl_clnt_t *clntp; 230 231 IBTF_DPRINTF_L3(ibtf, "ibt_attach(%p, %p, %p)", 232 mod_infop, arg, clnt_private); 233 234 if (mod_infop->mi_clnt_name == NULL) { 235 IBTF_DPRINTF_L1(ibtf, "ibt_attach: " 236 "IB client needs to specify its name"); 237 return (IBT_INVALID_PARAM); 238 } 239 240 /* 241 * Validate the Transport API version. 242 */ 243 if (mod_infop->mi_ibt_version != IBTI_V_CURR) { 244 IBTF_DPRINTF_L1(ibtf, "ibt_attach: IB client '%s' has an " 245 "invalid IB TI Version '%d'", mod_infop->mi_clnt_name, 246 mod_infop->mi_ibt_version); 247 return (IBT_NOT_SUPPORTED); 248 } 249 250 if (mod_infop->mi_async_handler == NULL) { 251 IBTF_DPRINTF_L2(ibtf, "ibt_attach: Client '%s' has not\n" 252 " provided an Asynchronous Event Handler.\n" 253 " This will be required soon.", 254 mod_infop->mi_clnt_name); 255 } 256 257 /* 258 * Check out Client's Class information. If it is not of mgmt class, 259 * we expect 'arg' to be Not NULL and point to client driver's 260 * device info struct. 261 */ 262 if ((!IBT_MISCMOD_CLIENTS(mod_infop->mi_clnt_class)) && 263 (arg == NULL)) { 264 IBTF_DPRINTF_L1(ibtf, "ibt_attach: " 265 "arg not set with driver's dip."); 266 return (IBT_INVALID_PARAM); 267 } 268 269 if (!IBT_MISCMOD_CLIENTS(mod_infop->mi_clnt_class)) { 270 pdip = ddi_get_parent(arg); 271 if (pdip == NULL || 272 ibtl_ibnex_valid_hca_parent(pdip) != IBT_SUCCESS) { 273 IBTF_DPRINTF_L2(ibtf, "ibt_attach: " 274 "client %s is not a child of IB nexus driver.", 275 ddi_driver_name(arg)); 276 return (IBT_INVALID_PARAM); 277 } 278 } 279 280 mutex_enter(&ibtl_clnt_list_mutex); 281 if (mod_infop->mi_clnt_class == IBT_CM) { 282 if (ibtl_cm_async_handler != NULL) { 283 IBTF_DPRINTF_L1(ibtf, "ibt_attach: " 284 "CM is already attached."); 285 mutex_exit(&ibtl_clnt_list_mutex); 286 return (IBT_INVALID_PARAM); 287 } 288 ibtl_cm_async_handler = mod_infop->mi_async_handler; 289 ibtl_cm_clnt_private = clnt_private; 290 } else if (mod_infop->mi_clnt_class == IBT_DM) { 291 if (ibtl_dm_async_handler != NULL) { 292 IBTF_DPRINTF_L1(ibtf, "ibt_attach: " 293 "DM is already attached."); 294 mutex_exit(&ibtl_clnt_list_mutex); 295 return (IBT_INVALID_PARAM); 296 } 297 ibtl_dm_async_handler = mod_infop->mi_async_handler; 298 ibtl_dm_clnt_private = clnt_private; 299 } else if (mod_infop->mi_clnt_class == IBT_IBMA) { 300 if (ibtl_ibma_async_handler != NULL) { 301 IBTF_DPRINTF_L1(ibtf, "ibt_attach: " 302 "IBMF is already attached."); 303 mutex_exit(&ibtl_clnt_list_mutex); 304 return (IBT_INVALID_PARAM); 305 } 306 ibtl_ibma_async_handler = mod_infop->mi_async_handler; 307 ibtl_ibma_clnt_private = clnt_private; 308 } 309 310 /* Allocate the memory for per-client-device info structure */ 311 clntp = kmem_zalloc(sizeof (ibtl_clnt_t), KM_SLEEP); 312 313 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(clntp->clnt_modinfop, 314 clntp->clnt_dip, clntp->clnt_name, clntp->clnt_async_cnt, 315 clntp->clnt_private)) 316 /* Update the Client info structure */ 317 clntp->clnt_modinfop = mod_infop; /* IBT Client's Mod Info */ 318 clntp->clnt_private = clnt_private; /* IBT Client's private */ 319 clntp->clnt_dip = arg; /* IBT Client's dip */ 320 clntp->clnt_async_cnt = 0; 321 /* using a count of 7 below guarantees it is NULL terminated */ 322 (void) strncpy(clntp->clnt_name, mod_infop->mi_clnt_name, 7); 323 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(clntp->clnt_modinfop, 324 clntp->clnt_dip, clntp->clnt_name, clntp->clnt_async_cnt, 325 clntp->clnt_private)) 326 327 /* 328 * Update Client Device Instance List. 329 */ 330 clntp->clnt_list_link = ibtl_clnt_list; 331 ibtl_clnt_list = clntp; 332 mutex_exit(&ibtl_clnt_list_mutex); 333 334 /* 335 * The ibt_hdl_p is a opaque handle which is the address of 336 * ibt_clnt_t structure passed back to the clients. 337 * The client will pass on this handle in its future calls to IBTF. 338 */ 339 *ibt_hdl_p = clntp; 340 341 return (IBT_SUCCESS); 342 } 343 344 345 /* 346 * Function: 347 * ibt_detach 348 * Input: 349 * ibt_hdl - IBT Handle as returned during ibt_attach call. 350 * Output: 351 * none 352 * Returns: 353 * IBT_SUCCESS 354 * IBT_INVALID_PARAM. 355 * Called by: 356 * IBTF Client module during its detach() to de-register its instance 357 * from IBTF. 358 * Description: 359 * Deregisters the IBTF client module instance from the IBTF. 360 * All resources and any reference to this ibt_hdl will be removed. 361 */ 362 ibt_status_t 363 ibt_detach(ibt_clnt_hdl_t ibt_hdl) 364 { 365 ibtl_clnt_t **clntpp; 366 367 IBTF_DPRINTF_L3(ibtf, "ibt_detach(%p)", ibt_hdl); 368 369 mutex_enter(&ibtl_clnt_list_mutex); 370 clntpp = &ibtl_clnt_list; 371 for (; *clntpp != NULL; clntpp = &(*clntpp)->clnt_list_link) 372 if (*clntpp == ibt_hdl) 373 break; 374 if (*clntpp == NULL) { 375 IBTF_DPRINTF_L1(ibtf, "ibt_detach: Client @ %p Not Found", 376 ibt_hdl); 377 mutex_exit(&ibtl_clnt_list_mutex); 378 return (IBT_INVALID_PARAM); 379 } 380 381 /* 382 * Check out whether the client has freed all its resources. 383 * If not done, then fail the detach. 384 * 385 * viz. A client has to close all the HCA they have opened, 386 * i.e. the HCA List maintained for clients has to be empty. 387 * If this list is not empty, then the client has not performed 388 * complete clean-up, so fail the detach. 389 */ 390 if (ibt_hdl->clnt_hca_list != NULL) { 391 mutex_exit(&ibtl_clnt_list_mutex); 392 393 IBTF_DPRINTF_L2(ibtf, "ibt_detach: " 394 "ERROR: Client '%s' has not closed all of its HCAs", 395 ibt_hdl->clnt_modinfop->mi_clnt_name); 396 return (IBT_HCA_RESOURCES_NOT_FREED); 397 } 398 399 if (ibt_hdl->clnt_srv_cnt != 0) { 400 mutex_exit(&ibtl_clnt_list_mutex); 401 IBTF_DPRINTF_L2(ibtf, "ibt_detach: client '%s' still has " 402 "services or subnet_notices registered", 403 ibt_hdl->clnt_modinfop->mi_clnt_name); 404 return (IBT_HCA_RESOURCES_NOT_FREED); 405 } 406 407 /* 408 * Delete the entry of this module from the ibtl_clnt_list List. 409 */ 410 *clntpp = ibt_hdl->clnt_list_link; /* remove us */ 411 412 /* make sure asyncs complete before freeing */ 413 ibtl_free_clnt_async_check(ibt_hdl); 414 415 if (ibt_hdl->clnt_modinfop->mi_clnt_class == IBT_CM) { 416 ibtl_cm_async_handler = NULL; 417 ibtl_cm_clnt_private = NULL; 418 } else if (ibt_hdl->clnt_modinfop->mi_clnt_class == IBT_DM) { 419 ibtl_dm_async_handler = NULL; 420 ibtl_dm_clnt_private = NULL; 421 } else if (ibt_hdl->clnt_modinfop->mi_clnt_class == IBT_IBMA) { 422 ibtl_ibma_async_handler = NULL; 423 ibtl_ibma_clnt_private = NULL; 424 } 425 mutex_exit(&ibtl_clnt_list_mutex); 426 427 /* Free up the memory of per-client info struct. */ 428 kmem_free(ibt_hdl, sizeof (ibtl_clnt_t)); 429 430 return (IBT_SUCCESS); 431 } 432 433 static void 434 ibtl_set_ibhw_status() 435 { 436 ib_hw_status++; 437 } 438 439 static void 440 ibtl_clear_ibhw_status() 441 { 442 ib_hw_status--; 443 } 444 445 /* 446 * Function: 447 * ibc_init 448 * Input: 449 * modlp - Pointer to IBC client module linkage structure 450 * Output: 451 * None 452 * Returns: 453 * 0 always for now 454 * Called by: 455 * CI client calls IBTF during its _init() to register HCA with 456 * Solaris I/O framework. 457 * Description: 458 * Initializes the CI clients module linkage structure with 459 * default bus_ops structure 460 */ 461 int 462 ibc_init(struct modlinkage *modlp) 463 { 464 ibtl_ibnex_cb_args_t cb_args; 465 466 mutex_enter(&ibtl_clnt_list_mutex); 467 cb_args.cb_flag = IBTL_IBNEX_IBC_INIT; 468 cb_args.cb_modlp = modlp; 469 if (ibtl_ibnex_callback_routine) { 470 (void) ((*ibtl_ibnex_callback_routine)(&cb_args)); 471 } 472 mutex_exit(&ibtl_clnt_list_mutex); 473 return (0); 474 } 475 476 477 /* 478 * Function: 479 * ibc_fini 480 * Input: 481 * modlp - Pointer to IBC client module linkage structure 482 * Output: 483 * None 484 * Returns: 485 * None 486 * Called by: 487 * CI client calls IBTF during its _fini() to remove HCA with 488 * Solaris I/O framework. 489 * Description: 490 * Undo what is done during ibc_init 491 */ 492 void 493 ibc_fini(struct modlinkage *modlp) 494 { 495 ibtl_ibnex_cb_args_t cb_args; 496 497 mutex_enter(&ibtl_clnt_list_mutex); 498 cb_args.cb_flag = IBTL_IBNEX_IBC_FINI; 499 cb_args.cb_modlp = modlp; 500 if (ibtl_ibnex_callback_routine) { 501 (void) ((*ibtl_ibnex_callback_routine)(&cb_args)); 502 } 503 mutex_exit(&ibtl_clnt_list_mutex); 504 } 505 506 /* 507 * Function: 508 * ibc_attach 509 * Input: 510 * info_p - IBC HCA Info. 511 * Output: 512 * ibc_hdl_p - IBC Client's HCA Handle. 513 * Returns: 514 * IBC_SUCCESS 515 * IBC_FAILURE 516 * Called by: 517 * CI calls IBTF during its attach() to register HCA Device with IBTF. 518 * Description: 519 * Registers the presence of HCA device by providing the HCA device info 520 * structure and provides an opaque HCA handler for future calls to this 521 * HCA device. 522 */ 523 ibc_status_t 524 ibc_attach(ibc_clnt_hdl_t *ibc_hdl_p, ibc_hca_info_t *info_p) 525 { 526 ibtl_hca_devinfo_t *hca_devp; 527 uint_t nports; 528 ibt_status_t status; 529 530 IBTF_DPRINTF_L2(ibtf, "ibc_attach(%p, %p)", ibc_hdl_p, info_p); 531 532 /* Validate the Transport API version */ 533 if (info_p->hca_ci_vers != IBCI_V3) { 534 IBTF_DPRINTF_L1(ibtf, "ibc_attach: Invalid IB CI Version '%d'", 535 info_p->hca_ci_vers); 536 return (IBC_FAILURE); 537 } 538 539 if (info_p->hca_attr == NULL) { 540 IBTF_DPRINTF_L1(ibtf, "ibc_attach: " 541 "HCA Attributes must be specified."); 542 return (IBC_FAILURE); 543 } 544 545 nports = info_p->hca_attr->hca_nports; 546 if (nports == 0) { 547 IBTF_DPRINTF_L1(ibtf, "ibc_attach: " 548 "Number of ports must be valid"); 549 return (IBC_FAILURE); 550 } 551 552 if (info_p->hca_attr->hca_max_port_pkey_tbl_sz == 0) { 553 IBTF_DPRINTF_L1(ibtf, "ibc_attach: " 554 "Number of Partitions must be at least 1"); 555 return (IBC_FAILURE); 556 } 557 558 if ((info_p->hca_attr->hca_flags & IBT_HCA_CURRENT_QP_STATE) == 0) { 559 IBTF_DPRINTF_L1(ibtf, "ibc_attach: " 560 "HCA driver must support QP current state checking"); 561 return (IBC_FAILURE); 562 } 563 564 if ((info_p->hca_attr->hca_flags & IBT_HCA_PORT_UP) == 0) { 565 IBTF_DPRINTF_L1(ibtf, "ibc_attach: " 566 "HCA driver must support PORT_UP async events"); 567 return (IBC_FAILURE); 568 } 569 570 /* 571 * Install IB nexus driver (if not installed already) 572 */ 573 ibtl_set_ibhw_status(); 574 if (ndi_devi_config_vhci("ib", 0) == NULL) { 575 IBTF_DPRINTF_L2(ibtf, "ibc_attach: IB nexus attach failed"); 576 ibtl_clear_ibhw_status(); 577 return (IBC_FAILURE); 578 } 579 580 ibtl_thread_init2(); 581 582 /* Allocate the memory for per-client info structure */ 583 hca_devp = kmem_zalloc(sizeof (ibtl_hca_devinfo_t) + 584 (nports - 1) * sizeof (ibtl_async_port_event_t), KM_SLEEP); 585 586 mutex_enter(&ibtl_clnt_list_mutex); 587 588 /* Update HCA dev info structure */ 589 hca_devp->hd_ibc_hca_hdl = info_p->hca_handle; 590 hca_devp->hd_ibc_ops = info_p->hca_ops; 591 hca_devp->hd_hca_attr = info_p->hca_attr; 592 hca_devp->hd_hca_dip = info_p->hca_dip; 593 594 status = ibtl_init_hca_portinfo(hca_devp); 595 if (status != IBT_SUCCESS) { 596 mutex_exit(&ibtl_clnt_list_mutex); 597 IBTF_DPRINTF_L1(ibtf, "ibc_attach: call to ibc_query_hca_ports " 598 "failed: status = %d", status); 599 kmem_free(hca_devp, sizeof (ibtl_hca_devinfo_t) + 600 (nports - 1) * sizeof (ibtl_async_port_event_t)); 601 return (IBC_FAILURE); 602 } 603 604 /* Register the with MPxIO as PHCI */ 605 if (ibtl_ibnex_phci_register(info_p->hca_dip) != IBT_SUCCESS) { 606 mutex_exit(&ibtl_clnt_list_mutex); 607 IBTF_DPRINTF_L1(ibtf, "ibc_attach: MPxIO register failed"); 608 kmem_free(hca_devp, sizeof (ibtl_hca_devinfo_t) + 609 (nports - 1) * sizeof (ibtl_async_port_event_t)); 610 return (IBC_FAILURE); 611 } 612 613 /* Initialize the Client List for this HCA. */ 614 hca_devp->hd_state = IBTL_HCA_DEV_ATTACHED; 615 616 /* lock out asyncs until after we announce the new HCA */ 617 hca_devp->hd_async_busy = 1; 618 619 cv_init(&hca_devp->hd_async_task_cv, NULL, CV_DEFAULT, NULL); 620 cv_init(&hca_devp->hd_async_busy_cv, NULL, CV_DEFAULT, NULL); 621 622 /* init portinfo locking variables */ 623 hca_devp->hd_portinfo_locked_port = 0; 624 cv_init(&hca_devp->hd_portinfo_cv, NULL, CV_DEFAULT, NULL); 625 626 ibtl_kstat_init(hca_devp); 627 628 mutex_exit(&ibtl_clnt_list_mutex); 629 630 /* 631 * The ibc_hdl_p points to an opaque handle which is the address 632 * of ibt_hca_devinfo_t structure passed back to the CI. 633 * The CI will pass on this handle in its future upcalls to IBTF. 634 */ 635 *ibc_hdl_p = hca_devp; 636 637 return (IBC_SUCCESS); 638 } 639 640 641 /* 642 * Function: 643 * ibc_post_attach 644 * Input: 645 * ibc_hdl - IBC Client's HCA Handle. 646 * Returns: 647 * none 648 * Called by: 649 * CI calls IBTF during its attach() after a successful ibc_attach(). 650 * Description: 651 * Announces to all known clients the existence of this HCA (by GUID). 652 */ 653 void 654 ibc_post_attach(ibc_clnt_hdl_t ibc_hdl) 655 { 656 IBTF_DPRINTF_L2(ibtf, "ibc_post_attach(%p)", ibc_hdl); 657 658 /* 659 * Update the HCA Device List. 660 */ 661 mutex_enter(&ibtl_clnt_list_mutex); 662 ibc_hdl->hd_hca_dev_link = ibtl_hca_list; 663 ibtl_hca_list = ibc_hdl; 664 mutex_exit(&ibtl_clnt_list_mutex); 665 666 /* notify all IBT Client Device Instances of the new HCA Device */ 667 ibtl_announce_new_hca(ibc_hdl); 668 } 669 670 671 /* 672 * Function: 673 * ibc_pre_detach 674 * Input: 675 * ibc_clnt_hdl - IBC HCA Handle as returned during ibc_attach call. 676 * cmd - DDI_DETACH/DDI_SUSPEND command. 677 * Output: 678 * none 679 * Returns: 680 * IBC_SUCCESS 681 * IBC_FAILURE. 682 * Called by: 683 * CI to try to get all IBTF clients to close the HCA device. 684 * Description: 685 * Attempts to deregister the HCA device entry from the IBTF. 686 * If all resources are freed by the IBTF clients and this HCA 687 * is closed, then IBC_SUCCESS is returned. 688 */ 689 ibc_status_t 690 ibc_pre_detach(ibc_clnt_hdl_t hca_devp, ddi_detach_cmd_t cmd) 691 { 692 ibtl_hca_devinfo_t **hcapp, *hcap; 693 694 IBTF_DPRINTF_L2(ibtf, "ibc_pre_detach(%p, 0x%x)", hca_devp, cmd); 695 696 /* 697 * Return failure, if command is not DDI_DETACH 698 */ 699 switch (cmd) { 700 case DDI_DETACH: 701 break; 702 default: 703 return (IBC_FAILURE); /* TBD: DDI_FAILURE */ 704 } 705 706 /* Make sure this HCA is on the HCA Device List. */ 707 mutex_enter(&ibtl_clnt_list_mutex); 708 hcap = ibtl_hca_list; 709 while (hcap != NULL) { 710 if (hcap == hca_devp) 711 break; 712 hcap = hcap->hd_hca_dev_link; 713 } 714 if (hcap == NULL) { 715 mutex_exit(&ibtl_clnt_list_mutex); 716 return (IBC_FAILURE); 717 } 718 719 /* 720 * Initially set the state to "Detaching". 721 */ 722 hca_devp->hd_state = IBTL_HCA_DEV_DETACHING; 723 724 /* 725 * Try to detach all IBTI clients, and continue only if all 726 * of the detaches succeed. 727 */ 728 if (ibtl_detach_all_clients(hca_devp)) { 729 hca_devp->hd_state = IBTL_HCA_DEV_ATTACHED; /* fix hd_state */ 730 mutex_exit(&ibtl_clnt_list_mutex); 731 732 return (IBC_FAILURE); 733 } 734 735 /* 736 * Check to see if all clients closed this HCA, or not. 737 * We only succeed if all clients cooperated. 738 */ 739 if (hca_devp->hd_clnt_list != NULL) { 740 hca_devp->hd_state = IBTL_HCA_DEV_ATTACHED; 741 mutex_exit(&ibtl_clnt_list_mutex); 742 IBTF_DPRINTF_L2(ibtf, "ibc_pre_detach: HCA still has attached " 743 "clients"); 744 return (IBC_FAILURE); 745 } 746 747 /* 748 * mark this device as detached 749 */ 750 hca_devp->hd_state = IBTL_HCA_DEV_DETACHED; 751 752 /* Delete the entry for this hca_devp from hca_head_list */ 753 hcapp = &ibtl_hca_list; 754 while (*hcapp != NULL) { 755 if (*hcapp == hca_devp) 756 break; 757 hcapp = &(*hcapp)->hd_hca_dev_link; 758 } 759 760 if (ibtl_ibnex_phci_unregister(hca_devp->hd_hca_dip) != IBT_SUCCESS) { 761 hca_devp->hd_state = IBTL_HCA_DEV_ATTACHED; /* fix hd_state */ 762 mutex_exit(&ibtl_clnt_list_mutex); 763 IBTF_DPRINTF_L1(ibtf, "ibc_pre_detach: PHCI unregister failed"); 764 return (IBC_FAILURE); 765 } 766 767 if (*hcapp == NULL) { 768 hca_devp->hd_state = IBTL_HCA_DEV_ATTACHED; /* fix hd_state */ 769 mutex_exit(&ibtl_clnt_list_mutex); 770 IBTF_DPRINTF_L1(ibtf, "ibc_pre_detach: HCA not attached"); 771 return (IBC_FAILURE); 772 } 773 *hcapp = hca_devp->hd_hca_dev_link; 774 ibtl_fast_gid_cache_valid = B_FALSE; /* invalidate fast_gid_cache */ 775 mutex_exit(&ibtl_clnt_list_mutex); 776 777 return (IBC_SUCCESS); 778 } 779 780 /* 781 * Function: 782 * ibc_detach 783 * Input: 784 * ibc_clnt_hdl - IBC HCA Handle as returned during ibc_attach call. 785 * Output: 786 * none 787 * Returns: 788 * None 789 * Called by: 790 * CI to detach the HCA device from IBTF. 791 * Description: 792 * Do the second step of detaching the HCA, which is required 793 * after a successful ibc_pre_detach. 794 */ 795 void 796 ibc_detach(ibc_clnt_hdl_t hca_devp) 797 { 798 IBTF_DPRINTF_L2(ibtf, "ibc_detach(%p)", hca_devp); 799 800 mutex_enter(&ibtl_clnt_list_mutex); 801 if (hca_devp->hd_state != IBTL_HCA_DEV_DETACHED) { 802 mutex_exit(&ibtl_clnt_list_mutex); 803 IBTF_DPRINTF_L0(ibtf, "ibc_detach: HCA has not successfully " 804 "pre-detached"); 805 return; 806 } 807 808 cv_destroy(&hca_devp->hd_async_task_cv); 809 cv_destroy(&hca_devp->hd_async_busy_cv); 810 cv_destroy(&hca_devp->hd_portinfo_cv); 811 812 kmem_free(hca_devp->hd_portinfop, hca_devp->hd_portinfo_len); 813 mutex_exit(&ibtl_clnt_list_mutex); 814 815 ibtl_kstat_fini(hca_devp); 816 817 /* Free up the memory of per-client info struct */ 818 kmem_free(hca_devp, sizeof (ibtl_hca_devinfo_t) + 819 (hca_devp->hd_hca_attr->hca_nports - 1) * 820 sizeof (ibtl_async_port_event_t)); 821 ibtl_clear_ibhw_status(); 822 } 823 824 /* 825 * Function: 826 * ibt_ci_data_in() 827 * 828 * Input: 829 * hca_hdl HCA Handle. 830 * flags IBT_COMPLETE_ALLOC - Finish a deferred alloc. 831 * object Identifies the type object pointed to by 832 * ibt_object_handle. 833 * 834 * ibt_object_handle The handle of the object to be associated with 835 * the data in/out 836 * 837 * data_p Pointer data passed in to the CI. The buffer 838 * should be allocated by the caller. 839 * 840 * data_sz The size of the buffer pointed to by 841 * data_p. 842 * Output: 843 * 844 * Returns: 845 * IBT_SUCCESS 846 * IBT_NOT_SUPPORTED Feature not supported. 847 * IBT_INVALID_PARAM Invalid object type specified. 848 * IBT_HCA_HDL_INVALID 849 * IBT_AH_HDL_INVALID/IBT_UD_DEST_HDL_INVALID 850 * IBT_CHAN_HDL_INVALID/IBT_QP_HDL_INVALID 851 * IBT_CQ_HDL_INVALID 852 * IBT_EEC_HDL_INVALID 853 * IBT_RDD_HDL_INVALID 854 * IBT_MW_HDL_INVALID 855 * IBT_PD_HDL_INVALID 856 * IBT_SRQ_HDL_INVALID 857 * 858 * Description: 859 * Exchange CI private data for the specified CI object. 860 */ 861 ibt_status_t 862 ibt_ci_data_in(ibt_hca_hdl_t hca, ibt_ci_data_flags_t flags, 863 ibt_object_type_t object, void *ibt_object_handle, void *data_p, 864 size_t data_sz) 865 { 866 ibt_status_t retval; 867 void *ci_obj_hdl; 868 869 IBTF_DPRINTF_L3(ibtf, "ibt_ci_data_in(%p, %x, %d, %p, %p, %d)", 870 hca, flags, object, ibt_object_handle, data_p, data_sz); 871 872 switch (object) { 873 case IBT_HDL_HCA: 874 ci_obj_hdl = (void *) 875 (IBTL_HCA2CIHCA(((ibt_hca_hdl_t)ibt_object_handle))); 876 break; 877 878 case IBT_HDL_CHANNEL: 879 ci_obj_hdl = (void *) 880 (IBTL_CHAN2CIQP(((ibt_channel_hdl_t)ibt_object_handle))); 881 break; 882 883 case IBT_HDL_CQ: 884 ci_obj_hdl = (void *) 885 (((ibt_cq_hdl_t)(ibt_object_handle))->cq_ibc_cq_hdl); 886 break; 887 888 case IBT_HDL_EEC: 889 ci_obj_hdl = (void *) 890 (((ibt_eec_hdl_t)(ibt_object_handle))->eec_ibc_eec_hdl); 891 break; 892 893 case IBT_HDL_UD_DEST: 894 ci_obj_hdl = (void *) 895 (((ibt_ud_dest_hdl_t)(ibt_object_handle))->ud_ah); 896 break; 897 898 case IBT_HDL_SRQ: 899 ci_obj_hdl = (void *) 900 (((ibt_srq_hdl_t)(ibt_object_handle))->srq_ibc_srq_hdl); 901 break; 902 903 default: 904 ci_obj_hdl = ibt_object_handle; 905 break; 906 } 907 908 retval = (IBTL_HCA2CIHCAOPS_P(hca)->ibc_ci_data_in)(IBTL_HCA2CIHCA(hca), 909 flags, object, ci_obj_hdl, data_p, data_sz); 910 911 if (retval != IBT_SUCCESS) { 912 IBTF_DPRINTF_L2(ibtf, "ibt_ci_data_in: Failed : %d", retval); 913 } 914 return (retval); 915 } 916 917 /* 918 * Function: 919 * ibt_ci_data_out() 920 * 921 * Input: 922 * hca_hdl HCA Handle. 923 * flags IBT_COMPLETE_ALLOC - Finish a deferred alloc. 924 * object Identifies the type object pointed to by 925 * ibt_object_handle. 926 * 927 * ibt_object_handle The handle of the object to be associated with 928 * the data in/out 929 * 930 * data_p Pointer to a buffer in which to return the CI 931 * private data. The buffer should be allocated 932 * by the caller. 933 * 934 * data_sz The size of the buffer pointed to by 935 * data_p. 936 * Output: 937 * 938 * Returns: 939 * IBT_SUCCESS 940 * IBT_NOT_SUPPORTED Feature not supported. 941 * IBT_INSUFF_RESOURCE The buffer pointed to by data_p was too 942 * small to hold the data. 943 * IBT_INVALID_PARAM Invalid object type specified. 944 * IBT_HCA_HDL_INVALID 945 * IBT_AH_HDL_INVALID/IBT_UD_DEST_HDL_INVALID 946 * IBT_CHAN_HDL_INVALID/IBT_QP_HDL_INVALID 947 * IBT_CQ_HDL_INVALID 948 * IBT_EEC_HDL_INVALID 949 * IBT_RDD_HDL_INVALID 950 * IBT_MW_HDL_INVALID 951 * IBT_PD_HDL_INVALID 952 * IBT_SRQ_HDL_INVALID 953 * 954 * Description: 955 * Exchange CI private data for the specified CI object. 956 */ 957 ibt_status_t 958 ibt_ci_data_out(ibt_hca_hdl_t hca, ibt_ci_data_flags_t flags, 959 ibt_object_type_t object, void *ibt_object_handle, void *data_p, 960 size_t data_sz) 961 { 962 ibt_status_t retval; 963 void *ci_obj_hdl; 964 965 IBTF_DPRINTF_L3(ibtf, "ibt_ci_data_out(%p, %x, %d, %p, %p, %d)", 966 hca, flags, object, ibt_object_handle, data_p, data_sz); 967 968 switch (object) { 969 case IBT_HDL_HCA: 970 ci_obj_hdl = (void *) 971 (IBTL_HCA2CIHCA(((ibt_hca_hdl_t)ibt_object_handle))); 972 break; 973 974 case IBT_HDL_CHANNEL: 975 ci_obj_hdl = (void *) 976 (IBTL_CHAN2CIQP(((ibt_channel_hdl_t)ibt_object_handle))); 977 break; 978 979 case IBT_HDL_CQ: 980 ci_obj_hdl = (void *) 981 (((ibt_cq_hdl_t)(ibt_object_handle))->cq_ibc_cq_hdl); 982 break; 983 984 case IBT_HDL_EEC: 985 ci_obj_hdl = (void *) 986 (((ibt_eec_hdl_t)(ibt_object_handle))->eec_ibc_eec_hdl); 987 break; 988 989 case IBT_HDL_UD_DEST: 990 ci_obj_hdl = (void *) 991 (((ibt_ud_dest_hdl_t)(ibt_object_handle))->ud_ah); 992 break; 993 994 case IBT_HDL_SRQ: 995 ci_obj_hdl = (void *) 996 (((ibt_srq_hdl_t)(ibt_object_handle))->srq_ibc_srq_hdl); 997 break; 998 999 default: 1000 ci_obj_hdl = ibt_object_handle; 1001 break; 1002 } 1003 1004 retval = (IBTL_HCA2CIHCAOPS_P(hca)->ibc_ci_data_out) 1005 (IBTL_HCA2CIHCA(hca), flags, object, ci_obj_hdl, data_p, data_sz); 1006 1007 if (retval != IBT_SUCCESS) { 1008 IBTF_DPRINTF_L2(ibtf, "ibt_ci_data_out: Failed : %d", retval); 1009 } 1010 return (retval); 1011 } 1012 1013 1014 /* 1015 * FMA Support functions. 1016 */ 1017 1018 #define IBTL_ENA_MASK 0xC0000000 1019 #define IBTL_ENA_POSSIBLE 0x80000000 1020 #define IBTL_TYPE_SHIFT 27 1021 1022 /* 1023 * Function: 1024 * ibt_get_module_failure() 1025 * 1026 * Input: 1027 * type Identifies the failing IB module. 1028 * ena '0' or the data for Fault Management 1029 * Architecture (ENA). 1030 * 1031 * Returns: 1032 * status Special IB failure status. 1033 * 1034 * Description: 1035 * XXX Just stubbed out to return failures with no data for Fault 1036 * Management Architecture (ENAs) at the moment XXX 1037 */ 1038 ibt_status_t 1039 ibt_get_module_failure(ibt_failure_type_t type, uint64_t ena) 1040 { 1041 ibt_status_t ret; 1042 1043 IBTF_DPRINTF_L3(ibtf, "ibt_get_module_failure(%d, 0x%llX)", type, ena); 1044 1045 switch (type) { 1046 case IBT_FAILURE_CI: 1047 case IBT_FAILURE_IBMF: 1048 case IBT_FAILURE_IBCM: 1049 case IBT_FAILURE_IBDM: 1050 case IBT_FAILURE_IBTL: 1051 case IBT_FAILURE_IBSM: 1052 ret = IBTL_ENA_POSSIBLE | (type << IBTL_TYPE_SHIFT); 1053 break; 1054 default: 1055 ret = IBT_FAILURE; 1056 } 1057 IBTF_DPRINTF_L3(ibtf, "ibt_get_module_failure: ret = 0x%lX", ret); 1058 return (ret); 1059 } 1060 1061 1062 /* 1063 * Function: 1064 * ibc_get_ci_failure() 1065 * 1066 * Input: 1067 * ena '0' or the data for Fault Management 1068 * Architecture (ENA). 1069 * 1070 * Returns: 1071 * status Special CI failure status. 1072 * 1073 * Description: 1074 * Just use the function above to do the job. 1075 */ 1076 ibt_status_t 1077 ibc_get_ci_failure(uint64_t ena) 1078 { 1079 return (ibt_get_module_failure(IBT_FAILURE_CI, ena)); 1080 } 1081 1082 1083 /* 1084 * ibt_check_failure() 1085 * Function to test for special case failures. 1086 * 1087 * status An ibt_status_t returned from an IBTF function call. 1088 * 1089 * reserved_p NULL, or a pointer to where we store the data for 1090 * Fault Management Architecture (ENA). 1091 * 1092 * Description: 1093 * XXX Still need to determine the data for Fault Management Architecture 1094 * (ENA), using 0 for now XXX 1095 */ 1096 ibt_failure_type_t 1097 ibt_check_failure(ibt_status_t status, uint64_t *reserved_p) 1098 { 1099 ibt_failure_type_t type; 1100 1101 IBTF_DPRINTF_L3(ibtf, "ibt_check_failure(%X)", status); 1102 1103 if ((status & IBTL_ENA_MASK) == IBTL_ENA_POSSIBLE) { 1104 type = status & ~IBTL_ENA_POSSIBLE >> IBTL_TYPE_SHIFT; 1105 1106 /* XXX Need more work here... */ 1107 if (reserved_p != NULL) 1108 *reserved_p = 0; 1109 } else { 1110 type = IBT_FAILURE_STANDARD; 1111 if (reserved_p != NULL) 1112 *reserved_p = 0; /* No FMA Data Available. */ 1113 } 1114 IBTF_DPRINTF_L3(ibtf, "ibt_check_failure: type = 0x%X", type); 1115 return (type); 1116 } 1117 1118 /* 1119 * Initialize and create kstats. 1120 * 1121 * We create the following kstats on all ports of the HCA: 1122 * <hca_driver_name><instance_number>/port<port_num>/stats 1123 * <hca_driver_name><instance_number>/port<port_num>/pkeys 1124 */ 1125 static void 1126 ibtl_kstat_init(ibtl_hca_devinfo_t *hca_devp) 1127 { 1128 uint_t nports = hca_devp->hd_hca_attr->hca_nports; 1129 ibtl_hca_port_kstat_t *pks; 1130 int i; 1131 1132 IBTF_DPRINTF_L3(ibtf, "ibtl_kstat_init(hca_devp = 0x%p)", hca_devp); 1133 1134 hca_devp->hd_hca_port_ks_info_len = 1135 sizeof (ibtl_hca_port_kstat_t) * nports; 1136 pks = kmem_zalloc(hca_devp->hd_hca_port_ks_info_len, KM_SLEEP); 1137 hca_devp->hd_hca_port_ks_info = pks; 1138 1139 for (i = 0; i < nports; i++, pks++) { 1140 pks->pks_hca_devp = hca_devp; 1141 pks->pks_port_num = i + 1; 1142 ibtl_kstat_stats_create(hca_devp, i + 1); 1143 ibtl_kstat_pkeys_create(hca_devp, i + 1); 1144 } 1145 } 1146 1147 /* 1148 * Delete kstats on all ports of the HCA. 1149 */ 1150 static void 1151 ibtl_kstat_fini(ibtl_hca_devinfo_t *hca_devp) 1152 { 1153 ibtl_hca_port_kstat_t *pks; 1154 int i; 1155 1156 IBTF_DPRINTF_L3(ibtf, "ibtl_kstat_fini(hca_devp = 0x%p)", hca_devp); 1157 1158 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*hca_devp)) 1159 1160 pks = hca_devp->hd_hca_port_ks_info; 1161 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*pks)) 1162 1163 if (pks == NULL) 1164 return; 1165 1166 for (i = 0; i < hca_devp->hd_hca_attr->hca_nports; i++, pks++) { 1167 if (pks->pks_stats_ksp) 1168 kstat_delete(pks->pks_stats_ksp); 1169 1170 if (pks->pks_pkeys_ksp) { 1171 ASSERT(!MUTEX_HELD(&ibtl_clnt_list_mutex)); 1172 kstat_delete(pks->pks_pkeys_ksp); 1173 } 1174 } 1175 1176 kmem_free(hca_devp->hd_hca_port_ks_info, 1177 hca_devp->hd_hca_port_ks_info_len); 1178 } 1179 1180 /* 1181 * Update "stats" kstat. 1182 * Called by kstat framework. 1183 */ 1184 static int 1185 ibtl_kstat_stats_update(kstat_t *ksp, int rw) 1186 { 1187 ibtl_hca_port_kstat_t *pks; 1188 ibtl_hca_devinfo_t *hca_devp; 1189 ibt_hca_portinfo_t *p; 1190 struct kstat_named *data; 1191 1192 IBTF_DPRINTF_L4(ibtf, "ibtl_kstat_stats_update(ksp = 0x%p, rw = %d)", 1193 ksp, rw); 1194 1195 if (rw == KSTAT_WRITE) 1196 return (EACCES); 1197 1198 mutex_enter(&ibtl_clnt_list_mutex); 1199 1200 /* 1201 * Update the link_state kstat using the value from portinfo cache. 1202 */ 1203 pks = ksp->ks_private; 1204 hca_devp = pks->pks_hca_devp; 1205 data = (struct kstat_named *)(ksp->ks_data); 1206 p = hca_devp->hd_portinfop + pks->pks_port_num - 1; 1207 data[0].value.ui32 = (uint32_t)p->p_linkstate; 1208 1209 mutex_exit(&ibtl_clnt_list_mutex); 1210 1211 return (0); 1212 } 1213 1214 /* 1215 * Create "stats" kstat for the specified HCA port in the form: 1216 * <hca_driver_name><instance_number>/port<port_num>/stats 1217 * At preset it contains only one named data of "link_state" 1218 */ 1219 static void 1220 ibtl_kstat_stats_create(ibtl_hca_devinfo_t *hca_devp, uint_t port_num) 1221 { 1222 struct kstat *ksp; 1223 struct kstat_named *named_data; 1224 char *drv_name; 1225 int drv_instance; 1226 ibtl_hca_port_kstat_t *pks; 1227 char kname[40]; 1228 1229 IBTF_DPRINTF_L3(ibtf, "ibtl_kstat_stats_create(hca_devp = 0x%p, " 1230 "port_num = 0x%u)", hca_devp, port_num); 1231 1232 drv_name = (char *)ddi_driver_name(hca_devp->hd_hca_dip); 1233 drv_instance = ddi_get_instance(hca_devp->hd_hca_dip); 1234 (void) snprintf(kname, sizeof (kname), "%s%d/port%d/stats", 1235 drv_name, drv_instance, port_num); 1236 1237 ksp = kstat_create("ibtf", 0, kname, "ib", KSTAT_TYPE_NAMED, 1, 0); 1238 if (ksp == NULL) { 1239 IBTF_DPRINTF_L2(ibtf, 1240 "ibtl_kstat_stats_create: kstat_create() failed"); 1241 return; 1242 } 1243 1244 named_data = (struct kstat_named *)(ksp->ks_data); 1245 kstat_named_init(&named_data[0], "link_state", KSTAT_DATA_UINT32); 1246 1247 pks = hca_devp->hd_hca_port_ks_info + port_num - 1; 1248 pks->pks_stats_ksp = ksp; 1249 1250 ksp->ks_private = pks; 1251 ksp->ks_update = ibtl_kstat_stats_update; 1252 1253 /* Install the kstat */ 1254 kstat_install(ksp); 1255 } 1256 1257 /* 1258 * Update "pkeys" kstat. 1259 * 1260 * Called by kstat framework. Since ks_lock was set to ibtl_clnt_list_mutex 1261 * at the time of the kstat creation, kstat framework will hold this lock 1262 * while calling this function. 1263 */ 1264 static int 1265 ibtl_kstat_pkeys_update(kstat_t *ksp, int rw) 1266 { 1267 ibtl_hca_port_kstat_t *pks; 1268 ibtl_hca_devinfo_t *hca_devp; 1269 ibt_hca_portinfo_t *p; 1270 1271 IBTF_DPRINTF_L4(ibtf, "ibtl_kstat_pkeys_update(ksp = 0x%p, rw = %d)", 1272 ksp, rw); 1273 1274 #ifndef __lock_lint 1275 ASSERT(MUTEX_HELD(&ibtl_clnt_list_mutex)); 1276 #endif 1277 1278 if (rw == KSTAT_WRITE) 1279 return (EACCES); 1280 1281 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*ksp)) 1282 1283 pks = ksp->ks_private; 1284 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*pks)) 1285 1286 hca_devp = pks->pks_hca_devp; 1287 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*hca_devp)) 1288 1289 /* 1290 * Point kstat data to the pkey table in the portinfo cache. 1291 */ 1292 1293 p = hca_devp->hd_portinfop + pks->pks_port_num - 1; 1294 1295 ksp->ks_data = p->p_pkey_tbl; 1296 ksp->ks_ndata = p->p_pkey_tbl_sz; 1297 ksp->ks_data_size = p->p_pkey_tbl_sz * sizeof (ib_pkey_t); 1298 1299 return (0); 1300 } 1301 1302 /* 1303 * Create "pkeys" kstat for the specified HCA port in the form: 1304 * <hca_driver_name><instance_number>/port<port_num>/pkeys 1305 * 1306 * Currently kstat framework allows only some fixed data types as named 1307 * data components under a named kstat. Due to this limitation it is not 1308 * possible to add "pkeys" as a named data under the "stats" kstat. 1309 */ 1310 static void 1311 ibtl_kstat_pkeys_create(ibtl_hca_devinfo_t *hca_devp, uint_t port_num) 1312 { 1313 struct kstat *ksp; 1314 char *drv_name; 1315 int drv_instance; 1316 char kname[40]; 1317 ibtl_hca_port_kstat_t *pks; 1318 1319 IBTF_DPRINTF_L3(ibtf, "ibtl_kstat_stats_create(hca_devp = 0x%p, " 1320 "port_num = 0x%u)", hca_devp, port_num); 1321 1322 drv_name = (char *)ddi_driver_name(hca_devp->hd_hca_dip); 1323 drv_instance = ddi_get_instance(hca_devp->hd_hca_dip); 1324 (void) snprintf(kname, sizeof (kname), "%s%d/port%d/pkeys", 1325 drv_name, drv_instance, port_num); 1326 1327 ksp = kstat_create("ibtf", 0, kname, "ib", KSTAT_TYPE_RAW, 0, 1328 KSTAT_FLAG_VAR_SIZE | KSTAT_FLAG_VIRTUAL); 1329 if (ksp == NULL) { 1330 IBTF_DPRINTF_L2(ibtf, 1331 "ibtl_kstat_pkeys_create: kstat_create() failed"); 1332 return; 1333 } 1334 1335 pks = hca_devp->hd_hca_port_ks_info + port_num - 1; 1336 pks->pks_pkeys_ksp = ksp; 1337 1338 ksp->ks_private = pks; 1339 ksp->ks_update = ibtl_kstat_pkeys_update; 1340 ksp->ks_lock = &ibtl_clnt_list_mutex; 1341 1342 /* 1343 * We just go with the default_kstat_snapshot(). 1344 * So there is no need to set ks_snapshot field. 1345 */ 1346 1347 /* Install the kstat */ 1348 kstat_install(ksp); 1349 } 1350