1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved. 23 */ 24 25 /* 26 * ibtl_impl.c 27 * 28 * This file contains the IBTF module's initialization and 29 * IBTF Clients/Modules registration routines. 30 */ 31 32 #include <sys/modctl.h> 33 #include <sys/sunndi.h> 34 #include <sys/ib/ibtl/impl/ibtl.h> 35 #include <sys/ib/ibtl/impl/ibtl_ibnex.h> 36 37 /* 38 * Globals. 39 */ 40 static char ibtf[] = "ibtl_impl"; 41 42 extern ibtl_ibnex_callback_t ibtl_ibnex_callback_routine; 43 44 /* 45 * ibtl_clnt_list: 46 * 47 * Head of the list of IBT Client Instances. The IBT Client List 48 * is modified by IBTF on an IBT client's ibt_attach/ibt_detach call. 49 * 50 * ibtl_hca_list: 51 * 52 * Head of the list of HCA devices. The HCA List is modified by IBTF on 53 * a CI's ibc_attach/ibc_detach call. 54 * The datap of the list elements points to an ibtl_hca_devinfo_s 55 * structure. 56 * 57 * (ibc_attach) 58 * ibtl_hca_list -> ibtl_hca_devinfo_t--> ... -->ibtl_hca_devinfo_t 59 * [per-hca_dev] | ^ {nth HCA Dev} 60 * | | 61 * | ibtl_hca_t (ibt_open_hca) 62 * | ^ | 63 * | | | 64 * v | V 65 * ibtl_clnt_list -> ibtl_clnt_t--> ...--> {n'th Module} 66 * [per-client_instance] (ibt_attach) 67 * 68 */ 69 70 /* Global List of IBT Client Instances, and associated mutex. */ 71 struct ibtl_clnt_s *ibtl_clnt_list = NULL; 72 kmutex_t ibtl_clnt_list_mutex; 73 74 /* Lock for the race between the client and CM to free QPs. */ 75 kmutex_t ibtl_free_qp_mutex; 76 77 /* Lock for the race between the client closing the HCA and QPN being freed. */ 78 kcondvar_t ibtl_close_hca_cv; 79 80 /* Global List of HCA Devices, and associated mutex. */ 81 struct ibtl_hca_devinfo_s *ibtl_hca_list = NULL; 82 83 /* Well-known async handlers and associated client private. */ 84 ibt_async_handler_t ibtl_cm_async_handler; 85 ibt_async_handler_t ibtl_dm_async_handler; 86 ibt_async_handler_t ibtl_ibma_async_handler; 87 void *ibtl_cm_clnt_private; 88 void *ibtl_dm_clnt_private; 89 void *ibtl_ibma_clnt_private; 90 91 extern int ib_hw_status; 92 _NOTE(SCHEME_PROTECTS_DATA("Scheme protects data", ib_hw_status)) 93 94 /* 95 * Misc Module Declarations. 96 */ 97 extern struct mod_ops mod_miscops; 98 static struct modlmisc modlmisc = { 99 &mod_miscops, /* Type of module - misc. */ 100 "IB Transport Layer" /* Name of the Module. */ 101 }; 102 103 static struct modlinkage modlinkage = { 104 MODREV_1, (void *)&modlmisc, NULL 105 }; 106 107 static void ibtl_kstat_init(ibtl_hca_devinfo_t *); 108 static void ibtl_kstat_fini(ibtl_hca_devinfo_t *); 109 static void ibtl_kstat_stats_create(ibtl_hca_devinfo_t *, uint_t); 110 static void ibtl_kstat_pkeys_create(ibtl_hca_devinfo_t *, uint_t); 111 112 extern kmutex_t ibtl_part_attr_mutex; 113 114 /* 115 * IBTF Loadable Module Routines. 116 */ 117 118 int 119 _init(void) 120 { 121 int rval; 122 123 if ((rval = mod_install(&modlinkage)) != 0) 124 return (rval); 125 126 /* 127 * initialize IBTL ib2usec table 128 */ 129 ibtl_ib2usec_init(); 130 131 /* 132 * Initialize Logging 133 */ 134 ibtl_logging_initialization(); 135 136 /* 137 * Initialize the Alloc QP States. 138 */ 139 ibtl_init_cep_states(); 140 141 /* 142 * Initialize all Global Link Lists. 143 */ 144 mutex_init(&ibtl_clnt_list_mutex, NULL, MUTEX_DEFAULT, NULL); 145 mutex_init(&ibtl_free_qp_mutex, NULL, MUTEX_DEFAULT, NULL); 146 cv_init(&ibtl_close_hca_cv, NULL, CV_DEFAULT, NULL); 147 148 mutex_init(&ibtl_qp_mutex, NULL, MUTEX_DEFAULT, NULL); 149 cv_init(&ibtl_qp_cv, NULL, CV_DEFAULT, NULL); 150 151 mutex_init(&ibtl_part_attr_mutex, NULL, MUTEX_DEFAULT, NULL); 152 153 ibtl_thread_init(); 154 155 return (rval); 156 } 157 158 159 int 160 _fini(void) 161 { 162 int rval; 163 164 if ((rval = mod_remove(&modlinkage)) != 0) { 165 return (rval); 166 } 167 168 ibtl_thread_fini(); 169 170 mutex_destroy(&ibtl_clnt_list_mutex); 171 mutex_destroy(&ibtl_free_qp_mutex); 172 cv_destroy(&ibtl_close_hca_cv); 173 mutex_destroy(&ibtl_qp_mutex); 174 cv_destroy(&ibtl_qp_cv); 175 mutex_destroy(&ibtl_part_attr_mutex); 176 177 /* 178 * Stop Logging 179 */ 180 ibtl_logging_destroy(); 181 182 return (rval); 183 } 184 185 186 int 187 _info(struct modinfo *modinfop) 188 { 189 /* Return the Module Information. */ 190 return (mod_info(&modlinkage, modinfop)); 191 } 192 193 194 /* 195 * IBTF Client Registration Routines. 196 */ 197 198 /* 199 * Function: 200 * ibt_attach 201 * Input: 202 * modinfop - Client Module info structure. 203 * arg - usually client's dip 204 * clnt_private - client's private data pointer. 205 * Output: 206 * ibt_hdl_p - pointer to client's specific IBT handle, 207 * which is opaque to clients. 208 * Returns: 209 * IBT_SUCCESS 210 * IBT_INVALID_PARAM 211 * Called by: 212 * IBTF Client module during its attach() to register its instance 213 * to IBTF. 214 * Description: 215 * Registers the IBTF client module instance and returns an opaque 216 * handler to the client to be used for future calls to IBTF. 217 * Adds this client module instance to ibtl_clnt_list list. 218 * Records well-known async handlers. 219 */ 220 ibt_status_t 221 ibt_attach(ibt_clnt_modinfo_t *mod_infop, dev_info_t *arg, void *clnt_private, 222 ibt_clnt_hdl_t *ibt_hdl_p) 223 { 224 dev_info_t *pdip; 225 ibtl_clnt_t *clntp; 226 227 IBTF_DPRINTF_L3(ibtf, "ibt_attach(%p, %p, %p)", 228 mod_infop, arg, clnt_private); 229 230 if (mod_infop->mi_clnt_name == NULL) { 231 IBTF_DPRINTF_L1(ibtf, "ibt_attach: " 232 "IB client needs to specify its name"); 233 return (IBT_INVALID_PARAM); 234 } 235 236 /* 237 * Validate the Transport API version. 238 */ 239 if (mod_infop->mi_ibt_version != IBTI_V_CURR) { 240 IBTF_DPRINTF_L1(ibtf, "ibt_attach: IB client '%s' has an " 241 "invalid IB TI Version '%d'", mod_infop->mi_clnt_name, 242 mod_infop->mi_ibt_version); 243 return (IBT_NOT_SUPPORTED); 244 } 245 246 if (mod_infop->mi_async_handler == NULL) { 247 IBTF_DPRINTF_L2(ibtf, "ibt_attach: Client '%s' has not\n" 248 " provided an Asynchronous Event Handler.\n" 249 " This will be required soon.", 250 mod_infop->mi_clnt_name); 251 } 252 253 /* 254 * Check out Client's Class information. If it is not of mgmt class, 255 * we expect 'arg' to be Not NULL and point to client driver's 256 * device info struct. 257 */ 258 if ((!IBT_MISCMOD_CLIENTS(mod_infop->mi_clnt_class)) && 259 (arg == NULL)) { 260 IBTF_DPRINTF_L1(ibtf, "ibt_attach: " 261 "arg not set with driver's dip."); 262 return (IBT_INVALID_PARAM); 263 } 264 265 if (!IBT_MISCMOD_CLIENTS(mod_infop->mi_clnt_class)) { 266 pdip = ddi_get_parent(arg); 267 if (pdip == NULL || 268 ibtl_ibnex_valid_hca_parent(pdip) != IBT_SUCCESS) { 269 IBTF_DPRINTF_L2(ibtf, "ibt_attach: " 270 "client %s is not a child of IB nexus driver.", 271 ddi_driver_name(arg)); 272 return (IBT_INVALID_PARAM); 273 } 274 } 275 276 mutex_enter(&ibtl_clnt_list_mutex); 277 if (mod_infop->mi_clnt_class == IBT_CM) { 278 if (ibtl_cm_async_handler != NULL) { 279 IBTF_DPRINTF_L1(ibtf, "ibt_attach: " 280 "CM is already attached."); 281 mutex_exit(&ibtl_clnt_list_mutex); 282 return (IBT_INVALID_PARAM); 283 } 284 ibtl_cm_async_handler = mod_infop->mi_async_handler; 285 ibtl_cm_clnt_private = clnt_private; 286 } else if (mod_infop->mi_clnt_class == IBT_DM) { 287 if (ibtl_dm_async_handler != NULL) { 288 IBTF_DPRINTF_L1(ibtf, "ibt_attach: " 289 "DM is already attached."); 290 mutex_exit(&ibtl_clnt_list_mutex); 291 return (IBT_INVALID_PARAM); 292 } 293 ibtl_dm_async_handler = mod_infop->mi_async_handler; 294 ibtl_dm_clnt_private = clnt_private; 295 } else if (mod_infop->mi_clnt_class == IBT_IBMA) { 296 if (ibtl_ibma_async_handler != NULL) { 297 IBTF_DPRINTF_L1(ibtf, "ibt_attach: " 298 "IBMF is already attached."); 299 mutex_exit(&ibtl_clnt_list_mutex); 300 return (IBT_INVALID_PARAM); 301 } 302 ibtl_ibma_async_handler = mod_infop->mi_async_handler; 303 ibtl_ibma_clnt_private = clnt_private; 304 } 305 306 /* Allocate the memory for per-client-device info structure */ 307 clntp = kmem_zalloc(sizeof (ibtl_clnt_t), KM_SLEEP); 308 309 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(clntp->clnt_modinfop, 310 clntp->clnt_dip, clntp->clnt_name, clntp->clnt_async_cnt, 311 clntp->clnt_private)) 312 /* Update the Client info structure */ 313 clntp->clnt_modinfop = mod_infop; /* IBT Client's Mod Info */ 314 clntp->clnt_private = clnt_private; /* IBT Client's private */ 315 clntp->clnt_dip = arg; /* IBT Client's dip */ 316 clntp->clnt_async_cnt = 0; 317 /* using a count of 7 below guarantees it is NULL terminated */ 318 (void) strncpy(clntp->clnt_name, mod_infop->mi_clnt_name, 7); 319 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(clntp->clnt_modinfop, 320 clntp->clnt_dip, clntp->clnt_name, clntp->clnt_async_cnt, 321 clntp->clnt_private)) 322 323 /* 324 * Update Client Device Instance List. 325 */ 326 clntp->clnt_list_link = ibtl_clnt_list; 327 ibtl_clnt_list = clntp; 328 mutex_exit(&ibtl_clnt_list_mutex); 329 330 /* 331 * The ibt_hdl_p is a opaque handle which is the address of 332 * ibt_clnt_t structure passed back to the clients. 333 * The client will pass on this handle in its future calls to IBTF. 334 */ 335 *ibt_hdl_p = clntp; 336 337 return (IBT_SUCCESS); 338 } 339 340 341 /* 342 * Function: 343 * ibt_detach 344 * Input: 345 * ibt_hdl - IBT Handle as returned during ibt_attach call. 346 * Output: 347 * none 348 * Returns: 349 * IBT_SUCCESS 350 * IBT_INVALID_PARAM. 351 * Called by: 352 * IBTF Client module during its detach() to de-register its instance 353 * from IBTF. 354 * Description: 355 * Deregisters the IBTF client module instance from the IBTF. 356 * All resources and any reference to this ibt_hdl will be removed. 357 */ 358 ibt_status_t 359 ibt_detach(ibt_clnt_hdl_t ibt_hdl) 360 { 361 ibtl_clnt_t **clntpp; 362 363 IBTF_DPRINTF_L3(ibtf, "ibt_detach(%p)", ibt_hdl); 364 365 mutex_enter(&ibtl_clnt_list_mutex); 366 clntpp = &ibtl_clnt_list; 367 for (; *clntpp != NULL; clntpp = &(*clntpp)->clnt_list_link) 368 if (*clntpp == ibt_hdl) 369 break; 370 if (*clntpp == NULL) { 371 IBTF_DPRINTF_L1(ibtf, "ibt_detach: Client @ %p Not Found", 372 ibt_hdl); 373 mutex_exit(&ibtl_clnt_list_mutex); 374 return (IBT_INVALID_PARAM); 375 } 376 377 /* 378 * Check out whether the client has freed all its resources. 379 * If not done, then fail the detach. 380 * 381 * viz. A client has to close all the HCA they have opened, 382 * i.e. the HCA List maintained for clients has to be empty. 383 * If this list is not empty, then the client has not performed 384 * complete clean-up, so fail the detach. 385 */ 386 if (ibt_hdl->clnt_hca_list != NULL) { 387 mutex_exit(&ibtl_clnt_list_mutex); 388 389 IBTF_DPRINTF_L2(ibtf, "ibt_detach: " 390 "ERROR: Client '%s' has not closed all of its HCAs", 391 ibt_hdl->clnt_modinfop->mi_clnt_name); 392 cmn_err(CE_CONT, "IBT DETACH failed: resources not yet " 393 "freed by client '%s'\n", 394 ibt_hdl->clnt_modinfop->mi_clnt_name); 395 return (IBT_HCA_RESOURCES_NOT_FREED); 396 } 397 398 if (ibt_hdl->clnt_srv_cnt != 0) { 399 mutex_exit(&ibtl_clnt_list_mutex); 400 IBTF_DPRINTF_L2(ibtf, "ibt_detach: client '%s' still has " 401 "services or subnet_notices registered", 402 ibt_hdl->clnt_modinfop->mi_clnt_name); 403 cmn_err(CE_CONT, "IBT DETACH failed: resources not yet " 404 "freed by client '%s'\n", 405 ibt_hdl->clnt_modinfop->mi_clnt_name); 406 return (IBT_HCA_RESOURCES_NOT_FREED); 407 } 408 409 /* 410 * Delete the entry of this module from the ibtl_clnt_list List. 411 */ 412 *clntpp = ibt_hdl->clnt_list_link; /* remove us */ 413 414 /* make sure asyncs complete before freeing */ 415 ibtl_free_clnt_async_check(ibt_hdl); 416 417 if (ibt_hdl->clnt_modinfop->mi_clnt_class == IBT_CM) { 418 ibtl_cm_async_handler = NULL; 419 ibtl_cm_clnt_private = NULL; 420 } else if (ibt_hdl->clnt_modinfop->mi_clnt_class == IBT_DM) { 421 ibtl_dm_async_handler = NULL; 422 ibtl_dm_clnt_private = NULL; 423 } else if (ibt_hdl->clnt_modinfop->mi_clnt_class == IBT_IBMA) { 424 ibtl_ibma_async_handler = NULL; 425 ibtl_ibma_clnt_private = NULL; 426 } 427 mutex_exit(&ibtl_clnt_list_mutex); 428 429 /* Free up the memory of per-client info struct. */ 430 kmem_free(ibt_hdl, sizeof (ibtl_clnt_t)); 431 432 return (IBT_SUCCESS); 433 } 434 435 static void 436 ibtl_set_ibhw_status() 437 { 438 ib_hw_status++; 439 } 440 441 static void 442 ibtl_clear_ibhw_status() 443 { 444 ib_hw_status--; 445 } 446 447 /* 448 * Function: 449 * ibc_init 450 * Input: 451 * modlp - Pointer to IBC client module linkage structure 452 * Output: 453 * None 454 * Returns: 455 * 0 always for now 456 * Called by: 457 * CI client calls IBTF during its _init() to register HCA with 458 * Solaris I/O framework. 459 * Description: 460 * Initializes the CI clients module linkage structure with 461 * default bus_ops structure 462 */ 463 int 464 ibc_init(struct modlinkage *modlp) 465 { 466 ibtl_ibnex_cb_args_t cb_args; 467 468 mutex_enter(&ibtl_clnt_list_mutex); 469 cb_args.cb_flag = IBTL_IBNEX_IBC_INIT; 470 cb_args.cb_modlp = modlp; 471 if (ibtl_ibnex_callback_routine) { 472 (void) ((*ibtl_ibnex_callback_routine)(&cb_args)); 473 } 474 mutex_exit(&ibtl_clnt_list_mutex); 475 return (0); 476 } 477 478 479 /* 480 * Function: 481 * ibc_fini 482 * Input: 483 * modlp - Pointer to IBC client module linkage structure 484 * Output: 485 * None 486 * Returns: 487 * None 488 * Called by: 489 * CI client calls IBTF during its _fini() to remove HCA with 490 * Solaris I/O framework. 491 * Description: 492 * Undo what is done during ibc_init 493 */ 494 void 495 ibc_fini(struct modlinkage *modlp) 496 { 497 ibtl_ibnex_cb_args_t cb_args; 498 499 mutex_enter(&ibtl_clnt_list_mutex); 500 cb_args.cb_flag = IBTL_IBNEX_IBC_FINI; 501 cb_args.cb_modlp = modlp; 502 if (ibtl_ibnex_callback_routine) { 503 (void) ((*ibtl_ibnex_callback_routine)(&cb_args)); 504 } 505 mutex_exit(&ibtl_clnt_list_mutex); 506 } 507 508 /* 509 * Function: 510 * ibc_attach 511 * Input: 512 * info_p - IBC HCA Info. 513 * Output: 514 * ibc_hdl_p - IBC Client's HCA Handle. 515 * Returns: 516 * IBC_SUCCESS 517 * IBC_FAILURE 518 * Called by: 519 * CI calls IBTF during its attach() to register HCA Device with IBTF. 520 * Description: 521 * Registers the presence of HCA device by providing the HCA device info 522 * structure and provides an opaque HCA handler for future calls to this 523 * HCA device. 524 */ 525 ibc_status_t 526 ibc_attach(ibc_clnt_hdl_t *ibc_hdl_p, ibc_hca_info_t *info_p) 527 { 528 ibtl_hca_devinfo_t *hca_devp; 529 uint_t nports; 530 ibt_status_t status; 531 532 IBTF_DPRINTF_L2(ibtf, "ibc_attach(%p, %p)", ibc_hdl_p, info_p); 533 534 /* Validate the Transport API version */ 535 if (info_p->hca_ci_vers != IBCI_V4) { 536 IBTF_DPRINTF_L1(ibtf, "ibc_attach: Invalid IB CI Version '%d'", 537 info_p->hca_ci_vers); 538 return (IBC_FAILURE); 539 } 540 541 if (info_p->hca_attr == NULL) { 542 IBTF_DPRINTF_L1(ibtf, "ibc_attach: " 543 "HCA Attributes must be specified."); 544 return (IBC_FAILURE); 545 } 546 547 nports = info_p->hca_attr->hca_nports; 548 if (nports == 0) { 549 IBTF_DPRINTF_L1(ibtf, "ibc_attach: " 550 "Number of ports must be valid"); 551 return (IBC_FAILURE); 552 } 553 554 if (info_p->hca_attr->hca_max_port_pkey_tbl_sz == 0) { 555 IBTF_DPRINTF_L1(ibtf, "ibc_attach: " 556 "Number of Partitions must be at least 1"); 557 return (IBC_FAILURE); 558 } 559 560 if ((info_p->hca_attr->hca_flags & IBT_HCA_CURRENT_QP_STATE) == 0) { 561 IBTF_DPRINTF_L1(ibtf, "ibc_attach: " 562 "HCA driver must support QP current state checking"); 563 return (IBC_FAILURE); 564 } 565 566 if ((info_p->hca_attr->hca_flags & IBT_HCA_PORT_UP) == 0) { 567 IBTF_DPRINTF_L1(ibtf, "ibc_attach: " 568 "HCA driver must support PORT_UP async events"); 569 return (IBC_FAILURE); 570 } 571 572 /* 573 * Install IB nexus driver (if not installed already) 574 */ 575 ibtl_set_ibhw_status(); 576 if (ndi_devi_config_vhci("ib", 0) == NULL) { 577 IBTF_DPRINTF_L2(ibtf, "ibc_attach: IB nexus attach failed"); 578 ibtl_clear_ibhw_status(); 579 return (IBC_FAILURE); 580 } 581 582 ibtl_thread_init2(); 583 584 /* Allocate the memory for per-client info structure */ 585 hca_devp = kmem_zalloc(sizeof (ibtl_hca_devinfo_t) + 586 (nports - 1) * sizeof (ibtl_async_port_event_t), KM_SLEEP); 587 588 mutex_enter(&ibtl_clnt_list_mutex); 589 590 /* Update HCA dev info structure */ 591 hca_devp->hd_ibc_hca_hdl = info_p->hca_handle; 592 hca_devp->hd_ibc_ops = info_p->hca_ops; 593 hca_devp->hd_hca_attr = info_p->hca_attr; 594 hca_devp->hd_hca_dip = info_p->hca_attr->hca_dip; 595 596 status = ibtl_init_hca_portinfo(hca_devp); 597 if (status != IBT_SUCCESS) { 598 mutex_exit(&ibtl_clnt_list_mutex); 599 IBTF_DPRINTF_L1(ibtf, "ibc_attach: call to ibc_query_hca_ports " 600 "failed: status = %d", status); 601 kmem_free(hca_devp, sizeof (ibtl_hca_devinfo_t) + 602 (nports - 1) * sizeof (ibtl_async_port_event_t)); 603 return (IBC_FAILURE); 604 } 605 606 /* Register the with MPxIO as PHCI */ 607 if (ibtl_ibnex_phci_register(hca_devp->hd_hca_dip) != IBT_SUCCESS) { 608 mutex_exit(&ibtl_clnt_list_mutex); 609 IBTF_DPRINTF_L1(ibtf, "ibc_attach: MPxIO register failed"); 610 kmem_free(hca_devp, sizeof (ibtl_hca_devinfo_t) + 611 (nports - 1) * sizeof (ibtl_async_port_event_t)); 612 return (IBC_FAILURE); 613 } 614 615 /* Initialize the Client List for this HCA. */ 616 hca_devp->hd_state = IBTL_HCA_DEV_ATTACHED; 617 618 /* lock out asyncs until after we announce the new HCA */ 619 hca_devp->hd_async_busy = 1; 620 621 cv_init(&hca_devp->hd_async_task_cv, NULL, CV_DEFAULT, NULL); 622 cv_init(&hca_devp->hd_async_busy_cv, NULL, CV_DEFAULT, NULL); 623 624 /* init portinfo locking variables */ 625 hca_devp->hd_portinfo_locked_port = 0; 626 cv_init(&hca_devp->hd_portinfo_cv, NULL, CV_DEFAULT, NULL); 627 628 ibtl_kstat_init(hca_devp); 629 630 mutex_exit(&ibtl_clnt_list_mutex); 631 632 /* 633 * The ibc_hdl_p points to an opaque handle which is the address 634 * of ibt_hca_devinfo_t structure passed back to the CI. 635 * The CI will pass on this handle in its future upcalls to IBTF. 636 */ 637 *ibc_hdl_p = hca_devp; 638 639 return (IBC_SUCCESS); 640 } 641 642 643 /* 644 * Function: 645 * ibc_post_attach 646 * Input: 647 * ibc_hdl - IBC Client's HCA Handle. 648 * Returns: 649 * none 650 * Called by: 651 * CI calls IBTF during its attach() after a successful ibc_attach(). 652 * Description: 653 * Announces to all known clients the existence of this HCA (by GUID). 654 */ 655 void 656 ibc_post_attach(ibc_clnt_hdl_t ibc_hdl) 657 { 658 IBTF_DPRINTF_L2(ibtf, "ibc_post_attach(%p)", ibc_hdl); 659 660 /* 661 * Update the HCA Device List. 662 */ 663 mutex_enter(&ibtl_clnt_list_mutex); 664 ibc_hdl->hd_hca_dev_link = ibtl_hca_list; 665 ibtl_hca_list = ibc_hdl; 666 mutex_exit(&ibtl_clnt_list_mutex); 667 668 /* notify all IBT Client Device Instances of the new HCA Device */ 669 ibtl_announce_new_hca(ibc_hdl); 670 } 671 672 673 /* 674 * Function: 675 * ibc_pre_detach 676 * Input: 677 * ibc_clnt_hdl - IBC HCA Handle as returned during ibc_attach call. 678 * cmd - DDI_DETACH/DDI_SUSPEND command. 679 * Output: 680 * none 681 * Returns: 682 * IBC_SUCCESS 683 * IBC_FAILURE. 684 * Called by: 685 * CI to try to get all IBTF clients to close the HCA device. 686 * Description: 687 * Attempts to deregister the HCA device entry from the IBTF. 688 * If all resources are freed by the IBTF clients and this HCA 689 * is closed, then IBC_SUCCESS is returned. 690 */ 691 ibc_status_t 692 ibc_pre_detach(ibc_clnt_hdl_t hca_devp, ddi_detach_cmd_t cmd) 693 { 694 ibtl_hca_devinfo_t **hcapp, *hcap; 695 696 IBTF_DPRINTF_L2(ibtf, "ibc_pre_detach(%p, 0x%x)", hca_devp, cmd); 697 698 /* 699 * Return failure, if command is not DDI_DETACH 700 */ 701 switch (cmd) { 702 case DDI_DETACH: 703 break; 704 default: 705 return (IBC_FAILURE); /* TBD: DDI_FAILURE */ 706 } 707 708 /* Make sure this HCA is on the HCA Device List. */ 709 mutex_enter(&ibtl_clnt_list_mutex); 710 hcap = ibtl_hca_list; 711 while (hcap != NULL) { 712 if (hcap == hca_devp) 713 break; 714 hcap = hcap->hd_hca_dev_link; 715 } 716 if (hcap == NULL) { 717 mutex_exit(&ibtl_clnt_list_mutex); 718 return (IBC_FAILURE); 719 } 720 721 /* 722 * Initially set the state to "Detaching". 723 */ 724 hca_devp->hd_state = IBTL_HCA_DEV_DETACHING; 725 726 /* 727 * Try to detach all IBTI clients, and continue only if all 728 * of the detaches succeed. 729 */ 730 if (ibtl_detach_all_clients(hca_devp)) { 731 hca_devp->hd_state = IBTL_HCA_DEV_ATTACHED; /* fix hd_state */ 732 mutex_exit(&ibtl_clnt_list_mutex); 733 734 return (IBC_FAILURE); 735 } 736 737 /* 738 * Check to see if all clients closed this HCA, or not. 739 * We only succeed if all clients cooperated. 740 */ 741 if (hca_devp->hd_clnt_list != NULL) { 742 hca_devp->hd_state = IBTL_HCA_DEV_ATTACHED; 743 mutex_exit(&ibtl_clnt_list_mutex); 744 IBTF_DPRINTF_L2(ibtf, "ibc_pre_detach: HCA still has attached " 745 "clients"); 746 return (IBC_FAILURE); 747 } 748 749 /* 750 * mark this device as detached 751 */ 752 hca_devp->hd_state = IBTL_HCA_DEV_DETACHED; 753 754 /* Delete the entry for this hca_devp from hca_head_list */ 755 hcapp = &ibtl_hca_list; 756 while (*hcapp != NULL) { 757 if (*hcapp == hca_devp) 758 break; 759 hcapp = &(*hcapp)->hd_hca_dev_link; 760 } 761 762 if (ibtl_ibnex_phci_unregister(hca_devp->hd_hca_dip) != IBT_SUCCESS) { 763 hca_devp->hd_state = IBTL_HCA_DEV_ATTACHED; /* fix hd_state */ 764 mutex_exit(&ibtl_clnt_list_mutex); 765 IBTF_DPRINTF_L1(ibtf, "ibc_pre_detach: PHCI unregister failed"); 766 return (IBC_FAILURE); 767 } 768 769 if (*hcapp == NULL) { 770 hca_devp->hd_state = IBTL_HCA_DEV_ATTACHED; /* fix hd_state */ 771 mutex_exit(&ibtl_clnt_list_mutex); 772 IBTF_DPRINTF_L1(ibtf, "ibc_pre_detach: HCA not attached"); 773 return (IBC_FAILURE); 774 } 775 *hcapp = hca_devp->hd_hca_dev_link; 776 ibtl_fast_gid_cache_valid = B_FALSE; /* invalidate fast_gid_cache */ 777 mutex_exit(&ibtl_clnt_list_mutex); 778 779 return (IBC_SUCCESS); 780 } 781 782 /* 783 * Function: 784 * ibc_detach 785 * Input: 786 * ibc_clnt_hdl - IBC HCA Handle as returned during ibc_attach call. 787 * Output: 788 * none 789 * Returns: 790 * None 791 * Called by: 792 * CI to detach the HCA device from IBTF. 793 * Description: 794 * Do the second step of detaching the HCA, which is required 795 * after a successful ibc_pre_detach. 796 */ 797 void 798 ibc_detach(ibc_clnt_hdl_t hca_devp) 799 { 800 IBTF_DPRINTF_L2(ibtf, "ibc_detach(%p)", hca_devp); 801 802 mutex_enter(&ibtl_clnt_list_mutex); 803 if (hca_devp->hd_state != IBTL_HCA_DEV_DETACHED) { 804 mutex_exit(&ibtl_clnt_list_mutex); 805 IBTF_DPRINTF_L0(ibtf, "ibc_detach: HCA has not successfully " 806 "pre-detached"); 807 return; 808 } 809 810 cv_destroy(&hca_devp->hd_async_task_cv); 811 cv_destroy(&hca_devp->hd_async_busy_cv); 812 cv_destroy(&hca_devp->hd_portinfo_cv); 813 814 kmem_free(hca_devp->hd_portinfop, hca_devp->hd_portinfo_len); 815 mutex_exit(&ibtl_clnt_list_mutex); 816 817 ibtl_kstat_fini(hca_devp); 818 819 /* Free up the memory of per-client info struct */ 820 kmem_free(hca_devp, sizeof (ibtl_hca_devinfo_t) + 821 (hca_devp->hd_hca_attr->hca_nports - 1) * 822 sizeof (ibtl_async_port_event_t)); 823 ibtl_clear_ibhw_status(); 824 } 825 826 /* 827 * Function: 828 * ibt_ci_data_in() 829 * 830 * Input: 831 * hca_hdl HCA Handle. 832 * flags IBT_COMPLETE_ALLOC - Finish a deferred alloc. 833 * object Identifies the type object pointed to by 834 * ibt_object_handle. 835 * 836 * ibt_object_handle The handle of the object to be associated with 837 * the data in/out 838 * 839 * data_p Pointer data passed in to the CI. The buffer 840 * should be allocated by the caller. 841 * 842 * data_sz The size of the buffer pointed to by 843 * data_p. 844 * Output: 845 * 846 * Returns: 847 * IBT_SUCCESS 848 * IBT_NOT_SUPPORTED Feature not supported. 849 * IBT_INVALID_PARAM Invalid object type specified. 850 * IBT_HCA_HDL_INVALID 851 * IBT_AH_HDL_INVALID/IBT_UD_DEST_HDL_INVALID 852 * IBT_CHAN_HDL_INVALID/IBT_QP_HDL_INVALID 853 * IBT_CQ_HDL_INVALID 854 * IBT_EEC_HDL_INVALID 855 * IBT_RDD_HDL_INVALID 856 * IBT_MW_HDL_INVALID 857 * IBT_PD_HDL_INVALID 858 * IBT_SRQ_HDL_INVALID 859 * 860 * Description: 861 * Exchange CI private data for the specified CI object. 862 */ 863 ibt_status_t 864 ibt_ci_data_in(ibt_hca_hdl_t hca, ibt_ci_data_flags_t flags, 865 ibt_object_type_t object, void *ibt_object_handle, void *data_p, 866 size_t data_sz) 867 { 868 ibt_status_t retval; 869 void *ci_obj_hdl; 870 871 IBTF_DPRINTF_L3(ibtf, "ibt_ci_data_in(%p, %x, %d, %p, %p, %d)", 872 hca, flags, object, ibt_object_handle, data_p, data_sz); 873 874 switch (object) { 875 case IBT_HDL_HCA: 876 ci_obj_hdl = (void *) 877 (IBTL_HCA2CIHCA(((ibt_hca_hdl_t)ibt_object_handle))); 878 break; 879 880 case IBT_HDL_CHANNEL: 881 ci_obj_hdl = (void *) 882 (IBTL_CHAN2CIQP(((ibt_channel_hdl_t)ibt_object_handle))); 883 break; 884 885 case IBT_HDL_CQ: 886 ci_obj_hdl = (void *) 887 (((ibt_cq_hdl_t)(ibt_object_handle))->cq_ibc_cq_hdl); 888 break; 889 890 case IBT_HDL_EEC: 891 ci_obj_hdl = (void *) 892 (((ibt_eec_hdl_t)(ibt_object_handle))->eec_ibc_eec_hdl); 893 break; 894 895 case IBT_HDL_UD_DEST: 896 ci_obj_hdl = (void *) 897 (((ibt_ud_dest_hdl_t)(ibt_object_handle))->ud_ah); 898 break; 899 900 case IBT_HDL_SRQ: 901 ci_obj_hdl = (void *) 902 (((ibt_srq_hdl_t)(ibt_object_handle))->srq_ibc_srq_hdl); 903 break; 904 905 default: 906 ci_obj_hdl = ibt_object_handle; 907 break; 908 } 909 910 retval = (IBTL_HCA2CIHCAOPS_P(hca)->ibc_ci_data_in)(IBTL_HCA2CIHCA(hca), 911 flags, object, ci_obj_hdl, data_p, data_sz); 912 913 if (retval != IBT_SUCCESS) { 914 IBTF_DPRINTF_L2(ibtf, "ibt_ci_data_in: Failed : %d", retval); 915 } 916 return (retval); 917 } 918 919 /* 920 * Function: 921 * ibt_ci_data_out() 922 * 923 * Input: 924 * hca_hdl HCA Handle. 925 * flags IBT_COMPLETE_ALLOC - Finish a deferred alloc. 926 * object Identifies the type object pointed to by 927 * ibt_object_handle. 928 * 929 * ibt_object_handle The handle of the object to be associated with 930 * the data in/out 931 * 932 * data_p Pointer to a buffer in which to return the CI 933 * private data. The buffer should be allocated 934 * by the caller. 935 * 936 * data_sz The size of the buffer pointed to by 937 * data_p. 938 * Output: 939 * 940 * Returns: 941 * IBT_SUCCESS 942 * IBT_NOT_SUPPORTED Feature not supported. 943 * IBT_INSUFF_RESOURCE The buffer pointed to by data_p was too 944 * small to hold the data. 945 * IBT_INVALID_PARAM Invalid object type specified. 946 * IBT_HCA_HDL_INVALID 947 * IBT_AH_HDL_INVALID/IBT_UD_DEST_HDL_INVALID 948 * IBT_CHAN_HDL_INVALID/IBT_QP_HDL_INVALID 949 * IBT_CQ_HDL_INVALID 950 * IBT_EEC_HDL_INVALID 951 * IBT_RDD_HDL_INVALID 952 * IBT_MW_HDL_INVALID 953 * IBT_PD_HDL_INVALID 954 * IBT_SRQ_HDL_INVALID 955 * 956 * Description: 957 * Exchange CI private data for the specified CI object. 958 */ 959 ibt_status_t 960 ibt_ci_data_out(ibt_hca_hdl_t hca, ibt_ci_data_flags_t flags, 961 ibt_object_type_t object, void *ibt_object_handle, void *data_p, 962 size_t data_sz) 963 { 964 ibt_status_t retval; 965 void *ci_obj_hdl; 966 967 IBTF_DPRINTF_L3(ibtf, "ibt_ci_data_out(%p, %x, %d, %p, %p, %d)", 968 hca, flags, object, ibt_object_handle, data_p, data_sz); 969 970 switch (object) { 971 case IBT_HDL_HCA: 972 ci_obj_hdl = (void *) 973 (IBTL_HCA2CIHCA(((ibt_hca_hdl_t)ibt_object_handle))); 974 break; 975 976 case IBT_HDL_CHANNEL: 977 ci_obj_hdl = (void *) 978 (IBTL_CHAN2CIQP(((ibt_channel_hdl_t)ibt_object_handle))); 979 break; 980 981 case IBT_HDL_CQ: 982 ci_obj_hdl = (void *) 983 (((ibt_cq_hdl_t)(ibt_object_handle))->cq_ibc_cq_hdl); 984 break; 985 986 case IBT_HDL_EEC: 987 ci_obj_hdl = (void *) 988 (((ibt_eec_hdl_t)(ibt_object_handle))->eec_ibc_eec_hdl); 989 break; 990 991 case IBT_HDL_UD_DEST: 992 ci_obj_hdl = (void *) 993 (((ibt_ud_dest_hdl_t)(ibt_object_handle))->ud_ah); 994 break; 995 996 case IBT_HDL_SRQ: 997 ci_obj_hdl = (void *) 998 (((ibt_srq_hdl_t)(ibt_object_handle))->srq_ibc_srq_hdl); 999 break; 1000 1001 default: 1002 ci_obj_hdl = ibt_object_handle; 1003 break; 1004 } 1005 1006 retval = (IBTL_HCA2CIHCAOPS_P(hca)->ibc_ci_data_out) 1007 (IBTL_HCA2CIHCA(hca), flags, object, ci_obj_hdl, data_p, data_sz); 1008 1009 if (retval != IBT_SUCCESS) { 1010 IBTF_DPRINTF_L2(ibtf, "ibt_ci_data_out: Failed : %d", retval); 1011 } 1012 return (retval); 1013 } 1014 1015 1016 /* 1017 * FMA Support functions. 1018 */ 1019 1020 #define IBTL_ENA_MASK 0xC0000000 1021 #define IBTL_ENA_POSSIBLE 0x80000000 1022 #define IBTL_TYPE_SHIFT 27 1023 1024 /* 1025 * Function: 1026 * ibt_get_module_failure() 1027 * 1028 * Input: 1029 * type Identifies the failing IB module. 1030 * ena '0' or the data for Fault Management 1031 * Architecture (ENA). 1032 * 1033 * Returns: 1034 * status Special IB failure status. 1035 * 1036 * Description: 1037 * XXX Just stubbed out to return failures with no data for Fault 1038 * Management Architecture (ENAs) at the moment XXX 1039 */ 1040 ibt_status_t 1041 ibt_get_module_failure(ibt_failure_type_t type, uint64_t ena) 1042 { 1043 ibt_status_t ret; 1044 1045 IBTF_DPRINTF_L3(ibtf, "ibt_get_module_failure(%d, 0x%llX)", type, ena); 1046 1047 switch (type) { 1048 case IBT_FAILURE_CI: 1049 case IBT_FAILURE_IBMF: 1050 case IBT_FAILURE_IBCM: 1051 case IBT_FAILURE_IBDM: 1052 case IBT_FAILURE_IBTL: 1053 case IBT_FAILURE_IBSM: 1054 ret = IBTL_ENA_POSSIBLE | (type << IBTL_TYPE_SHIFT); 1055 break; 1056 default: 1057 ret = IBT_FAILURE; 1058 } 1059 IBTF_DPRINTF_L3(ibtf, "ibt_get_module_failure: ret = 0x%lX", ret); 1060 return (ret); 1061 } 1062 1063 1064 /* 1065 * Function: 1066 * ibc_get_ci_failure() 1067 * 1068 * Input: 1069 * ena '0' or the data for Fault Management 1070 * Architecture (ENA). 1071 * 1072 * Returns: 1073 * status Special CI failure status. 1074 * 1075 * Description: 1076 * Just use the function above to do the job. 1077 */ 1078 ibt_status_t 1079 ibc_get_ci_failure(uint64_t ena) 1080 { 1081 return (ibt_get_module_failure(IBT_FAILURE_CI, ena)); 1082 } 1083 1084 1085 /* 1086 * ibt_check_failure() 1087 * Function to test for special case failures. 1088 * 1089 * status An ibt_status_t returned from an IBTF function call. 1090 * 1091 * reserved_p NULL, or a pointer to where we store the data for 1092 * Fault Management Architecture (ENA). 1093 * 1094 * Description: 1095 * XXX Still need to determine the data for Fault Management Architecture 1096 * (ENA), using 0 for now XXX 1097 */ 1098 ibt_failure_type_t 1099 ibt_check_failure(ibt_status_t status, uint64_t *reserved_p) 1100 { 1101 ibt_failure_type_t type; 1102 1103 IBTF_DPRINTF_L3(ibtf, "ibt_check_failure(%X)", status); 1104 1105 if ((status & IBTL_ENA_MASK) == IBTL_ENA_POSSIBLE) { 1106 type = status & ~IBTL_ENA_POSSIBLE >> IBTL_TYPE_SHIFT; 1107 1108 /* XXX Need more work here... */ 1109 if (reserved_p != NULL) 1110 *reserved_p = 0; 1111 } else { 1112 type = IBT_FAILURE_STANDARD; 1113 if (reserved_p != NULL) 1114 *reserved_p = 0; /* No FMA Data Available. */ 1115 } 1116 IBTF_DPRINTF_L3(ibtf, "ibt_check_failure: type = 0x%X", type); 1117 return (type); 1118 } 1119 1120 /* 1121 * Initialize and create kstats. 1122 * 1123 * We create the following kstats on all ports of the HCA: 1124 * <hca_driver_name><instance_number>/port<port_num>/stats 1125 * <hca_driver_name><instance_number>/port<port_num>/pkeys 1126 */ 1127 static void 1128 ibtl_kstat_init(ibtl_hca_devinfo_t *hca_devp) 1129 { 1130 uint_t nports = hca_devp->hd_hca_attr->hca_nports; 1131 ibtl_hca_port_kstat_t *pks; 1132 int i; 1133 1134 IBTF_DPRINTF_L3(ibtf, "ibtl_kstat_init(hca_devp = 0x%p)", hca_devp); 1135 1136 hca_devp->hd_hca_port_ks_info_len = 1137 sizeof (ibtl_hca_port_kstat_t) * nports; 1138 pks = kmem_zalloc(hca_devp->hd_hca_port_ks_info_len, KM_SLEEP); 1139 hca_devp->hd_hca_port_ks_info = pks; 1140 1141 for (i = 0; i < nports; i++, pks++) { 1142 pks->pks_hca_devp = hca_devp; 1143 pks->pks_port_num = i + 1; 1144 ibtl_kstat_stats_create(hca_devp, i + 1); 1145 ibtl_kstat_pkeys_create(hca_devp, i + 1); 1146 } 1147 } 1148 1149 /* 1150 * Delete kstats on all ports of the HCA. 1151 */ 1152 static void 1153 ibtl_kstat_fini(ibtl_hca_devinfo_t *hca_devp) 1154 { 1155 ibtl_hca_port_kstat_t *pks; 1156 int i; 1157 1158 IBTF_DPRINTF_L3(ibtf, "ibtl_kstat_fini(hca_devp = 0x%p)", hca_devp); 1159 1160 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*hca_devp)) 1161 1162 pks = hca_devp->hd_hca_port_ks_info; 1163 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*pks)) 1164 1165 if (pks == NULL) 1166 return; 1167 1168 for (i = 0; i < hca_devp->hd_hca_attr->hca_nports; i++, pks++) { 1169 if (pks->pks_stats_ksp) 1170 kstat_delete(pks->pks_stats_ksp); 1171 1172 if (pks->pks_pkeys_ksp) { 1173 ASSERT(!MUTEX_HELD(&ibtl_clnt_list_mutex)); 1174 kstat_delete(pks->pks_pkeys_ksp); 1175 } 1176 } 1177 1178 kmem_free(hca_devp->hd_hca_port_ks_info, 1179 hca_devp->hd_hca_port_ks_info_len); 1180 } 1181 1182 /* 1183 * Update "stats" kstat. 1184 * Called by kstat framework. 1185 */ 1186 static int 1187 ibtl_kstat_stats_update(kstat_t *ksp, int rw) 1188 { 1189 ibtl_hca_port_kstat_t *pks; 1190 ibtl_hca_devinfo_t *hca_devp; 1191 ibt_hca_portinfo_t *p; 1192 struct kstat_named *data; 1193 1194 IBTF_DPRINTF_L4(ibtf, "ibtl_kstat_stats_update(ksp = 0x%p, rw = %d)", 1195 ksp, rw); 1196 1197 if (rw == KSTAT_WRITE) 1198 return (EACCES); 1199 1200 mutex_enter(&ibtl_clnt_list_mutex); 1201 1202 /* 1203 * Update the link_state kstat using the value from portinfo cache. 1204 */ 1205 pks = ksp->ks_private; 1206 hca_devp = pks->pks_hca_devp; 1207 data = (struct kstat_named *)(ksp->ks_data); 1208 p = hca_devp->hd_portinfop + pks->pks_port_num - 1; 1209 data[0].value.ui32 = (uint32_t)p->p_linkstate; 1210 1211 mutex_exit(&ibtl_clnt_list_mutex); 1212 1213 return (0); 1214 } 1215 1216 /* 1217 * Create "stats" kstat for the specified HCA port in the form: 1218 * <hca_driver_name><instance_number>/port<port_num>/stats 1219 * At preset it contains only one named data of "link_state" 1220 */ 1221 static void 1222 ibtl_kstat_stats_create(ibtl_hca_devinfo_t *hca_devp, uint_t port_num) 1223 { 1224 struct kstat *ksp; 1225 struct kstat_named *named_data; 1226 char *drv_name; 1227 int drv_instance; 1228 ibtl_hca_port_kstat_t *pks; 1229 char kname[40]; 1230 1231 IBTF_DPRINTF_L3(ibtf, "ibtl_kstat_stats_create(hca_devp = 0x%p, " 1232 "port_num = 0x%u)", hca_devp, port_num); 1233 1234 drv_name = (char *)ddi_driver_name(hca_devp->hd_hca_dip); 1235 drv_instance = ddi_get_instance(hca_devp->hd_hca_dip); 1236 (void) snprintf(kname, sizeof (kname), "%s%d/port%d/stats", 1237 drv_name, drv_instance, port_num); 1238 1239 ksp = kstat_create("ibtf", 0, kname, "ib", KSTAT_TYPE_NAMED, 1, 0); 1240 if (ksp == NULL) { 1241 IBTF_DPRINTF_L2(ibtf, 1242 "ibtl_kstat_stats_create: kstat_create() failed"); 1243 return; 1244 } 1245 1246 named_data = (struct kstat_named *)(ksp->ks_data); 1247 kstat_named_init(&named_data[0], "link_state", KSTAT_DATA_UINT32); 1248 1249 pks = hca_devp->hd_hca_port_ks_info + port_num - 1; 1250 pks->pks_stats_ksp = ksp; 1251 1252 ksp->ks_private = pks; 1253 ksp->ks_update = ibtl_kstat_stats_update; 1254 1255 /* Install the kstat */ 1256 kstat_install(ksp); 1257 } 1258 1259 /* 1260 * Update "pkeys" kstat. 1261 * 1262 * Called by kstat framework. Since ks_lock was set to ibtl_clnt_list_mutex 1263 * at the time of the kstat creation, kstat framework will hold this lock 1264 * while calling this function. 1265 */ 1266 static int 1267 ibtl_kstat_pkeys_update(kstat_t *ksp, int rw) 1268 { 1269 ibtl_hca_port_kstat_t *pks; 1270 ibtl_hca_devinfo_t *hca_devp; 1271 ibt_hca_portinfo_t *p; 1272 1273 IBTF_DPRINTF_L4(ibtf, "ibtl_kstat_pkeys_update(ksp = 0x%p, rw = %d)", 1274 ksp, rw); 1275 1276 #ifndef __lock_lint 1277 ASSERT(MUTEX_HELD(&ibtl_clnt_list_mutex)); 1278 #endif 1279 1280 if (rw == KSTAT_WRITE) 1281 return (EACCES); 1282 1283 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*ksp)) 1284 1285 pks = ksp->ks_private; 1286 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*pks)) 1287 1288 hca_devp = pks->pks_hca_devp; 1289 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*hca_devp)) 1290 1291 /* 1292 * Point kstat data to the pkey table in the portinfo cache. 1293 */ 1294 1295 p = hca_devp->hd_portinfop + pks->pks_port_num - 1; 1296 1297 ksp->ks_data = p->p_pkey_tbl; 1298 ksp->ks_ndata = p->p_pkey_tbl_sz; 1299 ksp->ks_data_size = p->p_pkey_tbl_sz * sizeof (ib_pkey_t); 1300 1301 return (0); 1302 } 1303 1304 /* 1305 * Create "pkeys" kstat for the specified HCA port in the form: 1306 * <hca_driver_name><instance_number>/port<port_num>/pkeys 1307 * 1308 * Currently kstat framework allows only some fixed data types as named 1309 * data components under a named kstat. Due to this limitation it is not 1310 * possible to add "pkeys" as a named data under the "stats" kstat. 1311 */ 1312 static void 1313 ibtl_kstat_pkeys_create(ibtl_hca_devinfo_t *hca_devp, uint_t port_num) 1314 { 1315 struct kstat *ksp; 1316 char *drv_name; 1317 int drv_instance; 1318 char kname[40]; 1319 ibtl_hca_port_kstat_t *pks; 1320 1321 IBTF_DPRINTF_L3(ibtf, "ibtl_kstat_stats_create(hca_devp = 0x%p, " 1322 "port_num = 0x%u)", hca_devp, port_num); 1323 1324 drv_name = (char *)ddi_driver_name(hca_devp->hd_hca_dip); 1325 drv_instance = ddi_get_instance(hca_devp->hd_hca_dip); 1326 (void) snprintf(kname, sizeof (kname), "%s%d/port%d/pkeys", 1327 drv_name, drv_instance, port_num); 1328 1329 ksp = kstat_create("ibtf", 0, kname, "ib", KSTAT_TYPE_RAW, 0, 1330 KSTAT_FLAG_VAR_SIZE | KSTAT_FLAG_VIRTUAL); 1331 if (ksp == NULL) { 1332 IBTF_DPRINTF_L2(ibtf, 1333 "ibtl_kstat_pkeys_create: kstat_create() failed"); 1334 return; 1335 } 1336 1337 pks = hca_devp->hd_hca_port_ks_info + port_num - 1; 1338 pks->pks_pkeys_ksp = ksp; 1339 1340 ksp->ks_private = pks; 1341 ksp->ks_update = ibtl_kstat_pkeys_update; 1342 ksp->ks_lock = &ibtl_clnt_list_mutex; 1343 1344 /* 1345 * We just go with the default_kstat_snapshot(). 1346 * So there is no need to set ks_snapshot field. 1347 */ 1348 1349 /* Install the kstat */ 1350 kstat_install(ksp); 1351 } 1352