1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2006 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 #pragma ident "%Z%%M% %I% %E% SMI" 27 28 /* 29 * ibdm.c 30 * 31 * This file contains the InifiniBand Device Manager (IBDM) support functions. 32 * IB nexus driver will only be the client for the IBDM module. 33 * 34 * IBDM registers with IBTF for HCA arrival/removal notification. 35 * IBDM registers with SA access to send DM MADs to discover the IOC's behind 36 * the IOU's. 37 * 38 * IB nexus driver registers with IBDM to find the information about the 39 * HCA's and IOC's (behind the IOU) present on the IB fabric. 40 */ 41 42 #include <sys/systm.h> 43 #include <sys/taskq.h> 44 #include <sys/ib/mgt/ibdm/ibdm_impl.h> 45 #include <sys/modctl.h> 46 47 /* Function Prototype declarations */ 48 static int ibdm_free_iou_info(ibdm_dp_gidinfo_t *); 49 static int ibdm_fini(void); 50 static int ibdm_init(void); 51 static int ibdm_get_reachable_ports(ibdm_port_attr_t *, 52 ibdm_hca_list_t *); 53 static ibdm_dp_gidinfo_t *ibdm_check_dgid(ib_guid_t, ib_sn_prefix_t); 54 static ibdm_dp_gidinfo_t *ibdm_check_dest_nodeguid(ibdm_dp_gidinfo_t *); 55 static int ibdm_send_classportinfo(ibdm_dp_gidinfo_t *); 56 static int ibdm_send_iounitinfo(ibdm_dp_gidinfo_t *); 57 static int ibdm_is_dev_mgt_supported(ibdm_dp_gidinfo_t *); 58 static int ibdm_get_node_port_guids(ibmf_saa_handle_t, ib_lid_t, 59 ib_guid_t *, ib_guid_t *); 60 static int ibdm_retry_command(ibdm_timeout_cb_args_t *); 61 static int ibdm_get_diagcode(ibdm_dp_gidinfo_t *, int); 62 static int ibdm_verify_mad_status(ib_mad_hdr_t *); 63 static int ibdm_handle_redirection(ibmf_msg_t *, 64 ibdm_dp_gidinfo_t *, int *); 65 static void ibdm_wait_probe_completion(void); 66 static void ibdm_sweep_fabric(int); 67 static void ibdm_probe_gid_thread(void *); 68 static void ibdm_wakeup_probe_gid_cv(void); 69 static void ibdm_port_attr_ibmf_init(ibdm_port_attr_t *, ib_pkey_t, int); 70 static int ibdm_port_attr_ibmf_fini(ibdm_port_attr_t *, int); 71 static void ibdm_update_port_attr(ibdm_port_attr_t *); 72 static void ibdm_handle_hca_attach(ib_guid_t); 73 static void ibdm_handle_srventry_mad(ibmf_msg_t *, 74 ibdm_dp_gidinfo_t *, int *); 75 static void ibdm_ibmf_recv_cb(ibmf_handle_t, ibmf_msg_t *, void *); 76 static void ibdm_recv_incoming_mad(void *); 77 static void ibdm_process_incoming_mad(ibmf_handle_t, ibmf_msg_t *, void *); 78 static void ibdm_ibmf_send_cb(ibmf_handle_t, ibmf_msg_t *, void *); 79 static void ibdm_pkt_timeout_hdlr(void *arg); 80 static void ibdm_initialize_port(ibdm_port_attr_t *); 81 static void ibdm_handle_diagcode(ibmf_msg_t *, ibdm_dp_gidinfo_t *, int *); 82 static void ibdm_probe_gid(ibdm_dp_gidinfo_t *); 83 static void ibdm_alloc_send_buffers(ibmf_msg_t *); 84 static void ibdm_free_send_buffers(ibmf_msg_t *); 85 static void ibdm_handle_hca_detach(ib_guid_t); 86 static int ibdm_fini_port(ibdm_port_attr_t *); 87 static int ibdm_uninit_hca(ibdm_hca_list_t *); 88 static void ibdm_handle_iounitinfo(ibmf_handle_t, 89 ibmf_msg_t *, ibdm_dp_gidinfo_t *, int *); 90 static void ibdm_handle_ioc_profile(ibmf_handle_t, 91 ibmf_msg_t *, ibdm_dp_gidinfo_t *, int *); 92 static void ibdm_event_hdlr(void *, ibt_hca_hdl_t, 93 ibt_async_code_t, ibt_async_event_t *); 94 static void ibdm_handle_classportinfo(ibmf_handle_t, 95 ibmf_msg_t *, ibdm_dp_gidinfo_t *, int *); 96 static void ibdm_update_ioc_port_gidlist(ibdm_ioc_info_t *, 97 ibdm_dp_gidinfo_t *); 98 99 static ibdm_hca_list_t *ibdm_dup_hca_attr(ibdm_hca_list_t *); 100 static ibdm_ioc_info_t *ibdm_dup_ioc_info(ibdm_ioc_info_t *, 101 ibdm_dp_gidinfo_t *gid_list); 102 static void ibdm_probe_ioc(ib_guid_t, ib_guid_t, int); 103 static ibdm_ioc_info_t *ibdm_is_ioc_present(ib_guid_t, 104 ibdm_dp_gidinfo_t *, int *); 105 static ibdm_port_attr_t *ibdm_get_port_attr(ibt_async_event_t *, 106 ibdm_hca_list_t **); 107 static sa_node_record_t *ibdm_get_node_records(ibmf_saa_handle_t, 108 size_t *, ib_guid_t); 109 static sa_portinfo_record_t *ibdm_get_portinfo(ibmf_saa_handle_t, size_t *, 110 ib_lid_t); 111 static ibdm_dp_gidinfo_t *ibdm_create_gid_info(ibdm_port_attr_t *, 112 ib_gid_t, ib_gid_t); 113 static ibdm_dp_gidinfo_t *ibdm_find_gid(ib_guid_t, ib_guid_t); 114 static int ibdm_send_ioc_profile(ibdm_dp_gidinfo_t *, uint8_t); 115 static ibdm_ioc_info_t *ibdm_update_ioc_gidlist(ibdm_dp_gidinfo_t *, int); 116 static void ibdm_saa_event_cb(ibmf_saa_handle_t, ibmf_saa_subnet_event_t, 117 ibmf_saa_event_details_t *, void *); 118 static void ibdm_reprobe_update_port_srv(ibdm_ioc_info_t *, 119 ibdm_dp_gidinfo_t *); 120 static ibdm_dp_gidinfo_t *ibdm_handle_gid_rm(ibdm_dp_gidinfo_t *); 121 static void ibdm_rmfrom_glgid_list(ibdm_dp_gidinfo_t *, 122 ibdm_dp_gidinfo_t *); 123 static void ibdm_addto_gidlist(ibdm_gid_t **, ibdm_gid_t *); 124 static void ibdm_free_gid_list(ibdm_gid_t *); 125 static void ibdm_rescan_gidlist(ib_guid_t *ioc_guid); 126 static void ibdm_notify_newgid_iocs(ibdm_dp_gidinfo_t *); 127 static void ibdm_saa_event_taskq(void *); 128 static void ibdm_free_saa_event_arg(ibdm_saa_event_arg_t *); 129 static void ibdm_get_next_port(ibdm_hca_list_t **, 130 ibdm_port_attr_t **, int); 131 static void ibdm_add_to_gl_gid(ibdm_dp_gidinfo_t *, 132 ibdm_dp_gidinfo_t *); 133 static void ibdm_addto_glhcalist(ibdm_dp_gidinfo_t *, 134 ibdm_hca_list_t *); 135 static void ibdm_delete_glhca_list(ibdm_dp_gidinfo_t *); 136 static void ibdm_saa_handle_new_gid(void *); 137 static void ibdm_reset_all_dgids(ibmf_saa_handle_t); 138 static void ibdm_reset_gidinfo(ibdm_dp_gidinfo_t *); 139 static void ibdm_delete_gidinfo(ibdm_dp_gidinfo_t *); 140 static void ibdm_fill_srv_attr_mod(ib_mad_hdr_t *, ibdm_timeout_cb_args_t *); 141 static void ibdm_bump_transactionID(ibdm_dp_gidinfo_t *); 142 143 int ibdm_dft_timeout = IBDM_DFT_TIMEOUT; 144 int ibdm_dft_retry_cnt = IBDM_DFT_NRETRIES; 145 #ifdef DEBUG 146 int ibdm_ignore_saa_event = 0; 147 #endif 148 149 /* Modload support */ 150 static struct modlmisc ibdm_modlmisc = { 151 &mod_miscops, 152 "InfiniBand Device Manager %I%", 153 }; 154 155 struct modlinkage ibdm_modlinkage = { 156 MODREV_1, 157 (void *)&ibdm_modlmisc, 158 NULL 159 }; 160 161 static ibt_clnt_modinfo_t ibdm_ibt_modinfo = { 162 IBTI_V2, 163 IBT_DM, 164 ibdm_event_hdlr, 165 NULL, 166 "ibdm" 167 }; 168 169 /* Global variables */ 170 ibdm_t ibdm; 171 int ibdm_taskq_enable = IBDM_ENABLE_TASKQ_HANDLING; 172 char *ibdm_string = "ibdm"; 173 174 _NOTE(SCHEME_PROTECTS_DATA("Serialized access by cv", 175 ibdm.ibdm_dp_gidlist_head)) 176 177 /* 178 * _init 179 * Loadable module init, called before any other module. 180 * Initialize mutex 181 * Register with IBTF 182 */ 183 int 184 _init(void) 185 { 186 int err; 187 188 IBTF_DPRINTF_L4("ibdm", "\t_init: addr of ibdm %p", &ibdm); 189 190 if ((err = ibdm_init()) != IBDM_SUCCESS) { 191 IBTF_DPRINTF_L2("ibdm", "_init: ibdm_init failed 0x%x", err); 192 (void) ibdm_fini(); 193 return (DDI_FAILURE); 194 } 195 196 if ((err = mod_install(&ibdm_modlinkage)) != 0) { 197 IBTF_DPRINTF_L2("ibdm", "_init: mod_install failed 0x%x", err); 198 (void) ibdm_fini(); 199 } 200 return (err); 201 } 202 203 204 int 205 _fini(void) 206 { 207 int err; 208 209 if ((err = ibdm_fini()) != IBDM_SUCCESS) { 210 IBTF_DPRINTF_L2("ibdm", "_fini: ibdm_fini failed 0x%x", err); 211 (void) ibdm_init(); 212 return (EBUSY); 213 } 214 215 if ((err = mod_remove(&ibdm_modlinkage)) != 0) { 216 IBTF_DPRINTF_L2("ibdm", "_fini: mod_remove failed 0x%x", err); 217 (void) ibdm_init(); 218 } 219 return (err); 220 } 221 222 223 int 224 _info(struct modinfo *modinfop) 225 { 226 return (mod_info(&ibdm_modlinkage, modinfop)); 227 } 228 229 230 /* 231 * ibdm_init(): 232 * Register with IBTF 233 * Allocate memory for the HCAs 234 * Allocate minor-nodes for the HCAs 235 */ 236 static int 237 ibdm_init(void) 238 { 239 int i, hca_count; 240 ib_guid_t *hca_guids; 241 ibt_status_t status; 242 243 IBTF_DPRINTF_L4("ibdm", "\tibdm_init:"); 244 if (!(ibdm.ibdm_state & IBDM_LOCKS_ALLOCED)) { 245 mutex_init(&ibdm.ibdm_mutex, NULL, MUTEX_DEFAULT, NULL); 246 mutex_init(&ibdm.ibdm_hl_mutex, NULL, MUTEX_DEFAULT, NULL); 247 mutex_init(&ibdm.ibdm_ibnex_mutex, NULL, MUTEX_DEFAULT, NULL); 248 mutex_enter(&ibdm.ibdm_mutex); 249 ibdm.ibdm_state |= IBDM_LOCKS_ALLOCED; 250 } 251 252 if (!(ibdm.ibdm_state & IBDM_IBT_ATTACHED)) { 253 if ((status = ibt_attach(&ibdm_ibt_modinfo, NULL, NULL, 254 (void *)&ibdm.ibdm_ibt_clnt_hdl)) != IBT_SUCCESS) { 255 IBTF_DPRINTF_L2("ibdm", "ibdm_init: ibt_attach " 256 "failed %x", status); 257 mutex_exit(&ibdm.ibdm_mutex); 258 return (IBDM_FAILURE); 259 } 260 261 ibdm.ibdm_state |= IBDM_IBT_ATTACHED; 262 mutex_exit(&ibdm.ibdm_mutex); 263 } 264 265 266 if (!(ibdm.ibdm_state & IBDM_HCA_ATTACHED)) { 267 hca_count = ibt_get_hca_list(&hca_guids); 268 IBTF_DPRINTF_L4("ibdm", "ibdm_init: num_hcas = %d", hca_count); 269 for (i = 0; i < hca_count; i++) 270 (void) ibdm_handle_hca_attach(hca_guids[i]); 271 if (hca_count) 272 ibt_free_hca_list(hca_guids, hca_count); 273 274 mutex_enter(&ibdm.ibdm_mutex); 275 ibdm.ibdm_state |= IBDM_HCA_ATTACHED; 276 mutex_exit(&ibdm.ibdm_mutex); 277 } 278 279 if (!(ibdm.ibdm_state & IBDM_CVS_ALLOCED)) { 280 cv_init(&ibdm.ibdm_probe_cv, NULL, CV_DRIVER, NULL); 281 cv_init(&ibdm.ibdm_busy_cv, NULL, CV_DRIVER, NULL); 282 mutex_enter(&ibdm.ibdm_mutex); 283 ibdm.ibdm_state |= IBDM_CVS_ALLOCED; 284 mutex_exit(&ibdm.ibdm_mutex); 285 } 286 return (IBDM_SUCCESS); 287 } 288 289 290 static int 291 ibdm_free_iou_info(ibdm_dp_gidinfo_t *gid_info) 292 { 293 int ii, k, niocs; 294 size_t size; 295 ibdm_gid_t *delete, *head; 296 timeout_id_t timeout_id; 297 ibdm_ioc_info_t *ioc; 298 299 ASSERT(mutex_owned(&gid_info->gl_mutex)); 300 if (gid_info->gl_iou == NULL) { 301 IBTF_DPRINTF_L4("ibdm", "\tibdm_free_iou_info: No IOU"); 302 return (0); 303 } 304 305 niocs = gid_info->gl_iou->iou_info.iou_num_ctrl_slots; 306 IBTF_DPRINTF_L4("ibdm", "\tfree_iou_info: gid_info = %p, niocs %d", 307 gid_info, niocs); 308 309 for (ii = 0; ii < niocs; ii++) { 310 ioc = IBDM_GIDINFO2IOCINFO(gid_info, ii); 311 312 /* handle the case where an ioc_timeout_id is scheduled */ 313 if (ioc->ioc_timeout_id) { 314 timeout_id = ioc->ioc_timeout_id; 315 mutex_exit(&gid_info->gl_mutex); 316 IBTF_DPRINTF_L5("ibdm", "free_iou_info: " 317 "ioc_timeout_id = 0x%x", timeout_id); 318 if (untimeout(timeout_id) == -1) { 319 IBTF_DPRINTF_L2("ibdm", "free_iou_info: " 320 "untimeout ioc_timeout_id failed"); 321 mutex_enter(&gid_info->gl_mutex); 322 return (-1); 323 } 324 mutex_enter(&gid_info->gl_mutex); 325 ioc->ioc_timeout_id = 0; 326 } 327 328 /* handle the case where an ioc_dc_timeout_id is scheduled */ 329 if (ioc->ioc_dc_timeout_id) { 330 timeout_id = ioc->ioc_dc_timeout_id; 331 mutex_exit(&gid_info->gl_mutex); 332 IBTF_DPRINTF_L5("ibdm", "free_iou_info: " 333 "ioc_dc_timeout_id = 0x%x", timeout_id); 334 if (untimeout(timeout_id) == -1) { 335 IBTF_DPRINTF_L2("ibdm", "free_iou_info: " 336 "untimeout ioc_dc_timeout_id failed"); 337 mutex_enter(&gid_info->gl_mutex); 338 return (-1); 339 } 340 mutex_enter(&gid_info->gl_mutex); 341 ioc->ioc_dc_timeout_id = 0; 342 } 343 344 /* handle the case where serv[k].se_timeout_id is scheduled */ 345 for (k = 0; k < ioc->ioc_profile.ioc_service_entries; k++) { 346 if (ioc->ioc_serv[k].se_timeout_id) { 347 timeout_id = ioc->ioc_serv[k].se_timeout_id; 348 mutex_exit(&gid_info->gl_mutex); 349 IBTF_DPRINTF_L5("ibdm", "free_iou_info: " 350 "ioc->ioc_serv[%d].se_timeout_id = 0x%x", 351 k, timeout_id); 352 if (untimeout(timeout_id) == -1) { 353 IBTF_DPRINTF_L2("ibdm", "free_iou_info:" 354 " untimeout se_timeout_id failed"); 355 mutex_enter(&gid_info->gl_mutex); 356 return (-1); 357 } 358 mutex_enter(&gid_info->gl_mutex); 359 ioc->ioc_serv[k].se_timeout_id = 0; 360 } 361 } 362 363 /* delete GID list */ 364 head = ioc->ioc_gid_list; 365 while (head) { 366 IBTF_DPRINTF_L4("ibdm", "\tibdm_free_iou_info: " 367 "Deleting gid_list struct %p", head); 368 delete = head; 369 head = head->gid_next; 370 kmem_free(delete, sizeof (ibdm_gid_t)); 371 } 372 ioc->ioc_gid_list = NULL; 373 374 /* delete ioc_serv */ 375 size = ioc->ioc_profile.ioc_service_entries * 376 sizeof (ibdm_srvents_info_t); 377 if (ioc->ioc_serv && size) { 378 kmem_free(ioc->ioc_serv, size); 379 ioc->ioc_serv = NULL; 380 } 381 } 382 383 IBTF_DPRINTF_L4("ibdm", "\tibdm_free_iou_info: deleting IOU & IOC"); 384 size = sizeof (ibdm_iou_info_t) + niocs * sizeof (ibdm_ioc_info_t); 385 kmem_free(gid_info->gl_iou, size); 386 gid_info->gl_iou = NULL; 387 return (0); 388 } 389 390 391 /* 392 * ibdm_fini(): 393 * Un-register with IBTF 394 * De allocate memory for the GID info 395 */ 396 static int 397 ibdm_fini() 398 { 399 int ii; 400 ibdm_hca_list_t *hca_list, *temp; 401 ibdm_dp_gidinfo_t *gid_info, *tmp; 402 ibdm_gid_t *head, *delete; 403 404 IBTF_DPRINTF_L4("ibdm", "\tibdm_fini"); 405 406 mutex_enter(&ibdm.ibdm_hl_mutex); 407 if (ibdm.ibdm_state & IBDM_IBT_ATTACHED) { 408 if (ibt_detach(ibdm.ibdm_ibt_clnt_hdl) != IBT_SUCCESS) { 409 IBTF_DPRINTF_L2("ibdm", "\t_fini: ibt_detach failed"); 410 mutex_exit(&ibdm.ibdm_hl_mutex); 411 return (IBDM_FAILURE); 412 } 413 ibdm.ibdm_state &= ~IBDM_IBT_ATTACHED; 414 ibdm.ibdm_ibt_clnt_hdl = NULL; 415 } 416 417 hca_list = ibdm.ibdm_hca_list_head; 418 IBTF_DPRINTF_L4("ibdm", "\tibdm_fini: nhcas %d", ibdm.ibdm_hca_count); 419 for (ii = 0; ii < ibdm.ibdm_hca_count; ii++) { 420 temp = hca_list; 421 hca_list = hca_list->hl_next; 422 IBTF_DPRINTF_L4("ibdm", "\tibdm_fini: hca %p", temp); 423 if (ibdm_uninit_hca(temp) != IBDM_SUCCESS) { 424 IBTF_DPRINTF_L2("ibdm", "\tibdm_fini: " 425 "uninit_hca %p failed", temp); 426 mutex_exit(&ibdm.ibdm_hl_mutex); 427 return (IBDM_FAILURE); 428 } 429 } 430 mutex_exit(&ibdm.ibdm_hl_mutex); 431 432 mutex_enter(&ibdm.ibdm_mutex); 433 if (ibdm.ibdm_state & IBDM_HCA_ATTACHED) 434 ibdm.ibdm_state &= ~IBDM_HCA_ATTACHED; 435 436 gid_info = ibdm.ibdm_dp_gidlist_head; 437 while (gid_info) { 438 mutex_enter(&gid_info->gl_mutex); 439 (void) ibdm_free_iou_info(gid_info); 440 mutex_exit(&gid_info->gl_mutex); 441 ibdm_delete_glhca_list(gid_info); 442 443 tmp = gid_info; 444 gid_info = gid_info->gl_next; 445 mutex_destroy(&tmp->gl_mutex); 446 head = tmp->gl_gid; 447 while (head) { 448 IBTF_DPRINTF_L4("ibdm", 449 "\tibdm_fini: Deleting gid structs"); 450 delete = head; 451 head = head->gid_next; 452 kmem_free(delete, sizeof (ibdm_gid_t)); 453 } 454 kmem_free(tmp, sizeof (ibdm_dp_gidinfo_t)); 455 } 456 mutex_exit(&ibdm.ibdm_mutex); 457 458 if (ibdm.ibdm_state & IBDM_LOCKS_ALLOCED) { 459 ibdm.ibdm_state &= ~IBDM_LOCKS_ALLOCED; 460 mutex_destroy(&ibdm.ibdm_mutex); 461 mutex_destroy(&ibdm.ibdm_hl_mutex); 462 mutex_destroy(&ibdm.ibdm_ibnex_mutex); 463 } 464 if (ibdm.ibdm_state & IBDM_CVS_ALLOCED) { 465 ibdm.ibdm_state &= ~IBDM_CVS_ALLOCED; 466 cv_destroy(&ibdm.ibdm_probe_cv); 467 cv_destroy(&ibdm.ibdm_busy_cv); 468 } 469 return (IBDM_SUCCESS); 470 } 471 472 473 /* 474 * ibdm_event_hdlr() 475 * 476 * IBDM registers this asynchronous event handler at the time of 477 * ibt_attach. IBDM support the following async events. For other 478 * event, simply returns success. 479 * IBT_HCA_ATTACH_EVENT: 480 * Retrieves the information about all the port that are 481 * present on this HCA, allocates the port attributes 482 * structure and calls IB nexus callback routine with 483 * the port attributes structure as an input argument. 484 * IBT_HCA_DETACH_EVENT: 485 * Retrieves the information about all the ports that are 486 * present on this HCA and calls IB nexus callback with 487 * port guid as an argument 488 * IBT_EVENT_PORT_UP: 489 * Register with IBMF and SA access 490 * Setup IBMF receive callback routine 491 * IBT_EVENT_PORT_DOWN: 492 * Un-Register with IBMF and SA access 493 * Teardown IBMF receive callback routine 494 */ 495 /*ARGSUSED*/ 496 static void 497 ibdm_event_hdlr(void *clnt_hdl, 498 ibt_hca_hdl_t hca_hdl, ibt_async_code_t code, ibt_async_event_t *event) 499 { 500 ibdm_hca_list_t *hca_list; 501 ibdm_port_attr_t *port; 502 ibmf_saa_handle_t port_sa_hdl; 503 504 IBTF_DPRINTF_L4("ibdm", "\tevent_hdlr: async code 0x%x", code); 505 506 switch (code) { 507 case IBT_HCA_ATTACH_EVENT: /* New HCA registered with IBTF */ 508 ibdm_handle_hca_attach(event->ev_hca_guid); 509 break; 510 511 case IBT_HCA_DETACH_EVENT: /* HCA unregistered with IBTF */ 512 ibdm_handle_hca_detach(event->ev_hca_guid); 513 mutex_enter(&ibdm.ibdm_ibnex_mutex); 514 if (ibdm.ibdm_ibnex_callback != NULL) { 515 (*ibdm.ibdm_ibnex_callback)((void *) 516 &event->ev_hca_guid, IBDM_EVENT_HCA_REMOVED); 517 } 518 mutex_exit(&ibdm.ibdm_ibnex_mutex); 519 break; 520 521 case IBT_EVENT_PORT_UP: 522 IBTF_DPRINTF_L4("ibdm", "\tevent_hdlr: PORT_UP"); 523 mutex_enter(&ibdm.ibdm_hl_mutex); 524 port = ibdm_get_port_attr(event, &hca_list); 525 if (port == NULL) { 526 IBTF_DPRINTF_L2("ibdm", 527 "\tevent_hdlr: HCA not present"); 528 mutex_exit(&ibdm.ibdm_hl_mutex); 529 break; 530 } 531 ibdm_initialize_port(port); 532 hca_list->hl_nports_active++; 533 mutex_exit(&ibdm.ibdm_hl_mutex); 534 break; 535 536 case IBT_ERROR_PORT_DOWN: 537 IBTF_DPRINTF_L4("ibdm", "\tevent_hdlr: PORT_DOWN"); 538 mutex_enter(&ibdm.ibdm_hl_mutex); 539 port = ibdm_get_port_attr(event, &hca_list); 540 if (port == NULL) { 541 IBTF_DPRINTF_L2("ibdm", 542 "\tevent_hdlr: HCA not present"); 543 mutex_exit(&ibdm.ibdm_hl_mutex); 544 break; 545 } 546 hca_list->hl_nports_active--; 547 port_sa_hdl = port->pa_sa_hdl; 548 (void) ibdm_fini_port(port); 549 mutex_exit(&ibdm.ibdm_hl_mutex); 550 ibdm_reset_all_dgids(port_sa_hdl); 551 break; 552 553 default: /* Ignore all other events/errors */ 554 break; 555 } 556 } 557 558 559 /* 560 * ibdm_initialize_port() 561 * Register with IBMF 562 * Register with SA access 563 * Register a receive callback routine with IBMF. IBMF invokes 564 * this routine whenever a MAD arrives at this port. 565 * Update the port attributes 566 */ 567 static void 568 ibdm_initialize_port(ibdm_port_attr_t *port) 569 { 570 int ii; 571 uint_t nports, size; 572 uint_t pkey_idx; 573 ib_pkey_t pkey; 574 ibt_hca_portinfo_t *pinfop; 575 ibmf_register_info_t ibmf_reg; 576 ibmf_saa_subnet_event_args_t event_args; 577 578 IBTF_DPRINTF_L4("ibdm", "\tinitialize_port:"); 579 ASSERT(MUTEX_HELD(&ibdm.ibdm_hl_mutex)); 580 581 /* Check whether the port is active */ 582 if (ibt_get_port_state(port->pa_hca_hdl, port->pa_port_num, NULL, 583 NULL) != IBT_SUCCESS) 584 return; 585 586 if (port->pa_sa_hdl != NULL) 587 return; 588 589 if (ibt_query_hca_ports(port->pa_hca_hdl, port->pa_port_num, 590 &pinfop, &nports, &size) != IBT_SUCCESS) { 591 /* This should not occur */ 592 port->pa_npkeys = 0; 593 port->pa_pkey_tbl = NULL; 594 return; 595 } 596 port->pa_sn_prefix = pinfop->p_sgid_tbl[0].gid_prefix; 597 598 port->pa_state = pinfop->p_linkstate; 599 port->pa_npkeys = pinfop->p_pkey_tbl_sz; 600 port->pa_pkey_tbl = (ibdm_pkey_tbl_t *)kmem_zalloc( 601 port->pa_npkeys * sizeof (ibdm_pkey_tbl_t), KM_SLEEP); 602 603 for (pkey_idx = 0; pkey_idx < port->pa_npkeys; pkey_idx++) 604 port->pa_pkey_tbl[pkey_idx].pt_pkey = 605 pinfop->p_pkey_tbl[pkey_idx]; 606 607 ibt_free_portinfo(pinfop, size); 608 609 event_args.is_event_callback = ibdm_saa_event_cb; 610 event_args.is_event_callback_arg = port; 611 if (ibmf_sa_session_open(port->pa_port_guid, 0, &event_args, 612 IBMF_VERSION, 0, &port->pa_sa_hdl) != IBMF_SUCCESS) { 613 IBTF_DPRINTF_L2("ibdm", "\tinitialize_port: " 614 "sa access registration failed"); 615 return; 616 } 617 ibmf_reg.ir_ci_guid = port->pa_hca_guid; 618 ibmf_reg.ir_port_num = port->pa_port_num; 619 ibmf_reg.ir_client_class = DEV_MGT_MANAGER; 620 621 if (ibmf_register(&ibmf_reg, IBMF_VERSION, 0, NULL, NULL, 622 &port->pa_ibmf_hdl, &port->pa_ibmf_caps) != IBMF_SUCCESS) { 623 IBTF_DPRINTF_L2("ibdm", "\tinitialize_port: " 624 "IBMF registration failed"); 625 (void) ibdm_fini_port(port); 626 return; 627 } 628 if (ibmf_setup_async_cb(port->pa_ibmf_hdl, IBMF_QP_HANDLE_DEFAULT, 629 ibdm_ibmf_recv_cb, 0, 0) != IBMF_SUCCESS) { 630 IBTF_DPRINTF_L2("ibdm", "\tinitialize_port: " 631 "IBMF setup recv cb failed"); 632 (void) ibdm_fini_port(port); 633 return; 634 } 635 636 for (ii = 0; ii < port->pa_npkeys; ii++) { 637 pkey = port->pa_pkey_tbl[ii].pt_pkey; 638 if (IBDM_INVALID_PKEY(pkey)) { 639 port->pa_pkey_tbl[ii].pt_qp_hdl = NULL; 640 continue; 641 } 642 ibdm_port_attr_ibmf_init(port, pkey, ii); 643 } 644 } 645 646 647 /* 648 * ibdm_port_attr_ibmf_init: 649 * With IBMF - Alloc QP Handle and Setup Async callback 650 */ 651 static void 652 ibdm_port_attr_ibmf_init(ibdm_port_attr_t *port, ib_pkey_t pkey, int ii) 653 { 654 int ret; 655 656 if ((ret = ibmf_alloc_qp(port->pa_ibmf_hdl, pkey, IB_GSI_QKEY, 657 IBMF_ALT_QP_MAD_NO_RMPP, &port->pa_pkey_tbl[ii].pt_qp_hdl)) != 658 IBMF_SUCCESS) { 659 IBTF_DPRINTF_L2("ibdm", "\tport_attr_ibmf_init: " 660 "IBMF failed to alloc qp %d", ret); 661 port->pa_pkey_tbl[ii].pt_qp_hdl = NULL; 662 return; 663 } 664 665 IBTF_DPRINTF_L4("ibdm", "\tport_attr_ibmf_init: QP handle is %p", 666 port->pa_ibmf_hdl); 667 668 if ((ret = ibmf_setup_async_cb(port->pa_ibmf_hdl, 669 port->pa_pkey_tbl[ii].pt_qp_hdl, ibdm_ibmf_recv_cb, 0, 0)) != 670 IBMF_SUCCESS) { 671 IBTF_DPRINTF_L2("ibdm", "\tport_attr_ibmf_init: " 672 "IBMF setup recv cb failed %d", ret); 673 (void) ibmf_free_qp(port->pa_ibmf_hdl, 674 &port->pa_pkey_tbl[ii].pt_qp_hdl, 0); 675 port->pa_pkey_tbl[ii].pt_qp_hdl = NULL; 676 } 677 } 678 679 680 /* 681 * ibdm_get_port_attr() 682 * Get port attributes from HCA guid and port number 683 * Return pointer to ibdm_port_attr_t on Success 684 * and NULL on failure 685 */ 686 static ibdm_port_attr_t * 687 ibdm_get_port_attr(ibt_async_event_t *event, ibdm_hca_list_t **retval) 688 { 689 ibdm_hca_list_t *hca_list; 690 ibdm_port_attr_t *port_attr; 691 int ii; 692 693 IBTF_DPRINTF_L4("ibdm", "\tget_port_attr: port# %d", event->ev_port); 694 ASSERT(MUTEX_HELD(&ibdm.ibdm_hl_mutex)); 695 hca_list = ibdm.ibdm_hca_list_head; 696 while (hca_list) { 697 if (hca_list->hl_hca_guid == event->ev_hca_guid) { 698 for (ii = 0; ii < hca_list->hl_nports; ii++) { 699 port_attr = &hca_list->hl_port_attr[ii]; 700 if (port_attr->pa_port_num == event->ev_port) { 701 *retval = hca_list; 702 return (port_attr); 703 } 704 } 705 } 706 hca_list = hca_list->hl_next; 707 } 708 return (NULL); 709 } 710 711 712 /* 713 * ibdm_update_port_attr() 714 * Update the port attributes 715 */ 716 static void 717 ibdm_update_port_attr(ibdm_port_attr_t *port) 718 { 719 uint_t nports, size; 720 uint_t pkey_idx; 721 ibt_hca_portinfo_t *portinfop; 722 723 IBTF_DPRINTF_L4("ibdm", "\tupdate_port_attr: Begin"); 724 if (ibt_query_hca_ports(port->pa_hca_hdl, 725 port->pa_port_num, &portinfop, &nports, &size) != IBT_SUCCESS) { 726 /* This should not occur */ 727 port->pa_npkeys = 0; 728 port->pa_pkey_tbl = NULL; 729 return; 730 } 731 port->pa_sn_prefix = portinfop->p_sgid_tbl[0].gid_prefix; 732 733 port->pa_state = portinfop->p_linkstate; 734 735 /* 736 * PKey information in portinfo valid only if port is 737 * ACTIVE. Bail out if not. 738 */ 739 if (port->pa_state != IBT_PORT_ACTIVE) { 740 port->pa_npkeys = 0; 741 port->pa_pkey_tbl = NULL; 742 ibt_free_portinfo(portinfop, size); 743 return; 744 } 745 746 port->pa_npkeys = portinfop->p_pkey_tbl_sz; 747 port->pa_pkey_tbl = (ibdm_pkey_tbl_t *)kmem_zalloc( 748 port->pa_npkeys * sizeof (ibdm_pkey_tbl_t), KM_SLEEP); 749 750 for (pkey_idx = 0; pkey_idx < port->pa_npkeys; pkey_idx++) { 751 port->pa_pkey_tbl[pkey_idx].pt_pkey = 752 portinfop->p_pkey_tbl[pkey_idx]; 753 } 754 ibt_free_portinfo(portinfop, size); 755 } 756 757 758 /* 759 * ibdm_handle_hca_attach() 760 */ 761 static void 762 ibdm_handle_hca_attach(ib_guid_t hca_guid) 763 { 764 uint_t size; 765 uint_t ii, nports; 766 ibt_status_t status; 767 ibt_hca_hdl_t hca_hdl; 768 ibt_hca_attr_t *hca_attr; 769 ibdm_hca_list_t *hca_list, *temp; 770 ibdm_port_attr_t *port_attr; 771 ibt_hca_portinfo_t *portinfop; 772 773 IBTF_DPRINTF_L4("ibdm", 774 "\thandle_hca_attach: hca_guid = 0x%llX", hca_guid); 775 776 /* open the HCA first */ 777 if ((status = ibt_open_hca(ibdm.ibdm_ibt_clnt_hdl, hca_guid, 778 &hca_hdl)) != IBT_SUCCESS) { 779 IBTF_DPRINTF_L2("ibdm", "\thandle_hca_attach: " 780 "open_hca failed, status 0x%x", status); 781 return; 782 } 783 784 hca_attr = (ibt_hca_attr_t *) 785 kmem_alloc(sizeof (ibt_hca_attr_t), KM_SLEEP); 786 /* ibt_query_hca always returns IBT_SUCCESS */ 787 (void) ibt_query_hca(hca_hdl, hca_attr); 788 789 IBTF_DPRINTF_L4("ibdm", "\tvid: 0x%x, pid: 0x%x, ver: 0x%x," 790 " #ports: %d", hca_attr->hca_vendor_id, hca_attr->hca_device_id, 791 hca_attr->hca_version_id, hca_attr->hca_nports); 792 793 if ((status = ibt_query_hca_ports(hca_hdl, 0, &portinfop, &nports, 794 &size)) != IBT_SUCCESS) { 795 IBTF_DPRINTF_L2("ibdm", "\thandle_hca_attach: " 796 "ibt_query_hca_ports failed, status 0x%x", status); 797 kmem_free(hca_attr, sizeof (ibt_hca_attr_t)); 798 (void) ibt_close_hca(hca_hdl); 799 return; 800 } 801 hca_list = (ibdm_hca_list_t *) 802 kmem_zalloc((sizeof (ibdm_hca_list_t)), KM_SLEEP); 803 hca_list->hl_port_attr = (ibdm_port_attr_t *)kmem_zalloc( 804 (sizeof (ibdm_port_attr_t) * hca_attr->hca_nports), KM_SLEEP); 805 hca_list->hl_hca_guid = hca_attr->hca_node_guid; 806 hca_list->hl_nports = hca_attr->hca_nports; 807 hca_list->hl_attach_time = ddi_get_time(); 808 hca_list->hl_hca_hdl = hca_hdl; 809 810 /* 811 * Init a dummy port attribute for the HCA node 812 * This is for Per-HCA Node. Initialize port_attr : 813 * hca_guid & port_guid -> hca_guid 814 * npkeys, pkey_tbl is NULL 815 * port_num, sn_prefix is 0 816 * vendorid, product_id, dev_version from HCA 817 * pa_state is IBT_PORT_ACTIVE 818 */ 819 hca_list->hl_hca_port_attr = (ibdm_port_attr_t *)kmem_zalloc( 820 sizeof (ibdm_port_attr_t), KM_SLEEP); 821 port_attr = hca_list->hl_hca_port_attr; 822 port_attr->pa_vendorid = hca_attr->hca_vendor_id; 823 port_attr->pa_productid = hca_attr->hca_device_id; 824 port_attr->pa_dev_version = hca_attr->hca_version_id; 825 port_attr->pa_hca_guid = hca_attr->hca_node_guid; 826 port_attr->pa_hca_hdl = hca_list->hl_hca_hdl; 827 port_attr->pa_port_guid = hca_attr->hca_node_guid; 828 port_attr->pa_state = IBT_PORT_ACTIVE; 829 830 831 for (ii = 0; ii < nports; ii++) { 832 port_attr = &hca_list->hl_port_attr[ii]; 833 port_attr->pa_vendorid = hca_attr->hca_vendor_id; 834 port_attr->pa_productid = hca_attr->hca_device_id; 835 port_attr->pa_dev_version = hca_attr->hca_version_id; 836 port_attr->pa_hca_guid = hca_attr->hca_node_guid; 837 port_attr->pa_hca_hdl = hca_list->hl_hca_hdl; 838 port_attr->pa_port_guid = portinfop[ii].p_sgid_tbl->gid_guid; 839 port_attr->pa_sn_prefix = portinfop[ii].p_sgid_tbl->gid_prefix; 840 port_attr->pa_port_num = portinfop[ii].p_port_num; 841 port_attr->pa_state = portinfop[ii].p_linkstate; 842 843 /* 844 * Register with IBMF, SA access when the port is in 845 * ACTIVE state. Also register a callback routine 846 * with IBMF to receive incoming DM MAD's. 847 * The IBDM event handler takes care of registration of 848 * port which are not active. 849 */ 850 IBTF_DPRINTF_L4("ibdm", 851 "\thandle_hca_attach: port guid %llx Port state 0x%x", 852 port_attr->pa_port_guid, portinfop[ii].p_linkstate); 853 854 if (portinfop[ii].p_linkstate == IBT_PORT_ACTIVE) { 855 mutex_enter(&ibdm.ibdm_hl_mutex); 856 hca_list->hl_nports_active++; 857 ibdm_initialize_port(port_attr); 858 mutex_exit(&ibdm.ibdm_hl_mutex); 859 } 860 } 861 mutex_enter(&ibdm.ibdm_hl_mutex); 862 for (temp = ibdm.ibdm_hca_list_head; temp; temp = temp->hl_next) { 863 if (temp->hl_hca_guid == hca_guid) { 864 IBTF_DPRINTF_L2("ibdm", "hca_attach: HCA %llX " 865 "already seen by IBDM", hca_guid); 866 mutex_exit(&ibdm.ibdm_hl_mutex); 867 (void) ibdm_uninit_hca(hca_list); 868 return; 869 } 870 } 871 ibdm.ibdm_hca_count++; 872 if (ibdm.ibdm_hca_list_head == NULL) { 873 ibdm.ibdm_hca_list_head = hca_list; 874 ibdm.ibdm_hca_list_tail = hca_list; 875 } else { 876 ibdm.ibdm_hca_list_tail->hl_next = hca_list; 877 ibdm.ibdm_hca_list_tail = hca_list; 878 } 879 mutex_exit(&ibdm.ibdm_hl_mutex); 880 mutex_enter(&ibdm.ibdm_ibnex_mutex); 881 if (ibdm.ibdm_ibnex_callback != NULL) { 882 (*ibdm.ibdm_ibnex_callback)((void *) 883 &hca_guid, IBDM_EVENT_HCA_ADDED); 884 } 885 mutex_exit(&ibdm.ibdm_ibnex_mutex); 886 887 kmem_free(hca_attr, sizeof (ibt_hca_attr_t)); 888 ibt_free_portinfo(portinfop, size); 889 } 890 891 892 /* 893 * ibdm_handle_hca_detach() 894 */ 895 static void 896 ibdm_handle_hca_detach(ib_guid_t hca_guid) 897 { 898 ibdm_hca_list_t *head, *prev = NULL; 899 ibdm_dp_gidinfo_t *gidinfo; 900 901 IBTF_DPRINTF_L4("ibdm", 902 "\thandle_hca_detach: hca_guid = 0x%llx", hca_guid); 903 904 /* Make sure no probes are running */ 905 mutex_enter(&ibdm.ibdm_mutex); 906 while (ibdm.ibdm_busy & IBDM_BUSY) 907 cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex); 908 ibdm.ibdm_busy |= IBDM_BUSY; 909 mutex_exit(&ibdm.ibdm_mutex); 910 911 mutex_enter(&ibdm.ibdm_hl_mutex); 912 head = ibdm.ibdm_hca_list_head; 913 while (head) { 914 if (head->hl_hca_guid == hca_guid) { 915 if (prev == NULL) 916 ibdm.ibdm_hca_list_head = head->hl_next; 917 else 918 prev->hl_next = head->hl_next; 919 ibdm.ibdm_hca_count--; 920 break; 921 } 922 prev = head; 923 head = head->hl_next; 924 } 925 mutex_exit(&ibdm.ibdm_hl_mutex); 926 if (ibdm_uninit_hca(head) != IBDM_SUCCESS) 927 (void) ibdm_handle_hca_attach(hca_guid); 928 929 /* 930 * Now clean up the HCA lists in the gidlist. 931 */ 932 for (gidinfo = ibdm.ibdm_dp_gidlist_head; gidinfo; gidinfo = 933 gidinfo->gl_next) { 934 prev = NULL; 935 head = gidinfo->gl_hca_list; 936 while (head) { 937 if (head->hl_hca_guid == hca_guid) { 938 if (prev == NULL) 939 gidinfo->gl_hca_list = 940 head->hl_next; 941 else 942 prev->hl_next = head->hl_next; 943 944 break; 945 } 946 prev = head; 947 head = head->hl_next; 948 } 949 } 950 951 mutex_enter(&ibdm.ibdm_mutex); 952 ibdm.ibdm_busy &= ~IBDM_BUSY; 953 cv_broadcast(&ibdm.ibdm_busy_cv); 954 mutex_exit(&ibdm.ibdm_mutex); 955 } 956 957 958 static int 959 ibdm_uninit_hca(ibdm_hca_list_t *head) 960 { 961 int ii; 962 ibdm_port_attr_t *port_attr; 963 964 for (ii = 0; ii < head->hl_nports; ii++) { 965 port_attr = &head->hl_port_attr[ii]; 966 if (ibdm_fini_port(port_attr) != IBDM_SUCCESS) { 967 IBTF_DPRINTF_L2("ibdm", "uninit_hca: HCA %p port 0x%x " 968 "ibdm_fini_port() failed", head, ii); 969 return (IBDM_FAILURE); 970 } 971 } 972 if (head->hl_hca_hdl) 973 if (ibt_close_hca(head->hl_hca_hdl) != IBT_SUCCESS) 974 return (IBDM_FAILURE); 975 kmem_free(head->hl_port_attr, 976 head->hl_nports * sizeof (ibdm_port_attr_t)); 977 kmem_free(head->hl_hca_port_attr, sizeof (ibdm_port_attr_t)); 978 kmem_free(head, sizeof (ibdm_hca_list_t)); 979 return (IBDM_SUCCESS); 980 } 981 982 983 /* 984 * For each port on the HCA, 985 * 1) Teardown IBMF receive callback function 986 * 2) Unregister with IBMF 987 * 3) Unregister with SA access 988 */ 989 static int 990 ibdm_fini_port(ibdm_port_attr_t *port_attr) 991 { 992 int ii, ibmf_status; 993 994 for (ii = 0; ii < port_attr->pa_npkeys; ii++) { 995 if (port_attr->pa_pkey_tbl == NULL) 996 break; 997 if (!port_attr->pa_pkey_tbl[ii].pt_qp_hdl) 998 continue; 999 if (ibdm_port_attr_ibmf_fini(port_attr, ii) != IBDM_SUCCESS) { 1000 IBTF_DPRINTF_L4("ibdm", "\tfini_port: " 1001 "ibdm_port_attr_ibmf_fini failed for " 1002 "port pkey 0x%x", ii); 1003 return (IBDM_FAILURE); 1004 } 1005 } 1006 1007 if (port_attr->pa_ibmf_hdl) { 1008 ibmf_status = ibmf_tear_down_async_cb(port_attr->pa_ibmf_hdl, 1009 IBMF_QP_HANDLE_DEFAULT, 0); 1010 if (ibmf_status != IBMF_SUCCESS) { 1011 IBTF_DPRINTF_L4("ibdm", "\tfini_port: " 1012 "ibmf_tear_down_async_cb failed %d", ibmf_status); 1013 return (IBDM_FAILURE); 1014 } 1015 1016 ibmf_status = ibmf_unregister(&port_attr->pa_ibmf_hdl, 0); 1017 if (ibmf_status != IBMF_SUCCESS) { 1018 IBTF_DPRINTF_L4("ibdm", "\tfini_port: " 1019 "ibmf_unregister failed %d", ibmf_status); 1020 return (IBDM_FAILURE); 1021 } 1022 1023 port_attr->pa_ibmf_hdl = NULL; 1024 } 1025 1026 if (port_attr->pa_sa_hdl) { 1027 ibmf_status = ibmf_sa_session_close(&port_attr->pa_sa_hdl, 0); 1028 if (ibmf_status != IBMF_SUCCESS) { 1029 IBTF_DPRINTF_L4("ibdm", "\tfini_port: " 1030 "ibmf_sa_session_close failed %d", ibmf_status); 1031 return (IBDM_FAILURE); 1032 } 1033 port_attr->pa_sa_hdl = NULL; 1034 } 1035 1036 if (port_attr->pa_pkey_tbl != NULL) { 1037 kmem_free(port_attr->pa_pkey_tbl, 1038 port_attr->pa_npkeys * sizeof (ibdm_pkey_tbl_t)); 1039 port_attr->pa_pkey_tbl = NULL; 1040 port_attr->pa_npkeys = 0; 1041 } 1042 1043 return (IBDM_SUCCESS); 1044 } 1045 1046 1047 /* 1048 * ibdm_port_attr_ibmf_fini: 1049 * With IBMF - Tear down Async callback and free QP Handle 1050 */ 1051 static int 1052 ibdm_port_attr_ibmf_fini(ibdm_port_attr_t *port_attr, int ii) 1053 { 1054 int ibmf_status; 1055 1056 IBTF_DPRINTF_L5("ibdm", "\tport_attr_ibmf_fini:"); 1057 1058 if (port_attr->pa_pkey_tbl[ii].pt_qp_hdl) { 1059 ibmf_status = ibmf_tear_down_async_cb(port_attr->pa_ibmf_hdl, 1060 port_attr->pa_pkey_tbl[ii].pt_qp_hdl, 0); 1061 if (ibmf_status != IBMF_SUCCESS) { 1062 IBTF_DPRINTF_L4("ibdm", "\tport_attr_ibmf_fini: " 1063 "ibmf_tear_down_async_cb failed %d", ibmf_status); 1064 return (IBDM_FAILURE); 1065 } 1066 ibmf_status = ibmf_free_qp(port_attr->pa_ibmf_hdl, 1067 &port_attr->pa_pkey_tbl[ii].pt_qp_hdl, 0); 1068 if (ibmf_status != IBMF_SUCCESS) { 1069 IBTF_DPRINTF_L4("ibdm", "\tport_attr_ibmf_fini: " 1070 "ibmf_free_qp failed %d", ibmf_status); 1071 return (IBDM_FAILURE); 1072 } 1073 port_attr->pa_pkey_tbl[ii].pt_qp_hdl = NULL; 1074 } 1075 return (IBDM_SUCCESS); 1076 } 1077 1078 1079 /* 1080 * ibdm_gid_decr_pending: 1081 * decrement gl_pending_cmds. If zero wakeup sleeping threads 1082 */ 1083 static void 1084 ibdm_gid_decr_pending(ibdm_dp_gidinfo_t *gidinfo) 1085 { 1086 mutex_enter(&ibdm.ibdm_mutex); 1087 mutex_enter(&gidinfo->gl_mutex); 1088 if (--gidinfo->gl_pending_cmds == 0) { 1089 /* 1090 * Handle DGID getting removed. 1091 */ 1092 if (gidinfo->gl_disconnected) { 1093 mutex_exit(&gidinfo->gl_mutex); 1094 mutex_exit(&ibdm.ibdm_mutex); 1095 1096 IBTF_DPRINTF_L3(ibdm_string, "\tgid_decr_pending: " 1097 "gidinfo %p hot removal", gidinfo); 1098 ibdm_delete_gidinfo(gidinfo); 1099 1100 mutex_enter(&ibdm.ibdm_mutex); 1101 ibdm.ibdm_ngid_probes_in_progress--; 1102 ibdm_wait_probe_completion(); 1103 mutex_exit(&ibdm.ibdm_mutex); 1104 return; 1105 } 1106 mutex_exit(&gidinfo->gl_mutex); 1107 mutex_exit(&ibdm.ibdm_mutex); 1108 ibdm_notify_newgid_iocs(gidinfo); 1109 mutex_enter(&ibdm.ibdm_mutex); 1110 mutex_enter(&gidinfo->gl_mutex); 1111 1112 ibdm.ibdm_ngid_probes_in_progress--; 1113 ibdm_wait_probe_completion(); 1114 } 1115 mutex_exit(&gidinfo->gl_mutex); 1116 mutex_exit(&ibdm.ibdm_mutex); 1117 } 1118 1119 1120 /* 1121 * ibdm_wait_probe_completion: 1122 * wait for probing to complete 1123 */ 1124 static void 1125 ibdm_wait_probe_completion(void) 1126 { 1127 ASSERT(MUTEX_HELD(&ibdm.ibdm_mutex)); 1128 if (ibdm.ibdm_ngid_probes_in_progress) { 1129 IBTF_DPRINTF_L4("ibdm", "\twait for probe complete"); 1130 ibdm.ibdm_busy |= IBDM_PROBE_IN_PROGRESS; 1131 while (ibdm.ibdm_busy & IBDM_PROBE_IN_PROGRESS) 1132 cv_wait(&ibdm.ibdm_probe_cv, &ibdm.ibdm_mutex); 1133 } 1134 } 1135 1136 1137 /* 1138 * ibdm_wakeup_probe_gid_cv: 1139 * wakeup waiting threads (based on ibdm_ngid_probes_in_progress) 1140 */ 1141 static void 1142 ibdm_wakeup_probe_gid_cv(void) 1143 { 1144 ASSERT(MUTEX_HELD(&ibdm.ibdm_mutex)); 1145 if (!ibdm.ibdm_ngid_probes_in_progress) { 1146 IBTF_DPRINTF_L4("ibdm", "wakeup_probe_gid_thread: Wakeup"); 1147 ibdm.ibdm_busy &= ~IBDM_PROBE_IN_PROGRESS; 1148 cv_broadcast(&ibdm.ibdm_probe_cv); 1149 } 1150 1151 } 1152 1153 1154 /* 1155 * ibdm_sweep_fabric(reprobe_flag) 1156 * Find all possible Managed IOU's and their IOC's that are visible 1157 * to the host. The algorithm used is as follows 1158 * 1159 * Send a "bus walk" request for each port on the host HCA to SA access 1160 * SA returns complete set of GID's that are reachable from 1161 * source port. This is done in parallel. 1162 * 1163 * Initialize GID state to IBDM_GID_PROBE_NOT_DONE 1164 * 1165 * Sort the GID list and eliminate duplicate GID's 1166 * 1) Use DGID for sorting 1167 * 2) use PortGuid for sorting 1168 * Send SA query to retrieve NodeRecord and 1169 * extract PortGuid from that. 1170 * 1171 * Set GID state to IBDM_GID_PROBE_FAILED to all the ports that dont 1172 * support DM MAD's 1173 * Send a "Portinfo" query to get the port capabilities and 1174 * then check for DM MAD's support 1175 * 1176 * Send "ClassPortInfo" request for all the GID's in parallel, 1177 * set the GID state to IBDM_GET_CLASSPORTINFO and wait on the 1178 * cv_signal to complete. 1179 * 1180 * When DM agent on the remote GID sends back the response, IBMF 1181 * invokes DM callback routine. 1182 * 1183 * If the response is proper, send "IOUnitInfo" request and set 1184 * GID state to IBDM_GET_IOUNITINFO. 1185 * 1186 * If the response is proper, send "IocProfileInfo" request to 1187 * all the IOC simultaneously and set GID state to IBDM_GET_IOC_DETAILS. 1188 * 1189 * Send request to get Service entries simultaneously 1190 * 1191 * Signal the waiting thread when received response for all the commands. 1192 * 1193 * Set the GID state to IBDM_GID_PROBE_FAILED when received a error 1194 * response during the probing period. 1195 * 1196 * Note: 1197 * ibdm.ibdm_ngid_probes_in_progress and ibdm_gid_list_t:gl_pending_cmds 1198 * keep track of number commands in progress at any point of time. 1199 * MAD transaction ID is used to identify a particular GID 1200 * TBD: Consider registering the IBMF receive callback on demand 1201 * 1202 * Note: This routine must be called with ibdm.ibdm_mutex held 1203 * TBD: Re probe the failure GID (for certain failures) when requested 1204 * for fabric sweep next time 1205 * 1206 * Parameters : If reprobe_flag is set, All IOCs will be reprobed. 1207 */ 1208 static void 1209 ibdm_sweep_fabric(int reprobe_flag) 1210 { 1211 int ii; 1212 int new_paths = 0; 1213 uint8_t niocs; 1214 taskqid_t tid; 1215 ibdm_ioc_info_t *ioc; 1216 ibdm_hca_list_t *hca_list = NULL; 1217 ibdm_port_attr_t *port = NULL; 1218 ibdm_dp_gidinfo_t *gid_info; 1219 1220 IBTF_DPRINTF_L4("ibdm", "\tsweep_fabric: Enter"); 1221 ASSERT(MUTEX_HELD(&ibdm.ibdm_mutex)); 1222 1223 /* 1224 * Check whether a sweep already in progress. If so, just 1225 * wait for the fabric sweep to complete 1226 */ 1227 while (ibdm.ibdm_busy & IBDM_BUSY) 1228 cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex); 1229 ibdm.ibdm_busy |= IBDM_BUSY; 1230 mutex_exit(&ibdm.ibdm_mutex); 1231 1232 ibdm_dump_sweep_fabric_timestamp(0); 1233 1234 /* Rescan the GID list for any removed GIDs for reprobe */ 1235 if (reprobe_flag) 1236 ibdm_rescan_gidlist(NULL); 1237 1238 /* 1239 * Get list of all the ports reachable from the local known HCA 1240 * ports which are active 1241 */ 1242 mutex_enter(&ibdm.ibdm_hl_mutex); 1243 for (ibdm_get_next_port(&hca_list, &port, 1); port; 1244 ibdm_get_next_port(&hca_list, &port, 1)) { 1245 /* 1246 * Get PATHS to all the reachable ports from 1247 * SGID and update the global ibdm structure. 1248 */ 1249 new_paths = ibdm_get_reachable_ports(port, hca_list); 1250 ibdm.ibdm_ngids += new_paths; 1251 } 1252 mutex_exit(&ibdm.ibdm_hl_mutex); 1253 1254 mutex_enter(&ibdm.ibdm_mutex); 1255 ibdm.ibdm_ngid_probes_in_progress += ibdm.ibdm_ngids; 1256 mutex_exit(&ibdm.ibdm_mutex); 1257 1258 /* Send a request to probe GIDs asynchronously. */ 1259 for (gid_info = ibdm.ibdm_dp_gidlist_head; gid_info; 1260 gid_info = gid_info->gl_next) { 1261 mutex_enter(&gid_info->gl_mutex); 1262 gid_info->gl_reprobe_flag = reprobe_flag; 1263 mutex_exit(&gid_info->gl_mutex); 1264 1265 /* process newly encountered GIDs */ 1266 tid = taskq_dispatch(system_taskq, ibdm_probe_gid_thread, 1267 (void *)gid_info, TQ_NOSLEEP); 1268 IBTF_DPRINTF_L4("ibdm", "\tsweep_fabric: gid_info = %p" 1269 " taskq_id = %x", gid_info, tid); 1270 /* taskq failed to dispatch call it directly */ 1271 if (tid == NULL) 1272 ibdm_probe_gid_thread((void *)gid_info); 1273 } 1274 1275 mutex_enter(&ibdm.ibdm_mutex); 1276 ibdm_wait_probe_completion(); 1277 1278 /* 1279 * Update the properties, if reprobe_flag is set 1280 * Skip if gl_reprobe_flag is set, this will be 1281 * a re-inserted / new GID, for which notifications 1282 * have already been send. 1283 */ 1284 if (reprobe_flag) { 1285 for (gid_info = ibdm.ibdm_dp_gidlist_head; gid_info; 1286 gid_info = gid_info->gl_next) { 1287 if (gid_info->gl_iou == NULL) 1288 continue; 1289 if (gid_info->gl_reprobe_flag) { 1290 gid_info->gl_reprobe_flag = 0; 1291 continue; 1292 } 1293 1294 niocs = gid_info->gl_iou->iou_info.iou_num_ctrl_slots; 1295 for (ii = 0; ii < niocs; ii++) { 1296 ioc = IBDM_GIDINFO2IOCINFO(gid_info, ii); 1297 if (ioc) 1298 ibdm_reprobe_update_port_srv(ioc, 1299 gid_info); 1300 } 1301 } 1302 } 1303 ibdm_dump_sweep_fabric_timestamp(1); 1304 1305 ibdm.ibdm_busy &= ~IBDM_BUSY; 1306 cv_broadcast(&ibdm.ibdm_busy_cv); 1307 IBTF_DPRINTF_L5("ibdm", "\tsweep_fabric: EXIT"); 1308 } 1309 1310 1311 /* 1312 * ibdm_probe_gid_thread: 1313 * thread that does the actual work for sweeping the fabric 1314 * for a given GID 1315 */ 1316 static void 1317 ibdm_probe_gid_thread(void *args) 1318 { 1319 int reprobe_flag; 1320 ib_guid_t node_guid; 1321 ib_guid_t port_guid; 1322 ibdm_dp_gidinfo_t *gid_info; 1323 1324 gid_info = (ibdm_dp_gidinfo_t *)args; 1325 reprobe_flag = gid_info->gl_reprobe_flag; 1326 IBTF_DPRINTF_L4("ibdm", "\tprobe_gid_thread: gid_info = %p, flag = %d", 1327 gid_info, reprobe_flag); 1328 ASSERT(gid_info != NULL); 1329 ASSERT(gid_info->gl_pending_cmds == 0); 1330 1331 if (gid_info->gl_state != IBDM_GID_PROBE_NOT_DONE && 1332 reprobe_flag == 0) { 1333 /* 1334 * This GID may have been already probed. Send 1335 * in a CLP to check if IOUnitInfo changed? 1336 * Explicitly set gl_reprobe_flag to 0 so that 1337 * IBnex is not notified on completion 1338 */ 1339 if (gid_info->gl_state == IBDM_GID_PROBING_COMPLETE) { 1340 IBTF_DPRINTF_L4("ibdm", "\tprobe_gid_thread: " 1341 "get new IOCs information"); 1342 mutex_enter(&gid_info->gl_mutex); 1343 gid_info->gl_pending_cmds++; 1344 gid_info->gl_state = IBDM_GET_IOUNITINFO; 1345 gid_info->gl_reprobe_flag = 0; 1346 mutex_exit(&gid_info->gl_mutex); 1347 if (ibdm_send_iounitinfo(gid_info) != IBDM_SUCCESS) { 1348 mutex_enter(&gid_info->gl_mutex); 1349 gid_info->gl_pending_cmds = 0; 1350 mutex_exit(&gid_info->gl_mutex); 1351 mutex_enter(&ibdm.ibdm_mutex); 1352 --ibdm.ibdm_ngid_probes_in_progress; 1353 ibdm_wakeup_probe_gid_cv(); 1354 mutex_exit(&ibdm.ibdm_mutex); 1355 } 1356 } else { 1357 mutex_enter(&ibdm.ibdm_mutex); 1358 --ibdm.ibdm_ngid_probes_in_progress; 1359 ibdm_wakeup_probe_gid_cv(); 1360 mutex_exit(&ibdm.ibdm_mutex); 1361 } 1362 return; 1363 } else if (reprobe_flag && gid_info->gl_state == 1364 IBDM_GID_PROBING_COMPLETE) { 1365 /* 1366 * Reprobe all IOCs for the GID which has completed 1367 * probe. Skip other port GIDs to same IOU. 1368 * Explicitly set gl_reprobe_flag to 0 so that 1369 * IBnex is not notified on completion 1370 */ 1371 ibdm_ioc_info_t *ioc_info; 1372 uint8_t niocs, ii; 1373 1374 ASSERT(gid_info->gl_iou); 1375 mutex_enter(&gid_info->gl_mutex); 1376 niocs = gid_info->gl_iou->iou_info.iou_num_ctrl_slots; 1377 gid_info->gl_state = IBDM_GET_IOC_DETAILS; 1378 gid_info->gl_pending_cmds += niocs; 1379 gid_info->gl_reprobe_flag = 0; 1380 mutex_exit(&gid_info->gl_mutex); 1381 for (ii = 0; ii < niocs; ii++) { 1382 uchar_t slot_info; 1383 ib_dm_io_unitinfo_t *giou_info; 1384 1385 /* 1386 * Check whether IOC is present in the slot 1387 * Series of nibbles (in the field 1388 * iou_ctrl_list) represents a slot in the 1389 * IOU. 1390 * Byte format: 76543210 1391 * Bits 0-3 of first byte represent Slot 2 1392 * bits 4-7 of first byte represent slot 1, 1393 * bits 0-3 of second byte represent slot 4 1394 * and so on 1395 * Each 4-bit nibble has the following meaning 1396 * 0x0 : IOC not installed 1397 * 0x1 : IOC is present 1398 * 0xf : Slot does not exist 1399 * and all other values are reserved. 1400 */ 1401 ioc_info = IBDM_GIDINFO2IOCINFO(gid_info, ii); 1402 giou_info = &gid_info->gl_iou->iou_info; 1403 slot_info = giou_info->iou_ctrl_list[(ii/2)]; 1404 if ((ii % 2) == 0) 1405 slot_info = (slot_info >> 4); 1406 1407 if ((slot_info & 0xf) != 1) { 1408 ioc_info->ioc_state = 1409 IBDM_IOC_STATE_PROBE_FAILED; 1410 ibdm_gid_decr_pending(gid_info); 1411 continue; 1412 } 1413 1414 if (ibdm_send_ioc_profile(gid_info, ii) != 1415 IBDM_SUCCESS) { 1416 ibdm_gid_decr_pending(gid_info); 1417 } 1418 } 1419 1420 return; 1421 } else if (gid_info->gl_state != IBDM_GID_PROBE_NOT_DONE) { 1422 mutex_enter(&ibdm.ibdm_mutex); 1423 --ibdm.ibdm_ngid_probes_in_progress; 1424 ibdm_wakeup_probe_gid_cv(); 1425 mutex_exit(&ibdm.ibdm_mutex); 1426 return; 1427 } 1428 1429 mutex_enter(&gid_info->gl_mutex); 1430 gid_info->gl_pending_cmds++; 1431 gid_info->gl_state = IBDM_GET_CLASSPORTINFO; 1432 mutex_exit(&gid_info->gl_mutex); 1433 1434 /* 1435 * Check whether the destination GID supports DM agents. If 1436 * not, stop probing the GID and continue with the next GID 1437 * in the list. 1438 */ 1439 if (ibdm_is_dev_mgt_supported(gid_info) != IBDM_SUCCESS) { 1440 mutex_enter(&gid_info->gl_mutex); 1441 gid_info->gl_pending_cmds = 0; 1442 gid_info->gl_state = IBDM_GID_PROBING_FAILED; 1443 mutex_exit(&gid_info->gl_mutex); 1444 ibdm_delete_glhca_list(gid_info); 1445 mutex_enter(&ibdm.ibdm_mutex); 1446 --ibdm.ibdm_ngid_probes_in_progress; 1447 ibdm_wakeup_probe_gid_cv(); 1448 mutex_exit(&ibdm.ibdm_mutex); 1449 return; 1450 } 1451 1452 /* Get the nodeguid and portguid of the port */ 1453 if (ibdm_get_node_port_guids(gid_info->gl_sa_hdl, gid_info->gl_dlid, 1454 &node_guid, &port_guid) != IBDM_SUCCESS) { 1455 mutex_enter(&gid_info->gl_mutex); 1456 gid_info->gl_pending_cmds = 0; 1457 gid_info->gl_state = IBDM_GID_PROBING_FAILED; 1458 mutex_exit(&gid_info->gl_mutex); 1459 ibdm_delete_glhca_list(gid_info); 1460 mutex_enter(&ibdm.ibdm_mutex); 1461 --ibdm.ibdm_ngid_probes_in_progress; 1462 ibdm_wakeup_probe_gid_cv(); 1463 mutex_exit(&ibdm.ibdm_mutex); 1464 return; 1465 } 1466 1467 /* 1468 * Check whether we already knew about this NodeGuid 1469 * If so, do not probe the GID and continue with the 1470 * next GID in the gid list. Set the GID state to 1471 * probing done. 1472 */ 1473 mutex_enter(&ibdm.ibdm_mutex); 1474 gid_info->gl_nodeguid = node_guid; 1475 gid_info->gl_portguid = port_guid; 1476 if (ibdm_check_dest_nodeguid(gid_info) != NULL) { 1477 mutex_exit(&ibdm.ibdm_mutex); 1478 mutex_enter(&gid_info->gl_mutex); 1479 gid_info->gl_pending_cmds = 0; 1480 gid_info->gl_state = IBDM_GID_PROBING_SKIPPED; 1481 mutex_exit(&gid_info->gl_mutex); 1482 ibdm_delete_glhca_list(gid_info); 1483 mutex_enter(&ibdm.ibdm_mutex); 1484 --ibdm.ibdm_ngid_probes_in_progress; 1485 ibdm_wakeup_probe_gid_cv(); 1486 mutex_exit(&ibdm.ibdm_mutex); 1487 return; 1488 } 1489 ibdm_add_to_gl_gid(gid_info, gid_info); 1490 mutex_exit(&ibdm.ibdm_mutex); 1491 1492 /* 1493 * New or reinserted GID : Enable notification to IBnex 1494 */ 1495 mutex_enter(&gid_info->gl_mutex); 1496 gid_info->gl_reprobe_flag = 1; 1497 mutex_exit(&gid_info->gl_mutex); 1498 1499 /* 1500 * Send ClassPortInfo request to the GID asynchronously. 1501 */ 1502 if (ibdm_send_classportinfo(gid_info) != IBDM_SUCCESS) { 1503 mutex_enter(&gid_info->gl_mutex); 1504 gid_info->gl_state = IBDM_GID_PROBING_FAILED; 1505 gid_info->gl_pending_cmds = 0; 1506 mutex_exit(&gid_info->gl_mutex); 1507 ibdm_delete_glhca_list(gid_info); 1508 mutex_enter(&ibdm.ibdm_mutex); 1509 --ibdm.ibdm_ngid_probes_in_progress; 1510 ibdm_wakeup_probe_gid_cv(); 1511 mutex_exit(&ibdm.ibdm_mutex); 1512 return; 1513 } 1514 } 1515 1516 1517 /* 1518 * ibdm_check_dest_nodeguid 1519 * Searches for the NodeGuid in the GID list 1520 * Returns matching gid_info if found and otherwise NULL 1521 * 1522 * This function is called to handle new GIDs discovered 1523 * during device sweep / probe or for GID_AVAILABLE event. 1524 * 1525 * Parameter : 1526 * gid_info GID to check 1527 */ 1528 static ibdm_dp_gidinfo_t * 1529 ibdm_check_dest_nodeguid(ibdm_dp_gidinfo_t *gid_info) 1530 { 1531 ibdm_dp_gidinfo_t *gid_list; 1532 ibdm_gid_t *tmp; 1533 1534 IBTF_DPRINTF_L4("ibdm", "\tcheck_dest_nodeguid"); 1535 1536 gid_list = ibdm.ibdm_dp_gidlist_head; 1537 while (gid_list) { 1538 if ((gid_list != gid_info) && 1539 (gid_info->gl_nodeguid == gid_list->gl_nodeguid)) { 1540 IBTF_DPRINTF_L4("ibdm", 1541 "\tcheck_dest_nodeguid: NodeGuid is present"); 1542 1543 /* Add to gid_list */ 1544 tmp = kmem_zalloc(sizeof (ibdm_gid_t), 1545 KM_SLEEP); 1546 tmp->gid_dgid_hi = gid_info->gl_dgid_hi; 1547 tmp->gid_dgid_lo = gid_info->gl_dgid_lo; 1548 tmp->gid_next = gid_list->gl_gid; 1549 gid_list->gl_gid = tmp; 1550 gid_list->gl_ngids++; 1551 return (gid_list); 1552 } 1553 1554 gid_list = gid_list->gl_next; 1555 } 1556 1557 return (NULL); 1558 } 1559 1560 1561 /* 1562 * ibdm_is_dev_mgt_supported 1563 * Get the PortInfo attribute (SA Query) 1564 * Check "CompatabilityMask" field in the Portinfo. 1565 * Return IBDM_SUCCESS if DM MAD's supported (if bit 19 set) 1566 * by the port, otherwise IBDM_FAILURE 1567 */ 1568 static int 1569 ibdm_is_dev_mgt_supported(ibdm_dp_gidinfo_t *gid_info) 1570 { 1571 int ret; 1572 size_t length = 0; 1573 sa_portinfo_record_t req, *resp = NULL; 1574 ibmf_saa_access_args_t qargs; 1575 1576 bzero(&req, sizeof (sa_portinfo_record_t)); 1577 req.EndportLID = gid_info->gl_dlid; 1578 1579 qargs.sq_attr_id = SA_PORTINFORECORD_ATTRID; 1580 qargs.sq_access_type = IBMF_SAA_RETRIEVE; 1581 qargs.sq_component_mask = SA_PORTINFO_COMPMASK_PORTLID; 1582 qargs.sq_template = &req; 1583 qargs.sq_callback = NULL; 1584 qargs.sq_callback_arg = NULL; 1585 1586 ret = ibmf_sa_access(gid_info->gl_sa_hdl, 1587 &qargs, 0, &length, (void **)&resp); 1588 1589 if ((ret != IBMF_SUCCESS) || (length == 0) || (resp == NULL)) { 1590 IBTF_DPRINTF_L2("ibdm", "\tis_dev_mgt_supported:" 1591 "failed to get PORTINFO attribute %d", ret); 1592 return (IBDM_FAILURE); 1593 } 1594 1595 if (resp->PortInfo.CapabilityMask & SM_CAP_MASK_IS_DM_SUPPD) { 1596 IBTF_DPRINTF_L4("ibdm", "\tis_dev_mgt_supported: SUPPD !!"); 1597 ret = IBDM_SUCCESS; 1598 } else { 1599 IBTF_DPRINTF_L4("ibdm", "\tis_dev_mgt_supported: " 1600 "Not SUPPD !!, cap 0x%x", resp->PortInfo.CapabilityMask); 1601 ret = IBDM_FAILURE; 1602 } 1603 kmem_free(resp, length); 1604 return (ret); 1605 } 1606 1607 1608 /* 1609 * ibdm_get_node_port_guids() 1610 * Get the NodeInfoRecord of the port 1611 * Save NodeGuid and PortGUID values in the GID list structure. 1612 * Return IBDM_SUCCESS/IBDM_FAILURE 1613 */ 1614 static int 1615 ibdm_get_node_port_guids(ibmf_saa_handle_t sa_hdl, ib_lid_t dlid, 1616 ib_guid_t *node_guid, ib_guid_t *port_guid) 1617 { 1618 int ret; 1619 size_t length = 0; 1620 sa_node_record_t req, *resp = NULL; 1621 ibmf_saa_access_args_t qargs; 1622 1623 IBTF_DPRINTF_L4("ibdm", "\tget_node_port_guids"); 1624 1625 bzero(&req, sizeof (sa_node_record_t)); 1626 req.LID = dlid; 1627 1628 qargs.sq_attr_id = SA_NODERECORD_ATTRID; 1629 qargs.sq_access_type = IBMF_SAA_RETRIEVE; 1630 qargs.sq_component_mask = SA_NODEINFO_COMPMASK_NODELID; 1631 qargs.sq_template = &req; 1632 qargs.sq_callback = NULL; 1633 qargs.sq_callback_arg = NULL; 1634 1635 ret = ibmf_sa_access(sa_hdl, &qargs, 0, &length, (void **)&resp); 1636 if ((ret != IBMF_SUCCESS) || (length == 0) || (resp == NULL)) { 1637 IBTF_DPRINTF_L2("ibdm", "\tget_node_port_guids:" 1638 " SA Retrieve Failed: %d", ret); 1639 return (IBDM_FAILURE); 1640 } 1641 IBTF_DPRINTF_L4("ibdm", "\tget_node_port_guids: NodeGuid %llx Port" 1642 "GUID %llx", resp->NodeInfo.NodeGUID, resp->NodeInfo.NodeGUID); 1643 1644 *node_guid = resp->NodeInfo.NodeGUID; 1645 *port_guid = resp->NodeInfo.PortGUID; 1646 kmem_free(resp, length); 1647 return (IBDM_SUCCESS); 1648 } 1649 1650 1651 /* 1652 * ibdm_get_reachable_ports() 1653 * Get list of the destination GID (and its path records) by 1654 * querying the SA access. 1655 * 1656 * Returns Number paths 1657 */ 1658 static int 1659 ibdm_get_reachable_ports(ibdm_port_attr_t *portinfo, ibdm_hca_list_t *hca) 1660 { 1661 uint_t ii, jj, nrecs; 1662 uint_t npaths = 0; 1663 size_t length; 1664 ib_gid_t sgid; 1665 ibdm_pkey_tbl_t *pkey_tbl; 1666 sa_path_record_t *result; 1667 sa_path_record_t *precp; 1668 ibdm_dp_gidinfo_t *gid_info; 1669 1670 ASSERT(MUTEX_HELD(&ibdm.ibdm_hl_mutex)); 1671 IBTF_DPRINTF_L4("ibdm", "\tget_reachable_ports: portinfo %p", portinfo); 1672 1673 sgid.gid_prefix = portinfo->pa_sn_prefix; 1674 sgid.gid_guid = portinfo->pa_port_guid; 1675 1676 /* get reversible paths */ 1677 if (portinfo->pa_sa_hdl && ibmf_saa_paths_from_gid(portinfo->pa_sa_hdl, 1678 sgid, IBMF_SAA_PKEY_WC, B_TRUE, 0, &nrecs, &length, &result) 1679 != IBMF_SUCCESS) { 1680 IBTF_DPRINTF_L2("ibdm", 1681 "\tget_reachable_ports: Getting path records failed"); 1682 return (0); 1683 } 1684 1685 for (ii = 0; ii < nrecs; ii++) { 1686 precp = &result[ii]; 1687 if ((gid_info = ibdm_check_dgid(precp->DGID.gid_guid, 1688 precp->DGID.gid_prefix)) != NULL) { 1689 IBTF_DPRINTF_L2("ibdm", "\tget_reachable_ports: " 1690 "Already exists nrecs %d, ii %d", nrecs, ii); 1691 ibdm_addto_glhcalist(gid_info, hca); 1692 continue; 1693 } 1694 /* 1695 * This is a new GID. Allocate a GID structure and 1696 * initialize the structure 1697 * gl_state is initialized to IBDM_GID_PROBE_NOT_DONE (0) 1698 * by kmem_zalloc call 1699 */ 1700 gid_info = kmem_zalloc(sizeof (ibdm_dp_gidinfo_t), KM_SLEEP); 1701 mutex_init(&gid_info->gl_mutex, NULL, MUTEX_DEFAULT, NULL); 1702 gid_info->gl_dgid_hi = precp->DGID.gid_prefix; 1703 gid_info->gl_dgid_lo = precp->DGID.gid_guid; 1704 gid_info->gl_sgid_hi = precp->SGID.gid_prefix; 1705 gid_info->gl_sgid_lo = precp->SGID.gid_guid; 1706 gid_info->gl_p_key = precp->P_Key; 1707 gid_info->gl_sa_hdl = portinfo->pa_sa_hdl; 1708 gid_info->gl_ibmf_hdl = portinfo->pa_ibmf_hdl; 1709 gid_info->gl_slid = precp->SLID; 1710 gid_info->gl_dlid = precp->DLID; 1711 gid_info->gl_transactionID = (++ibdm.ibdm_transactionID) 1712 << IBDM_GID_TRANSACTIONID_SHIFT; 1713 gid_info->gl_min_transactionID = gid_info->gl_transactionID; 1714 gid_info->gl_max_transactionID = (ibdm.ibdm_transactionID +1) 1715 << IBDM_GID_TRANSACTIONID_SHIFT; 1716 ibdm_addto_glhcalist(gid_info, hca); 1717 1718 ibdm_dump_path_info(precp); 1719 1720 gid_info->gl_qp_hdl = NULL; 1721 ASSERT(portinfo->pa_pkey_tbl != NULL && 1722 portinfo->pa_npkeys != 0); 1723 1724 for (jj = 0; jj < portinfo->pa_npkeys; jj++) { 1725 pkey_tbl = &portinfo->pa_pkey_tbl[jj]; 1726 if ((gid_info->gl_p_key == pkey_tbl->pt_pkey) && 1727 (pkey_tbl->pt_qp_hdl != NULL)) { 1728 gid_info->gl_qp_hdl = pkey_tbl->pt_qp_hdl; 1729 break; 1730 } 1731 } 1732 1733 /* 1734 * QP handle for GID not initialized. No matching Pkey 1735 * was found!! ibdm should *not* hit this case. Flag an 1736 * error and drop the GID if ibdm does encounter this. 1737 */ 1738 if (gid_info->gl_qp_hdl == NULL) { 1739 IBTF_DPRINTF_L2(ibdm_string, 1740 "\tget_reachable_ports: No matching Pkey"); 1741 ibdm_delete_gidinfo(gid_info); 1742 continue; 1743 } 1744 if (ibdm.ibdm_dp_gidlist_head == NULL) { 1745 ibdm.ibdm_dp_gidlist_head = gid_info; 1746 ibdm.ibdm_dp_gidlist_tail = gid_info; 1747 } else { 1748 ibdm.ibdm_dp_gidlist_tail->gl_next = gid_info; 1749 gid_info->gl_prev = ibdm.ibdm_dp_gidlist_tail; 1750 ibdm.ibdm_dp_gidlist_tail = gid_info; 1751 } 1752 npaths++; 1753 } 1754 kmem_free(result, length); 1755 IBTF_DPRINTF_L4("ibdm", "\tget_reachable_ports: npaths = %d", npaths); 1756 return (npaths); 1757 } 1758 1759 1760 /* 1761 * ibdm_check_dgid() 1762 * Look in the global list to check whether we know this DGID already 1763 * Return IBDM_GID_PRESENT/IBDM_GID_NOT_PRESENT 1764 */ 1765 static ibdm_dp_gidinfo_t * 1766 ibdm_check_dgid(ib_guid_t guid, ib_sn_prefix_t prefix) 1767 { 1768 ibdm_dp_gidinfo_t *gid_list; 1769 1770 for (gid_list = ibdm.ibdm_dp_gidlist_head; gid_list; 1771 gid_list = gid_list->gl_next) { 1772 if ((guid == gid_list->gl_dgid_lo) && 1773 (prefix == gid_list->gl_dgid_hi)) { 1774 break; 1775 } 1776 } 1777 return (gid_list); 1778 } 1779 1780 1781 /* 1782 * ibdm_find_gid() 1783 * Look in the global list to find a GID entry with matching 1784 * port & node GUID. 1785 * Return pointer to gidinfo if found, else return NULL 1786 */ 1787 static ibdm_dp_gidinfo_t * 1788 ibdm_find_gid(ib_guid_t nodeguid, ib_guid_t portguid) 1789 { 1790 ibdm_dp_gidinfo_t *gid_list; 1791 1792 IBTF_DPRINTF_L4("ibdm", "ibdm_find_gid(%llx, %llx)\n", 1793 nodeguid, portguid); 1794 1795 for (gid_list = ibdm.ibdm_dp_gidlist_head; gid_list; 1796 gid_list = gid_list->gl_next) { 1797 if ((portguid == gid_list->gl_portguid) && 1798 (nodeguid == gid_list->gl_nodeguid)) { 1799 break; 1800 } 1801 } 1802 1803 IBTF_DPRINTF_L4("ibdm", "ibdm_find_gid : returned %p\n", 1804 gid_list); 1805 return (gid_list); 1806 } 1807 1808 1809 /* 1810 * ibdm_send_classportinfo() 1811 * Send classportinfo request. When the request is completed 1812 * IBMF calls ibdm_classportinfo_cb routine to inform about 1813 * the completion. 1814 * Returns IBDM_SUCCESS/IBDM_FAILURE 1815 */ 1816 static int 1817 ibdm_send_classportinfo(ibdm_dp_gidinfo_t *gid_info) 1818 { 1819 ibmf_msg_t *msg; 1820 ib_mad_hdr_t *hdr; 1821 ibdm_timeout_cb_args_t *cb_args; 1822 1823 IBTF_DPRINTF_L4("ibdm", 1824 "\tsend_classportinfo: gid info 0x%p", gid_info); 1825 1826 /* 1827 * Send command to get classportinfo attribute. Allocate a IBMF 1828 * packet and initialize the packet. 1829 */ 1830 if (ibmf_alloc_msg(gid_info->gl_ibmf_hdl, IBMF_ALLOC_SLEEP, 1831 &msg) != IBMF_SUCCESS) { 1832 IBTF_DPRINTF_L4("ibdm", "\tsend_classportinfo: pkt alloc fail"); 1833 return (IBDM_FAILURE); 1834 } 1835 1836 ibdm_alloc_send_buffers(msg); 1837 1838 msg->im_local_addr.ia_local_lid = gid_info->gl_slid; 1839 msg->im_local_addr.ia_remote_lid = gid_info->gl_dlid; 1840 msg->im_local_addr.ia_remote_qno = 1; 1841 msg->im_local_addr.ia_p_key = gid_info->gl_p_key; 1842 msg->im_local_addr.ia_q_key = IB_GSI_QKEY; 1843 1844 hdr = IBDM_OUT_IBMFMSG_MADHDR(msg); 1845 hdr->BaseVersion = MAD_CLASS_BASE_VERS_1; 1846 hdr->MgmtClass = MAD_MGMT_CLASS_DEV_MGT; 1847 hdr->ClassVersion = IB_DM_CLASS_VERSION_1; 1848 hdr->R_Method = IB_DM_DEVMGT_METHOD_GET; 1849 hdr->Status = 0; 1850 hdr->TransactionID = h2b64(gid_info->gl_transactionID); 1851 hdr->AttributeID = h2b16(IB_DM_ATTR_CLASSPORTINFO); 1852 hdr->AttributeModifier = 0; 1853 1854 cb_args = &gid_info->gl_cpi_cb_args; 1855 cb_args->cb_gid_info = gid_info; 1856 cb_args->cb_retry_count = ibdm_dft_retry_cnt; 1857 cb_args->cb_req_type = IBDM_REQ_TYPE_CLASSPORTINFO; 1858 1859 gid_info->gl_timeout_id = timeout(ibdm_pkt_timeout_hdlr, 1860 cb_args, IBDM_TIMEOUT_VALUE(ibdm_dft_timeout)); 1861 1862 IBTF_DPRINTF_L5("ibdm", "\tsend_classportinfo: " 1863 "timeout id %x", gid_info->gl_timeout_id); 1864 1865 if (ibmf_msg_transport(gid_info->gl_ibmf_hdl, gid_info->gl_qp_hdl, 1866 msg, NULL, ibdm_ibmf_send_cb, cb_args, 0) != IBMF_SUCCESS) { 1867 IBTF_DPRINTF_L2("ibdm", 1868 "\tsend_classportinfo: ibmf send failed"); 1869 ibdm_ibmf_send_cb(gid_info->gl_ibmf_hdl, msg, cb_args); 1870 } 1871 1872 return (IBDM_SUCCESS); 1873 } 1874 1875 1876 /* 1877 * ibdm_handle_classportinfo() 1878 * Invoked by the IBMF when the classportinfo request is completed. 1879 */ 1880 static void 1881 ibdm_handle_classportinfo(ibmf_handle_t ibmf_hdl, 1882 ibmf_msg_t *msg, ibdm_dp_gidinfo_t *gid_info, int *flag) 1883 { 1884 void *data; 1885 timeout_id_t timeout_id; 1886 ib_mad_hdr_t *hdr; 1887 ibdm_mad_classportinfo_t *cpi; 1888 1889 IBTF_DPRINTF_L4("ibdm", "\thandle_classportinfo:ibmf hdl " 1890 "%p msg %p gid info %p", ibmf_hdl, msg, gid_info); 1891 1892 if (IBDM_IN_IBMFMSG_ATTR(msg) != IB_DM_ATTR_CLASSPORTINFO) { 1893 IBTF_DPRINTF_L4("ibdm", "\thandle_classportinfo: " 1894 "Not a ClassPortInfo resp"); 1895 *flag |= IBDM_IBMF_PKT_UNEXP_RESP; 1896 return; 1897 } 1898 1899 /* 1900 * Verify whether timeout handler is created/active. 1901 * If created/ active, cancel the timeout handler 1902 */ 1903 mutex_enter(&gid_info->gl_mutex); 1904 ibdm_bump_transactionID(gid_info); 1905 if (gid_info->gl_state != IBDM_GET_CLASSPORTINFO) { 1906 IBTF_DPRINTF_L2("ibdm", "\thandle_classportinfo:DUP resp"); 1907 *flag |= IBDM_IBMF_PKT_DUP_RESP; 1908 mutex_exit(&gid_info->gl_mutex); 1909 return; 1910 } 1911 gid_info->gl_iou_cb_args.cb_req_type = 0; 1912 if (gid_info->gl_timeout_id) { 1913 timeout_id = gid_info->gl_timeout_id; 1914 mutex_exit(&gid_info->gl_mutex); 1915 IBTF_DPRINTF_L5("ibdm", "handle_ioclassportinfo: " 1916 "gl_timeout_id = 0x%x", timeout_id); 1917 if (untimeout(timeout_id) == -1) { 1918 IBTF_DPRINTF_L2("ibdm", "handle_classportinfo: " 1919 "untimeout gl_timeout_id failed"); 1920 } 1921 mutex_enter(&gid_info->gl_mutex); 1922 gid_info->gl_timeout_id = 0; 1923 } 1924 gid_info->gl_state = IBDM_GET_IOUNITINFO; 1925 gid_info->gl_pending_cmds++; 1926 mutex_exit(&gid_info->gl_mutex); 1927 1928 data = msg->im_msgbufs_recv.im_bufs_cl_data; 1929 cpi = (ibdm_mad_classportinfo_t *)data; 1930 1931 /* 1932 * Cache the "RespTimeValue" and redirection information in the 1933 * global gid list data structure. This cached information will 1934 * be used to send any further requests to the GID. 1935 */ 1936 gid_info->gl_resp_timeout = 1937 (b2h32(cpi->RespTimeValue) & 0x1F); 1938 1939 gid_info->gl_redirected = ((IBDM_IN_IBMFMSG_STATUS(msg) & 1940 MAD_STATUS_REDIRECT_REQUIRED) ? B_TRUE: B_FALSE); 1941 gid_info->gl_redirect_dlid = b2h16(cpi->RedirectLID); 1942 gid_info->gl_redirect_QP = (b2h32(cpi->RedirectQP) & 0xffffff); 1943 gid_info->gl_redirect_pkey = b2h16(cpi->RedirectP_Key); 1944 gid_info->gl_redirect_qkey = b2h32(cpi->RedirectQ_Key); 1945 gid_info->gl_redirectGID_hi = b2h64(cpi->RedirectGID_hi); 1946 gid_info->gl_redirectGID_lo = b2h64(cpi->RedirectGID_lo); 1947 1948 ibdm_dump_classportinfo(cpi); 1949 1950 /* 1951 * Send IOUnitInfo request 1952 * Reuse previously allocated IBMF packet for sending ClassPortInfo 1953 * Check whether DM agent on the remote node requested redirection 1954 * If so, send the request to the redirect DGID/DLID/PKEY/QP. 1955 */ 1956 ibdm_alloc_send_buffers(msg); 1957 msg->im_local_addr.ia_local_lid = gid_info->gl_slid; 1958 msg->im_local_addr.ia_remote_lid = gid_info->gl_dlid; 1959 1960 if (gid_info->gl_redirected == B_TRUE) { 1961 if (gid_info->gl_redirect_dlid != 0) { 1962 msg->im_local_addr.ia_remote_lid = 1963 gid_info->gl_redirect_dlid; 1964 } 1965 msg->im_local_addr.ia_remote_qno = gid_info->gl_redirect_QP; 1966 msg->im_local_addr.ia_p_key = gid_info->gl_redirect_pkey; 1967 msg->im_local_addr.ia_q_key = gid_info->gl_redirect_qkey; 1968 } else { 1969 msg->im_local_addr.ia_remote_qno = 1; 1970 msg->im_local_addr.ia_p_key = gid_info->gl_p_key; 1971 msg->im_local_addr.ia_q_key = IB_GSI_QKEY; 1972 } 1973 1974 hdr = IBDM_OUT_IBMFMSG_MADHDR(msg); 1975 hdr->BaseVersion = MAD_CLASS_BASE_VERS_1; 1976 hdr->MgmtClass = MAD_MGMT_CLASS_DEV_MGT; 1977 hdr->ClassVersion = IB_DM_CLASS_VERSION_1; 1978 hdr->R_Method = IB_DM_DEVMGT_METHOD_GET; 1979 hdr->Status = 0; 1980 hdr->TransactionID = h2b64(gid_info->gl_transactionID); 1981 hdr->AttributeID = h2b16(IB_DM_ATTR_IO_UNITINFO); 1982 hdr->AttributeModifier = 0; 1983 1984 gid_info->gl_iou_cb_args.cb_req_type = IBDM_REQ_TYPE_IOUINFO; 1985 gid_info->gl_iou_cb_args.cb_gid_info = gid_info; 1986 gid_info->gl_iou_cb_args.cb_retry_count = ibdm_dft_retry_cnt; 1987 1988 gid_info->gl_timeout_id = timeout(ibdm_pkt_timeout_hdlr, 1989 &gid_info->gl_iou_cb_args, IBDM_TIMEOUT_VALUE(ibdm_dft_timeout)); 1990 1991 IBTF_DPRINTF_L5("ibdm", "handle_classportinfo:" 1992 "timeout %x", gid_info->gl_timeout_id); 1993 1994 if (ibmf_msg_transport(ibmf_hdl, gid_info->gl_qp_hdl, msg, NULL, 1995 ibdm_ibmf_send_cb, &gid_info->gl_iou_cb_args, 0) != IBMF_SUCCESS) { 1996 IBTF_DPRINTF_L2("ibdm", 1997 "\thandle_classportinfo: msg transport failed"); 1998 ibdm_ibmf_send_cb(ibmf_hdl, msg, &gid_info->gl_iou_cb_args); 1999 } 2000 (*flag) |= IBDM_IBMF_PKT_REUSED; 2001 } 2002 2003 2004 /* 2005 * ibdm_send_iounitinfo: 2006 * Sends a DM request to get IOU unitinfo. 2007 */ 2008 static int 2009 ibdm_send_iounitinfo(ibdm_dp_gidinfo_t *gid_info) 2010 { 2011 ibmf_msg_t *msg; 2012 ib_mad_hdr_t *hdr; 2013 2014 IBTF_DPRINTF_L4("ibdm", "\tsend_iounitinfo: gid info 0x%p", gid_info); 2015 2016 /* 2017 * Send command to get iounitinfo attribute. Allocate a IBMF 2018 * packet and initialize the packet. 2019 */ 2020 if (ibmf_alloc_msg(gid_info->gl_ibmf_hdl, IBMF_ALLOC_SLEEP, &msg) != 2021 IBMF_SUCCESS) { 2022 IBTF_DPRINTF_L4("ibdm", "\tsend_iounitinfo: pkt alloc fail"); 2023 return (IBDM_FAILURE); 2024 } 2025 2026 mutex_enter(&gid_info->gl_mutex); 2027 ibdm_bump_transactionID(gid_info); 2028 mutex_exit(&gid_info->gl_mutex); 2029 2030 2031 ibdm_alloc_send_buffers(msg); 2032 msg->im_local_addr.ia_local_lid = gid_info->gl_slid; 2033 msg->im_local_addr.ia_remote_lid = gid_info->gl_dlid; 2034 msg->im_local_addr.ia_remote_qno = 1; 2035 msg->im_local_addr.ia_p_key = gid_info->gl_p_key; 2036 msg->im_local_addr.ia_q_key = IB_GSI_QKEY; 2037 2038 hdr = IBDM_OUT_IBMFMSG_MADHDR(msg); 2039 hdr->BaseVersion = MAD_CLASS_BASE_VERS_1; 2040 hdr->MgmtClass = MAD_MGMT_CLASS_DEV_MGT; 2041 hdr->ClassVersion = IB_DM_CLASS_VERSION_1; 2042 hdr->R_Method = IB_DM_DEVMGT_METHOD_GET; 2043 hdr->Status = 0; 2044 hdr->TransactionID = h2b64(gid_info->gl_transactionID); 2045 hdr->AttributeID = h2b16(IB_DM_ATTR_IO_UNITINFO); 2046 hdr->AttributeModifier = 0; 2047 2048 gid_info->gl_iou_cb_args.cb_gid_info = gid_info; 2049 gid_info->gl_iou_cb_args.cb_retry_count = ibdm_dft_retry_cnt; 2050 gid_info->gl_iou_cb_args.cb_req_type = IBDM_REQ_TYPE_IOUINFO; 2051 2052 gid_info->gl_timeout_id = timeout(ibdm_pkt_timeout_hdlr, 2053 &gid_info->gl_iou_cb_args, IBDM_TIMEOUT_VALUE(ibdm_dft_timeout)); 2054 2055 IBTF_DPRINTF_L5("ibdm", "send_iouunitinfo:" 2056 "timeout %x", gid_info->gl_timeout_id); 2057 2058 if (ibmf_msg_transport(gid_info->gl_ibmf_hdl, gid_info->gl_qp_hdl, msg, 2059 NULL, ibdm_ibmf_send_cb, &gid_info->gl_iou_cb_args, 0) != 2060 IBMF_SUCCESS) { 2061 IBTF_DPRINTF_L2("ibdm", "\tsend_iounitinfo: ibmf send failed"); 2062 ibdm_ibmf_send_cb(gid_info->gl_ibmf_hdl, 2063 msg, &gid_info->gl_iou_cb_args); 2064 } 2065 return (IBDM_SUCCESS); 2066 } 2067 2068 /* 2069 * ibdm_handle_iounitinfo() 2070 * Invoked by the IBMF when IO Unitinfo request is completed. 2071 */ 2072 static void 2073 ibdm_handle_iounitinfo(ibmf_handle_t ibmf_hdl, 2074 ibmf_msg_t *msg, ibdm_dp_gidinfo_t *gid_info, int *flag) 2075 { 2076 int ii, first = B_TRUE; 2077 int num_iocs; 2078 size_t size; 2079 uchar_t slot_info; 2080 timeout_id_t timeout_id; 2081 ib_mad_hdr_t *hdr; 2082 ibdm_ioc_info_t *ioc_info; 2083 ib_dm_io_unitinfo_t *iou_info; 2084 ib_dm_io_unitinfo_t *giou_info; 2085 ibdm_timeout_cb_args_t *cb_args; 2086 2087 IBTF_DPRINTF_L4("ibdm", "\thandle_iouintinfo:" 2088 " ibmf hdl %p pkt %p gid info %p", ibmf_hdl, msg, gid_info); 2089 2090 if (IBDM_IN_IBMFMSG_ATTR(msg) != IB_DM_ATTR_IO_UNITINFO) { 2091 IBTF_DPRINTF_L4("ibdm", "\thandle_iounitinfo: " 2092 "Unexpected response"); 2093 (*flag) |= IBDM_IBMF_PKT_UNEXP_RESP; 2094 return; 2095 } 2096 2097 mutex_enter(&gid_info->gl_mutex); 2098 if (gid_info->gl_state != IBDM_GET_IOUNITINFO) { 2099 IBTF_DPRINTF_L4("ibdm", 2100 "\thandle_iounitinfo: DUP resp"); 2101 mutex_exit(&gid_info->gl_mutex); 2102 (*flag) = IBDM_IBMF_PKT_DUP_RESP; 2103 return; 2104 } 2105 gid_info->gl_iou_cb_args.cb_req_type = 0; 2106 if (gid_info->gl_timeout_id) { 2107 timeout_id = gid_info->gl_timeout_id; 2108 mutex_exit(&gid_info->gl_mutex); 2109 IBTF_DPRINTF_L5("ibdm", "handle_iounitinfo: " 2110 "gl_timeout_id = 0x%x", timeout_id); 2111 if (untimeout(timeout_id) == -1) { 2112 IBTF_DPRINTF_L2("ibdm", "handle_iounitinfo: " 2113 "untimeout gl_timeout_id failed"); 2114 } 2115 mutex_enter(&gid_info->gl_mutex); 2116 gid_info->gl_timeout_id = 0; 2117 } 2118 gid_info->gl_state = IBDM_GET_IOC_DETAILS; 2119 2120 iou_info = IBDM_IN_IBMFMSG2IOU(msg); 2121 ibdm_dump_iounitinfo(iou_info); 2122 num_iocs = iou_info->iou_num_ctrl_slots; 2123 /* 2124 * check if number of IOCs reported is zero? if yes, return. 2125 * when num_iocs are reported zero internal IOC database needs 2126 * to be updated. To ensure that save the number of IOCs in 2127 * the new field "gl_num_iocs". Use a new field instead of 2128 * "giou_info->iou_num_ctrl_slots" as that would prevent 2129 * an unnecessary kmem_alloc/kmem_free when num_iocs is 0. 2130 */ 2131 if (num_iocs == 0 && gid_info->gl_num_iocs == 0) { 2132 IBTF_DPRINTF_L4("ibdm", "\thandle_iounitinfo: no IOC's"); 2133 mutex_exit(&gid_info->gl_mutex); 2134 return; 2135 } 2136 IBTF_DPRINTF_L4("ibdm", "\thandle_iounitinfo: num_iocs = %d", num_iocs); 2137 2138 /* 2139 * if there is an existing gl_iou (IOU has been probed before) 2140 * check if the "iou_changeid" is same as saved entry in 2141 * "giou_info->iou_changeid". 2142 * (note: this logic can prevent IOC enumeration if a given 2143 * vendor doesn't support setting iou_changeid field for its IOU) 2144 * 2145 * if there is an existing gl_iou and iou_changeid has changed : 2146 * free up existing gl_iou info and its related structures. 2147 * reallocate gl_iou info all over again. 2148 * if we donot free this up; then this leads to memory leaks 2149 */ 2150 if (gid_info->gl_iou) { 2151 giou_info = &gid_info->gl_iou->iou_info; 2152 if (iou_info->iou_changeid == giou_info->iou_changeid) { 2153 IBTF_DPRINTF_L3("ibdm", 2154 "\thandle_iounitinfo: no IOCs changed"); 2155 gid_info->gl_state = IBDM_GID_PROBING_COMPLETE; 2156 mutex_exit(&gid_info->gl_mutex); 2157 return; 2158 } 2159 if (ibdm_free_iou_info(gid_info)) { 2160 IBTF_DPRINTF_L3("ibdm", 2161 "\thandle_iounitinfo: failed to cleanup resources"); 2162 gid_info->gl_state = IBDM_GID_PROBING_COMPLETE; 2163 mutex_exit(&gid_info->gl_mutex); 2164 return; 2165 } 2166 } 2167 2168 size = sizeof (ibdm_iou_info_t) + num_iocs * sizeof (ibdm_ioc_info_t); 2169 gid_info->gl_iou = (ibdm_iou_info_t *)kmem_zalloc(size, KM_SLEEP); 2170 giou_info = &gid_info->gl_iou->iou_info; 2171 gid_info->gl_iou->iou_ioc_info = (ibdm_ioc_info_t *) 2172 ((char *)gid_info->gl_iou + sizeof (ibdm_iou_info_t)); 2173 2174 giou_info->iou_num_ctrl_slots = gid_info->gl_num_iocs = num_iocs; 2175 giou_info->iou_flag = iou_info->iou_flag; 2176 bcopy(iou_info->iou_ctrl_list, giou_info->iou_ctrl_list, 128); 2177 giou_info->iou_changeid = b2h16(iou_info->iou_changeid); 2178 gid_info->gl_pending_cmds += num_iocs; 2179 gid_info->gl_pending_cmds += 1; /* for diag code */ 2180 mutex_exit(&gid_info->gl_mutex); 2181 2182 if (ibdm_get_diagcode(gid_info, 0) != IBDM_SUCCESS) { 2183 mutex_enter(&gid_info->gl_mutex); 2184 gid_info->gl_pending_cmds--; 2185 mutex_exit(&gid_info->gl_mutex); 2186 } 2187 /* 2188 * Parallelize getting IOC controller profiles from here. 2189 * Allocate IBMF packets and send commands to get IOC profile for 2190 * each IOC present on the IOU. 2191 */ 2192 for (ii = 0; ii < num_iocs; ii++) { 2193 /* 2194 * Check whether IOC is present in the slot 2195 * Series of nibbles (in the field iou_ctrl_list) represents 2196 * a slot in the IOU. 2197 * Byte format: 76543210 2198 * Bits 0-3 of first byte represent Slot 2 2199 * bits 4-7 of first byte represent slot 1, 2200 * bits 0-3 of second byte represent slot 4 and so on 2201 * Each 4-bit nibble has the following meaning 2202 * 0x0 : IOC not installed 2203 * 0x1 : IOC is present 2204 * 0xf : Slot does not exist 2205 * and all other values are reserved. 2206 */ 2207 ioc_info = IBDM_GIDINFO2IOCINFO(gid_info, ii); 2208 slot_info = giou_info->iou_ctrl_list[(ii/2)]; 2209 if ((ii % 2) == 0) 2210 slot_info = (slot_info >> 4); 2211 2212 if ((slot_info & 0xf) != 1) { 2213 IBTF_DPRINTF_L4("ibdm", "\thandle_iouintinfo: " 2214 "No IOC is present in the slot = %d", ii); 2215 ioc_info->ioc_state = IBDM_IOC_STATE_PROBE_FAILED; 2216 mutex_enter(&gid_info->gl_mutex); 2217 gid_info->gl_pending_cmds--; 2218 mutex_exit(&gid_info->gl_mutex); 2219 continue; 2220 } 2221 2222 mutex_enter(&gid_info->gl_mutex); 2223 ibdm_bump_transactionID(gid_info); 2224 mutex_exit(&gid_info->gl_mutex); 2225 2226 /* 2227 * Re use the already allocated packet (for IOUnitinfo) to 2228 * send the first IOC controller attribute. Allocate new 2229 * IBMF packets for the rest of the IOC's 2230 */ 2231 if (first != B_TRUE) { 2232 msg = NULL; 2233 if (ibmf_alloc_msg(ibmf_hdl, IBMF_ALLOC_SLEEP, 2234 &msg) != IBMF_SUCCESS) { 2235 IBTF_DPRINTF_L4("ibdm", "\thandle_iouintinfo: " 2236 "IBMF packet allocation failed"); 2237 mutex_enter(&gid_info->gl_mutex); 2238 gid_info->gl_pending_cmds--; 2239 mutex_exit(&gid_info->gl_mutex); 2240 continue; 2241 } 2242 2243 } 2244 2245 /* allocate send buffers for all messages */ 2246 ibdm_alloc_send_buffers(msg); 2247 2248 msg->im_local_addr.ia_local_lid = gid_info->gl_slid; 2249 msg->im_local_addr.ia_remote_lid = gid_info->gl_dlid; 2250 if (gid_info->gl_redirected == B_TRUE) { 2251 if (gid_info->gl_redirect_dlid != 0) { 2252 msg->im_local_addr.ia_remote_lid = 2253 gid_info->gl_redirect_dlid; 2254 } 2255 msg->im_local_addr.ia_remote_qno = 2256 gid_info->gl_redirect_QP; 2257 msg->im_local_addr.ia_p_key = 2258 gid_info->gl_redirect_pkey; 2259 msg->im_local_addr.ia_q_key = 2260 gid_info->gl_redirect_qkey; 2261 } else { 2262 msg->im_local_addr.ia_remote_qno = 1; 2263 msg->im_local_addr.ia_p_key = gid_info->gl_p_key; 2264 msg->im_local_addr.ia_q_key = IB_GSI_QKEY; 2265 } 2266 2267 hdr = IBDM_OUT_IBMFMSG_MADHDR(msg); 2268 hdr->BaseVersion = MAD_CLASS_BASE_VERS_1; 2269 hdr->MgmtClass = MAD_MGMT_CLASS_DEV_MGT; 2270 hdr->ClassVersion = IB_DM_CLASS_VERSION_1; 2271 hdr->R_Method = IB_DM_DEVMGT_METHOD_GET; 2272 hdr->Status = 0; 2273 hdr->TransactionID = h2b64(gid_info->gl_transactionID); 2274 hdr->AttributeID = h2b16(IB_DM_ATTR_IOC_CTRL_PROFILE); 2275 hdr->AttributeModifier = h2b32(ii + 1); 2276 2277 ioc_info->ioc_state = IBDM_IOC_STATE_PROBE_INVALID; 2278 cb_args = &ioc_info->ioc_cb_args; 2279 cb_args->cb_gid_info = gid_info; 2280 cb_args->cb_retry_count = ibdm_dft_retry_cnt; 2281 cb_args->cb_req_type = IBDM_REQ_TYPE_IOCINFO; 2282 cb_args->cb_ioc_num = ii; 2283 2284 ioc_info->ioc_timeout_id = timeout(ibdm_pkt_timeout_hdlr, 2285 cb_args, IBDM_TIMEOUT_VALUE(ibdm_dft_timeout)); 2286 2287 IBTF_DPRINTF_L5("ibdm", "\thandle_iounitinfo:" 2288 "timeout 0x%x, ioc_num %d", ioc_info->ioc_timeout_id, ii); 2289 2290 if (ibmf_msg_transport(ibmf_hdl, gid_info->gl_qp_hdl, msg, 2291 NULL, ibdm_ibmf_send_cb, cb_args, 0) != IBMF_SUCCESS) { 2292 IBTF_DPRINTF_L2("ibdm", 2293 "\thandle_iounitinfo: msg transport failed"); 2294 ibdm_ibmf_send_cb(ibmf_hdl, msg, cb_args); 2295 } 2296 (*flag) |= IBDM_IBMF_PKT_REUSED; 2297 first = B_FALSE; 2298 gid_info->gl_iou->iou_niocs_probe_in_progress++; 2299 } 2300 } 2301 2302 2303 /* 2304 * ibdm_handle_ioc_profile() 2305 * Invoked by the IBMF when the IOCControllerProfile request 2306 * gets completed 2307 */ 2308 static void 2309 ibdm_handle_ioc_profile(ibmf_handle_t ibmf_hdl, 2310 ibmf_msg_t *msg, ibdm_dp_gidinfo_t *gid_info, int *flag) 2311 { 2312 int first = B_TRUE, reprobe = 0; 2313 uint_t ii, ioc_no, srv_start; 2314 uint_t nserv_entries; 2315 timeout_id_t timeout_id; 2316 ib_mad_hdr_t *hdr; 2317 ibdm_ioc_info_t *ioc_info; 2318 ibdm_timeout_cb_args_t *cb_args; 2319 ib_dm_ioc_ctrl_profile_t *ioc, *gioc; 2320 2321 IBTF_DPRINTF_L4("ibdm", "\thandle_ioc_profile:" 2322 " ibmf hdl %p msg %p gid info %p", ibmf_hdl, msg, gid_info); 2323 2324 ioc = IBDM_IN_IBMFMSG2IOC(msg); 2325 /* 2326 * Check whether we know this IOC already 2327 * This will return NULL if reprobe is in progress 2328 * IBDM_IOC_STATE_REPROBE_PROGRESS will be set. 2329 * Do not hold mutexes here. 2330 */ 2331 if (ibdm_is_ioc_present(ioc->ioc_guid, gid_info, flag) != NULL) { 2332 IBTF_DPRINTF_L4("ibdm", "\thandle_ioc_profile:" 2333 "IOC guid %llx is present", ioc->ioc_guid); 2334 return; 2335 } 2336 ioc_no = IBDM_IN_IBMFMSG_ATTRMOD(msg); 2337 IBTF_DPRINTF_L4("ibdm", "\thandle_ioc_profile: ioc_no = %d", ioc_no-1); 2338 2339 /* Make sure that IOC index is with the valid range */ 2340 if (IBDM_IS_IOC_NUM_INVALID(ioc_no, gid_info)) { 2341 IBTF_DPRINTF_L2("ibdm", "\thandle_ioc_profile: " 2342 "IOC index Out of range, index %d", ioc); 2343 (*flag) |= IBDM_IBMF_PKT_UNEXP_RESP; 2344 return; 2345 } 2346 ioc_info = &gid_info->gl_iou->iou_ioc_info[ioc_no - 1]; 2347 ioc_info->ioc_iou_info = gid_info->gl_iou; 2348 2349 mutex_enter(&gid_info->gl_mutex); 2350 if (ioc_info->ioc_state == IBDM_IOC_STATE_REPROBE_PROGRESS) { 2351 reprobe = 1; 2352 ioc_info->ioc_prev_serv = ioc_info->ioc_serv; 2353 ioc_info->ioc_serv = NULL; 2354 ioc_info->ioc_prev_serv_cnt = 2355 ioc_info->ioc_profile.ioc_service_entries; 2356 } else if (ioc_info->ioc_state != IBDM_IOC_STATE_PROBE_INVALID) { 2357 IBTF_DPRINTF_L2("ibdm", "\thandle_ioc_profile: DUP response" 2358 "ioc %d, ioc_state %x", ioc_no - 1, ioc_info->ioc_state); 2359 mutex_exit(&gid_info->gl_mutex); 2360 (*flag) |= IBDM_IBMF_PKT_DUP_RESP; 2361 return; 2362 } 2363 ioc_info->ioc_cb_args.cb_req_type = 0; 2364 if (ioc_info->ioc_timeout_id) { 2365 timeout_id = ioc_info->ioc_timeout_id; 2366 mutex_exit(&gid_info->gl_mutex); 2367 IBTF_DPRINTF_L5("ibdm", "handle_ioc_profile: " 2368 "ioc_timeout_id = 0x%x", timeout_id); 2369 if (untimeout(timeout_id) == -1) { 2370 IBTF_DPRINTF_L2("ibdm", "handle_ioc_profile: " 2371 "untimeout ioc_timeout_id failed"); 2372 } 2373 mutex_enter(&gid_info->gl_mutex); 2374 ioc_info->ioc_timeout_id = 0; 2375 } 2376 2377 ioc_info->ioc_state = IBDM_IOC_STATE_PROBE_SUCCESS; 2378 if (reprobe == 0) { 2379 ioc_info->ioc_iou_guid = gid_info->gl_nodeguid; 2380 ioc_info->ioc_nodeguid = gid_info->gl_nodeguid; 2381 } 2382 2383 /* 2384 * Save all the IOC information in the global structures. 2385 * Note the wire format is Big Endian and the Sparc process also 2386 * big endian. So, there is no need to convert the data fields 2387 * The conversion routines used below are ineffective on Sparc 2388 * machines where as they will be effective on little endian 2389 * machines such as Intel processors. 2390 */ 2391 gioc = (ib_dm_ioc_ctrl_profile_t *)&ioc_info->ioc_profile; 2392 2393 /* 2394 * Restrict updates to onlyport GIDs and service entries during reprobe 2395 */ 2396 if (reprobe == 0) { 2397 gioc->ioc_guid = b2h64(ioc->ioc_guid); 2398 gioc->ioc_vendorid = 2399 ((b2h32(ioc->ioc_vendorid) & IB_DM_VENDORID_MASK) 2400 >> IB_DM_VENDORID_SHIFT); 2401 gioc->ioc_deviceid = b2h32(ioc->ioc_deviceid); 2402 gioc->ioc_device_ver = b2h16(ioc->ioc_device_ver); 2403 gioc->ioc_subsys_vendorid = 2404 ((b2h32(ioc->ioc_subsys_vendorid) & IB_DM_VENDORID_MASK) 2405 >> IB_DM_VENDORID_SHIFT); 2406 gioc->ioc_subsys_id = b2h32(ioc->ioc_subsys_id); 2407 gioc->ioc_io_class = b2h16(ioc->ioc_io_class); 2408 gioc->ioc_io_subclass = b2h16(ioc->ioc_io_subclass); 2409 gioc->ioc_protocol = b2h16(ioc->ioc_protocol); 2410 gioc->ioc_protocol_ver = b2h16(ioc->ioc_protocol_ver); 2411 gioc->ioc_send_msg_qdepth = 2412 b2h16(ioc->ioc_send_msg_qdepth); 2413 gioc->ioc_rdma_read_qdepth = 2414 b2h16(ioc->ioc_rdma_read_qdepth); 2415 gioc->ioc_send_msg_sz = b2h32(ioc->ioc_send_msg_sz); 2416 gioc->ioc_rdma_xfer_sz = b2h32(ioc->ioc_rdma_xfer_sz); 2417 gioc->ioc_ctrl_opcap_mask = ioc->ioc_ctrl_opcap_mask; 2418 bcopy(ioc->ioc_id_string, gioc->ioc_id_string, 2419 IB_DM_IOC_ID_STRING_LEN); 2420 2421 ioc_info->ioc_iou_diagcode = gid_info->gl_iou->iou_diagcode; 2422 ioc_info->ioc_iou_dc_valid = gid_info->gl_iou->iou_dc_valid; 2423 ioc_info->ioc_diagdeviceid = (IB_DM_IOU_DEVICEID_MASK & 2424 gid_info->gl_iou->iou_info.iou_flag) ? B_TRUE : B_FALSE; 2425 2426 if (ioc_info->ioc_diagdeviceid == B_TRUE) 2427 gid_info->gl_pending_cmds++; 2428 } 2429 gioc->ioc_service_entries = ioc->ioc_service_entries; 2430 gid_info->gl_pending_cmds += (gioc->ioc_service_entries/4); 2431 if (gioc->ioc_service_entries % 4) 2432 gid_info->gl_pending_cmds++; 2433 2434 mutex_exit(&gid_info->gl_mutex); 2435 2436 ibdm_dump_ioc_profile(gioc); 2437 2438 if ((ioc_info->ioc_diagdeviceid == B_TRUE) && (reprobe == 0)) { 2439 if (ibdm_get_diagcode(gid_info, ioc_no) != IBDM_SUCCESS) { 2440 mutex_enter(&gid_info->gl_mutex); 2441 gid_info->gl_pending_cmds--; 2442 mutex_exit(&gid_info->gl_mutex); 2443 } 2444 } 2445 ioc_info->ioc_serv = (ibdm_srvents_info_t *)kmem_zalloc( 2446 (gioc->ioc_service_entries * sizeof (ibdm_srvents_info_t)), 2447 KM_SLEEP); 2448 2449 /* 2450 * In one single request, maximum number of requests that can be 2451 * obtained is 4. If number of service entries are more than four, 2452 * calculate number requests needed and send them parallelly. 2453 */ 2454 nserv_entries = ioc->ioc_service_entries; 2455 ii = 0; 2456 while (nserv_entries) { 2457 mutex_enter(&gid_info->gl_mutex); 2458 ibdm_bump_transactionID(gid_info); 2459 mutex_exit(&gid_info->gl_mutex); 2460 2461 if (first != B_TRUE) { 2462 if (ibmf_alloc_msg(ibmf_hdl, IBMF_ALLOC_SLEEP, 2463 &msg) != IBMF_SUCCESS) { 2464 continue; 2465 } 2466 2467 } 2468 ibdm_alloc_send_buffers(msg); 2469 msg->im_local_addr.ia_local_lid = gid_info->gl_slid; 2470 msg->im_local_addr.ia_remote_lid = gid_info->gl_dlid; 2471 if (gid_info->gl_redirected == B_TRUE) { 2472 if (gid_info->gl_redirect_dlid != 0) { 2473 msg->im_local_addr.ia_remote_lid = 2474 gid_info->gl_redirect_dlid; 2475 } 2476 msg->im_local_addr.ia_remote_qno = 2477 gid_info->gl_redirect_QP; 2478 msg->im_local_addr.ia_p_key = 2479 gid_info->gl_redirect_pkey; 2480 msg->im_local_addr.ia_q_key = 2481 gid_info->gl_redirect_qkey; 2482 } else { 2483 msg->im_local_addr.ia_remote_qno = 1; 2484 msg->im_local_addr.ia_p_key = gid_info->gl_p_key; 2485 msg->im_local_addr.ia_q_key = IB_GSI_QKEY; 2486 } 2487 2488 hdr = IBDM_OUT_IBMFMSG_MADHDR(msg); 2489 hdr->BaseVersion = MAD_CLASS_BASE_VERS_1; 2490 hdr->MgmtClass = MAD_MGMT_CLASS_DEV_MGT; 2491 hdr->ClassVersion = IB_DM_CLASS_VERSION_1; 2492 hdr->R_Method = IB_DM_DEVMGT_METHOD_GET; 2493 hdr->Status = 0; 2494 hdr->TransactionID = h2b64(gid_info->gl_transactionID); 2495 hdr->AttributeID = h2b16(IB_DM_ATTR_SERVICE_ENTRIES); 2496 2497 srv_start = ii * 4; 2498 cb_args = &ioc_info->ioc_serv[srv_start].se_cb_args; 2499 cb_args->cb_gid_info = gid_info; 2500 cb_args->cb_retry_count = ibdm_dft_retry_cnt; 2501 cb_args->cb_req_type = IBDM_REQ_TYPE_SRVENTS; 2502 cb_args->cb_srvents_start = srv_start; 2503 cb_args->cb_ioc_num = ioc_no - 1; 2504 2505 if (nserv_entries >= IBDM_MAX_SERV_ENTRIES_PER_REQ) { 2506 nserv_entries -= IBDM_MAX_SERV_ENTRIES_PER_REQ; 2507 cb_args->cb_srvents_end = (cb_args->cb_srvents_start + 2508 IBDM_MAX_SERV_ENTRIES_PER_REQ - 1); 2509 } else { 2510 cb_args->cb_srvents_end = 2511 (cb_args->cb_srvents_start + nserv_entries - 1); 2512 nserv_entries = 0; 2513 } 2514 ibdm_fill_srv_attr_mod(hdr, cb_args); 2515 2516 ioc_info->ioc_serv[srv_start].se_timeout_id = timeout( 2517 ibdm_pkt_timeout_hdlr, cb_args, 2518 IBDM_TIMEOUT_VALUE(ibdm_dft_timeout)); 2519 2520 IBTF_DPRINTF_L5("ibdm", "\thandle_ioc_profile:" 2521 "timeout %x, ioc %d srv %d", 2522 ioc_info->ioc_serv[srv_start].se_timeout_id, 2523 ioc_no - 1, srv_start); 2524 2525 if (ibmf_msg_transport(ibmf_hdl, gid_info->gl_qp_hdl, msg, 2526 NULL, ibdm_ibmf_send_cb, cb_args, 0) != IBMF_SUCCESS) { 2527 IBTF_DPRINTF_L2("ibdm", 2528 "\thandle_ioc_profile: msg send failed"); 2529 ibdm_ibmf_send_cb(ibmf_hdl, msg, cb_args); 2530 } 2531 (*flag) |= IBDM_IBMF_PKT_REUSED; 2532 first = B_FALSE; 2533 ii++; 2534 } 2535 } 2536 2537 2538 /* 2539 * ibdm_handle_srventry_mad() 2540 */ 2541 static void 2542 ibdm_handle_srventry_mad(ibmf_msg_t *msg, 2543 ibdm_dp_gidinfo_t *gid_info, int *flag) 2544 { 2545 uint_t ii, ioc_no, attrmod; 2546 uint_t nentries, start, end; 2547 timeout_id_t timeout_id; 2548 ib_dm_srv_t *srv_ents; 2549 ibdm_ioc_info_t *ioc_info; 2550 ibdm_srvents_info_t *gsrv_ents; 2551 2552 IBTF_DPRINTF_L4("ibdm", "\thandle_srventry_mad:" 2553 " IBMF msg %p gid info %p", msg, gid_info); 2554 2555 srv_ents = IBDM_IN_IBMFMSG2SRVENT(msg); 2556 /* 2557 * Get the start and end index of the service entries 2558 * Upper 16 bits identify the IOC 2559 * Lower 16 bits specify the range of service entries 2560 * LSB specifies (Big endian) end of the range 2561 * MSB specifies (Big endian) start of the range 2562 */ 2563 attrmod = IBDM_IN_IBMFMSG_ATTRMOD(msg); 2564 ioc_no = ((attrmod >> 16) & IBDM_16_BIT_MASK); 2565 end = ((attrmod >> 8) & IBDM_8_BIT_MASK); 2566 start = (attrmod & IBDM_8_BIT_MASK); 2567 2568 /* Make sure that IOC index is with the valid range */ 2569 if ((ioc_no < 1) | 2570 (ioc_no > gid_info->gl_iou->iou_info.iou_num_ctrl_slots)) { 2571 IBTF_DPRINTF_L2("ibdm", "\thandle_srventry_mad: " 2572 "IOC index Out of range, index %d", ioc_no); 2573 (*flag) |= IBDM_IBMF_PKT_UNEXP_RESP; 2574 return; 2575 } 2576 ioc_info = IBDM_GIDINFO2IOCINFO(gid_info, (ioc_no -1)); 2577 2578 /* 2579 * Make sure that the "start" and "end" service indexes are 2580 * with in the valid range 2581 */ 2582 nentries = ioc_info->ioc_profile.ioc_service_entries; 2583 if ((start > end) | (start >= nentries) | (end >= nentries)) { 2584 IBTF_DPRINTF_L2("ibdm", "\thandle_srventry_mad: " 2585 "Attr modifier 0x%x, #Serv entries %d", attrmod, nentries); 2586 (*flag) |= IBDM_IBMF_PKT_UNEXP_RESP; 2587 return; 2588 } 2589 gsrv_ents = &ioc_info->ioc_serv[start]; 2590 mutex_enter(&gid_info->gl_mutex); 2591 if (gsrv_ents->se_state != IBDM_SE_INVALID) { 2592 IBTF_DPRINTF_L2("ibdm", "\thandle_srventry_mad: " 2593 "already known, ioc %d, srv %d, se_state %x", 2594 ioc_no - 1, start, gsrv_ents->se_state); 2595 mutex_exit(&gid_info->gl_mutex); 2596 (*flag) |= IBDM_IBMF_PKT_DUP_RESP; 2597 return; 2598 } 2599 ioc_info->ioc_serv[start].se_cb_args.cb_req_type = 0; 2600 if (ioc_info->ioc_serv[start].se_timeout_id) { 2601 IBTF_DPRINTF_L2("ibdm", 2602 "\thandle_srventry_mad: ioc %d start %d", ioc_no, start); 2603 timeout_id = ioc_info->ioc_serv[start].se_timeout_id; 2604 mutex_exit(&gid_info->gl_mutex); 2605 IBTF_DPRINTF_L5("ibdm", "handle_srverntry_mad: " 2606 "se_timeout_id = 0x%x", timeout_id); 2607 if (untimeout(timeout_id) == -1) { 2608 IBTF_DPRINTF_L2("ibdm", "handle_srventry_mad: " 2609 "untimeout se_timeout_id failed"); 2610 } 2611 mutex_enter(&gid_info->gl_mutex); 2612 ioc_info->ioc_serv[start].se_timeout_id = 0; 2613 } 2614 2615 gsrv_ents->se_state = IBDM_SE_VALID; 2616 mutex_exit(&gid_info->gl_mutex); 2617 for (ii = start; ii <= end; ii++, srv_ents++, gsrv_ents++) { 2618 gsrv_ents->se_attr.srv_id = b2h64(srv_ents->srv_id); 2619 bcopy(srv_ents->srv_name, 2620 gsrv_ents->se_attr.srv_name, IB_DM_MAX_SVC_NAME_LEN); 2621 ibdm_dump_service_entries(&gsrv_ents->se_attr); 2622 } 2623 } 2624 2625 2626 /* 2627 * ibdm_get_diagcode: 2628 * Send request to get IOU/IOC diag code 2629 * Returns IBDM_SUCCESS/IBDM_FAILURE 2630 */ 2631 static int 2632 ibdm_get_diagcode(ibdm_dp_gidinfo_t *gid_info, int attr) 2633 { 2634 ibmf_msg_t *msg; 2635 ib_mad_hdr_t *hdr; 2636 ibdm_ioc_info_t *ioc; 2637 ibdm_timeout_cb_args_t *cb_args; 2638 timeout_id_t *timeout_id; 2639 2640 IBTF_DPRINTF_L4("ibdm", "\tget_diagcode: gid info %p, attr = %d", 2641 gid_info, attr); 2642 2643 if (ibmf_alloc_msg(gid_info->gl_ibmf_hdl, IBMF_ALLOC_SLEEP, 2644 &msg) != IBMF_SUCCESS) { 2645 IBTF_DPRINTF_L4("ibdm", "\tget_diagcode: pkt alloc fail"); 2646 return (IBDM_FAILURE); 2647 } 2648 2649 ibdm_alloc_send_buffers(msg); 2650 2651 mutex_enter(&gid_info->gl_mutex); 2652 ibdm_bump_transactionID(gid_info); 2653 mutex_exit(&gid_info->gl_mutex); 2654 2655 msg->im_local_addr.ia_local_lid = gid_info->gl_slid; 2656 msg->im_local_addr.ia_remote_lid = gid_info->gl_dlid; 2657 if (gid_info->gl_redirected == B_TRUE) { 2658 if (gid_info->gl_redirect_dlid != 0) { 2659 msg->im_local_addr.ia_remote_lid = 2660 gid_info->gl_redirect_dlid; 2661 } 2662 2663 msg->im_local_addr.ia_remote_qno = gid_info->gl_redirect_QP; 2664 msg->im_local_addr.ia_p_key = gid_info->gl_redirect_pkey; 2665 msg->im_local_addr.ia_q_key = gid_info->gl_redirect_qkey; 2666 } else { 2667 msg->im_local_addr.ia_remote_qno = 1; 2668 msg->im_local_addr.ia_p_key = gid_info->gl_p_key; 2669 msg->im_local_addr.ia_q_key = IB_GSI_QKEY; 2670 } 2671 2672 hdr = IBDM_OUT_IBMFMSG_MADHDR(msg); 2673 hdr->BaseVersion = MAD_CLASS_BASE_VERS_1; 2674 hdr->MgmtClass = MAD_MGMT_CLASS_DEV_MGT; 2675 hdr->ClassVersion = IB_DM_CLASS_VERSION_1; 2676 hdr->R_Method = IB_DM_DEVMGT_METHOD_GET; 2677 hdr->Status = 0; 2678 hdr->TransactionID = h2b64(gid_info->gl_transactionID); 2679 2680 hdr->AttributeID = h2b16(IB_DM_ATTR_DIAG_CODE); 2681 hdr->AttributeModifier = h2b32(attr); 2682 2683 if (attr == 0) { 2684 cb_args = &gid_info->gl_iou_cb_args; 2685 gid_info->gl_iou->iou_dc_valid = B_FALSE; 2686 cb_args->cb_ioc_num = 0; 2687 cb_args->cb_req_type = IBDM_REQ_TYPE_IOU_DIAGCODE; 2688 timeout_id = &gid_info->gl_timeout_id; 2689 } else { 2690 ioc = IBDM_GIDINFO2IOCINFO(gid_info, (attr - 1)); 2691 ioc->ioc_dc_valid = B_FALSE; 2692 cb_args = &ioc->ioc_dc_cb_args; 2693 cb_args->cb_ioc_num = attr - 1; 2694 cb_args->cb_req_type = IBDM_REQ_TYPE_IOC_DIAGCODE; 2695 timeout_id = &ioc->ioc_dc_timeout_id; 2696 } 2697 cb_args->cb_gid_info = gid_info; 2698 cb_args->cb_retry_count = ibdm_dft_retry_cnt; 2699 cb_args->cb_srvents_start = 0; 2700 2701 2702 *timeout_id = timeout(ibdm_pkt_timeout_hdlr, 2703 cb_args, IBDM_TIMEOUT_VALUE(ibdm_dft_timeout)); 2704 2705 IBTF_DPRINTF_L5("ibdm", "\tget_diagcode:" 2706 "timeout %x, ioc %d", *timeout_id, cb_args->cb_ioc_num); 2707 2708 if (ibmf_msg_transport(gid_info->gl_ibmf_hdl, gid_info->gl_qp_hdl, 2709 msg, NULL, ibdm_ibmf_send_cb, cb_args, 0) != IBMF_SUCCESS) { 2710 IBTF_DPRINTF_L2("ibdm", "\tget_diagcode: ibmf send failed"); 2711 ibdm_ibmf_send_cb(gid_info->gl_ibmf_hdl, msg, cb_args); 2712 } 2713 return (IBDM_SUCCESS); 2714 } 2715 2716 /* 2717 * ibdm_handle_diagcode: 2718 * Process the DiagCode MAD response and update local DM 2719 * data structure. 2720 */ 2721 static void 2722 ibdm_handle_diagcode(ibmf_msg_t *ibmf_msg, 2723 ibdm_dp_gidinfo_t *gid_info, int *flag) 2724 { 2725 uint16_t attrmod, *diagcode; 2726 ibdm_iou_info_t *iou; 2727 ibdm_ioc_info_t *ioc; 2728 timeout_id_t timeout_id; 2729 ibdm_timeout_cb_args_t *cb_args; 2730 2731 diagcode = (uint16_t *)ibmf_msg->im_msgbufs_recv.im_bufs_cl_data; 2732 2733 mutex_enter(&gid_info->gl_mutex); 2734 attrmod = IBDM_IN_IBMFMSG_ATTRMOD(ibmf_msg); 2735 iou = gid_info->gl_iou; 2736 if (attrmod == 0) { 2737 if (iou->iou_dc_valid != B_FALSE) { 2738 (*flag) |= IBDM_IBMF_PKT_DUP_RESP; 2739 IBTF_DPRINTF_L4("ibdm", 2740 "\thandle_diagcode: Duplicate IOU DiagCode"); 2741 mutex_exit(&gid_info->gl_mutex); 2742 return; 2743 } 2744 cb_args = &gid_info->gl_iou_cb_args; 2745 cb_args->cb_req_type = 0; 2746 iou->iou_diagcode = b2h16(*diagcode); 2747 iou->iou_dc_valid = B_TRUE; 2748 if (gid_info->gl_timeout_id) { 2749 timeout_id = gid_info->gl_timeout_id; 2750 mutex_exit(&gid_info->gl_mutex); 2751 IBTF_DPRINTF_L5("ibdm", "\thandle_diagcode: " 2752 "gl_timeout_id = 0x%x", timeout_id); 2753 if (untimeout(timeout_id) == -1) { 2754 IBTF_DPRINTF_L2("ibdm", "handle_diagcode: " 2755 "untimeout gl_timeout_id failed"); 2756 } 2757 mutex_enter(&gid_info->gl_mutex); 2758 gid_info->gl_timeout_id = 0; 2759 } 2760 } else { 2761 ioc = IBDM_GIDINFO2IOCINFO(gid_info, (attrmod - 1)); 2762 if (ioc->ioc_dc_valid != B_FALSE) { 2763 (*flag) |= IBDM_IBMF_PKT_DUP_RESP; 2764 IBTF_DPRINTF_L4("ibdm", 2765 "\thandle_diagcode: Duplicate IOC DiagCode"); 2766 mutex_exit(&gid_info->gl_mutex); 2767 return; 2768 } 2769 cb_args = &ioc->ioc_dc_cb_args; 2770 cb_args->cb_req_type = 0; 2771 ioc->ioc_diagcode = b2h16(*diagcode); 2772 ioc->ioc_dc_valid = B_TRUE; 2773 timeout_id = iou->iou_ioc_info[attrmod - 1].ioc_dc_timeout_id; 2774 if (timeout_id) { 2775 mutex_exit(&gid_info->gl_mutex); 2776 IBTF_DPRINTF_L5("ibdm", "handle_diagcode: " 2777 "timeout_id = 0x%x", timeout_id); 2778 if (untimeout(timeout_id) == -1) { 2779 IBTF_DPRINTF_L2("ibdm", "\thandle_diagcode: " 2780 "untimeout ioc_dc_timeout_id failed"); 2781 } 2782 mutex_enter(&gid_info->gl_mutex); 2783 iou->iou_ioc_info[attrmod - 1].ioc_dc_timeout_id = 0; 2784 } 2785 } 2786 mutex_exit(&gid_info->gl_mutex); 2787 2788 IBTF_DPRINTF_L4("ibdm", "\thandle_diagcode: DiagCode : 0x%x" 2789 "attrmod : 0x%x", b2h16(*diagcode), attrmod); 2790 } 2791 2792 2793 /* 2794 * ibdm_is_ioc_present() 2795 * Return ibdm_ioc_info_t if IOC guid is found in the global gid list 2796 */ 2797 static ibdm_ioc_info_t * 2798 ibdm_is_ioc_present(ib_guid_t ioc_guid, 2799 ibdm_dp_gidinfo_t *gid_info, int *flag) 2800 { 2801 int ii; 2802 ibdm_ioc_info_t *ioc; 2803 ibdm_dp_gidinfo_t *head; 2804 ib_dm_io_unitinfo_t *iou; 2805 2806 mutex_enter(&ibdm.ibdm_mutex); 2807 head = ibdm.ibdm_dp_gidlist_head; 2808 while (head) { 2809 mutex_enter(&head->gl_mutex); 2810 if (head->gl_iou == NULL) { 2811 mutex_exit(&head->gl_mutex); 2812 head = head->gl_next; 2813 continue; 2814 } 2815 iou = &head->gl_iou->iou_info; 2816 for (ii = 0; ii < iou->iou_num_ctrl_slots; ii++) { 2817 ioc = IBDM_GIDINFO2IOCINFO(head, ii); 2818 if ((ioc->ioc_state == IBDM_IOC_STATE_PROBE_SUCCESS) && 2819 (ioc->ioc_profile.ioc_guid == ioc_guid)) { 2820 if (gid_info == head) { 2821 *flag |= IBDM_IBMF_PKT_DUP_RESP; 2822 } else if (ibdm_check_dgid(head->gl_dgid_lo, 2823 head->gl_dgid_hi) != NULL) { 2824 IBTF_DPRINTF_L4("ibdm", "\tis_ioc_" 2825 "present: gid not present"); 2826 ibdm_add_to_gl_gid(gid_info, head); 2827 } 2828 mutex_exit(&head->gl_mutex); 2829 mutex_exit(&ibdm.ibdm_mutex); 2830 return (ioc); 2831 } 2832 } 2833 mutex_exit(&head->gl_mutex); 2834 head = head->gl_next; 2835 } 2836 mutex_exit(&ibdm.ibdm_mutex); 2837 return (NULL); 2838 } 2839 2840 2841 /* 2842 * ibdm_ibmf_send_cb() 2843 * IBMF invokes this callback routine after posting the DM MAD to 2844 * the HCA. 2845 */ 2846 /*ARGSUSED*/ 2847 static void 2848 ibdm_ibmf_send_cb(ibmf_handle_t ibmf_hdl, ibmf_msg_t *ibmf_msg, void *arg) 2849 { 2850 ibdm_dump_ibmf_msg(ibmf_msg, 1); 2851 ibdm_free_send_buffers(ibmf_msg); 2852 if (ibmf_free_msg(ibmf_hdl, &ibmf_msg) != IBMF_SUCCESS) { 2853 IBTF_DPRINTF_L4("ibdm", 2854 "\tibmf_send_cb: IBMF free msg failed"); 2855 } 2856 } 2857 2858 2859 /* 2860 * ibdm_ibmf_recv_cb() 2861 * Invoked by the IBMF when a response to the one of the DM requests 2862 * is received. 2863 */ 2864 /*ARGSUSED*/ 2865 static void 2866 ibdm_ibmf_recv_cb(ibmf_handle_t ibmf_hdl, ibmf_msg_t *msg, void *arg) 2867 { 2868 ibdm_taskq_args_t *taskq_args; 2869 2870 /* 2871 * If the taskq enable is set then dispatch a taskq to process 2872 * the MAD, otherwise just process it on this thread 2873 */ 2874 if (ibdm_taskq_enable != IBDM_ENABLE_TASKQ_HANDLING) { 2875 ibdm_process_incoming_mad(ibmf_hdl, msg, arg); 2876 return; 2877 } 2878 2879 /* 2880 * create a taskq and dispatch it to process the incoming MAD 2881 */ 2882 taskq_args = kmem_alloc(sizeof (ibdm_taskq_args_t), KM_NOSLEEP); 2883 if (taskq_args == NULL) { 2884 IBTF_DPRINTF_L2("ibdm", "ibmf_recv_cb: kmem_alloc failed for" 2885 "taskq_args"); 2886 if (ibmf_free_msg(ibmf_hdl, &msg) != IBMF_SUCCESS) { 2887 IBTF_DPRINTF_L4("ibmf_recv_cb", 2888 "\tibmf_recv_cb: IBMF free msg failed"); 2889 } 2890 return; 2891 } 2892 taskq_args->tq_ibmf_handle = ibmf_hdl; 2893 taskq_args->tq_ibmf_msg = msg; 2894 taskq_args->tq_args = arg; 2895 2896 if (taskq_dispatch(system_taskq, ibdm_recv_incoming_mad, taskq_args, 2897 TQ_NOSLEEP) == 0) { 2898 IBTF_DPRINTF_L2("ibdm", "ibmf_recv_cb: taskq_dispatch failed"); 2899 if (ibmf_free_msg(ibmf_hdl, &msg) != IBMF_SUCCESS) { 2900 IBTF_DPRINTF_L4("ibmf_recv_cb", 2901 "\tibmf_recv_cb: IBMF free msg failed"); 2902 } 2903 kmem_free(taskq_args, sizeof (ibdm_taskq_args_t)); 2904 return; 2905 } 2906 2907 /* taskq_args are deleted in ibdm_recv_incoming_mad() */ 2908 } 2909 2910 2911 void 2912 ibdm_recv_incoming_mad(void *args) 2913 { 2914 ibdm_taskq_args_t *taskq_args; 2915 2916 taskq_args = (ibdm_taskq_args_t *)args; 2917 2918 IBTF_DPRINTF_L4("ibdm", "\tibdm_recv_incoming_mad: " 2919 "Processing incoming MAD via taskq"); 2920 2921 ibdm_process_incoming_mad(taskq_args->tq_ibmf_handle, 2922 taskq_args->tq_ibmf_msg, taskq_args->tq_args); 2923 2924 kmem_free(taskq_args, sizeof (ibdm_taskq_args_t)); 2925 } 2926 2927 2928 /* 2929 * Calls ibdm_process_incoming_mad with all function arguments extracted 2930 * from args 2931 */ 2932 /*ARGSUSED*/ 2933 static void 2934 ibdm_process_incoming_mad(ibmf_handle_t ibmf_hdl, ibmf_msg_t *msg, void *arg) 2935 { 2936 int flag = 0; 2937 int ret; 2938 uint64_t transaction_id; 2939 ib_mad_hdr_t *hdr; 2940 ibdm_dp_gidinfo_t *gid_info = NULL; 2941 2942 IBTF_DPRINTF_L4("ibdm", 2943 "\tprocess_incoming_mad: ibmf hdl %p pkt %p", ibmf_hdl, msg); 2944 ibdm_dump_ibmf_msg(msg, 0); 2945 2946 /* 2947 * IBMF calls this routine for every DM MAD that arrives at this port. 2948 * But we handle only the responses for requests we sent. We drop all 2949 * the DM packets that does not have response bit set in the MAD 2950 * header(this eliminates all the requests sent to this port). 2951 * We handle only DM class version 1 MAD's 2952 */ 2953 hdr = IBDM_IN_IBMFMSG_MADHDR(msg); 2954 if (ibdm_verify_mad_status(hdr) != IBDM_SUCCESS) { 2955 if (ibmf_free_msg(ibmf_hdl, &msg) != IBMF_SUCCESS) { 2956 IBTF_DPRINTF_L2("ibdm", "\tprocess_incoming_mad: " 2957 "IBMF free msg failed DM request drop it"); 2958 } 2959 return; 2960 } 2961 2962 transaction_id = b2h64(hdr->TransactionID); 2963 2964 mutex_enter(&ibdm.ibdm_mutex); 2965 gid_info = ibdm.ibdm_dp_gidlist_head; 2966 while (gid_info) { 2967 if ((gid_info->gl_transactionID & 2968 IBDM_GID_TRANSACTIONID_MASK) == 2969 (transaction_id & IBDM_GID_TRANSACTIONID_MASK)) 2970 break; 2971 gid_info = gid_info->gl_next; 2972 } 2973 mutex_exit(&ibdm.ibdm_mutex); 2974 2975 if (gid_info == NULL) { 2976 /* Drop the packet */ 2977 IBTF_DPRINTF_L2("ibdm", "process_incoming_mad: transaction ID" 2978 " does not match: 0x%llx", transaction_id); 2979 if (ibmf_free_msg(ibmf_hdl, &msg) != IBMF_SUCCESS) { 2980 IBTF_DPRINTF_L2("ibdm", "process_incoming_mad: " 2981 "IBMF free msg failed DM request drop it"); 2982 } 2983 return; 2984 } 2985 2986 /* Handle redirection for all the MAD's, except ClassPortInfo */ 2987 if (((IBDM_IN_IBMFMSG_STATUS(msg) & MAD_STATUS_REDIRECT_REQUIRED)) && 2988 (IBDM_IN_IBMFMSG_ATTR(msg) != IB_DM_ATTR_CLASSPORTINFO)) { 2989 ret = ibdm_handle_redirection(msg, gid_info, &flag); 2990 if (ret == IBDM_SUCCESS) { 2991 return; 2992 } 2993 } else { 2994 uint_t gl_state; 2995 2996 mutex_enter(&gid_info->gl_mutex); 2997 gl_state = gid_info->gl_state; 2998 mutex_exit(&gid_info->gl_mutex); 2999 3000 switch (gl_state) { 3001 case IBDM_GET_CLASSPORTINFO: 3002 ibdm_handle_classportinfo( 3003 ibmf_hdl, msg, gid_info, &flag); 3004 break; 3005 3006 case IBDM_GET_IOUNITINFO: 3007 ibdm_handle_iounitinfo(ibmf_hdl, msg, gid_info, &flag); 3008 break; 3009 3010 case IBDM_GET_IOC_DETAILS: 3011 switch (IBDM_IN_IBMFMSG_ATTR(msg)) { 3012 3013 case IB_DM_ATTR_SERVICE_ENTRIES: 3014 ibdm_handle_srventry_mad(msg, gid_info, &flag); 3015 break; 3016 3017 case IB_DM_ATTR_IOC_CTRL_PROFILE: 3018 ibdm_handle_ioc_profile( 3019 ibmf_hdl, msg, gid_info, &flag); 3020 break; 3021 3022 case IB_DM_ATTR_DIAG_CODE: 3023 ibdm_handle_diagcode(msg, gid_info, &flag); 3024 break; 3025 3026 default: 3027 IBTF_DPRINTF_L2("ibdm", "process_incoming_mad: " 3028 "Error state, wrong attribute :-("); 3029 (void) ibmf_free_msg(ibmf_hdl, &msg); 3030 return; 3031 } 3032 break; 3033 default: 3034 IBTF_DPRINTF_L2("ibdm", 3035 "process_incoming_mad: Dropping the packet" 3036 " gl_state %x", gl_state); 3037 if (ibmf_free_msg(ibmf_hdl, &msg) != IBMF_SUCCESS) { 3038 IBTF_DPRINTF_L2("ibdm", "process_incoming_mad: " 3039 "IBMF free msg failed DM request drop it"); 3040 } 3041 return; 3042 } 3043 } 3044 3045 if ((flag & IBDM_IBMF_PKT_DUP_RESP) || 3046 (flag & IBDM_IBMF_PKT_UNEXP_RESP)) { 3047 IBTF_DPRINTF_L2("ibdm", 3048 "\tprocess_incoming_mad:Dup/unexp resp : 0x%x", flag); 3049 if (ibmf_free_msg(ibmf_hdl, &msg) != IBMF_SUCCESS) { 3050 IBTF_DPRINTF_L2("ibdm", "process_incoming_mad: " 3051 "IBMF free msg failed DM request drop it"); 3052 } 3053 return; 3054 } 3055 3056 mutex_enter(&gid_info->gl_mutex); 3057 if (gid_info->gl_pending_cmds < 1) { 3058 IBTF_DPRINTF_L2("ibdm", 3059 "\tprocess_incoming_mad: pending commands negative"); 3060 } 3061 if (--gid_info->gl_pending_cmds) { 3062 IBTF_DPRINTF_L4("ibdm", "\tprocess_incoming_mad: " 3063 "gid_info %p pending cmds %d", 3064 gid_info, gid_info->gl_pending_cmds); 3065 mutex_exit(&gid_info->gl_mutex); 3066 } else { 3067 IBTF_DPRINTF_L4("ibdm", "\tprocess_incoming_mad: Probing DONE"); 3068 gid_info->gl_state = IBDM_GID_PROBING_COMPLETE; 3069 mutex_exit(&gid_info->gl_mutex); 3070 ibdm_notify_newgid_iocs(gid_info); 3071 mutex_enter(&ibdm.ibdm_mutex); 3072 if (--ibdm.ibdm_ngid_probes_in_progress == 0) { 3073 IBTF_DPRINTF_L4("ibdm", 3074 "\tprocess_incoming_mad: Wakeup"); 3075 ibdm.ibdm_busy &= ~IBDM_PROBE_IN_PROGRESS; 3076 cv_broadcast(&ibdm.ibdm_probe_cv); 3077 } 3078 mutex_exit(&ibdm.ibdm_mutex); 3079 } 3080 3081 /* 3082 * Do not deallocate the IBMF packet if atleast one request 3083 * is posted. IBMF packet is reused. 3084 */ 3085 if (!(flag & IBDM_IBMF_PKT_REUSED)) { 3086 if (ibmf_free_msg(ibmf_hdl, &msg) != IBMF_SUCCESS) { 3087 IBTF_DPRINTF_L2("ibdm", "\tprocess_incoming_mad: " 3088 "IBMF free msg failed DM request drop it"); 3089 } 3090 } 3091 } 3092 3093 3094 /* 3095 * ibdm_verify_mad_status() 3096 * Verifies the MAD status 3097 * Returns IBDM_SUCCESS if status is correct 3098 * Returns IBDM_FAILURE for bogus MAD status 3099 */ 3100 static int 3101 ibdm_verify_mad_status(ib_mad_hdr_t *hdr) 3102 { 3103 int ret = 0; 3104 3105 if ((hdr->R_Method != IB_DM_DEVMGT_METHOD_GET_RESP) || 3106 (hdr->ClassVersion != IB_DM_CLASS_VERSION_1)) { 3107 return (IBDM_FAILURE); 3108 } 3109 3110 if (b2h16(hdr->Status) == 0) 3111 ret = IBDM_SUCCESS; 3112 else if ((b2h16(hdr->Status) & 0x1f) == MAD_STATUS_REDIRECT_REQUIRED) 3113 ret = IBDM_SUCCESS; 3114 else { 3115 IBTF_DPRINTF_L4("ibdm", 3116 "\tverify_mad_status: Stauts : 0x%x", b2h16(hdr->Status)); 3117 ret = IBDM_FAILURE; 3118 } 3119 return (ret); 3120 } 3121 3122 3123 3124 /* 3125 * ibdm_handle_redirection() 3126 * Returns IBDM_SUCCESS/IBDM_FAILURE 3127 */ 3128 static int 3129 ibdm_handle_redirection(ibmf_msg_t *msg, 3130 ibdm_dp_gidinfo_t *gid_info, int *flag) 3131 { 3132 int attrmod, ioc_no, start; 3133 void *data; 3134 timeout_id_t *timeout_id; 3135 ib_mad_hdr_t *hdr; 3136 ibdm_ioc_info_t *ioc = NULL; 3137 ibdm_timeout_cb_args_t *cb_args; 3138 ibdm_mad_classportinfo_t *cpi; 3139 3140 IBTF_DPRINTF_L4("ibdm", "\thandle_redirection: Enter"); 3141 mutex_enter(&gid_info->gl_mutex); 3142 switch (gid_info->gl_state) { 3143 case IBDM_GET_IOUNITINFO: 3144 cb_args = &gid_info->gl_iou_cb_args; 3145 timeout_id = &gid_info->gl_timeout_id; 3146 break; 3147 3148 case IBDM_GET_IOC_DETAILS: 3149 attrmod = IBDM_IN_IBMFMSG_ATTRMOD(msg); 3150 switch (IBDM_IN_IBMFMSG_ATTR(msg)) { 3151 3152 case IB_DM_ATTR_DIAG_CODE: 3153 if (attrmod == 0) { 3154 cb_args = &gid_info->gl_iou_cb_args; 3155 timeout_id = &gid_info->gl_timeout_id; 3156 break; 3157 } 3158 if (IBDM_IS_IOC_NUM_INVALID(attrmod, gid_info)) { 3159 IBTF_DPRINTF_L2("ibdm", "\thandle_redirction:" 3160 "IOC# Out of range %d", attrmod); 3161 (*flag) |= IBDM_IBMF_PKT_UNEXP_RESP; 3162 mutex_exit(&gid_info->gl_mutex); 3163 return (IBDM_FAILURE); 3164 } 3165 ioc = IBDM_GIDINFO2IOCINFO(gid_info, (attrmod -1)); 3166 cb_args = &ioc->ioc_dc_cb_args; 3167 timeout_id = &ioc->ioc_dc_timeout_id; 3168 break; 3169 3170 case IB_DM_ATTR_IOC_CTRL_PROFILE: 3171 if (IBDM_IS_IOC_NUM_INVALID(attrmod, gid_info)) { 3172 IBTF_DPRINTF_L2("ibdm", "\thandle_redirction:" 3173 "IOC# Out of range %d", attrmod); 3174 (*flag) |= IBDM_IBMF_PKT_UNEXP_RESP; 3175 mutex_exit(&gid_info->gl_mutex); 3176 return (IBDM_FAILURE); 3177 } 3178 ioc = IBDM_GIDINFO2IOCINFO(gid_info, (attrmod -1)); 3179 cb_args = &ioc->ioc_cb_args; 3180 timeout_id = &ioc->ioc_timeout_id; 3181 break; 3182 3183 case IB_DM_ATTR_SERVICE_ENTRIES: 3184 ioc_no = ((attrmod >> 16) & IBDM_16_BIT_MASK); 3185 if (IBDM_IS_IOC_NUM_INVALID(ioc_no, gid_info)) { 3186 IBTF_DPRINTF_L2("ibdm", "\thandle_redirction:" 3187 "IOC# Out of range %d", ioc_no); 3188 (*flag) |= IBDM_IBMF_PKT_UNEXP_RESP; 3189 mutex_exit(&gid_info->gl_mutex); 3190 return (IBDM_FAILURE); 3191 } 3192 start = (attrmod & IBDM_8_BIT_MASK); 3193 ioc = IBDM_GIDINFO2IOCINFO(gid_info, (ioc_no -1)); 3194 if (start > ioc->ioc_profile.ioc_service_entries) { 3195 IBTF_DPRINTF_L2("ibdm", "\thandle_redirction:" 3196 " SE index Out of range %d", start); 3197 (*flag) |= IBDM_IBMF_PKT_UNEXP_RESP; 3198 mutex_exit(&gid_info->gl_mutex); 3199 return (IBDM_FAILURE); 3200 } 3201 cb_args = &ioc->ioc_serv[start].se_cb_args; 3202 timeout_id = &ioc->ioc_serv[start].se_timeout_id; 3203 break; 3204 3205 default: 3206 /* ERROR State */ 3207 IBTF_DPRINTF_L2("ibdm", 3208 "\thandle_redirection: wrong attribute :-("); 3209 (*flag) |= IBDM_IBMF_PKT_UNEXP_RESP; 3210 mutex_exit(&gid_info->gl_mutex); 3211 return (IBDM_FAILURE); 3212 } 3213 break; 3214 default: 3215 /* ERROR State */ 3216 IBTF_DPRINTF_L2("ibdm", 3217 "\thandle_redirection: Error state :-("); 3218 (*flag) |= IBDM_IBMF_PKT_UNEXP_RESP; 3219 mutex_exit(&gid_info->gl_mutex); 3220 return (IBDM_FAILURE); 3221 } 3222 if ((*timeout_id) != 0) { 3223 mutex_exit(&gid_info->gl_mutex); 3224 if (untimeout(*timeout_id) == -1) { 3225 IBTF_DPRINTF_L2("ibdm", "\thandle_redirection: " 3226 "untimeout failed %x", *timeout_id); 3227 } else { 3228 IBTF_DPRINTF_L5("ibdm", 3229 "\thandle_redirection: timeout %x", *timeout_id); 3230 } 3231 mutex_enter(&gid_info->gl_mutex); 3232 *timeout_id = 0; 3233 } 3234 3235 data = msg->im_msgbufs_recv.im_bufs_cl_data; 3236 cpi = (ibdm_mad_classportinfo_t *)data; 3237 3238 gid_info->gl_resp_timeout = 3239 (b2h32(cpi->RespTimeValue) & 0x1F); 3240 3241 gid_info->gl_redirected = B_TRUE; 3242 gid_info->gl_redirect_dlid = b2h16(cpi->RedirectLID); 3243 gid_info->gl_redirect_QP = (b2h32(cpi->RedirectQP) & 0xffffff); 3244 gid_info->gl_redirect_pkey = b2h16(cpi->RedirectP_Key); 3245 gid_info->gl_redirect_qkey = b2h32(cpi->RedirectQ_Key); 3246 gid_info->gl_redirectGID_hi = b2h64(cpi->RedirectGID_hi); 3247 gid_info->gl_redirectGID_lo = b2h64(cpi->RedirectGID_lo); 3248 3249 if (gid_info->gl_redirect_dlid != 0) { 3250 msg->im_local_addr.ia_remote_lid = 3251 gid_info->gl_redirect_dlid; 3252 } 3253 ibdm_bump_transactionID(gid_info); 3254 mutex_exit(&gid_info->gl_mutex); 3255 3256 ibdm_alloc_send_buffers(msg); 3257 3258 hdr = IBDM_OUT_IBMFMSG_MADHDR(msg); 3259 hdr->BaseVersion = MAD_CLASS_BASE_VERS_1; 3260 hdr->MgmtClass = MAD_MGMT_CLASS_DEV_MGT; 3261 hdr->ClassVersion = IB_DM_CLASS_VERSION_1; 3262 hdr->R_Method = IB_DM_DEVMGT_METHOD_GET; 3263 hdr->Status = 0; 3264 hdr->TransactionID = h2b64(gid_info->gl_transactionID); 3265 hdr->AttributeID = 3266 msg->im_msgbufs_recv.im_bufs_mad_hdr->AttributeID; 3267 hdr->AttributeModifier = 3268 msg->im_msgbufs_recv.im_bufs_mad_hdr->AttributeModifier; 3269 3270 msg->im_local_addr.ia_remote_qno = gid_info->gl_redirect_QP; 3271 msg->im_local_addr.ia_p_key = gid_info->gl_redirect_pkey; 3272 msg->im_local_addr.ia_q_key = gid_info->gl_redirect_qkey; 3273 3274 *timeout_id = timeout(ibdm_pkt_timeout_hdlr, 3275 cb_args, IBDM_TIMEOUT_VALUE(ibdm_dft_timeout)); 3276 3277 IBTF_DPRINTF_L5("ibdm", "\thandle_redirect:" 3278 "timeout %x", *timeout_id); 3279 3280 if (ibmf_msg_transport(gid_info->gl_ibmf_hdl, gid_info->gl_qp_hdl, 3281 msg, NULL, ibdm_ibmf_send_cb, cb_args, 0) != IBMF_SUCCESS) { 3282 IBTF_DPRINTF_L4("ibdm", "\thandle_redirection:" 3283 "message transport failed"); 3284 ibdm_ibmf_send_cb(gid_info->gl_ibmf_hdl, msg, cb_args); 3285 } 3286 (*flag) |= IBDM_IBMF_PKT_REUSED; 3287 IBTF_DPRINTF_L4("ibdm", "\thandle_redirection: Exit"); 3288 return (IBDM_SUCCESS); 3289 } 3290 3291 3292 /* 3293 * ibdm_pkt_timeout_hdlr 3294 * This timeout handler is registed for every IBMF packet that is 3295 * sent through the IBMF. It gets called when no response is received 3296 * within the specified time for the packet. No retries for the failed 3297 * commands currently. Drops the failed IBMF packet and update the 3298 * pending list commands. 3299 */ 3300 static void 3301 ibdm_pkt_timeout_hdlr(void *arg) 3302 { 3303 int probe_done = B_FALSE; 3304 ibdm_iou_info_t *iou; 3305 ibdm_ioc_info_t *ioc; 3306 ibdm_timeout_cb_args_t *cb_args = arg; 3307 ibdm_dp_gidinfo_t *gid_info; 3308 int srv_ent; 3309 uint_t new_gl_state; 3310 3311 IBTF_DPRINTF_L2("ibdm", "\tpkt_timeout_hdlr: gid_info: %p " 3312 "rtype 0x%x iocidx 0x%x srvidx %d", cb_args->cb_gid_info, 3313 cb_args->cb_req_type, cb_args->cb_ioc_num, 3314 cb_args->cb_srvents_start); 3315 3316 gid_info = cb_args->cb_gid_info; 3317 mutex_enter(&gid_info->gl_mutex); 3318 3319 if ((gid_info->gl_state == IBDM_GID_PROBING_COMPLETE) || 3320 (cb_args->cb_req_type == 0)) { 3321 3322 IBTF_DPRINTF_L2("ibdm", "\tpkt_timeout_hdlr: req completed" 3323 "rtype 0x%x iocidx 0x%x srvidx %d", cb_args->cb_req_type, 3324 cb_args->cb_ioc_num, cb_args->cb_srvents_start); 3325 3326 if (gid_info->gl_timeout_id) 3327 gid_info->gl_timeout_id = 0; 3328 mutex_exit(&gid_info->gl_mutex); 3329 return; 3330 } 3331 if (cb_args->cb_retry_count) { 3332 cb_args->cb_retry_count--; 3333 if (ibdm_retry_command(cb_args) == IBDM_SUCCESS) { 3334 if (gid_info->gl_timeout_id) 3335 gid_info->gl_timeout_id = 0; 3336 mutex_exit(&gid_info->gl_mutex); 3337 return; 3338 } 3339 cb_args->cb_retry_count = 0; 3340 } 3341 3342 IBTF_DPRINTF_L2("ibdm", "\tpkt_timeout_hdlr: command failed: gid %p" 3343 " rtype 0x%x iocidx 0x%x srvidx %d", cb_args->cb_gid_info, 3344 cb_args->cb_req_type, cb_args->cb_ioc_num, 3345 cb_args->cb_srvents_start); 3346 3347 new_gl_state = IBDM_GID_PROBING_COMPLETE; 3348 switch (cb_args->cb_req_type) { 3349 3350 case IBDM_REQ_TYPE_CLASSPORTINFO: 3351 case IBDM_REQ_TYPE_IOUINFO: 3352 new_gl_state = IBDM_GID_PROBING_FAILED; 3353 if (--gid_info->gl_pending_cmds == 0) 3354 probe_done = B_TRUE; 3355 if (gid_info->gl_timeout_id) 3356 gid_info->gl_timeout_id = 0; 3357 mutex_exit(&gid_info->gl_mutex); 3358 ibdm_delete_glhca_list(gid_info); 3359 mutex_enter(&gid_info->gl_mutex); 3360 break; 3361 case IBDM_REQ_TYPE_IOCINFO: 3362 iou = gid_info->gl_iou; 3363 ioc = &iou->iou_ioc_info[cb_args->cb_ioc_num]; 3364 ioc->ioc_state = IBDM_IOC_STATE_PROBE_FAILED; 3365 if (--gid_info->gl_pending_cmds == 0) 3366 probe_done = B_TRUE; 3367 #ifndef __lock_lint 3368 if (ioc->ioc_timeout_id) 3369 ioc->ioc_timeout_id = 0; 3370 #endif 3371 break; 3372 case IBDM_REQ_TYPE_SRVENTS: 3373 iou = gid_info->gl_iou; 3374 ioc = &iou->iou_ioc_info[cb_args->cb_ioc_num]; 3375 ioc->ioc_state = IBDM_IOC_STATE_PROBE_FAILED; 3376 if (--gid_info->gl_pending_cmds == 0) 3377 probe_done = B_TRUE; 3378 srv_ent = cb_args->cb_srvents_start; 3379 #ifndef __lock_lint 3380 if (ioc->ioc_serv[srv_ent].se_timeout_id) 3381 ioc->ioc_serv[srv_ent].se_timeout_id = 0; 3382 #endif 3383 break; 3384 case IBDM_REQ_TYPE_IOU_DIAGCODE: 3385 iou = gid_info->gl_iou; 3386 iou->iou_dc_valid = B_FALSE; 3387 if (--gid_info->gl_pending_cmds == 0) 3388 probe_done = B_TRUE; 3389 if (gid_info->gl_timeout_id) 3390 gid_info->gl_timeout_id = 0; 3391 break; 3392 case IBDM_REQ_TYPE_IOC_DIAGCODE: 3393 iou = gid_info->gl_iou; 3394 ioc = &iou->iou_ioc_info[cb_args->cb_ioc_num]; 3395 ioc->ioc_dc_valid = B_FALSE; 3396 if (--gid_info->gl_pending_cmds == 0) 3397 probe_done = B_TRUE; 3398 #ifndef __lock_lint 3399 if (ioc->ioc_dc_timeout_id) 3400 ioc->ioc_dc_timeout_id = 0; 3401 #endif 3402 break; 3403 } 3404 if (probe_done == B_TRUE) { 3405 gid_info->gl_state = new_gl_state; 3406 mutex_exit(&gid_info->gl_mutex); 3407 ibdm_notify_newgid_iocs(gid_info); 3408 mutex_enter(&ibdm.ibdm_mutex); 3409 if (--ibdm.ibdm_ngid_probes_in_progress == 0) { 3410 IBTF_DPRINTF_L4("ibdm", "\tpkt_timeout_hdlr: Wakeup"); 3411 ibdm.ibdm_busy &= ~IBDM_PROBE_IN_PROGRESS; 3412 cv_broadcast(&ibdm.ibdm_probe_cv); 3413 } 3414 mutex_exit(&ibdm.ibdm_mutex); 3415 } else 3416 mutex_exit(&gid_info->gl_mutex); 3417 } 3418 3419 3420 /* 3421 * ibdm_retry_command() 3422 * Retries the failed command. 3423 * Returns IBDM_FAILURE/IBDM_SUCCESS 3424 */ 3425 static int 3426 ibdm_retry_command(ibdm_timeout_cb_args_t *cb_args) 3427 { 3428 int ret, rval = IBDM_SUCCESS; 3429 ibmf_msg_t *msg; 3430 ib_mad_hdr_t *hdr; 3431 ibdm_dp_gidinfo_t *gid_info = cb_args->cb_gid_info; 3432 timeout_id_t *timeout_id; 3433 ibdm_ioc_info_t *ioc; 3434 int ioc_no; 3435 3436 IBTF_DPRINTF_L2("ibdm", "\tretry_command: gid_info: %p " 3437 "rtype 0x%x iocidx 0x%x srvidx %d", cb_args->cb_gid_info, 3438 cb_args->cb_req_type, cb_args->cb_ioc_num, 3439 cb_args->cb_srvents_start); 3440 3441 ret = ibmf_alloc_msg(gid_info->gl_ibmf_hdl, IBMF_ALLOC_NOSLEEP, &msg); 3442 3443 3444 /* 3445 * Reset the gid if alloc_msg failed with BAD_HANDLE 3446 * ibdm_reset_gidinfo reinits the gid_info 3447 */ 3448 if (ret == IBMF_BAD_HANDLE) { 3449 IBTF_DPRINTF_L3(ibdm_string, "\tretry_command: gid %p hdl bad", 3450 gid_info); 3451 3452 mutex_exit(&gid_info->gl_mutex); 3453 ibdm_reset_gidinfo(gid_info); 3454 mutex_enter(&gid_info->gl_mutex); 3455 3456 /* Retry alloc */ 3457 ret = ibmf_alloc_msg(gid_info->gl_ibmf_hdl, IBMF_ALLOC_NOSLEEP, 3458 &msg); 3459 } 3460 3461 if (ret != IBDM_SUCCESS) { 3462 IBTF_DPRINTF_L2("ibdm", "\tretry_command: alloc failed: %p " 3463 "rtype 0x%x iocidx 0x%x srvidx %d", cb_args->cb_gid_info, 3464 cb_args->cb_req_type, cb_args->cb_ioc_num, 3465 cb_args->cb_srvents_start); 3466 return (IBDM_FAILURE); 3467 } 3468 3469 ibdm_alloc_send_buffers(msg); 3470 3471 ibdm_bump_transactionID(gid_info); 3472 3473 msg->im_local_addr.ia_local_lid = gid_info->gl_slid; 3474 msg->im_local_addr.ia_remote_lid = gid_info->gl_dlid; 3475 if (gid_info->gl_redirected == B_TRUE) { 3476 if (gid_info->gl_redirect_dlid != 0) { 3477 msg->im_local_addr.ia_remote_lid = 3478 gid_info->gl_redirect_dlid; 3479 } 3480 msg->im_local_addr.ia_remote_qno = gid_info->gl_redirect_QP; 3481 msg->im_local_addr.ia_p_key = gid_info->gl_redirect_pkey; 3482 msg->im_local_addr.ia_q_key = gid_info->gl_redirect_qkey; 3483 } else { 3484 msg->im_local_addr.ia_remote_qno = 1; 3485 msg->im_local_addr.ia_p_key = gid_info->gl_p_key; 3486 msg->im_local_addr.ia_q_key = IB_GSI_QKEY; 3487 } 3488 hdr = IBDM_OUT_IBMFMSG_MADHDR(msg); 3489 hdr->BaseVersion = MAD_CLASS_BASE_VERS_1; 3490 hdr->MgmtClass = MAD_MGMT_CLASS_DEV_MGT; 3491 hdr->ClassVersion = IB_DM_CLASS_VERSION_1; 3492 hdr->R_Method = IB_DM_DEVMGT_METHOD_GET; 3493 hdr->Status = 0; 3494 hdr->TransactionID = h2b64(gid_info->gl_transactionID); 3495 3496 switch (cb_args->cb_req_type) { 3497 case IBDM_REQ_TYPE_CLASSPORTINFO: 3498 hdr->AttributeID = h2b16(IB_DM_ATTR_CLASSPORTINFO); 3499 hdr->AttributeModifier = 0; 3500 timeout_id = &gid_info->gl_timeout_id; 3501 break; 3502 case IBDM_REQ_TYPE_IOUINFO: 3503 hdr->AttributeID = h2b16(IB_DM_ATTR_IO_UNITINFO); 3504 hdr->AttributeModifier = 0; 3505 timeout_id = &gid_info->gl_timeout_id; 3506 break; 3507 case IBDM_REQ_TYPE_IOCINFO: 3508 hdr->AttributeID = h2b16(IB_DM_ATTR_IOC_CTRL_PROFILE); 3509 hdr->AttributeModifier = h2b32(cb_args->cb_ioc_num + 1); 3510 ioc = IBDM_GIDINFO2IOCINFO(gid_info, cb_args->cb_ioc_num); 3511 timeout_id = &ioc->ioc_timeout_id; 3512 break; 3513 case IBDM_REQ_TYPE_SRVENTS: 3514 hdr->AttributeID = h2b16(IB_DM_ATTR_SERVICE_ENTRIES); 3515 ibdm_fill_srv_attr_mod(hdr, cb_args); 3516 ioc = IBDM_GIDINFO2IOCINFO(gid_info, cb_args->cb_ioc_num); 3517 timeout_id = 3518 &ioc->ioc_serv[cb_args->cb_srvents_start].se_timeout_id; 3519 break; 3520 case IBDM_REQ_TYPE_IOU_DIAGCODE: 3521 hdr->AttributeID = h2b16(IB_DM_ATTR_DIAG_CODE); 3522 hdr->AttributeModifier = 0; 3523 timeout_id = &gid_info->gl_timeout_id; 3524 break; 3525 case IBDM_REQ_TYPE_IOC_DIAGCODE: 3526 hdr->AttributeID = h2b16(IB_DM_ATTR_DIAG_CODE); 3527 hdr->AttributeModifier = h2b32(cb_args->cb_ioc_num + 1); 3528 ioc_no = cb_args->cb_ioc_num; 3529 ioc = &gid_info->gl_iou->iou_ioc_info[ioc_no]; 3530 timeout_id = &ioc->ioc_dc_timeout_id; 3531 break; 3532 } 3533 3534 mutex_exit(&gid_info->gl_mutex); 3535 3536 *timeout_id = timeout(ibdm_pkt_timeout_hdlr, 3537 cb_args, IBDM_TIMEOUT_VALUE(ibdm_dft_timeout)); 3538 3539 IBTF_DPRINTF_L5("ibdm", "\tretry_command: %p,%x,%d,%d:" 3540 "timeout %x", cb_args->cb_req_type, cb_args->cb_ioc_num, 3541 cb_args->cb_srvents_start, *timeout_id); 3542 3543 if ((rval = ibmf_msg_transport(gid_info->gl_ibmf_hdl, 3544 gid_info->gl_qp_hdl, msg, NULL, ibdm_ibmf_send_cb, 3545 cb_args, 0)) != IBMF_SUCCESS) { 3546 IBTF_DPRINTF_L2("ibdm", "\tretry_command: send failed: %p " 3547 "rtype 0x%x iocidx 0x%x srvidx %d", cb_args->cb_gid_info, 3548 cb_args->cb_req_type, cb_args->cb_ioc_num, 3549 cb_args->cb_srvents_start); 3550 ibdm_ibmf_send_cb(gid_info->gl_ibmf_hdl, msg, cb_args); 3551 rval = IBDM_FAILURE; 3552 } 3553 mutex_enter(&gid_info->gl_mutex); 3554 return (rval); 3555 } 3556 3557 3558 /* 3559 * ibdm_update_ioc_port_gidlist() 3560 */ 3561 static void 3562 ibdm_update_ioc_port_gidlist(ibdm_ioc_info_t *dest, 3563 ibdm_dp_gidinfo_t *gid_info) 3564 { 3565 int ii, ngid_ents; 3566 ibdm_gid_t *tmp; 3567 ibdm_hca_list_t *gid_hca_head, *temp; 3568 ibdm_hca_list_t *ioc_head = NULL; 3569 3570 IBTF_DPRINTF_L5("ibdm", "\tupdate_ioc_port_gidlist: Enter"); 3571 3572 ngid_ents = gid_info->gl_ngids; 3573 dest->ioc_nportgids = ngid_ents; 3574 dest->ioc_gid_list = kmem_zalloc(sizeof (ibdm_gid_t) * 3575 ngid_ents, KM_SLEEP); 3576 tmp = gid_info->gl_gid; 3577 for (ii = 0; (ii < ngid_ents) && (tmp); ii++) { 3578 dest->ioc_gid_list[ii].gid_dgid_hi = tmp->gid_dgid_hi; 3579 dest->ioc_gid_list[ii].gid_dgid_lo = tmp->gid_dgid_lo; 3580 tmp = tmp->gid_next; 3581 } 3582 3583 gid_hca_head = gid_info->gl_hca_list; 3584 while (gid_hca_head) { 3585 temp = ibdm_dup_hca_attr(gid_hca_head); 3586 temp->hl_next = ioc_head; 3587 ioc_head = temp; 3588 gid_hca_head = gid_hca_head->hl_next; 3589 } 3590 dest->ioc_hca_list = ioc_head; 3591 } 3592 3593 3594 /* 3595 * ibdm_alloc_send_buffers() 3596 * Allocates memory for the IBMF send buffer 3597 */ 3598 static void 3599 ibdm_alloc_send_buffers(ibmf_msg_t *msgp) 3600 { 3601 msgp->im_msgbufs_send.im_bufs_mad_hdr = 3602 kmem_zalloc(IBDM_MAD_SIZE, KM_SLEEP); 3603 msgp->im_msgbufs_send.im_bufs_cl_data = (uchar_t *) 3604 msgp->im_msgbufs_send.im_bufs_mad_hdr + sizeof (ib_mad_hdr_t); 3605 msgp->im_msgbufs_send.im_bufs_cl_data_len = 3606 IBDM_MAD_SIZE - sizeof (ib_mad_hdr_t); 3607 } 3608 3609 3610 /* 3611 * ibdm_alloc_send_buffers() 3612 * De-allocates memory for the IBMF send buffer 3613 */ 3614 static void 3615 ibdm_free_send_buffers(ibmf_msg_t *msgp) 3616 { 3617 if (msgp->im_msgbufs_send.im_bufs_mad_hdr != NULL) 3618 kmem_free(msgp->im_msgbufs_send.im_bufs_mad_hdr, IBDM_MAD_SIZE); 3619 } 3620 3621 /* 3622 * ibdm_probe_ioc() 3623 * 1. Gets the node records for the port GUID. This detects all the port 3624 * to the IOU. 3625 * 2. Selectively probes all the IOC, given it's node GUID 3626 * 3. In case of reprobe, only the IOC to be reprobed is send the IOC 3627 * Controller Profile asynchronously 3628 */ 3629 /*ARGSUSED*/ 3630 static void 3631 ibdm_probe_ioc(ib_guid_t nodeguid, ib_guid_t ioc_guid, int reprobe_flag) 3632 { 3633 int ii, nrecords; 3634 size_t nr_len = 0, pi_len = 0; 3635 ib_gid_t sgid, dgid; 3636 ibdm_hca_list_t *hca_list = NULL; 3637 sa_node_record_t *nr, *tmp; 3638 ibdm_port_attr_t *port = NULL; 3639 ibdm_dp_gidinfo_t *reprobe_gid, *new_gid, *node_gid; 3640 ibdm_dp_gidinfo_t *temp_gidinfo; 3641 ibdm_gid_t *temp_gid; 3642 sa_portinfo_record_t *pi; 3643 3644 IBTF_DPRINTF_L4("ibdm", "\tprobe_ioc(%x, %x, %x): Begin", 3645 nodeguid, ioc_guid, reprobe_flag); 3646 3647 /* Rescan the GID list for any removed GIDs for reprobe */ 3648 if (reprobe_flag) 3649 ibdm_rescan_gidlist(&ioc_guid); 3650 3651 mutex_enter(&ibdm.ibdm_hl_mutex); 3652 for (ibdm_get_next_port(&hca_list, &port, 1); port; 3653 ibdm_get_next_port(&hca_list, &port, 1)) { 3654 reprobe_gid = new_gid = node_gid = NULL; 3655 3656 nr = ibdm_get_node_records(port->pa_sa_hdl, &nr_len, nodeguid); 3657 if (nr == NULL) { 3658 IBTF_DPRINTF_L4("ibdm", "\tprobe_ioc: no records"); 3659 continue; 3660 } 3661 nrecords = (nr_len / sizeof (sa_node_record_t)); 3662 for (tmp = nr, ii = 0; (ii < nrecords); ii++, tmp++) { 3663 pi = ibdm_get_portinfo( 3664 port->pa_sa_hdl, &pi_len, tmp->LID); 3665 3666 if ((pi) && (pi->PortInfo.CapabilityMask & 3667 SM_CAP_MASK_IS_DM_SUPPD)) { 3668 /* 3669 * For reprobes: Check if GID, already in 3670 * the list. If so, set the state to SKIPPED 3671 */ 3672 if (((temp_gidinfo = ibdm_find_gid(nodeguid, 3673 tmp->NodeInfo.PortGUID)) != NULL) && 3674 temp_gidinfo->gl_state == 3675 IBDM_GID_PROBING_COMPLETE) { 3676 ASSERT(reprobe_gid == NULL); 3677 ibdm_addto_glhcalist(temp_gidinfo, 3678 hca_list); 3679 reprobe_gid = temp_gidinfo; 3680 kmem_free(pi, pi_len); 3681 continue; 3682 } else if (temp_gidinfo != NULL) { 3683 kmem_free(pi, pi_len); 3684 ibdm_addto_glhcalist(temp_gidinfo, 3685 hca_list); 3686 continue; 3687 } 3688 3689 IBTF_DPRINTF_L4("ibdm", "\tprobe_ioc : " 3690 "create_gid : prefix %llx, guid %llx\n", 3691 pi->PortInfo.GidPrefix, 3692 tmp->NodeInfo.PortGUID); 3693 3694 sgid.gid_prefix = port->pa_sn_prefix; 3695 sgid.gid_guid = port->pa_port_guid; 3696 dgid.gid_prefix = pi->PortInfo.GidPrefix; 3697 dgid.gid_guid = tmp->NodeInfo.PortGUID; 3698 new_gid = ibdm_create_gid_info(port, sgid, 3699 dgid); 3700 if (new_gid == NULL) { 3701 IBTF_DPRINTF_L2("ibdm", "\tprobe_ioc: " 3702 "create_gid_info failed\n"); 3703 kmem_free(pi, pi_len); 3704 continue; 3705 } 3706 if (node_gid == NULL) { 3707 node_gid = new_gid; 3708 ibdm_add_to_gl_gid(node_gid, node_gid); 3709 } else { 3710 IBTF_DPRINTF_L4("ibdm", 3711 "\tprobe_ioc: new gid"); 3712 temp_gid = kmem_zalloc( 3713 sizeof (ibdm_gid_t), KM_SLEEP); 3714 temp_gid->gid_dgid_hi = 3715 new_gid->gl_dgid_hi; 3716 temp_gid->gid_dgid_lo = 3717 new_gid->gl_dgid_lo; 3718 temp_gid->gid_next = node_gid->gl_gid; 3719 node_gid->gl_gid = temp_gid; 3720 node_gid->gl_ngids++; 3721 } 3722 new_gid->gl_nodeguid = nodeguid; 3723 new_gid->gl_portguid = dgid.gid_guid; 3724 ibdm_addto_glhcalist(new_gid, hca_list); 3725 3726 /* 3727 * Set the state to skipped as all these 3728 * gids point to the same node. 3729 * We (re)probe only one GID below and reset 3730 * state appropriately 3731 */ 3732 new_gid->gl_state = IBDM_GID_PROBING_SKIPPED; 3733 kmem_free(pi, pi_len); 3734 } 3735 } 3736 kmem_free(nr, nr_len); 3737 3738 IBTF_DPRINTF_L4("ibdm", "\tprobe_ioc : reprobe_flag %d " 3739 "reprobe_gid %p new_gid %p node_gid %p", 3740 reprobe_flag, reprobe_gid, new_gid, node_gid); 3741 3742 if (reprobe_flag != 0 && reprobe_gid != NULL) { 3743 int niocs, jj; 3744 ibdm_ioc_info_t *tmp_ioc; 3745 int ioc_matched = 0; 3746 3747 mutex_exit(&ibdm.ibdm_hl_mutex); 3748 mutex_enter(&reprobe_gid->gl_mutex); 3749 reprobe_gid->gl_state = IBDM_GET_IOC_DETAILS; 3750 niocs = 3751 reprobe_gid->gl_iou->iou_info.iou_num_ctrl_slots; 3752 reprobe_gid->gl_pending_cmds++; 3753 mutex_exit(&reprobe_gid->gl_mutex); 3754 3755 for (jj = 0; jj < niocs; jj++) { 3756 tmp_ioc = 3757 IBDM_GIDINFO2IOCINFO(reprobe_gid, jj); 3758 if (tmp_ioc->ioc_profile.ioc_guid != ioc_guid) 3759 continue; 3760 3761 ioc_matched = 1; 3762 3763 /* 3764 * Explicitly set gl_reprobe_flag to 0 so that 3765 * IBnex is not notified on completion 3766 */ 3767 mutex_enter(&reprobe_gid->gl_mutex); 3768 reprobe_gid->gl_reprobe_flag = 0; 3769 mutex_exit(&reprobe_gid->gl_mutex); 3770 3771 mutex_enter(&ibdm.ibdm_mutex); 3772 ibdm.ibdm_ngid_probes_in_progress++; 3773 mutex_exit(&ibdm.ibdm_mutex); 3774 if (ibdm_send_ioc_profile(reprobe_gid, jj) != 3775 IBDM_SUCCESS) { 3776 IBTF_DPRINTF_L4("ibdm", 3777 "\tprobe_ioc: " 3778 "send_ioc_profile failed " 3779 "for ioc %d", jj); 3780 ibdm_gid_decr_pending(reprobe_gid); 3781 break; 3782 } 3783 mutex_enter(&ibdm.ibdm_mutex); 3784 ibdm_wait_probe_completion(); 3785 mutex_exit(&ibdm.ibdm_mutex); 3786 break; 3787 } 3788 if (ioc_matched == 0) 3789 ibdm_gid_decr_pending(reprobe_gid); 3790 else { 3791 mutex_enter(&ibdm.ibdm_hl_mutex); 3792 break; 3793 } 3794 } else if (new_gid != NULL) { 3795 mutex_exit(&ibdm.ibdm_hl_mutex); 3796 node_gid = node_gid ? node_gid : new_gid; 3797 3798 /* 3799 * New or reinserted GID : Enable notification 3800 * to IBnex 3801 */ 3802 mutex_enter(&node_gid->gl_mutex); 3803 node_gid->gl_reprobe_flag = 1; 3804 mutex_exit(&node_gid->gl_mutex); 3805 3806 ibdm_probe_gid(node_gid); 3807 3808 mutex_enter(&ibdm.ibdm_hl_mutex); 3809 } else { 3810 IBTF_DPRINTF_L2("ibdm", "\tibdm_probe_ioc " 3811 "Invalid state!"); 3812 } 3813 3814 } 3815 mutex_exit(&ibdm.ibdm_hl_mutex); 3816 IBTF_DPRINTF_L4("ibdm", "\tprobe_ioc : End\n"); 3817 } 3818 3819 3820 /* 3821 * ibdm_probe_gid() 3822 * Selectively probes the GID 3823 */ 3824 static void 3825 ibdm_probe_gid(ibdm_dp_gidinfo_t *gid_info) 3826 { 3827 IBTF_DPRINTF_L4("ibdm", "\tprobe_gid:"); 3828 mutex_enter(&gid_info->gl_mutex); 3829 gid_info->gl_pending_cmds++; 3830 gid_info->gl_state = IBDM_GET_CLASSPORTINFO; 3831 mutex_exit(&gid_info->gl_mutex); 3832 if (ibdm_send_classportinfo(gid_info) != IBDM_SUCCESS) { 3833 mutex_enter(&gid_info->gl_mutex); 3834 gid_info->gl_state = IBDM_GID_PROBING_FAILED; 3835 --gid_info->gl_pending_cmds; 3836 mutex_exit(&gid_info->gl_mutex); 3837 ibdm_delete_glhca_list(gid_info); 3838 gid_info = gid_info->gl_next; 3839 return; 3840 } 3841 mutex_enter(&ibdm.ibdm_mutex); 3842 ibdm.ibdm_ngid_probes_in_progress++; 3843 gid_info = gid_info->gl_next; 3844 3845 ibdm_wait_probe_completion(); 3846 mutex_exit(&ibdm.ibdm_mutex); 3847 IBTF_DPRINTF_L4("ibdm", "\tprobe_gid: Wakeup signal received"); 3848 } 3849 3850 3851 /* 3852 * ibdm_create_gid_info() 3853 * Allocates a gid_info structure and initializes 3854 * Returns pointer to the structure on success 3855 * and NULL on failure 3856 */ 3857 static ibdm_dp_gidinfo_t * 3858 ibdm_create_gid_info(ibdm_port_attr_t *port, ib_gid_t sgid, ib_gid_t dgid) 3859 { 3860 uint8_t ii, npaths; 3861 sa_path_record_t *path; 3862 size_t len; 3863 ibdm_pkey_tbl_t *pkey_tbl; 3864 ibdm_dp_gidinfo_t *gid_info = NULL; 3865 int ret; 3866 3867 IBTF_DPRINTF_L4("ibdm", "\tcreate_gid_info: Begin"); 3868 npaths = 1; 3869 3870 /* query for reversible paths */ 3871 if (port->pa_sa_hdl) 3872 ret = ibmf_saa_gid_to_pathrecords(port->pa_sa_hdl, 3873 sgid, dgid, IBMF_SAA_PKEY_WC, 0, B_TRUE, &npaths, 0, 3874 &len, &path); 3875 else 3876 return (NULL); 3877 3878 if (ret == IBMF_SUCCESS && path) { 3879 ibdm_dump_path_info(path); 3880 3881 gid_info = kmem_zalloc( 3882 sizeof (ibdm_dp_gidinfo_t), KM_SLEEP); 3883 mutex_init(&gid_info->gl_mutex, NULL, MUTEX_DEFAULT, NULL); 3884 gid_info->gl_dgid_hi = path->DGID.gid_prefix; 3885 gid_info->gl_dgid_lo = path->DGID.gid_guid; 3886 gid_info->gl_sgid_hi = path->SGID.gid_prefix; 3887 gid_info->gl_sgid_lo = path->SGID.gid_guid; 3888 gid_info->gl_p_key = path->P_Key; 3889 gid_info->gl_sa_hdl = port->pa_sa_hdl; 3890 gid_info->gl_ibmf_hdl = port->pa_ibmf_hdl; 3891 gid_info->gl_slid = path->SLID; 3892 gid_info->gl_dlid = path->DLID; 3893 gid_info->gl_transactionID = (++ibdm.ibdm_transactionID) 3894 << IBDM_GID_TRANSACTIONID_SHIFT; 3895 gid_info->gl_min_transactionID = gid_info->gl_transactionID; 3896 gid_info->gl_max_transactionID = (ibdm.ibdm_transactionID +1) 3897 << IBDM_GID_TRANSACTIONID_SHIFT; 3898 3899 gid_info->gl_qp_hdl = IBMF_QP_HANDLE_DEFAULT; 3900 for (ii = 0; ii < port->pa_npkeys; ii++) { 3901 if (port->pa_pkey_tbl == NULL) 3902 break; 3903 3904 pkey_tbl = &port->pa_pkey_tbl[ii]; 3905 if ((gid_info->gl_p_key == pkey_tbl->pt_pkey) && 3906 (pkey_tbl->pt_qp_hdl != NULL)) { 3907 gid_info->gl_qp_hdl = pkey_tbl->pt_qp_hdl; 3908 break; 3909 } 3910 } 3911 kmem_free(path, len); 3912 3913 /* 3914 * QP handle for GID not initialized. No matching Pkey 3915 * was found!! ibdm should *not* hit this case. Flag an 3916 * error and drop the GID if ibdm does encounter this. 3917 */ 3918 if (gid_info->gl_qp_hdl == NULL) { 3919 IBTF_DPRINTF_L2(ibdm_string, 3920 "\tcreate_gid_info: No matching Pkey"); 3921 ibdm_delete_gidinfo(gid_info); 3922 return (NULL); 3923 } 3924 3925 ibdm.ibdm_ngids++; 3926 if (ibdm.ibdm_dp_gidlist_head == NULL) { 3927 ibdm.ibdm_dp_gidlist_head = gid_info; 3928 ibdm.ibdm_dp_gidlist_tail = gid_info; 3929 } else { 3930 ibdm.ibdm_dp_gidlist_tail->gl_next = gid_info; 3931 gid_info->gl_prev = ibdm.ibdm_dp_gidlist_tail; 3932 ibdm.ibdm_dp_gidlist_tail = gid_info; 3933 } 3934 } 3935 3936 return (gid_info); 3937 } 3938 3939 3940 /* 3941 * ibdm_get_node_records 3942 * Sends a SA query to get the NODE record 3943 * Returns pointer to the sa_node_record_t on success 3944 * and NULL on failure 3945 */ 3946 static sa_node_record_t * 3947 ibdm_get_node_records(ibmf_saa_handle_t sa_hdl, size_t *length, ib_guid_t guid) 3948 { 3949 sa_node_record_t req, *resp = NULL; 3950 ibmf_saa_access_args_t args; 3951 int ret; 3952 3953 IBTF_DPRINTF_L4("ibdm", "\tget_node_records: Begin"); 3954 3955 bzero(&req, sizeof (sa_node_record_t)); 3956 req.NodeInfo.NodeGUID = guid; 3957 3958 args.sq_attr_id = SA_NODERECORD_ATTRID; 3959 args.sq_access_type = IBMF_SAA_RETRIEVE; 3960 args.sq_component_mask = SA_NODEINFO_COMPMASK_NODEGUID; 3961 args.sq_template = &req; 3962 args.sq_callback = NULL; 3963 args.sq_callback_arg = NULL; 3964 3965 ret = ibmf_sa_access(sa_hdl, &args, 0, length, (void **) &resp); 3966 if (ret != IBMF_SUCCESS) { 3967 IBTF_DPRINTF_L2("ibdm", "\tget_node_records:" 3968 " SA Retrieve Failed: %d", ret); 3969 return (NULL); 3970 } 3971 if ((resp == NULL) || (*length == 0)) { 3972 IBTF_DPRINTF_L2("ibdm", "\tget_node_records: No records"); 3973 return (NULL); 3974 } 3975 3976 IBTF_DPRINTF_L4("ibdm", "\tget_node_records: NodeGuid %llx " 3977 "PortGUID %llx", resp->NodeInfo.NodeGUID, resp->NodeInfo.PortGUID); 3978 3979 return (resp); 3980 } 3981 3982 3983 /* 3984 * ibdm_get_portinfo() 3985 * Sends a SA query to get the PortInfo record 3986 * Returns pointer to the sa_portinfo_record_t on success 3987 * and NULL on failure 3988 */ 3989 static sa_portinfo_record_t * 3990 ibdm_get_portinfo(ibmf_saa_handle_t sa_hdl, size_t *length, ib_lid_t lid) 3991 { 3992 sa_portinfo_record_t req, *resp = NULL; 3993 ibmf_saa_access_args_t args; 3994 int ret; 3995 3996 IBTF_DPRINTF_L4("ibdm", "\tget_portinfo: Begin"); 3997 3998 bzero(&req, sizeof (sa_portinfo_record_t)); 3999 req.EndportLID = lid; 4000 4001 args.sq_attr_id = SA_PORTINFORECORD_ATTRID; 4002 args.sq_access_type = IBMF_SAA_RETRIEVE; 4003 args.sq_component_mask = SA_PORTINFO_COMPMASK_PORTLID; 4004 args.sq_template = &req; 4005 args.sq_callback = NULL; 4006 args.sq_callback_arg = NULL; 4007 4008 ret = ibmf_sa_access(sa_hdl, &args, 0, length, (void **) &resp); 4009 if (ret != IBMF_SUCCESS) { 4010 IBTF_DPRINTF_L2("ibdm", "\tget_portinfo:" 4011 " SA Retrieve Failed: 0x%X", ret); 4012 return (NULL); 4013 } 4014 if ((*length == 0) || (resp == NULL)) 4015 return (NULL); 4016 4017 IBTF_DPRINTF_L4("ibdm", "\tget_portinfo: GidPrefix %llx Cap 0x%x", 4018 resp->PortInfo.GidPrefix, resp->PortInfo.CapabilityMask); 4019 return (resp); 4020 } 4021 4022 4023 /* 4024 * ibdm_ibnex_register_callback 4025 * IB nexus callback routine for HCA attach and detach notification 4026 */ 4027 void 4028 ibdm_ibnex_register_callback(ibdm_callback_t ibnex_dm_callback) 4029 { 4030 IBTF_DPRINTF_L4("ibdm", "\tibnex_register_callbacks"); 4031 mutex_enter(&ibdm.ibdm_ibnex_mutex); 4032 ibdm.ibdm_ibnex_callback = ibnex_dm_callback; 4033 mutex_exit(&ibdm.ibdm_ibnex_mutex); 4034 } 4035 4036 4037 /* 4038 * ibdm_ibnex_unregister_callbacks 4039 */ 4040 void 4041 ibdm_ibnex_unregister_callback() 4042 { 4043 IBTF_DPRINTF_L4("ibdm", "\tibnex_unregister_callbacks"); 4044 mutex_enter(&ibdm.ibdm_ibnex_mutex); 4045 ibdm.ibdm_ibnex_callback = NULL; 4046 mutex_exit(&ibdm.ibdm_ibnex_mutex); 4047 } 4048 4049 4050 /* 4051 * ibdm_ibnex_get_waittime() 4052 * Calculates the wait time based on the last HCA attach time 4053 */ 4054 time_t 4055 ibdm_ibnex_get_waittime(ib_guid_t hca_guid, int *dft_wait) 4056 { 4057 int ii; 4058 time_t temp, wait_time = 0; 4059 ibdm_hca_list_t *hca; 4060 4061 IBTF_DPRINTF_L4("ibdm", "\tibnex_get_waittime hcaguid:%llx" 4062 "\tport settling time %d", hca_guid, *dft_wait); 4063 4064 mutex_enter(&ibdm.ibdm_hl_mutex); 4065 hca = ibdm.ibdm_hca_list_head; 4066 4067 if (hca_guid) { 4068 for (ii = 0; ii < ibdm.ibdm_hca_count; ii++) { 4069 if ((hca_guid == hca->hl_hca_guid) && 4070 (hca->hl_nports != hca->hl_nports_active)) { 4071 wait_time = 4072 ddi_get_time() - hca->hl_attach_time; 4073 wait_time = ((wait_time >= *dft_wait) ? 4074 0 : (*dft_wait - wait_time)); 4075 break; 4076 } 4077 hca = hca->hl_next; 4078 } 4079 mutex_exit(&ibdm.ibdm_hl_mutex); 4080 IBTF_DPRINTF_L4("ibdm", "\tibnex_get_waittime %llx", wait_time); 4081 return (wait_time); 4082 } 4083 4084 for (ii = 0; ii < ibdm.ibdm_hca_count; ii++) { 4085 if (hca->hl_nports != hca->hl_nports_active) { 4086 temp = ddi_get_time() - hca->hl_attach_time; 4087 temp = ((temp >= *dft_wait) ? 0 : (*dft_wait - temp)); 4088 wait_time = (temp > wait_time) ? temp : wait_time; 4089 } 4090 } 4091 mutex_exit(&ibdm.ibdm_hl_mutex); 4092 IBTF_DPRINTF_L4("ibdm", "\tibnex_get_waittime %llx", wait_time); 4093 return (wait_time); 4094 } 4095 4096 4097 /* 4098 * ibdm_ibnex_probe_hcaport 4099 * Probes the presence of HCA port (with HCA dip and port number) 4100 * Returns port attributes structure on SUCCESS 4101 */ 4102 ibdm_port_attr_t * 4103 ibdm_ibnex_probe_hcaport(ib_guid_t hca_guid, uint8_t port_num) 4104 { 4105 int ii, jj; 4106 ibdm_hca_list_t *hca_list; 4107 ibdm_port_attr_t *port_attr; 4108 4109 IBTF_DPRINTF_L4("ibdm", "\tibnex_probe_hcaport:"); 4110 4111 mutex_enter(&ibdm.ibdm_hl_mutex); 4112 hca_list = ibdm.ibdm_hca_list_head; 4113 for (ii = 0; ii < ibdm.ibdm_hca_count; ii++) { 4114 if (hca_list->hl_hca_guid == hca_guid) { 4115 for (jj = 0; jj < hca_list->hl_nports; jj++) { 4116 if (hca_list->hl_port_attr[jj].pa_port_num == 4117 port_num) { 4118 break; 4119 } 4120 } 4121 if (jj != hca_list->hl_nports) 4122 break; 4123 } 4124 hca_list = hca_list->hl_next; 4125 } 4126 if (ii == ibdm.ibdm_hca_count) { 4127 IBTF_DPRINTF_L2("ibdm", "\tibnex_probe_hcaport: not found"); 4128 mutex_exit(&ibdm.ibdm_hl_mutex); 4129 return (NULL); 4130 } 4131 port_attr = (ibdm_port_attr_t *)kmem_zalloc( 4132 sizeof (ibdm_port_attr_t), KM_SLEEP); 4133 bcopy((char *)&hca_list->hl_port_attr[jj], 4134 port_attr, sizeof (ibdm_port_attr_t)); 4135 ibdm_update_port_attr(port_attr); 4136 4137 mutex_exit(&ibdm.ibdm_hl_mutex); 4138 return (port_attr); 4139 } 4140 4141 4142 /* 4143 * ibdm_ibnex_get_port_attrs 4144 * Scan all HCAs for a matching port_guid. 4145 * Returns "port attributes" structure on success. 4146 */ 4147 ibdm_port_attr_t * 4148 ibdm_ibnex_get_port_attrs(ib_guid_t port_guid) 4149 { 4150 int ii, jj; 4151 ibdm_hca_list_t *hca_list; 4152 ibdm_port_attr_t *port_attr; 4153 4154 IBTF_DPRINTF_L4("ibdm", "\tibnex_get_port_attrs:"); 4155 4156 mutex_enter(&ibdm.ibdm_hl_mutex); 4157 hca_list = ibdm.ibdm_hca_list_head; 4158 4159 for (ii = 0; ii < ibdm.ibdm_hca_count; ii++) { 4160 for (jj = 0; jj < hca_list->hl_nports; jj++) { 4161 if (hca_list->hl_port_attr[jj].pa_port_guid == 4162 port_guid) { 4163 break; 4164 } 4165 } 4166 if (jj != hca_list->hl_nports) 4167 break; 4168 hca_list = hca_list->hl_next; 4169 } 4170 4171 if (ii == ibdm.ibdm_hca_count) { 4172 IBTF_DPRINTF_L2("ibdm", "\tibnex_get_port_attrs: not found"); 4173 mutex_exit(&ibdm.ibdm_hl_mutex); 4174 return (NULL); 4175 } 4176 4177 port_attr = (ibdm_port_attr_t *)kmem_alloc(sizeof (ibdm_port_attr_t), 4178 KM_SLEEP); 4179 bcopy((char *)&hca_list->hl_port_attr[jj], port_attr, 4180 sizeof (ibdm_port_attr_t)); 4181 ibdm_update_port_attr(port_attr); 4182 4183 mutex_exit(&ibdm.ibdm_hl_mutex); 4184 return (port_attr); 4185 } 4186 4187 4188 /* 4189 * ibdm_ibnex_free_port_attr() 4190 */ 4191 void 4192 ibdm_ibnex_free_port_attr(ibdm_port_attr_t *port_attr) 4193 { 4194 IBTF_DPRINTF_L4("ibdm", "\tibnex_free_port_attr:"); 4195 if (port_attr) { 4196 if (port_attr->pa_pkey_tbl != NULL) { 4197 kmem_free(port_attr->pa_pkey_tbl, 4198 (port_attr->pa_npkeys * sizeof (ibdm_pkey_tbl_t))); 4199 } 4200 kmem_free(port_attr, sizeof (ibdm_port_attr_t)); 4201 } 4202 } 4203 4204 4205 /* 4206 * ibdm_ibnex_get_hca_list() 4207 * Returns portinfo for all the port for all the HCA's 4208 */ 4209 void 4210 ibdm_ibnex_get_hca_list(ibdm_hca_list_t **hca, int *count) 4211 { 4212 ibdm_hca_list_t *head = NULL, *temp, *temp1; 4213 int ii; 4214 4215 IBTF_DPRINTF_L4("ibdm", "\tibnex_get_hca_list:"); 4216 4217 mutex_enter(&ibdm.ibdm_hl_mutex); 4218 temp = ibdm.ibdm_hca_list_head; 4219 for (ii = 0; ii < ibdm.ibdm_hca_count; ii++) { 4220 temp1 = ibdm_dup_hca_attr(temp); 4221 temp1->hl_next = head; 4222 head = temp1; 4223 temp = temp->hl_next; 4224 } 4225 *count = ibdm.ibdm_hca_count; 4226 *hca = head; 4227 mutex_exit(&ibdm.ibdm_hl_mutex); 4228 } 4229 4230 4231 /* 4232 * ibdm_ibnex_get_hca_info_by_guid() 4233 */ 4234 ibdm_hca_list_t * 4235 ibdm_ibnex_get_hca_info_by_guid(ib_guid_t hca_guid) 4236 { 4237 ibdm_hca_list_t *head = NULL, *hca = NULL; 4238 4239 IBTF_DPRINTF_L4("ibdm", "\tibnex_get_hca_info_by_dip"); 4240 4241 mutex_enter(&ibdm.ibdm_hl_mutex); 4242 head = ibdm.ibdm_hca_list_head; 4243 while (head) { 4244 if (head->hl_hca_guid == hca_guid) { 4245 hca = ibdm_dup_hca_attr(head); 4246 hca->hl_next = NULL; 4247 break; 4248 } 4249 head = head->hl_next; 4250 } 4251 mutex_exit(&ibdm.ibdm_hl_mutex); 4252 IBTF_DPRINTF_L4("ibdm", "\tibnex_get_hca_info_by_dip %p", hca); 4253 return (hca); 4254 } 4255 4256 4257 /* 4258 * ibdm_dup_hca_attr() 4259 * Allocate a new HCA attribute strucuture and initialize 4260 * hca attribute structure with the incoming HCA attributes 4261 * returned the allocated hca attributes. 4262 */ 4263 static ibdm_hca_list_t * 4264 ibdm_dup_hca_attr(ibdm_hca_list_t *in_hca) 4265 { 4266 int len; 4267 ibdm_hca_list_t *out_hca; 4268 4269 len = sizeof (ibdm_hca_list_t) + 4270 (in_hca->hl_nports * sizeof (ibdm_port_attr_t)); 4271 IBTF_DPRINTF_L4("ibdm", "\tdup_hca_attr len %d", len); 4272 out_hca = (ibdm_hca_list_t *)kmem_alloc(len, KM_SLEEP); 4273 bcopy((char *)in_hca, 4274 (char *)out_hca, sizeof (ibdm_hca_list_t)); 4275 if (in_hca->hl_nports) { 4276 out_hca->hl_port_attr = (ibdm_port_attr_t *) 4277 ((char *)out_hca + sizeof (ibdm_hca_list_t)); 4278 bcopy((char *)in_hca->hl_port_attr, 4279 (char *)out_hca->hl_port_attr, 4280 (in_hca->hl_nports * sizeof (ibdm_port_attr_t))); 4281 for (len = 0; len < out_hca->hl_nports; len++) 4282 ibdm_update_port_attr(&out_hca->hl_port_attr[len]); 4283 } 4284 return (out_hca); 4285 } 4286 4287 4288 /* 4289 * ibdm_ibnex_free_hca_list() 4290 * Free one/more HCA lists 4291 */ 4292 void 4293 ibdm_ibnex_free_hca_list(ibdm_hca_list_t *hca_list) 4294 { 4295 int ii, len; 4296 ibdm_hca_list_t *temp; 4297 ibdm_port_attr_t *port; 4298 4299 IBTF_DPRINTF_L4("ibdm", "\tibnex_free_hca_list:"); 4300 ASSERT(hca_list); 4301 while (hca_list) { 4302 temp = hca_list; 4303 hca_list = hca_list->hl_next; 4304 for (ii = 0; ii < temp->hl_nports; ii++) { 4305 port = &temp->hl_port_attr[ii]; 4306 len = (port->pa_npkeys * sizeof (ibdm_pkey_tbl_t)); 4307 if (len != 0) 4308 kmem_free(port->pa_pkey_tbl, len); 4309 } 4310 len = sizeof (ibdm_hca_list_t) + (temp->hl_nports * 4311 sizeof (ibdm_port_attr_t)); 4312 kmem_free(temp, len); 4313 } 4314 } 4315 4316 4317 /* 4318 * ibdm_ibnex_probe_iocguid() 4319 * Probes the IOC on the fabric and returns the IOC information 4320 * if present. Otherwise, NULL is returned 4321 */ 4322 /* ARGSUSED */ 4323 ibdm_ioc_info_t * 4324 ibdm_ibnex_probe_ioc(ib_guid_t iou, ib_guid_t ioc_guid, int reprobe_flag) 4325 { 4326 int k; 4327 ibdm_ioc_info_t *ioc_info; 4328 ibdm_dp_gidinfo_t *gid_info; 4329 4330 IBTF_DPRINTF_L4("ibdm", "\tibnex_probe_ioc: (%llX, %llX, %d) Begin", 4331 iou, ioc_guid, reprobe_flag); 4332 /* Check whether we know this already */ 4333 ioc_info = ibdm_ibnex_get_ioc_info(ioc_guid); 4334 if (ioc_info == NULL) { 4335 mutex_enter(&ibdm.ibdm_mutex); 4336 while (ibdm.ibdm_busy & IBDM_BUSY) 4337 cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex); 4338 ibdm.ibdm_busy |= IBDM_BUSY; 4339 mutex_exit(&ibdm.ibdm_mutex); 4340 ibdm_probe_ioc(iou, ioc_guid, 0); 4341 mutex_enter(&ibdm.ibdm_mutex); 4342 ibdm.ibdm_busy &= ~IBDM_BUSY; 4343 cv_broadcast(&ibdm.ibdm_busy_cv); 4344 mutex_exit(&ibdm.ibdm_mutex); 4345 ioc_info = ibdm_ibnex_get_ioc_info(ioc_guid); 4346 } else if (reprobe_flag) { /* Handle Reprobe for the IOC */ 4347 /* Free the ioc_list before reprobe; and cancel any timers */ 4348 mutex_enter(&ibdm.ibdm_mutex); 4349 if (ioc_info->ioc_timeout_id) { 4350 IBTF_DPRINTF_L5("ibdm", "\tprobe_ioc: " 4351 "ioc_timeout_id = 0x%x", 4352 ioc_info->ioc_timeout_id); 4353 if (untimeout(ioc_info->ioc_timeout_id) == -1) { 4354 IBTF_DPRINTF_L2("ibdm", "\tprobe_ioc: " 4355 "untimeout ioc_timeout_id failed"); 4356 } 4357 ioc_info->ioc_timeout_id = 0; 4358 } 4359 if (ioc_info->ioc_dc_timeout_id) { 4360 IBTF_DPRINTF_L5("ibdm", "\tprobe_ioc: " 4361 "ioc_dc_timeout_id = 0x%x", 4362 ioc_info->ioc_dc_timeout_id); 4363 if (untimeout(ioc_info->ioc_dc_timeout_id) == -1) { 4364 IBTF_DPRINTF_L2("ibdm", "\tprobe_ioc: " 4365 "untimeout ioc_dc_timeout_id failed"); 4366 } 4367 ioc_info->ioc_dc_timeout_id = 0; 4368 } 4369 for (k = 0; k < ioc_info->ioc_profile.ioc_service_entries; k++) 4370 if (ioc_info->ioc_serv[k].se_timeout_id) { 4371 IBTF_DPRINTF_L5("ibdm", "\tprobe_ioc: " 4372 "ioc_info->ioc_serv[k].se_timeout_id = %x", 4373 k, ioc_info->ioc_serv[k].se_timeout_id); 4374 if (untimeout(ioc_info->ioc_serv[k]. 4375 se_timeout_id) == -1) { 4376 IBTF_DPRINTF_L2("ibdm", "\tprobe_ioc: " 4377 "untimeout se_timeout_id %d " 4378 "failed", k); 4379 } 4380 ioc_info->ioc_serv[k].se_timeout_id = 0; 4381 } 4382 mutex_exit(&ibdm.ibdm_mutex); 4383 ibdm_ibnex_free_ioc_list(ioc_info); 4384 4385 mutex_enter(&ibdm.ibdm_mutex); 4386 while (ibdm.ibdm_busy & IBDM_BUSY) 4387 cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex); 4388 ibdm.ibdm_busy |= IBDM_BUSY; 4389 mutex_exit(&ibdm.ibdm_mutex); 4390 4391 ibdm_probe_ioc(iou, ioc_guid, 1); 4392 4393 /* 4394 * Skip if gl_reprobe_flag is set, this will be 4395 * a re-inserted / new GID, for which notifications 4396 * have already been send. 4397 */ 4398 for (gid_info = ibdm.ibdm_dp_gidlist_head; gid_info; 4399 gid_info = gid_info->gl_next) { 4400 uint8_t ii, niocs; 4401 ibdm_ioc_info_t *ioc; 4402 4403 if (gid_info->gl_iou == NULL) 4404 continue; 4405 4406 if (gid_info->gl_reprobe_flag) { 4407 gid_info->gl_reprobe_flag = 0; 4408 continue; 4409 } 4410 4411 niocs = gid_info->gl_iou->iou_info.iou_num_ctrl_slots; 4412 for (ii = 0; ii < niocs; ii++) { 4413 ioc = IBDM_GIDINFO2IOCINFO(gid_info, ii); 4414 if (ioc->ioc_profile.ioc_guid == ioc_guid) { 4415 mutex_enter(&ibdm.ibdm_mutex); 4416 ibdm_reprobe_update_port_srv(ioc, 4417 gid_info); 4418 mutex_exit(&ibdm.ibdm_mutex); 4419 } 4420 } 4421 } 4422 mutex_enter(&ibdm.ibdm_mutex); 4423 ibdm.ibdm_busy &= ~IBDM_BUSY; 4424 cv_broadcast(&ibdm.ibdm_busy_cv); 4425 mutex_exit(&ibdm.ibdm_mutex); 4426 4427 ioc_info = ibdm_ibnex_get_ioc_info(ioc_guid); 4428 } 4429 return (ioc_info); 4430 } 4431 4432 4433 /* 4434 * ibdm_ibnex_get_ioc_info() 4435 * Returns pointer to ibdm_ioc_info_t if it finds 4436 * matching record for the ioc_guid, otherwise NULL 4437 * is returned 4438 */ 4439 ibdm_ioc_info_t * 4440 ibdm_ibnex_get_ioc_info(ib_guid_t ioc_guid) 4441 { 4442 int ii; 4443 ibdm_ioc_info_t *ioc = NULL, *tmp = NULL; 4444 ibdm_dp_gidinfo_t *gid_list; 4445 ib_dm_io_unitinfo_t *iou; 4446 4447 IBTF_DPRINTF_L4("ibdm", "\tibnex_get_ioc_info: GUID %llx", ioc_guid); 4448 4449 mutex_enter(&ibdm.ibdm_mutex); 4450 while (ibdm.ibdm_busy & IBDM_BUSY) 4451 cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex); 4452 ibdm.ibdm_busy |= IBDM_BUSY; 4453 4454 gid_list = ibdm.ibdm_dp_gidlist_head; 4455 while (gid_list) { 4456 mutex_enter(&gid_list->gl_mutex); 4457 if (gid_list->gl_state != IBDM_GID_PROBING_COMPLETE) { 4458 mutex_exit(&gid_list->gl_mutex); 4459 gid_list = gid_list->gl_next; 4460 continue; 4461 } 4462 if (gid_list->gl_iou == NULL) { 4463 IBTF_DPRINTF_L2("ibdm", 4464 "\tget_ioc_info: No IOU info"); 4465 mutex_exit(&gid_list->gl_mutex); 4466 gid_list = gid_list->gl_next; 4467 continue; 4468 } 4469 iou = &gid_list->gl_iou->iou_info; 4470 for (ii = 0; ii < iou->iou_num_ctrl_slots; ii++) { 4471 tmp = IBDM_GIDINFO2IOCINFO(gid_list, ii); 4472 if ((tmp->ioc_profile.ioc_guid == ioc_guid) && 4473 (tmp->ioc_state == IBDM_IOC_STATE_PROBE_SUCCESS)) { 4474 ioc = ibdm_dup_ioc_info(tmp, gid_list); 4475 mutex_exit(&gid_list->gl_mutex); 4476 ibdm.ibdm_busy &= ~IBDM_BUSY; 4477 cv_broadcast(&ibdm.ibdm_busy_cv); 4478 mutex_exit(&ibdm.ibdm_mutex); 4479 IBTF_DPRINTF_L4("ibdm", "\tget_ioc_info: End"); 4480 return (ioc); 4481 } 4482 } 4483 if (ii == iou->iou_num_ctrl_slots) 4484 ioc = NULL; 4485 4486 mutex_exit(&gid_list->gl_mutex); 4487 gid_list = gid_list->gl_next; 4488 } 4489 4490 ibdm.ibdm_busy &= ~IBDM_BUSY; 4491 cv_broadcast(&ibdm.ibdm_busy_cv); 4492 mutex_exit(&ibdm.ibdm_mutex); 4493 IBTF_DPRINTF_L4("ibdm", "\tget_ioc_info: failure End"); 4494 return (ioc); 4495 } 4496 4497 4498 /* 4499 * ibdm_ibnex_get_ioc_count() 4500 * Returns number of ibdm_ioc_info_t it finds 4501 */ 4502 int 4503 ibdm_ibnex_get_ioc_count(void) 4504 { 4505 int count = 0, k; 4506 ibdm_ioc_info_t *ioc; 4507 ibdm_dp_gidinfo_t *gid_list; 4508 4509 mutex_enter(&ibdm.ibdm_mutex); 4510 ibdm_sweep_fabric(0); 4511 4512 while (ibdm.ibdm_busy & IBDM_BUSY) 4513 cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex); 4514 ibdm.ibdm_busy |= IBDM_BUSY; 4515 4516 for (gid_list = ibdm.ibdm_dp_gidlist_head; gid_list; 4517 gid_list = gid_list->gl_next) { 4518 mutex_enter(&gid_list->gl_mutex); 4519 if ((gid_list->gl_state != IBDM_GID_PROBING_COMPLETE) || 4520 (gid_list->gl_iou == NULL)) { 4521 mutex_exit(&gid_list->gl_mutex); 4522 continue; 4523 } 4524 for (k = 0; k < gid_list->gl_iou->iou_info.iou_num_ctrl_slots; 4525 k++) { 4526 ioc = IBDM_GIDINFO2IOCINFO(gid_list, k); 4527 if (ioc->ioc_state == IBDM_IOC_STATE_PROBE_SUCCESS) 4528 ++count; 4529 } 4530 mutex_exit(&gid_list->gl_mutex); 4531 } 4532 ibdm.ibdm_busy &= ~IBDM_BUSY; 4533 cv_broadcast(&ibdm.ibdm_busy_cv); 4534 mutex_exit(&ibdm.ibdm_mutex); 4535 4536 IBTF_DPRINTF_L4("ibdm", "\tget_ioc_count: count = %d", count); 4537 return (count); 4538 } 4539 4540 4541 /* 4542 * ibdm_ibnex_get_ioc_list() 4543 * Returns information about all the IOCs present on the fabric. 4544 * Reprobes the IOCs and the GID list if list_flag is set to REPROBE_ALL. 4545 * Does not sweep fabric if DONOT_PROBE is set 4546 */ 4547 ibdm_ioc_info_t * 4548 ibdm_ibnex_get_ioc_list(ibdm_ibnex_get_ioclist_mtd_t list_flag) 4549 { 4550 int ii; 4551 ibdm_ioc_info_t *ioc_list = NULL, *tmp, *ioc; 4552 ibdm_dp_gidinfo_t *gid_list; 4553 ib_dm_io_unitinfo_t *iou; 4554 4555 IBTF_DPRINTF_L4("ibdm", "\tget_ioc_list: Enter"); 4556 4557 mutex_enter(&ibdm.ibdm_mutex); 4558 if (list_flag != IBDM_IBNEX_DONOT_PROBE) 4559 ibdm_sweep_fabric(list_flag == IBDM_IBNEX_REPROBE_ALL); 4560 4561 while (ibdm.ibdm_busy & IBDM_BUSY) 4562 cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex); 4563 ibdm.ibdm_busy |= IBDM_BUSY; 4564 4565 gid_list = ibdm.ibdm_dp_gidlist_head; 4566 while (gid_list) { 4567 mutex_enter(&gid_list->gl_mutex); 4568 if (gid_list->gl_state != IBDM_GID_PROBING_COMPLETE) { 4569 mutex_exit(&gid_list->gl_mutex); 4570 gid_list = gid_list->gl_next; 4571 continue; 4572 } 4573 if (gid_list->gl_iou == NULL) { 4574 IBTF_DPRINTF_L2("ibdm", 4575 "\tget_ioc_list: No IOU info"); 4576 mutex_exit(&gid_list->gl_mutex); 4577 gid_list = gid_list->gl_next; 4578 continue; 4579 } 4580 iou = &gid_list->gl_iou->iou_info; 4581 for (ii = 0; ii < iou->iou_num_ctrl_slots; ii++) { 4582 ioc = IBDM_GIDINFO2IOCINFO(gid_list, ii); 4583 if (ioc->ioc_state == IBDM_IOC_STATE_PROBE_SUCCESS) { 4584 tmp = ibdm_dup_ioc_info(ioc, gid_list); 4585 tmp->ioc_next = ioc_list; 4586 ioc_list = tmp; 4587 } 4588 } 4589 mutex_exit(&gid_list->gl_mutex); 4590 gid_list = gid_list->gl_next; 4591 } 4592 ibdm.ibdm_busy &= ~IBDM_BUSY; 4593 cv_broadcast(&ibdm.ibdm_busy_cv); 4594 mutex_exit(&ibdm.ibdm_mutex); 4595 4596 IBTF_DPRINTF_L4("ibdm", "\tget_ioc_list: End"); 4597 return (ioc_list); 4598 } 4599 4600 /* 4601 * ibdm_dup_ioc_info() 4602 * Duplicate the IOC information and return the IOC 4603 * information. 4604 */ 4605 static ibdm_ioc_info_t * 4606 ibdm_dup_ioc_info(ibdm_ioc_info_t *in_ioc, ibdm_dp_gidinfo_t *gid_list) 4607 { 4608 ibdm_ioc_info_t *out_ioc; 4609 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*out_ioc)); 4610 4611 out_ioc = kmem_alloc(sizeof (ibdm_ioc_info_t), KM_SLEEP); 4612 bcopy(in_ioc, out_ioc, sizeof (ibdm_ioc_info_t)); 4613 ibdm_update_ioc_port_gidlist(out_ioc, gid_list); 4614 out_ioc->ioc_iou_dc_valid = gid_list->gl_iou->iou_dc_valid; 4615 out_ioc->ioc_iou_diagcode = gid_list->gl_iou->iou_diagcode; 4616 4617 return (out_ioc); 4618 } 4619 4620 4621 /* 4622 * ibdm_free_ioc_list() 4623 * Deallocate memory for IOC list structure 4624 */ 4625 void 4626 ibdm_ibnex_free_ioc_list(ibdm_ioc_info_t *ioc) 4627 { 4628 ibdm_ioc_info_t *temp; 4629 4630 IBTF_DPRINTF_L4("ibdm", "\tibnex_free_ioc_list:"); 4631 while (ioc) { 4632 temp = ioc; 4633 ioc = ioc->ioc_next; 4634 kmem_free(temp->ioc_gid_list, 4635 (sizeof (ibdm_gid_t) * temp->ioc_nportgids)); 4636 if (temp->ioc_hca_list) 4637 ibdm_ibnex_free_hca_list(temp->ioc_hca_list); 4638 kmem_free(temp, sizeof (ibdm_ioc_info_t)); 4639 } 4640 } 4641 4642 4643 /* 4644 * ibdm_ibnex_update_pkey_tbls 4645 * Updates the DM P_Key database. 4646 * NOTE: Two cases are handled here: P_Key being added or removed. 4647 * 4648 * Arguments : NONE 4649 * Return Values : NONE 4650 */ 4651 void 4652 ibdm_ibnex_update_pkey_tbls(void) 4653 { 4654 int h, pp, pidx; 4655 uint_t nports; 4656 uint_t size; 4657 ib_pkey_t new_pkey; 4658 ib_pkey_t *orig_pkey; 4659 ibdm_hca_list_t *hca_list; 4660 ibdm_port_attr_t *port; 4661 ibt_hca_portinfo_t *pinfop; 4662 4663 IBTF_DPRINTF_L4("ibdm", "\tibnex_update_pkey_tbls:"); 4664 4665 mutex_enter(&ibdm.ibdm_hl_mutex); 4666 hca_list = ibdm.ibdm_hca_list_head; 4667 4668 for (h = 0; h < ibdm.ibdm_hca_count; h++) { 4669 4670 /* This updates P_Key Tables for all ports of this HCA */ 4671 (void) ibt_query_hca_ports(hca_list->hl_hca_hdl, 0, &pinfop, 4672 &nports, &size); 4673 4674 /* number of ports shouldn't have changed */ 4675 ASSERT(nports == hca_list->hl_nports); 4676 4677 for (pp = 0; pp < hca_list->hl_nports; pp++) { 4678 port = &hca_list->hl_port_attr[pp]; 4679 4680 /* 4681 * First figure out the P_Keys from IBTL. 4682 * Three things could have happened: 4683 * New P_Keys added 4684 * Existing P_Keys removed 4685 * Both of the above two 4686 * 4687 * Loop through the P_Key Indices and check if a 4688 * give P_Key_Ix matches that of the one seen by 4689 * IBDM. If they match no action is needed. 4690 * 4691 * If they don't match: 4692 * 1. if orig_pkey is invalid and new_pkey is valid 4693 * ---> add new_pkey to DM database 4694 * 2. if orig_pkey is valid and new_pkey is invalid 4695 * ---> remove orig_pkey from DM database 4696 * 3. if orig_pkey and new_pkey are both valid: 4697 * ---> remov orig_pkey from DM database 4698 * ---> add new_pkey to DM database 4699 * 4. if orig_pkey and new_pkey are both invalid: 4700 * ---> do nothing. Updated DM database. 4701 */ 4702 4703 for (pidx = 0; pidx < port->pa_npkeys; pidx++) { 4704 new_pkey = pinfop[pp].p_pkey_tbl[pidx]; 4705 orig_pkey = &port->pa_pkey_tbl[pidx].pt_pkey; 4706 4707 /* keys match - do nothing */ 4708 if (*orig_pkey == new_pkey) 4709 continue; 4710 4711 if (IBDM_INVALID_PKEY(*orig_pkey) && 4712 !IBDM_INVALID_PKEY(new_pkey)) { 4713 /* P_Key was added */ 4714 IBTF_DPRINTF_L5("ibdm", 4715 "\tibnex_update_pkey_tbls: new " 4716 "P_Key added = 0x%x", new_pkey); 4717 *orig_pkey = new_pkey; 4718 ibdm_port_attr_ibmf_init(port, 4719 new_pkey, pp); 4720 } else if (!IBDM_INVALID_PKEY(*orig_pkey) && 4721 IBDM_INVALID_PKEY(new_pkey)) { 4722 /* P_Key was removed */ 4723 IBTF_DPRINTF_L5("ibdm", 4724 "\tibnex_update_pkey_tbls: P_Key " 4725 "removed = 0x%x", *orig_pkey); 4726 *orig_pkey = new_pkey; 4727 (void) ibdm_port_attr_ibmf_fini(port, 4728 pidx); 4729 } else if (!IBDM_INVALID_PKEY(*orig_pkey) && 4730 !IBDM_INVALID_PKEY(new_pkey)) { 4731 /* P_Key were replaced */ 4732 IBTF_DPRINTF_L5("ibdm", 4733 "\tibnex_update_pkey_tbls: P_Key " 4734 "replaced 0x%x with 0x%x", 4735 *orig_pkey, new_pkey); 4736 (void) ibdm_port_attr_ibmf_fini(port, 4737 pidx); 4738 *orig_pkey = new_pkey; 4739 ibdm_port_attr_ibmf_init(port, 4740 new_pkey, pp); 4741 } else { 4742 /* 4743 * P_Keys are invalid 4744 * set anyway to reflect if 4745 * INVALID_FULL was changed to 4746 * INVALID_LIMITED or vice-versa. 4747 */ 4748 *orig_pkey = new_pkey; 4749 } /* end of else */ 4750 4751 } /* loop of p_key index */ 4752 4753 } /* loop of #ports of HCA */ 4754 4755 ibt_free_portinfo(pinfop, size); 4756 hca_list = hca_list->hl_next; 4757 4758 } /* loop for all HCAs in the system */ 4759 4760 mutex_exit(&ibdm.ibdm_hl_mutex); 4761 } 4762 4763 4764 /* 4765 * ibdm_send_ioc_profile() 4766 * Send IOC Controller Profile request. When the request is completed 4767 * IBMF calls ibdm_process_incoming_mad routine to inform about 4768 * the completion. 4769 */ 4770 static int 4771 ibdm_send_ioc_profile(ibdm_dp_gidinfo_t *gid_info, uint8_t ioc_no) 4772 { 4773 ibmf_msg_t *msg; 4774 ib_mad_hdr_t *hdr; 4775 ibdm_ioc_info_t *ioc_info = &(gid_info->gl_iou->iou_ioc_info[ioc_no]); 4776 ibdm_timeout_cb_args_t *cb_args; 4777 4778 IBTF_DPRINTF_L4("ibdm", "\tsend_ioc_profile: " 4779 "gid info 0x%p, ioc_no = %d", gid_info, ioc_no); 4780 4781 /* 4782 * Send command to get IOC profile. 4783 * Allocate a IBMF packet and initialize the packet. 4784 */ 4785 if (ibmf_alloc_msg(gid_info->gl_ibmf_hdl, IBMF_ALLOC_SLEEP, 4786 &msg) != IBMF_SUCCESS) { 4787 IBTF_DPRINTF_L4("ibdm", "\tsend_ioc_profile: pkt alloc fail"); 4788 return (IBDM_FAILURE); 4789 } 4790 4791 ibdm_alloc_send_buffers(msg); 4792 4793 mutex_enter(&gid_info->gl_mutex); 4794 ibdm_bump_transactionID(gid_info); 4795 mutex_exit(&gid_info->gl_mutex); 4796 4797 msg->im_local_addr.ia_local_lid = gid_info->gl_slid; 4798 msg->im_local_addr.ia_remote_lid = gid_info->gl_dlid; 4799 if (gid_info->gl_redirected == B_TRUE) { 4800 if (gid_info->gl_redirect_dlid != 0) { 4801 msg->im_local_addr.ia_remote_lid = 4802 gid_info->gl_redirect_dlid; 4803 } 4804 msg->im_local_addr.ia_remote_qno = gid_info->gl_redirect_QP; 4805 msg->im_local_addr.ia_p_key = gid_info->gl_redirect_pkey; 4806 msg->im_local_addr.ia_q_key = gid_info->gl_redirect_qkey; 4807 } else { 4808 msg->im_local_addr.ia_remote_qno = 1; 4809 msg->im_local_addr.ia_p_key = gid_info->gl_p_key; 4810 msg->im_local_addr.ia_q_key = IB_GSI_QKEY; 4811 } 4812 4813 hdr = IBDM_OUT_IBMFMSG_MADHDR(msg); 4814 hdr->BaseVersion = MAD_CLASS_BASE_VERS_1; 4815 hdr->MgmtClass = MAD_MGMT_CLASS_DEV_MGT; 4816 hdr->ClassVersion = IB_DM_CLASS_VERSION_1; 4817 hdr->R_Method = IB_DM_DEVMGT_METHOD_GET; 4818 hdr->Status = 0; 4819 hdr->TransactionID = h2b64(gid_info->gl_transactionID); 4820 hdr->AttributeID = h2b16(IB_DM_ATTR_IOC_CTRL_PROFILE); 4821 hdr->AttributeModifier = h2b32(ioc_no + 1); 4822 4823 ioc_info->ioc_state = IBDM_IOC_STATE_REPROBE_PROGRESS; 4824 cb_args = &ioc_info->ioc_cb_args; 4825 cb_args->cb_gid_info = gid_info; 4826 cb_args->cb_retry_count = ibdm_dft_retry_cnt; 4827 cb_args->cb_req_type = IBDM_REQ_TYPE_IOCINFO; 4828 cb_args->cb_ioc_num = ioc_no; 4829 4830 ioc_info->ioc_timeout_id = timeout(ibdm_pkt_timeout_hdlr, 4831 cb_args, IBDM_TIMEOUT_VALUE(ibdm_dft_timeout)); 4832 4833 IBTF_DPRINTF_L5("ibdm", "\tsend_ioc_profile:" 4834 "timeout %x", ioc_info->ioc_timeout_id); 4835 4836 if (ibmf_msg_transport(gid_info->gl_ibmf_hdl, gid_info->gl_qp_hdl, msg, 4837 NULL, ibdm_ibmf_send_cb, cb_args, 0) != IBMF_SUCCESS) { 4838 IBTF_DPRINTF_L2("ibdm", 4839 "\tsend_ioc_profile: msg transport failed"); 4840 ibdm_ibmf_send_cb(gid_info->gl_ibmf_hdl, msg, cb_args); 4841 } 4842 ioc_info->ioc_state = IBDM_IOC_STATE_REPROBE_PROGRESS; 4843 return (IBDM_SUCCESS); 4844 } 4845 4846 4847 /* 4848 * ibdm_port_reachable 4849 * Sends a SA query to get the NODE record for port GUID 4850 * Returns IBDM_SUCCESS if the port GID is reachable 4851 */ 4852 static int 4853 ibdm_port_reachable(ibmf_saa_handle_t sa_hdl, ib_guid_t guid, 4854 ib_guid_t *node_guid) 4855 { 4856 sa_node_record_t req, *resp = NULL; 4857 ibmf_saa_access_args_t args; 4858 int ret; 4859 size_t length; 4860 4861 IBTF_DPRINTF_L4("ibdm", "\tport_reachable: port_guid %llx", 4862 guid); 4863 4864 bzero(&req, sizeof (sa_node_record_t)); 4865 req.NodeInfo.PortGUID = guid; 4866 4867 args.sq_attr_id = SA_NODERECORD_ATTRID; 4868 args.sq_access_type = IBMF_SAA_RETRIEVE; 4869 args.sq_component_mask = SA_NODEINFO_COMPMASK_PORTGUID; 4870 args.sq_template = &req; 4871 args.sq_callback = NULL; 4872 args.sq_callback_arg = NULL; 4873 4874 ret = ibmf_sa_access(sa_hdl, &args, 0, &length, (void **) &resp); 4875 if (ret != IBMF_SUCCESS) { 4876 IBTF_DPRINTF_L2("ibdm", "\tport_reachable:" 4877 " SA Retrieve Failed: %d", ret); 4878 return (IBDM_FAILURE); 4879 } 4880 4881 if ((resp == NULL) || (length == 0)) { 4882 IBTF_DPRINTF_L2("ibdm", "\tport_reachable: No records"); 4883 return (IBDM_FAILURE); 4884 } 4885 4886 if (node_guid != NULL) 4887 *node_guid = resp->NodeInfo.NodeGUID; 4888 4889 kmem_free(resp, length); 4890 4891 return (IBDM_SUCCESS); 4892 } 4893 4894 /* 4895 * Update the gidlist for all affected IOCs when GID becomes 4896 * available/unavailable. 4897 * 4898 * Parameters : 4899 * gidinfo - Incoming / Outgoing GID. 4900 * add_flag - 1 for GID added, 0 for GID removed. 4901 * - (-1) : IOC gid list updated, ioc_list required. 4902 * 4903 * This function gets the GID for the node GUID corresponding to the 4904 * port GID. Gets the IOU info 4905 */ 4906 static ibdm_ioc_info_t * 4907 ibdm_update_ioc_gidlist(ibdm_dp_gidinfo_t *gid_info, int avail_flag) 4908 { 4909 ibdm_dp_gidinfo_t *node_gid = NULL; 4910 uint8_t niocs, ii; 4911 ibdm_ioc_info_t *ioc, *ioc_list = NULL, *tmp; 4912 4913 IBTF_DPRINTF_L4("ibdm", "\tupdate_ioc_gidlist"); 4914 4915 switch (avail_flag) { 4916 case 1 : 4917 node_gid = ibdm_check_dest_nodeguid(gid_info); 4918 break; 4919 case 0 : 4920 node_gid = ibdm_handle_gid_rm(gid_info); 4921 break; 4922 case -1 : 4923 node_gid = gid_info; 4924 break; 4925 default : 4926 break; 4927 } 4928 4929 if (node_gid == NULL) { 4930 IBTF_DPRINTF_L4("ibdm", "\tupdate_ioc_gidlist: " 4931 "No node GID found, port gid 0x%p, avail_flag %d", 4932 gid_info, avail_flag); 4933 return (NULL); 4934 } 4935 4936 mutex_enter(&node_gid->gl_mutex); 4937 if ((node_gid->gl_state != IBDM_GID_PROBING_COMPLETE && 4938 node_gid->gl_state != IBDM_GID_PROBING_SKIPPED) || 4939 node_gid->gl_iou == NULL) { 4940 IBTF_DPRINTF_L4("ibdm", "\tupdate_ioc_gidlist " 4941 "gl_state %x, gl_iou %p", node_gid->gl_state, 4942 node_gid->gl_iou); 4943 mutex_exit(&node_gid->gl_mutex); 4944 return (NULL); 4945 } 4946 4947 niocs = node_gid->gl_iou->iou_info.iou_num_ctrl_slots; 4948 IBTF_DPRINTF_L4("ibdm", "\tupdate_ioc_gidlist : niocs %x", 4949 niocs); 4950 for (ii = 0; ii < niocs; ii++) { 4951 ioc = IBDM_GIDINFO2IOCINFO(node_gid, ii); 4952 /* 4953 * Skip IOCs for which probe is not complete or 4954 * reprobe is progress 4955 */ 4956 if (ioc->ioc_state == IBDM_IOC_STATE_PROBE_SUCCESS) { 4957 tmp = ibdm_dup_ioc_info(ioc, node_gid); 4958 tmp->ioc_info_updated.ib_gid_prop_updated = 1; 4959 tmp->ioc_next = ioc_list; 4960 ioc_list = tmp; 4961 } 4962 } 4963 mutex_exit(&node_gid->gl_mutex); 4964 4965 IBTF_DPRINTF_L4("ibdm", "\tupdate_ioc_gidlist : return %p", 4966 ioc_list); 4967 return (ioc_list); 4968 } 4969 4970 /* 4971 * ibdm_saa_event_cb : 4972 * Event handling which does *not* require ibdm_hl_mutex to be 4973 * held are executed in the same thread. This is to prevent 4974 * deadlocks with HCA port down notifications which hold the 4975 * ibdm_hl_mutex. 4976 * 4977 * GID_AVAILABLE event is handled here. A taskq is spawned to 4978 * handle GID_UNAVAILABLE. 4979 * 4980 * A new mutex ibdm_ibnex_mutex has been introduced to protect 4981 * ibnex_callback. This has been done to prevent any possible 4982 * deadlock (described above) while handling GID_AVAILABLE. 4983 * 4984 * IBMF calls the event callback for a HCA port. The SA handle 4985 * for this port would be valid, till the callback returns. 4986 * IBDM calling IBDM using the above SA handle should be valid. 4987 * 4988 * IBDM will additionally check (SA handle != NULL), before 4989 * calling IBMF. 4990 */ 4991 /*ARGSUSED*/ 4992 static void 4993 ibdm_saa_event_cb(ibmf_saa_handle_t ibmf_saa_handle, 4994 ibmf_saa_subnet_event_t ibmf_saa_event, 4995 ibmf_saa_event_details_t *event_details, void *callback_arg) 4996 { 4997 ibdm_saa_event_arg_t *event_arg; 4998 ib_gid_t sgid, dgid; 4999 ibdm_port_attr_t *hca_port; 5000 ibdm_dp_gidinfo_t *gid_info, *node_gid_info = NULL; 5001 ib_guid_t nodeguid; 5002 5003 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*event_arg)); 5004 5005 hca_port = (ibdm_port_attr_t *)callback_arg; 5006 5007 IBTF_DPRINTF_L4("ibdm", "\tsaa_event_cb(%x, %x, %x, %x)\n", 5008 ibmf_saa_handle, ibmf_saa_event, event_details, 5009 callback_arg); 5010 #ifdef DEBUG 5011 if (ibdm_ignore_saa_event) 5012 return; 5013 #endif 5014 5015 if (ibmf_saa_event == IBMF_SAA_EVENT_GID_AVAILABLE) { 5016 /* 5017 * Ensure no other probe / sweep fabric is in 5018 * progress. 5019 */ 5020 mutex_enter(&ibdm.ibdm_mutex); 5021 while (ibdm.ibdm_busy & IBDM_BUSY) 5022 cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex); 5023 ibdm.ibdm_busy |= IBDM_BUSY; 5024 mutex_exit(&ibdm.ibdm_mutex); 5025 5026 /* 5027 * If we already know about this GID, return. 5028 * GID_AVAILABLE may be reported for multiple HCA 5029 * ports. 5030 */ 5031 if ((ibdm_check_dgid(event_details->ie_gid.gid_guid, 5032 event_details->ie_gid.gid_prefix)) != NULL) { 5033 mutex_enter(&ibdm.ibdm_mutex); 5034 ibdm.ibdm_busy &= ~IBDM_BUSY; 5035 cv_broadcast(&ibdm.ibdm_busy_cv); 5036 mutex_exit(&ibdm.ibdm_mutex); 5037 return; 5038 } 5039 5040 IBTF_DPRINTF_L4("ibdm", "\tGID (prefix %x, guid %llx) " 5041 "Insertion notified", 5042 event_details->ie_gid.gid_prefix, 5043 event_details->ie_gid.gid_guid); 5044 5045 /* This is a new gid, insert it to GID list */ 5046 sgid.gid_prefix = hca_port->pa_sn_prefix; 5047 sgid.gid_guid = hca_port->pa_port_guid; 5048 dgid.gid_prefix = event_details->ie_gid.gid_prefix; 5049 dgid.gid_guid = event_details->ie_gid.gid_guid; 5050 gid_info = ibdm_create_gid_info(hca_port, sgid, dgid); 5051 if (gid_info == NULL) { 5052 IBTF_DPRINTF_L4("ibdm", "\tGID_AVAILABLE: " 5053 "create_gid_info returned NULL"); 5054 mutex_enter(&ibdm.ibdm_mutex); 5055 ibdm.ibdm_busy &= ~IBDM_BUSY; 5056 cv_broadcast(&ibdm.ibdm_busy_cv); 5057 mutex_exit(&ibdm.ibdm_mutex); 5058 return; 5059 } 5060 mutex_enter(&gid_info->gl_mutex); 5061 gid_info->gl_state = IBDM_GID_PROBING_SKIPPED; 5062 mutex_exit(&gid_info->gl_mutex); 5063 5064 /* Get the node GUID */ 5065 if (ibdm_port_reachable(ibmf_saa_handle, dgid.gid_guid, 5066 &nodeguid) != IBDM_SUCCESS) { 5067 /* 5068 * Set the state to PROBE_NOT_DONE for the 5069 * next sweep to probe it 5070 */ 5071 IBTF_DPRINTF_L2("ibdm", "\tsaa_event_taskq: " 5072 "Skipping GID : port GUID not found"); 5073 mutex_enter(&gid_info->gl_mutex); 5074 gid_info->gl_state = IBDM_GID_PROBE_NOT_DONE; 5075 mutex_exit(&gid_info->gl_mutex); 5076 mutex_enter(&ibdm.ibdm_mutex); 5077 ibdm.ibdm_busy &= ~IBDM_BUSY; 5078 cv_broadcast(&ibdm.ibdm_busy_cv); 5079 mutex_exit(&ibdm.ibdm_mutex); 5080 return; 5081 } 5082 5083 gid_info->gl_nodeguid = nodeguid; 5084 gid_info->gl_portguid = dgid.gid_guid; 5085 5086 /* 5087 * Get the gid info with the same node GUID. 5088 */ 5089 mutex_enter(&ibdm.ibdm_mutex); 5090 node_gid_info = ibdm.ibdm_dp_gidlist_head; 5091 while (node_gid_info) { 5092 if (node_gid_info->gl_nodeguid == 5093 gid_info->gl_nodeguid && 5094 node_gid_info->gl_iou != NULL) { 5095 break; 5096 } 5097 node_gid_info = node_gid_info->gl_next; 5098 } 5099 mutex_exit(&ibdm.ibdm_mutex); 5100 5101 /* 5102 * Handling a new GID requires filling of gl_hca_list. 5103 * This require ibdm hca_list to be parsed and hence 5104 * holding the ibdm_hl_mutex. Spawning a new thread to 5105 * handle this. 5106 */ 5107 if (node_gid_info == NULL) { 5108 if (taskq_dispatch(system_taskq, 5109 ibdm_saa_handle_new_gid, (void *)gid_info, 5110 TQ_NOSLEEP) == NULL) { 5111 IBTF_DPRINTF_L2("ibdm", "\tsaa_event_cb: " 5112 "new_gid taskq_dispatch failed"); 5113 return; 5114 } 5115 } 5116 5117 mutex_enter(&ibdm.ibdm_mutex); 5118 ibdm.ibdm_busy &= ~IBDM_BUSY; 5119 cv_broadcast(&ibdm.ibdm_busy_cv); 5120 mutex_exit(&ibdm.ibdm_mutex); 5121 return; 5122 } 5123 5124 if (ibmf_saa_event != IBMF_SAA_EVENT_GID_UNAVAILABLE) 5125 return; 5126 5127 event_arg = (ibdm_saa_event_arg_t *)kmem_alloc( 5128 sizeof (ibdm_saa_event_arg_t), KM_SLEEP); 5129 event_arg->ibmf_saa_handle = ibmf_saa_handle; 5130 event_arg->ibmf_saa_event = ibmf_saa_event; 5131 bcopy(event_details, &event_arg->event_details, 5132 sizeof (ibmf_saa_event_details_t)); 5133 event_arg->callback_arg = callback_arg; 5134 5135 if (taskq_dispatch(system_taskq, ibdm_saa_event_taskq, 5136 (void *)event_arg, TQ_NOSLEEP) == NULL) { 5137 IBTF_DPRINTF_L2("ibdm", "\tsaa_event_cb: " 5138 "taskq_dispatch failed"); 5139 ibdm_free_saa_event_arg(event_arg); 5140 return; 5141 } 5142 } 5143 5144 /* 5145 * Handle a new GID discovered by GID_AVAILABLE saa event. 5146 */ 5147 void 5148 ibdm_saa_handle_new_gid(void *arg) 5149 { 5150 ibdm_dp_gidinfo_t *gid_info; 5151 ibdm_hca_list_t *hca_list = NULL; 5152 ibdm_port_attr_t *port = NULL; 5153 ibdm_ioc_info_t *ioc_list = NULL; 5154 5155 IBTF_DPRINTF_L4(ibdm_string, "\tsaa_handle_new_gid(%p)", arg); 5156 5157 gid_info = (ibdm_dp_gidinfo_t *)arg; 5158 5159 /* 5160 * Ensure that no other sweep / probe has completed 5161 * probing this gid. 5162 */ 5163 mutex_enter(&gid_info->gl_mutex); 5164 if (gid_info->gl_state != IBDM_GID_PROBE_NOT_DONE) { 5165 mutex_exit(&gid_info->gl_mutex); 5166 return; 5167 } 5168 mutex_exit(&gid_info->gl_mutex); 5169 5170 /* 5171 * Parse HCAs to fill gl_hca_list 5172 */ 5173 mutex_enter(&ibdm.ibdm_hl_mutex); 5174 for (ibdm_get_next_port(&hca_list, &port, 1); port; 5175 ibdm_get_next_port(&hca_list, &port, 1)) { 5176 if (ibdm_port_reachable(port->pa_sa_hdl, 5177 gid_info->gl_portguid, NULL) == 5178 IBDM_SUCCESS) { 5179 ibdm_addto_glhcalist(gid_info, hca_list); 5180 } 5181 } 5182 mutex_exit(&ibdm.ibdm_hl_mutex); 5183 5184 /* 5185 * Ensure no other probe / sweep fabric is in 5186 * progress. 5187 */ 5188 mutex_enter(&ibdm.ibdm_mutex); 5189 while (ibdm.ibdm_busy & IBDM_BUSY) 5190 cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex); 5191 ibdm.ibdm_busy |= IBDM_BUSY; 5192 mutex_exit(&ibdm.ibdm_mutex); 5193 5194 /* 5195 * New IOU probe it, to check if new IOCs 5196 */ 5197 IBTF_DPRINTF_L4(ibdm_string, "\tsaa_handle_new_gid: " 5198 "new GID : probing"); 5199 mutex_enter(&ibdm.ibdm_mutex); 5200 ibdm.ibdm_ngid_probes_in_progress++; 5201 mutex_exit(&ibdm.ibdm_mutex); 5202 mutex_enter(&gid_info->gl_mutex); 5203 gid_info->gl_reprobe_flag = 0; 5204 gid_info->gl_state = IBDM_GID_PROBE_NOT_DONE; 5205 mutex_exit(&gid_info->gl_mutex); 5206 ibdm_probe_gid_thread((void *)gid_info); 5207 5208 mutex_enter(&ibdm.ibdm_mutex); 5209 ibdm_wait_probe_completion(); 5210 mutex_exit(&ibdm.ibdm_mutex); 5211 5212 if (gid_info->gl_iou == NULL) { 5213 mutex_enter(&ibdm.ibdm_mutex); 5214 ibdm.ibdm_busy &= ~IBDM_BUSY; 5215 cv_broadcast(&ibdm.ibdm_busy_cv); 5216 mutex_exit(&ibdm.ibdm_mutex); 5217 return; 5218 } 5219 5220 /* 5221 * Update GID list in all IOCs affected by this 5222 */ 5223 ioc_list = ibdm_update_ioc_gidlist(gid_info, 1); 5224 5225 /* 5226 * Pass on the IOCs with updated GIDs to IBnexus 5227 */ 5228 if (ioc_list) { 5229 mutex_enter(&ibdm.ibdm_ibnex_mutex); 5230 if (ibdm.ibdm_ibnex_callback != NULL) { 5231 (*ibdm.ibdm_ibnex_callback)((void *)ioc_list, 5232 IBDM_EVENT_IOC_PROP_UPDATE); 5233 } 5234 mutex_exit(&ibdm.ibdm_ibnex_mutex); 5235 } 5236 5237 mutex_enter(&ibdm.ibdm_mutex); 5238 ibdm.ibdm_busy &= ~IBDM_BUSY; 5239 cv_broadcast(&ibdm.ibdm_busy_cv); 5240 mutex_exit(&ibdm.ibdm_mutex); 5241 } 5242 5243 /* 5244 * ibdm_saa_event_taskq : 5245 * GID_UNAVAILABLE Event handling requires ibdm_hl_mutex to be 5246 * held. The GID_UNAVAILABLE handling is done in a taskq to 5247 * prevent deadlocks with HCA port down notifications which hold 5248 * ibdm_hl_mutex. 5249 */ 5250 void 5251 ibdm_saa_event_taskq(void *arg) 5252 { 5253 ibdm_saa_event_arg_t *event_arg; 5254 ibmf_saa_handle_t ibmf_saa_handle; 5255 ibmf_saa_subnet_event_t ibmf_saa_event; 5256 ibmf_saa_event_details_t *event_details; 5257 void *callback_arg; 5258 5259 ibdm_dp_gidinfo_t *gid_info; 5260 ibdm_port_attr_t *hca_port, *port = NULL; 5261 ibdm_hca_list_t *hca_list = NULL; 5262 int sa_handle_valid = 0; 5263 ibdm_ioc_info_t *ioc_list = NULL; 5264 5265 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*event_arg)); 5266 5267 event_arg = (ibdm_saa_event_arg_t *)arg; 5268 ibmf_saa_handle = event_arg->ibmf_saa_handle; 5269 ibmf_saa_event = event_arg->ibmf_saa_event; 5270 event_details = &event_arg->event_details; 5271 callback_arg = event_arg->callback_arg; 5272 5273 ASSERT(callback_arg != NULL); 5274 ASSERT(ibmf_saa_event == IBMF_SAA_EVENT_GID_UNAVAILABLE); 5275 IBTF_DPRINTF_L4("ibdm", "\tsaa_event_taskq(%x, %x, %x, %x)", 5276 ibmf_saa_handle, ibmf_saa_event, event_details, 5277 callback_arg); 5278 5279 hca_port = (ibdm_port_attr_t *)callback_arg; 5280 5281 /* Check if the port_attr is still valid */ 5282 mutex_enter(&ibdm.ibdm_hl_mutex); 5283 for (ibdm_get_next_port(&hca_list, &port, 0); port; 5284 ibdm_get_next_port(&hca_list, &port, 0)) { 5285 if (port == hca_port && port->pa_port_guid == 5286 hca_port->pa_port_guid) { 5287 if (ibmf_saa_handle == hca_port->pa_sa_hdl) 5288 sa_handle_valid = 1; 5289 break; 5290 } 5291 } 5292 mutex_exit(&ibdm.ibdm_hl_mutex); 5293 if (sa_handle_valid == 0) { 5294 ibdm_free_saa_event_arg(event_arg); 5295 return; 5296 } 5297 5298 if (hca_port && (hca_port->pa_sa_hdl == NULL || 5299 ibmf_saa_handle != hca_port->pa_sa_hdl)) { 5300 ibdm_free_saa_event_arg(event_arg); 5301 return; 5302 } 5303 hca_list = NULL; 5304 port = NULL; 5305 5306 /* 5307 * Check if the GID is visible to other HCA ports. 5308 * Return if so. 5309 */ 5310 mutex_enter(&ibdm.ibdm_hl_mutex); 5311 for (ibdm_get_next_port(&hca_list, &port, 1); port; 5312 ibdm_get_next_port(&hca_list, &port, 1)) { 5313 if (ibdm_port_reachable(port->pa_sa_hdl, 5314 event_details->ie_gid.gid_guid, NULL) == 5315 IBDM_SUCCESS) { 5316 mutex_exit(&ibdm.ibdm_hl_mutex); 5317 ibdm_free_saa_event_arg(event_arg); 5318 return; 5319 } 5320 } 5321 mutex_exit(&ibdm.ibdm_hl_mutex); 5322 5323 /* 5324 * Ensure no other probe / sweep fabric is in 5325 * progress. 5326 */ 5327 mutex_enter(&ibdm.ibdm_mutex); 5328 while (ibdm.ibdm_busy & IBDM_BUSY) 5329 cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex); 5330 ibdm.ibdm_busy |= IBDM_BUSY; 5331 mutex_exit(&ibdm.ibdm_mutex); 5332 5333 /* 5334 * If this GID is no longer in GID list, return 5335 * GID_UNAVAILABLE may be reported for multiple HCA 5336 * ports. 5337 */ 5338 mutex_enter(&ibdm.ibdm_mutex); 5339 gid_info = ibdm.ibdm_dp_gidlist_head; 5340 while (gid_info) { 5341 if (gid_info->gl_portguid == 5342 event_details->ie_gid.gid_guid) { 5343 break; 5344 } 5345 gid_info = gid_info->gl_next; 5346 } 5347 mutex_exit(&ibdm.ibdm_mutex); 5348 if (gid_info == NULL) { 5349 mutex_enter(&ibdm.ibdm_mutex); 5350 ibdm.ibdm_busy &= ~IBDM_BUSY; 5351 cv_broadcast(&ibdm.ibdm_busy_cv); 5352 mutex_exit(&ibdm.ibdm_mutex); 5353 ibdm_free_saa_event_arg(event_arg); 5354 return; 5355 } 5356 5357 IBTF_DPRINTF_L4("ibdm", "\tGID (prefix %x, guid %llx) " 5358 "Unavailable notification", 5359 event_details->ie_gid.gid_prefix, 5360 event_details->ie_gid.gid_guid); 5361 5362 /* 5363 * Update GID list in all IOCs affected by this 5364 */ 5365 if (gid_info->gl_state == IBDM_GID_PROBING_SKIPPED || 5366 gid_info->gl_state == IBDM_GID_PROBING_COMPLETE) 5367 ioc_list = ibdm_update_ioc_gidlist(gid_info, 0); 5368 5369 /* 5370 * Remove GID from the global GID list 5371 * Handle the case where all port GIDs for an 5372 * IOU have been hot-removed. Check both gid_info 5373 * & ioc_info for checking ngids. 5374 */ 5375 mutex_enter(&ibdm.ibdm_mutex); 5376 if (gid_info->gl_iou != NULL && gid_info->gl_ngids == 0) { 5377 mutex_enter(&gid_info->gl_mutex); 5378 (void) ibdm_free_iou_info(gid_info); 5379 mutex_exit(&gid_info->gl_mutex); 5380 } 5381 if (gid_info->gl_prev != NULL) 5382 gid_info->gl_prev->gl_next = gid_info->gl_next; 5383 if (gid_info->gl_next != NULL) 5384 gid_info->gl_next->gl_prev = gid_info->gl_prev; 5385 5386 if (gid_info == ibdm.ibdm_dp_gidlist_head) 5387 ibdm.ibdm_dp_gidlist_head = gid_info->gl_next; 5388 if (gid_info == ibdm.ibdm_dp_gidlist_tail) 5389 ibdm.ibdm_dp_gidlist_tail = gid_info->gl_prev; 5390 ibdm.ibdm_ngids--; 5391 5392 ibdm.ibdm_busy &= ~IBDM_BUSY; 5393 cv_broadcast(&ibdm.ibdm_busy_cv); 5394 mutex_exit(&ibdm.ibdm_mutex); 5395 5396 mutex_destroy(&gid_info->gl_mutex); 5397 kmem_free(gid_info, sizeof (ibdm_dp_gidinfo_t)); 5398 5399 /* 5400 * Pass on the IOCs with updated GIDs to IBnexus 5401 */ 5402 if (ioc_list) { 5403 IBTF_DPRINTF_L4("ibdm", "\tGID_UNAVAILABLE " 5404 "IOC_PROP_UPDATE for %p\n", ioc_list); 5405 mutex_enter(&ibdm.ibdm_ibnex_mutex); 5406 if (ibdm.ibdm_ibnex_callback != NULL) { 5407 (*ibdm.ibdm_ibnex_callback)((void *) 5408 ioc_list, IBDM_EVENT_IOC_PROP_UPDATE); 5409 } 5410 mutex_exit(&ibdm.ibdm_ibnex_mutex); 5411 } 5412 5413 ibdm_free_saa_event_arg(event_arg); 5414 } 5415 5416 5417 static int 5418 ibdm_cmp_gid_list(ibdm_gid_t *new, ibdm_gid_t *prev) 5419 { 5420 ibdm_gid_t *scan_new, *scan_prev; 5421 int cmp_failed = 0; 5422 5423 ASSERT(new != NULL); 5424 ASSERT(prev != NULL); 5425 5426 /* 5427 * Search for each new gid anywhere in the prev GID list. 5428 * Note that the gid list could have been re-ordered. 5429 */ 5430 for (scan_new = new; scan_new; scan_new = scan_new->gid_next) { 5431 for (scan_prev = prev, cmp_failed = 1; scan_prev; 5432 scan_prev = scan_prev->gid_next) { 5433 if (scan_prev->gid_dgid_hi == scan_new->gid_dgid_hi && 5434 scan_prev->gid_dgid_lo == scan_new->gid_dgid_lo) { 5435 cmp_failed = 0; 5436 break; 5437 } 5438 } 5439 5440 if (cmp_failed) 5441 return (1); 5442 } 5443 return (0); 5444 } 5445 5446 /* 5447 * This is always called in a single thread 5448 * This function updates the gid_list and serv_list of IOC 5449 * The current gid_list is in ioc_info_t(contains only port 5450 * guids for which probe is done) & gidinfo_t(other port gids) 5451 * The gids in both locations are used for comparision. 5452 */ 5453 static void 5454 ibdm_reprobe_update_port_srv(ibdm_ioc_info_t *ioc, ibdm_dp_gidinfo_t *gidinfo) 5455 { 5456 ibdm_gid_t *cur_gid_list; 5457 uint_t cur_nportgids; 5458 5459 ASSERT(MUTEX_HELD(&ibdm.ibdm_mutex)); 5460 5461 ioc->ioc_info_updated.ib_prop_updated = 0; 5462 5463 5464 /* Current GID list in gid_info only */ 5465 cur_gid_list = gidinfo->gl_gid; 5466 cur_nportgids = gidinfo->gl_ngids; 5467 5468 /* 5469 * Service entry names and IDs are not compared currently. 5470 * This may require change. 5471 */ 5472 if (ioc->ioc_prev_serv_cnt != ioc->ioc_profile.ioc_service_entries) 5473 ioc->ioc_info_updated.ib_srv_prop_updated = 1; 5474 5475 if (ioc->ioc_prev_nportgids != cur_nportgids || 5476 ioc->ioc_prev_gid_list == NULL || cur_gid_list == NULL) { 5477 ioc->ioc_info_updated.ib_gid_prop_updated = 1; 5478 } else if (ibdm_cmp_gid_list(ioc->ioc_prev_gid_list, cur_gid_list)) { 5479 ioc->ioc_info_updated.ib_gid_prop_updated = 1; 5480 } 5481 5482 /* Zero out previous entries */ 5483 ibdm_free_gid_list(ioc->ioc_prev_gid_list); 5484 if (ioc->ioc_prev_serv) 5485 kmem_free(ioc->ioc_prev_serv, ioc->ioc_prev_serv_cnt * 5486 sizeof (ibdm_srvents_info_t)); 5487 ioc->ioc_prev_serv_cnt = 0; 5488 ioc->ioc_prev_nportgids = 0; 5489 ioc->ioc_prev_serv = NULL; 5490 ioc->ioc_prev_gid_list = NULL; 5491 } 5492 5493 /* 5494 * Handle GID removal. This returns gid_info of an GID for the same 5495 * node GUID, if found. For an GID with IOU information, the same 5496 * gid_info is returned if no gid_info with same node_guid is found. 5497 */ 5498 static ibdm_dp_gidinfo_t * 5499 ibdm_handle_gid_rm(ibdm_dp_gidinfo_t *rm_gid) 5500 { 5501 ibdm_dp_gidinfo_t *gid_list; 5502 5503 IBTF_DPRINTF_L4("ibdm", "\thandle_gid_rm(0x%p)", rm_gid); 5504 5505 if (rm_gid->gl_iou == NULL) { 5506 IBTF_DPRINTF_L4("ibdm", "\thandle_gid_rm NO iou"); 5507 /* 5508 * Search for a GID with same node_guid and 5509 * gl_iou != NULL 5510 */ 5511 for (gid_list = ibdm.ibdm_dp_gidlist_head; gid_list; 5512 gid_list = gid_list->gl_next) { 5513 if (gid_list->gl_iou != NULL && (gid_list->gl_nodeguid 5514 == rm_gid->gl_nodeguid)) 5515 break; 5516 } 5517 5518 if (gid_list) 5519 ibdm_rmfrom_glgid_list(gid_list, rm_gid); 5520 5521 IBTF_DPRINTF_L4("ibdm", "\thandle_gid_rm ret %p", gid_list); 5522 return (gid_list); 5523 } else { 5524 /* 5525 * Search for a GID with same node_guid and 5526 * gl_iou == NULL 5527 */ 5528 IBTF_DPRINTF_L4("ibdm", "\thandle_gid_rm with iou"); 5529 for (gid_list = ibdm.ibdm_dp_gidlist_head; gid_list; 5530 gid_list = gid_list->gl_next) { 5531 if (gid_list->gl_iou == NULL && (gid_list->gl_nodeguid 5532 == rm_gid->gl_nodeguid)) 5533 break; 5534 } 5535 5536 if (gid_list) { 5537 /* 5538 * Copy the following fields from rm_gid : 5539 * 1. gl_state 5540 * 2. gl_iou 5541 * 3. gl_gid & gl_ngids 5542 * 5543 * Note : Function is synchronized by 5544 * ibdm_busy flag. 5545 * 5546 * Note : Redirect info is initialized if 5547 * any MADs for the GID fail 5548 */ 5549 IBTF_DPRINTF_L4("ibdm", "\thandle_gid_rm " 5550 "copying info to GID with gl_iou != NULl"); 5551 gid_list->gl_state = rm_gid->gl_state; 5552 gid_list->gl_iou = rm_gid->gl_iou; 5553 gid_list->gl_gid = rm_gid->gl_gid; 5554 gid_list->gl_ngids = rm_gid->gl_ngids; 5555 5556 /* Remove the GID from gl_gid list */ 5557 ibdm_rmfrom_glgid_list(gid_list, rm_gid); 5558 } else { 5559 /* 5560 * Handle a case where all GIDs to the IOU have 5561 * been removed. 5562 */ 5563 IBTF_DPRINTF_L4("ibdm", "\thandle_gid_rm 0 GID " 5564 "to IOU"); 5565 5566 ibdm_rmfrom_glgid_list(rm_gid, rm_gid); 5567 return (rm_gid); 5568 } 5569 IBTF_DPRINTF_L4("ibdm", "\thandle_gid_rm ret %p", gid_list); 5570 return (gid_list); 5571 } 5572 } 5573 5574 static void 5575 ibdm_rmfrom_glgid_list(ibdm_dp_gidinfo_t *gid_info, 5576 ibdm_dp_gidinfo_t *rm_gid) 5577 { 5578 ibdm_gid_t *tmp, *prev; 5579 5580 IBTF_DPRINTF_L4("ibdm", "\trmfrom_glgid (%p, %p)", 5581 gid_info, rm_gid); 5582 5583 for (tmp = gid_info->gl_gid, prev = NULL; tmp; ) { 5584 if (tmp->gid_dgid_hi == rm_gid->gl_dgid_hi && 5585 tmp->gid_dgid_lo == rm_gid->gl_dgid_lo) { 5586 if (prev == NULL) 5587 gid_info->gl_gid = tmp->gid_next; 5588 else 5589 prev->gid_next = tmp->gid_next; 5590 5591 kmem_free(tmp, sizeof (ibdm_gid_t)); 5592 gid_info->gl_ngids--; 5593 break; 5594 } else { 5595 prev = tmp; 5596 tmp = tmp->gid_next; 5597 } 5598 } 5599 } 5600 5601 static void 5602 ibdm_addto_gidlist(ibdm_gid_t **src_ptr, ibdm_gid_t *dest) 5603 { 5604 ibdm_gid_t *head = NULL, *new, *tail; 5605 5606 /* First copy the destination */ 5607 for (; dest; dest = dest->gid_next) { 5608 new = kmem_zalloc(sizeof (ibdm_gid_t), KM_SLEEP); 5609 new->gid_dgid_hi = dest->gid_dgid_hi; 5610 new->gid_dgid_lo = dest->gid_dgid_lo; 5611 new->gid_next = head; 5612 head = new; 5613 } 5614 5615 /* Insert this to the source */ 5616 if (*src_ptr == NULL) 5617 *src_ptr = head; 5618 else { 5619 for (tail = *src_ptr; tail->gid_next != NULL; 5620 tail = tail->gid_next) 5621 ; 5622 5623 tail->gid_next = head; 5624 } 5625 } 5626 5627 static void 5628 ibdm_free_gid_list(ibdm_gid_t *head) 5629 { 5630 ibdm_gid_t *delete; 5631 5632 for (delete = head; delete; ) { 5633 head = delete->gid_next; 5634 kmem_free(delete, sizeof (ibdm_gid_t)); 5635 delete = head; 5636 } 5637 } 5638 5639 /* 5640 * This function rescans the DM capable GIDs (gl_state is 5641 * GID_PROBE_COMPLETE or IBDM_GID_PROBING_SKIPPED.This 5642 * basically checks if the DM capable GID is reachable. If 5643 * not this is handled the same way as GID_UNAVAILABLE, 5644 * except that notifications are not send to IBnexus. 5645 * 5646 * This function also initializes the ioc_prev_list for 5647 * a particular IOC (when called from probe_ioc, with 5648 * ioc_guidp != NULL) or all IOCs for the gid (called from 5649 * sweep_fabric, ioc_guidp == NULL). 5650 */ 5651 static void 5652 ibdm_rescan_gidlist(ib_guid_t *ioc_guidp) 5653 { 5654 ibdm_dp_gidinfo_t *gid_info, *tmp; 5655 int ii, niocs, found; 5656 ibdm_hca_list_t *hca_list = NULL; 5657 ibdm_port_attr_t *port = NULL; 5658 ibdm_ioc_info_t *ioc_list; 5659 5660 for (gid_info = ibdm.ibdm_dp_gidlist_head; gid_info; ) { 5661 found = 0; 5662 if (gid_info->gl_state != IBDM_GID_PROBING_SKIPPED && 5663 gid_info->gl_state != IBDM_GID_PROBING_COMPLETE) { 5664 gid_info = gid_info->gl_next; 5665 continue; 5666 } 5667 5668 /* 5669 * Check if the GID is visible to any HCA ports. 5670 * Return if so. 5671 */ 5672 mutex_enter(&ibdm.ibdm_hl_mutex); 5673 for (ibdm_get_next_port(&hca_list, &port, 1); port; 5674 ibdm_get_next_port(&hca_list, &port, 1)) { 5675 if (ibdm_port_reachable(port->pa_sa_hdl, 5676 gid_info->gl_dgid_lo, NULL) == IBDM_SUCCESS) { 5677 found = 1; 5678 break; 5679 } 5680 } 5681 mutex_exit(&ibdm.ibdm_hl_mutex); 5682 5683 if (found) { 5684 if (gid_info->gl_iou == NULL) { 5685 gid_info = gid_info->gl_next; 5686 continue; 5687 } 5688 5689 /* Intialize the ioc_prev_gid_list */ 5690 niocs = 5691 gid_info->gl_iou->iou_info.iou_num_ctrl_slots; 5692 for (ii = 0; ii < niocs; ii++) { 5693 ioc_list = IBDM_GIDINFO2IOCINFO(gid_info, ii); 5694 5695 if (ioc_guidp == NULL || (*ioc_guidp == 5696 ioc_list->ioc_profile.ioc_guid)) { 5697 /* Add info of GIDs in gid_info also */ 5698 ibdm_addto_gidlist( 5699 &ioc_list->ioc_prev_gid_list, 5700 gid_info->gl_gid); 5701 ioc_list->ioc_prev_nportgids = 5702 gid_info->gl_ngids; 5703 } 5704 } 5705 gid_info = gid_info->gl_next; 5706 continue; 5707 } 5708 5709 IBTF_DPRINTF_L4("ibdm", "\trescan_gidlist " 5710 "deleted port GUID %llx", 5711 gid_info->gl_dgid_lo); 5712 5713 /* 5714 * Update GID list in all IOCs affected by this 5715 */ 5716 ioc_list = ibdm_update_ioc_gidlist(gid_info, 0); 5717 5718 /* 5719 * Remove GID from the global GID list 5720 * Handle the case where all port GIDs for an 5721 * IOU have been hot-removed. 5722 */ 5723 mutex_enter(&ibdm.ibdm_mutex); 5724 if (gid_info->gl_iou != NULL && gid_info->gl_ngids == 0) { 5725 mutex_enter(&gid_info->gl_mutex); 5726 (void) ibdm_free_iou_info(gid_info); 5727 mutex_exit(&gid_info->gl_mutex); 5728 } 5729 tmp = gid_info->gl_next; 5730 if (gid_info->gl_prev != NULL) 5731 gid_info->gl_prev->gl_next = gid_info->gl_next; 5732 if (gid_info->gl_next != NULL) 5733 gid_info->gl_next->gl_prev = gid_info->gl_prev; 5734 5735 if (gid_info == ibdm.ibdm_dp_gidlist_head) 5736 ibdm.ibdm_dp_gidlist_head = gid_info->gl_next; 5737 if (gid_info == ibdm.ibdm_dp_gidlist_tail) 5738 ibdm.ibdm_dp_gidlist_tail = gid_info->gl_prev; 5739 ibdm.ibdm_ngids--; 5740 5741 mutex_destroy(&gid_info->gl_mutex); 5742 kmem_free(gid_info, sizeof (ibdm_dp_gidinfo_t)); 5743 gid_info = tmp; 5744 5745 mutex_exit(&ibdm.ibdm_mutex); 5746 5747 /* 5748 * Pass on the IOCs with updated GIDs to IBnexus 5749 */ 5750 if (ioc_list) { 5751 IBTF_DPRINTF_L4("ibdm", "\trescan_gidlist " 5752 "IOC_PROP_UPDATE for %p\n", ioc_list); 5753 mutex_enter(&ibdm.ibdm_ibnex_mutex); 5754 if (ibdm.ibdm_ibnex_callback != NULL) { 5755 (*ibdm.ibdm_ibnex_callback)((void *) 5756 ioc_list, IBDM_EVENT_IOC_PROP_UPDATE); 5757 } 5758 mutex_exit(&ibdm.ibdm_ibnex_mutex); 5759 } 5760 } 5761 } 5762 5763 /* 5764 * This function notifies IBnex of IOCs on this GID. 5765 * Notification is for GIDs with gl_reprobe_flag set. 5766 * The flag is set when IOC probe / fabric sweep 5767 * probes a GID starting from CLASS port info. 5768 * 5769 * IBnexus will have information of a reconnected IOC 5770 * if it had probed it before. If this is a new IOC, 5771 * IBnexus ignores the notification. 5772 * 5773 * This function should be called with no locks held. 5774 */ 5775 static void 5776 ibdm_notify_newgid_iocs(ibdm_dp_gidinfo_t *gid_info) 5777 { 5778 ibdm_ioc_info_t *ioc_list; 5779 5780 if (gid_info->gl_reprobe_flag == 0 || 5781 gid_info->gl_iou == NULL) 5782 return; 5783 5784 ioc_list = ibdm_update_ioc_gidlist(gid_info, -1); 5785 5786 /* 5787 * Pass on the IOCs with updated GIDs to IBnexus 5788 */ 5789 if (ioc_list) { 5790 mutex_enter(&ibdm.ibdm_ibnex_mutex); 5791 if (ibdm.ibdm_ibnex_callback != NULL) { 5792 (*ibdm.ibdm_ibnex_callback)((void *)ioc_list, 5793 IBDM_EVENT_IOC_PROP_UPDATE); 5794 } 5795 mutex_exit(&ibdm.ibdm_ibnex_mutex); 5796 } 5797 } 5798 5799 5800 static void 5801 ibdm_free_saa_event_arg(ibdm_saa_event_arg_t *arg) 5802 { 5803 if (arg != NULL) 5804 kmem_free(arg, sizeof (ibdm_saa_event_arg_t)); 5805 } 5806 5807 /* 5808 * This function parses the list of HCAs and HCA ports 5809 * to return the port_attr of the next HCA port. A port 5810 * connected to IB fabric (port_state active) is returned, 5811 * if connected_flag is set. 5812 */ 5813 static void 5814 ibdm_get_next_port(ibdm_hca_list_t **inp_hcap, 5815 ibdm_port_attr_t **inp_portp, int connect_flag) 5816 { 5817 int ii; 5818 ibdm_port_attr_t *port, *next_port = NULL; 5819 ibdm_port_attr_t *inp_port; 5820 ibdm_hca_list_t *hca_list; 5821 int found = 0; 5822 5823 ASSERT(MUTEX_HELD(&ibdm.ibdm_hl_mutex)); 5824 IBTF_DPRINTF_L4(ibdm_string, "\tget_next_port(%p, %p, %x)", 5825 inp_hcap, inp_portp, connect_flag); 5826 5827 hca_list = *inp_hcap; 5828 inp_port = *inp_portp; 5829 5830 if (hca_list == NULL) 5831 hca_list = ibdm.ibdm_hca_list_head; 5832 5833 for (; hca_list; hca_list = hca_list->hl_next) { 5834 for (ii = 0; ii < hca_list->hl_nports; ii++) { 5835 port = &hca_list->hl_port_attr[ii]; 5836 5837 /* 5838 * inp_port != NULL; 5839 * Skip till we find the matching port 5840 */ 5841 if (inp_port && !found) { 5842 if (inp_port == port) 5843 found = 1; 5844 continue; 5845 } 5846 5847 if (!connect_flag) { 5848 next_port = port; 5849 break; 5850 } 5851 5852 if (port->pa_sa_hdl == NULL) 5853 ibdm_initialize_port(port); 5854 if (port->pa_sa_hdl == NULL) 5855 (void) ibdm_fini_port(port); 5856 else if (next_port == NULL && 5857 port->pa_sa_hdl != NULL && 5858 port->pa_state == IBT_PORT_ACTIVE) { 5859 next_port = port; 5860 break; 5861 } 5862 } 5863 5864 if (next_port) 5865 break; 5866 } 5867 5868 IBTF_DPRINTF_L4(ibdm_string, "\tget_next_port : " 5869 "returns hca_list %p port %p", hca_list, next_port); 5870 *inp_hcap = hca_list; 5871 *inp_portp = next_port; 5872 } 5873 5874 static void 5875 ibdm_add_to_gl_gid(ibdm_dp_gidinfo_t *nodegid, ibdm_dp_gidinfo_t *addgid) 5876 { 5877 ibdm_gid_t *tmp; 5878 5879 tmp = kmem_zalloc(sizeof (ibdm_gid_t), KM_SLEEP); 5880 tmp->gid_dgid_hi = addgid->gl_dgid_hi; 5881 tmp->gid_dgid_lo = addgid->gl_dgid_lo; 5882 5883 mutex_enter(&nodegid->gl_mutex); 5884 tmp->gid_next = nodegid->gl_gid; 5885 nodegid->gl_gid = tmp; 5886 nodegid->gl_ngids++; 5887 mutex_exit(&nodegid->gl_mutex); 5888 } 5889 5890 static void 5891 ibdm_addto_glhcalist(ibdm_dp_gidinfo_t *gid_info, 5892 ibdm_hca_list_t *hca) 5893 { 5894 ibdm_hca_list_t *head, *prev = NULL, *temp; 5895 5896 IBTF_DPRINTF_L4(ibdm_string, "\taddto_glhcalist(%p, %p) " 5897 ": gl_hca_list %p", gid_info, hca, gid_info->gl_hca_list); 5898 ASSERT(!MUTEX_HELD(&gid_info->gl_mutex)); 5899 mutex_enter(&gid_info->gl_mutex); 5900 head = gid_info->gl_hca_list; 5901 if (head == NULL) { 5902 head = ibdm_dup_hca_attr(hca); 5903 head->hl_next = NULL; 5904 gid_info->gl_hca_list = head; 5905 mutex_exit(&gid_info->gl_mutex); 5906 IBTF_DPRINTF_L4(ibdm_string, "\tadd_to_glhcalist: " 5907 "gid %p, gl_hca_list %p", gid_info, 5908 gid_info->gl_hca_list); 5909 return; 5910 } 5911 5912 /* Check if already in the list */ 5913 while (head) { 5914 if (head->hl_hca_guid == hca->hl_hca_guid) { 5915 mutex_exit(&gid_info->gl_mutex); 5916 IBTF_DPRINTF_L4(ibdm_string, 5917 "\taddto_glhcalist : gid %x hca %x dup", 5918 gid_info, hca); 5919 return; 5920 } 5921 prev = head; 5922 head = head->hl_next; 5923 } 5924 5925 /* Add this HCA to gl_hca_list */ 5926 temp = ibdm_dup_hca_attr(hca); 5927 temp->hl_next = NULL; 5928 prev->hl_next = temp; 5929 5930 mutex_exit(&gid_info->gl_mutex); 5931 5932 IBTF_DPRINTF_L4(ibdm_string, "\tadd_to_glhcalist: " 5933 "gid %p, gl_hca_list %p", gid_info, gid_info->gl_hca_list); 5934 } 5935 5936 static void 5937 ibdm_delete_glhca_list(ibdm_dp_gidinfo_t *gid_info) 5938 { 5939 ASSERT(!MUTEX_HELD(&gid_info->gl_mutex)); 5940 ASSERT(!MUTEX_HELD(&ibdm.ibdm_mutex)); 5941 5942 mutex_enter(&gid_info->gl_mutex); 5943 if (gid_info->gl_hca_list) 5944 ibdm_ibnex_free_hca_list(gid_info->gl_hca_list); 5945 gid_info->gl_hca_list = NULL; 5946 mutex_exit(&gid_info->gl_mutex); 5947 } 5948 5949 5950 static void 5951 ibdm_reset_all_dgids(ibmf_saa_handle_t port_sa_hdl) 5952 { 5953 IBTF_DPRINTF_L4(ibdm_string, "\treset_all_dgids(%X)", 5954 port_sa_hdl); 5955 5956 ASSERT(!MUTEX_HELD(&ibdm.ibdm_mutex)); 5957 ASSERT(!MUTEX_HELD(&ibdm.ibdm_hl_mutex)); 5958 5959 /* Check : Not busy in another probe / sweep */ 5960 mutex_enter(&ibdm.ibdm_mutex); 5961 if ((ibdm.ibdm_busy & IBDM_BUSY) == 0) { 5962 ibdm_dp_gidinfo_t *gid_info; 5963 5964 ibdm.ibdm_busy |= IBDM_BUSY; 5965 mutex_exit(&ibdm.ibdm_mutex); 5966 5967 /* 5968 * Check if any GID is using the SA & IBMF handle 5969 * of HCA port going down. Reset ibdm_dp_gidinfo_t 5970 * using another HCA port which can reach the GID. 5971 * This is for DM capable GIDs only, no need to do 5972 * this for others 5973 * 5974 * Delete the GID if no alternate HCA port to reach 5975 * it is found. 5976 */ 5977 for (gid_info = ibdm.ibdm_dp_gidlist_head; gid_info; ) { 5978 ibdm_dp_gidinfo_t *tmp; 5979 5980 IBTF_DPRINTF_L4(ibdm_string, "\tevent_hdlr " 5981 "checking gidinfo %p", gid_info); 5982 5983 if (gid_info->gl_nodeguid != 0 && 5984 gid_info->gl_sa_hdl == port_sa_hdl) { 5985 IBTF_DPRINTF_L3(ibdm_string, 5986 "\tevent_hdlr: down HCA port hdl " 5987 "matches gid %p", gid_info); 5988 5989 ibdm_reset_gidinfo(gid_info); 5990 if (gid_info->gl_disconnected) { 5991 IBTF_DPRINTF_L3(ibdm_string, 5992 "\tevent_hdlr: deleting" 5993 " gid %p", gid_info); 5994 tmp = gid_info; 5995 gid_info = gid_info->gl_next; 5996 ibdm_delete_gidinfo(tmp); 5997 } else 5998 gid_info = gid_info->gl_next; 5999 } else 6000 gid_info = gid_info->gl_next; 6001 } 6002 6003 mutex_enter(&ibdm.ibdm_mutex); 6004 ibdm.ibdm_busy &= ~IBDM_BUSY; 6005 cv_signal(&ibdm.ibdm_busy_cv); 6006 } 6007 mutex_exit(&ibdm.ibdm_mutex); 6008 } 6009 6010 static void 6011 ibdm_reset_gidinfo(ibdm_dp_gidinfo_t *gidinfo) 6012 { 6013 ibdm_hca_list_t *hca_list = NULL; 6014 ibdm_port_attr_t *port = NULL; 6015 int gid_reinited = 0; 6016 sa_node_record_t *nr, *tmp; 6017 sa_portinfo_record_t *pi; 6018 size_t nr_len = 0, pi_len = 0; 6019 size_t path_len; 6020 ib_gid_t sgid, dgid; 6021 int ret, ii, nrecords; 6022 sa_path_record_t *path; 6023 uint8_t npaths = 1; 6024 6025 IBTF_DPRINTF_L4(ibdm_string, "\treset_gidinfo(%p)", gidinfo); 6026 6027 /* 6028 * Get list of all the ports reachable from the local known HCA 6029 * ports which are active 6030 */ 6031 mutex_enter(&ibdm.ibdm_hl_mutex); 6032 for (ibdm_get_next_port(&hca_list, &port, 1); port; 6033 ibdm_get_next_port(&hca_list, &port, 1)) { 6034 6035 6036 /* 6037 * Get the path and re-populate the gidinfo. 6038 * Getting the path is the same probe_ioc 6039 * Init the gid info as in ibdm_create_gidinfo() 6040 */ 6041 nr = ibdm_get_node_records(port->pa_sa_hdl, &nr_len, 6042 gidinfo->gl_nodeguid); 6043 if (nr == NULL) { 6044 IBTF_DPRINTF_L4(ibdm_string, 6045 "\treset_gidinfo : no records"); 6046 continue; 6047 } 6048 6049 nrecords = (nr_len / sizeof (sa_node_record_t)); 6050 for (tmp = nr, ii = 0; (ii < nrecords); ii++, tmp++) { 6051 if (tmp->NodeInfo.PortGUID == gidinfo->gl_portguid) 6052 break; 6053 } 6054 6055 if (ii == nrecords) { 6056 IBTF_DPRINTF_L4(ibdm_string, 6057 "\treset_gidinfo : no record for portguid"); 6058 kmem_free(nr, nr_len); 6059 continue; 6060 } 6061 6062 pi = ibdm_get_portinfo(port->pa_sa_hdl, &pi_len, tmp->LID); 6063 if (pi == NULL) { 6064 IBTF_DPRINTF_L4(ibdm_string, 6065 "\treset_gidinfo : no portinfo"); 6066 kmem_free(nr, nr_len); 6067 continue; 6068 } 6069 6070 sgid.gid_prefix = port->pa_sn_prefix; 6071 sgid.gid_guid = port->pa_port_guid; 6072 dgid.gid_prefix = pi->PortInfo.GidPrefix; 6073 dgid.gid_guid = tmp->NodeInfo.PortGUID; 6074 6075 ret = ibmf_saa_gid_to_pathrecords(port->pa_sa_hdl, sgid, dgid, 6076 IBMF_SAA_PKEY_WC, 0, B_TRUE, &npaths, 0, &path_len, &path); 6077 6078 if ((ret != IBMF_SUCCESS) || path == NULL) { 6079 IBTF_DPRINTF_L4(ibdm_string, 6080 "\treset_gidinfo : no paths"); 6081 kmem_free(pi, pi_len); 6082 kmem_free(nr, nr_len); 6083 continue; 6084 } 6085 6086 gidinfo->gl_dgid_hi = path->DGID.gid_prefix; 6087 gidinfo->gl_dgid_lo = path->DGID.gid_guid; 6088 gidinfo->gl_sgid_hi = path->SGID.gid_prefix; 6089 gidinfo->gl_sgid_lo = path->SGID.gid_guid; 6090 gidinfo->gl_p_key = path->P_Key; 6091 gidinfo->gl_sa_hdl = port->pa_sa_hdl; 6092 gidinfo->gl_ibmf_hdl = port->pa_ibmf_hdl; 6093 gidinfo->gl_slid = path->SLID; 6094 gidinfo->gl_dlid = path->DLID; 6095 6096 /* Reset redirect info, next MAD will set if redirected */ 6097 gidinfo->gl_redirected = 0; 6098 6099 gid_reinited = 1; 6100 6101 kmem_free(path, path_len); 6102 kmem_free(pi, pi_len); 6103 kmem_free(nr, nr_len); 6104 break; 6105 } 6106 mutex_exit(&ibdm.ibdm_hl_mutex); 6107 6108 if (!gid_reinited) 6109 gidinfo->gl_disconnected = 1; 6110 } 6111 6112 static void 6113 ibdm_delete_gidinfo(ibdm_dp_gidinfo_t *gidinfo) 6114 { 6115 ibdm_ioc_info_t *ioc_list; 6116 int in_gidlist = 0; 6117 6118 /* 6119 * Check if gidinfo has been inserted into the 6120 * ibdm_dp_gidlist_head list. gl_next or gl_prev 6121 * != NULL, if gidinfo is the list. 6122 */ 6123 if (gidinfo->gl_prev != NULL || 6124 gidinfo->gl_next != NULL) 6125 in_gidlist = 1; 6126 6127 ioc_list = ibdm_update_ioc_gidlist(gidinfo, 0); 6128 6129 /* 6130 * Remove GID from the global GID list 6131 * Handle the case where all port GIDs for an 6132 * IOU have been hot-removed. 6133 */ 6134 mutex_enter(&ibdm.ibdm_mutex); 6135 if (gidinfo->gl_iou != NULL && gidinfo->gl_ngids == 0) { 6136 mutex_enter(&gidinfo->gl_mutex); 6137 (void) ibdm_free_iou_info(gidinfo); 6138 mutex_exit(&gidinfo->gl_mutex); 6139 } 6140 6141 if (in_gidlist) { 6142 if (gidinfo->gl_prev != NULL) 6143 gidinfo->gl_prev->gl_next = gidinfo->gl_next; 6144 if (gidinfo->gl_next != NULL) 6145 gidinfo->gl_next->gl_prev = gidinfo->gl_prev; 6146 6147 if (gidinfo == ibdm.ibdm_dp_gidlist_head) 6148 ibdm.ibdm_dp_gidlist_head = gidinfo->gl_next; 6149 if (gidinfo == ibdm.ibdm_dp_gidlist_tail) 6150 ibdm.ibdm_dp_gidlist_tail = gidinfo->gl_prev; 6151 ibdm.ibdm_ngids--; 6152 } 6153 mutex_exit(&ibdm.ibdm_mutex); 6154 6155 mutex_destroy(&gidinfo->gl_mutex); 6156 kmem_free(gidinfo, sizeof (ibdm_dp_gidinfo_t)); 6157 6158 /* 6159 * Pass on the IOCs with updated GIDs to IBnexus 6160 */ 6161 if (ioc_list) { 6162 IBTF_DPRINTF_L4("ibdm", "\tdelete_gidinfo " 6163 "IOC_PROP_UPDATE for %p\n", ioc_list); 6164 mutex_enter(&ibdm.ibdm_ibnex_mutex); 6165 if (ibdm.ibdm_ibnex_callback != NULL) { 6166 (*ibdm.ibdm_ibnex_callback)((void *) 6167 ioc_list, IBDM_EVENT_IOC_PROP_UPDATE); 6168 } 6169 mutex_exit(&ibdm.ibdm_ibnex_mutex); 6170 } 6171 } 6172 6173 6174 static void 6175 ibdm_fill_srv_attr_mod(ib_mad_hdr_t *hdr, ibdm_timeout_cb_args_t *cb_args) 6176 { 6177 uint32_t attr_mod; 6178 6179 attr_mod = (cb_args->cb_ioc_num + 1) << 16; 6180 attr_mod |= cb_args->cb_srvents_start; 6181 attr_mod |= (cb_args->cb_srvents_end) << 8; 6182 hdr->AttributeModifier = h2b32(attr_mod); 6183 } 6184 6185 static void 6186 ibdm_bump_transactionID(ibdm_dp_gidinfo_t *gid_info) 6187 { 6188 ASSERT(MUTEX_HELD(&gid_info->gl_mutex)); 6189 gid_info->gl_transactionID++; 6190 if (gid_info->gl_transactionID == gid_info->gl_max_transactionID) { 6191 IBTF_DPRINTF_L4(ibdm_string, 6192 "\tbump_transactionID(%p), wrapup", gid_info); 6193 gid_info->gl_transactionID = gid_info->gl_min_transactionID; 6194 } 6195 } 6196 6197 /* For debugging purpose only */ 6198 #ifdef DEBUG 6199 void 6200 ibdm_dump_ibmf_msg(ibmf_msg_t *ibmf_msg, int flag) 6201 { 6202 ib_mad_hdr_t *mad_hdr; 6203 6204 IBTF_DPRINTF_L4("ibdm", "\t\t(IBMF_PKT): Local address info"); 6205 IBTF_DPRINTF_L4("ibdm", "\t\t ------------------"); 6206 6207 IBTF_DPRINTF_L4("ibdm", "\tLocal Lid : 0x%x\tRemote Lid : 0x%x" 6208 " Remote Qp : 0x%x", ibmf_msg->im_local_addr.ia_local_lid, 6209 ibmf_msg->im_local_addr.ia_remote_lid, 6210 ibmf_msg->im_local_addr.ia_remote_qno); 6211 IBTF_DPRINTF_L4("ibdm", "\tP_key : 0x%x\tQ_key : 0x%x", 6212 ibmf_msg->im_local_addr.ia_p_key, ibmf_msg->im_local_addr.ia_q_key); 6213 6214 if (flag) 6215 mad_hdr = (ib_mad_hdr_t *)IBDM_OUT_IBMFMSG_MADHDR(ibmf_msg); 6216 else 6217 mad_hdr = IBDM_IN_IBMFMSG_MADHDR(ibmf_msg); 6218 6219 IBTF_DPRINTF_L4("ibdm", "\t\t MAD Header info"); 6220 IBTF_DPRINTF_L4("ibdm", "\t\t ---------------"); 6221 6222 IBTF_DPRINTF_L4("ibdm", "\tBase version : 0x%x" 6223 "\tMgmt Class : 0x%x", mad_hdr->BaseVersion, mad_hdr->MgmtClass); 6224 IBTF_DPRINTF_L4("ibdm", "\tClass version : 0x%x" 6225 "\tR Method : 0x%x", 6226 mad_hdr->ClassVersion, mad_hdr->R_Method); 6227 IBTF_DPRINTF_L4("ibdm", "\tMAD Status : 0x%x" 6228 "\tTransaction ID : 0x%llx", 6229 mad_hdr->Status, mad_hdr->TransactionID); 6230 IBTF_DPRINTF_L4("ibdm", "\t Attribute ID : 0x%x" 6231 "\tAttribute Modified : 0x%lx", 6232 mad_hdr->AttributeID, mad_hdr->AttributeModifier); 6233 } 6234 6235 void 6236 ibdm_dump_path_info(sa_path_record_t *path) 6237 { 6238 IBTF_DPRINTF_L4("ibdm", "\t\t Path information"); 6239 IBTF_DPRINTF_L4("ibdm", "\t\t ----------------"); 6240 6241 IBTF_DPRINTF_L4("ibdm", "\t DGID hi : %llx\tDGID lo : %llx", 6242 path->DGID.gid_prefix, path->DGID.gid_guid); 6243 IBTF_DPRINTF_L4("ibdm", "\t SGID hi : %llx\tSGID lo : %llx", 6244 path->SGID.gid_prefix, path->SGID.gid_guid); 6245 IBTF_DPRINTF_L4("ibdm", "\t SLID : %x\tDlID : %x", 6246 path->SLID, path->DLID); 6247 IBTF_DPRINTF_L4("ibdm", "\t P Key : %x", path->P_Key); 6248 } 6249 6250 6251 void 6252 ibdm_dump_classportinfo(ibdm_mad_classportinfo_t *classportinfo) 6253 { 6254 IBTF_DPRINTF_L4("ibdm", "\t\t CLASSPORT INFO"); 6255 IBTF_DPRINTF_L4("ibdm", "\t\t --------------"); 6256 6257 IBTF_DPRINTF_L4("ibdm", "\t Response Time Value : 0x%x", 6258 ((b2h32(classportinfo->RespTimeValue)) & 0x1F)); 6259 6260 IBTF_DPRINTF_L4("ibdm", "\t Redirected QP : 0x%x", 6261 (b2h32(classportinfo->RedirectQP))); 6262 IBTF_DPRINTF_L4("ibdm", "\t Redirected P KEY : 0x%x", 6263 b2h16(classportinfo->RedirectP_Key)); 6264 IBTF_DPRINTF_L4("ibdm", "\t Redirected Q KEY : 0x%x", 6265 b2h16(classportinfo->RedirectQ_Key)); 6266 IBTF_DPRINTF_L4("ibdm", "\t Redirected GID hi : 0x%x", 6267 b2h64(classportinfo->RedirectGID_hi)); 6268 IBTF_DPRINTF_L4("ibdm", "\t Redirected GID lo : 0x%x", 6269 b2h64(classportinfo->RedirectGID_lo)); 6270 } 6271 6272 6273 void 6274 ibdm_dump_iounitinfo(ib_dm_io_unitinfo_t *iou_info) 6275 { 6276 IBTF_DPRINTF_L4("ibdm", "\t\t I/O UnitInfo"); 6277 IBTF_DPRINTF_L4("ibdm", "\t\t ------------"); 6278 6279 IBTF_DPRINTF_L4("ibdm", "\tChange ID : 0x%x", 6280 b2h16(iou_info->iou_changeid)); 6281 IBTF_DPRINTF_L4("ibdm", "\t#of ctrl slots : %d", 6282 iou_info->iou_num_ctrl_slots); 6283 IBTF_DPRINTF_L4("ibdm", "\tIOU flag : 0x%x", 6284 iou_info->iou_flag); 6285 IBTF_DPRINTF_L4("ibdm", "\tContrl list byte 0 : 0x%x", 6286 iou_info->iou_ctrl_list[0]); 6287 IBTF_DPRINTF_L4("ibdm", "\tContrl list byte 1 : 0x%x", 6288 iou_info->iou_ctrl_list[1]); 6289 IBTF_DPRINTF_L4("ibdm", "\tContrl list byte 2 : 0x%x", 6290 iou_info->iou_ctrl_list[2]); 6291 } 6292 6293 6294 void 6295 ibdm_dump_ioc_profile(ib_dm_ioc_ctrl_profile_t *ioc) 6296 { 6297 IBTF_DPRINTF_L4("ibdm", "\t\t IOC Controller Profile"); 6298 IBTF_DPRINTF_L4("ibdm", "\t\t ----------------------"); 6299 6300 IBTF_DPRINTF_L4("ibdm", "\tIOC Guid : %llx", ioc->ioc_guid); 6301 IBTF_DPRINTF_L4("ibdm", "\tVendorID : 0x%x", ioc->ioc_vendorid); 6302 IBTF_DPRINTF_L4("ibdm", "\tDevice Id : 0x%x", ioc->ioc_deviceid); 6303 IBTF_DPRINTF_L4("ibdm", "\tDevice Ver : 0x%x", ioc->ioc_device_ver); 6304 IBTF_DPRINTF_L4("ibdm", "\tSubsys ID : 0x%x", ioc->ioc_subsys_id); 6305 IBTF_DPRINTF_L4("ibdm", "\tIO class : 0x%x", ioc->ioc_io_class); 6306 IBTF_DPRINTF_L4("ibdm", "\tIO subclass : 0x%x", ioc->ioc_io_subclass); 6307 IBTF_DPRINTF_L4("ibdm", "\tProtocol : 0x%x", ioc->ioc_protocol); 6308 IBTF_DPRINTF_L4("ibdm", "\tProtocolV : 0x%x", ioc->ioc_protocol_ver); 6309 IBTF_DPRINTF_L4("ibdm", "\tmsg qdepth : %d", ioc->ioc_send_msg_qdepth); 6310 IBTF_DPRINTF_L4("ibdm", "\trdma qdepth : %d", 6311 ioc->ioc_rdma_read_qdepth); 6312 IBTF_DPRINTF_L4("ibdm", "\tsndmsg sz : %d", ioc->ioc_send_msg_sz); 6313 IBTF_DPRINTF_L4("ibdm", "\trdma xfersz : %d", ioc->ioc_rdma_xfer_sz); 6314 IBTF_DPRINTF_L4("ibdm", "\topcal mask : 0x%x", 6315 ioc->ioc_ctrl_opcap_mask); 6316 IBTF_DPRINTF_L4("ibdm", "\tsrventries : %x", ioc->ioc_service_entries); 6317 } 6318 6319 6320 void 6321 ibdm_dump_service_entries(ib_dm_srv_t *srv_ents) 6322 { 6323 IBTF_DPRINTF_L4("ibdm", 6324 "\thandle_srventry_mad: service id : %llx", srv_ents->srv_id); 6325 6326 IBTF_DPRINTF_L4("ibdm", "\thandle_srventry_mad: " 6327 "Service Name : %s", srv_ents->srv_name); 6328 } 6329 6330 int ibdm_allow_sweep_fabric_timestamp = 1; 6331 6332 void 6333 ibdm_dump_sweep_fabric_timestamp(int flag) 6334 { 6335 static hrtime_t x; 6336 if (flag) { 6337 if (ibdm_allow_sweep_fabric_timestamp) { 6338 IBTF_DPRINTF_L4("ibdm", "\tTime taken to complete " 6339 "sweep %lld ms", ((gethrtime() - x)/ 1000000)); 6340 } 6341 x = 0; 6342 } else 6343 x = gethrtime(); 6344 } 6345 #endif 6346