1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved. 23 */ 24 25 /* 26 * ibdm.c 27 * 28 * This file contains the InifiniBand Device Manager (IBDM) support functions. 29 * IB nexus driver will only be the client for the IBDM module. 30 * 31 * IBDM registers with IBTF for HCA arrival/removal notification. 32 * IBDM registers with SA access to send DM MADs to discover the IOC's behind 33 * the IOU's. 34 * 35 * IB nexus driver registers with IBDM to find the information about the 36 * HCA's and IOC's (behind the IOU) present on the IB fabric. 37 */ 38 39 #include <sys/systm.h> 40 #include <sys/taskq.h> 41 #include <sys/ib/mgt/ibdm/ibdm_impl.h> 42 #include <sys/ib/mgt/ibmf/ibmf_impl.h> 43 #include <sys/ib/ibtl/impl/ibtl_ibnex.h> 44 #include <sys/modctl.h> 45 46 /* Function Prototype declarations */ 47 static int ibdm_free_iou_info(ibdm_dp_gidinfo_t *, ibdm_iou_info_t **); 48 static int ibdm_fini(void); 49 static int ibdm_init(void); 50 static int ibdm_get_reachable_ports(ibdm_port_attr_t *, 51 ibdm_hca_list_t *); 52 static ibdm_dp_gidinfo_t *ibdm_check_dgid(ib_guid_t, ib_sn_prefix_t); 53 static ibdm_dp_gidinfo_t *ibdm_check_dest_nodeguid(ibdm_dp_gidinfo_t *); 54 static boolean_t ibdm_is_cisco(ib_guid_t); 55 static boolean_t ibdm_is_cisco_switch(ibdm_dp_gidinfo_t *); 56 static void ibdm_wait_cisco_probe_completion(ibdm_dp_gidinfo_t *); 57 static int ibdm_set_classportinfo(ibdm_dp_gidinfo_t *); 58 static int ibdm_send_classportinfo(ibdm_dp_gidinfo_t *); 59 static int ibdm_send_iounitinfo(ibdm_dp_gidinfo_t *); 60 static int ibdm_is_dev_mgt_supported(ibdm_dp_gidinfo_t *); 61 static int ibdm_get_node_port_guids(ibmf_saa_handle_t, ib_lid_t, 62 ib_guid_t *, ib_guid_t *); 63 static int ibdm_retry_command(ibdm_timeout_cb_args_t *); 64 static int ibdm_get_diagcode(ibdm_dp_gidinfo_t *, int); 65 static int ibdm_verify_mad_status(ib_mad_hdr_t *); 66 static int ibdm_handle_redirection(ibmf_msg_t *, 67 ibdm_dp_gidinfo_t *, int *); 68 static void ibdm_wait_probe_completion(void); 69 static void ibdm_sweep_fabric(int); 70 static void ibdm_probe_gid_thread(void *); 71 static void ibdm_wakeup_probe_gid_cv(void); 72 static void ibdm_port_attr_ibmf_init(ibdm_port_attr_t *, ib_pkey_t, int); 73 static int ibdm_port_attr_ibmf_fini(ibdm_port_attr_t *, int); 74 static void ibdm_update_port_attr(ibdm_port_attr_t *); 75 static void ibdm_handle_hca_attach(ib_guid_t); 76 static void ibdm_handle_srventry_mad(ibmf_msg_t *, 77 ibdm_dp_gidinfo_t *, int *); 78 static void ibdm_ibmf_recv_cb(ibmf_handle_t, ibmf_msg_t *, void *); 79 static void ibdm_recv_incoming_mad(void *); 80 static void ibdm_process_incoming_mad(ibmf_handle_t, ibmf_msg_t *, void *); 81 static void ibdm_ibmf_send_cb(ibmf_handle_t, ibmf_msg_t *, void *); 82 static void ibdm_pkt_timeout_hdlr(void *arg); 83 static void ibdm_initialize_port(ibdm_port_attr_t *); 84 static void ibdm_update_port_pkeys(ibdm_port_attr_t *port); 85 static void ibdm_handle_diagcode(ibmf_msg_t *, ibdm_dp_gidinfo_t *, int *); 86 static void ibdm_probe_gid(ibdm_dp_gidinfo_t *); 87 static void ibdm_alloc_send_buffers(ibmf_msg_t *); 88 static void ibdm_free_send_buffers(ibmf_msg_t *); 89 static void ibdm_handle_hca_detach(ib_guid_t); 90 static void ibdm_handle_port_change_event(ibt_async_event_t *); 91 static int ibdm_fini_port(ibdm_port_attr_t *); 92 static int ibdm_uninit_hca(ibdm_hca_list_t *); 93 static void ibdm_handle_setclassportinfo(ibmf_handle_t, ibmf_msg_t *, 94 ibdm_dp_gidinfo_t *, int *); 95 static void ibdm_handle_iounitinfo(ibmf_handle_t, 96 ibmf_msg_t *, ibdm_dp_gidinfo_t *, int *); 97 static void ibdm_handle_ioc_profile(ibmf_handle_t, 98 ibmf_msg_t *, ibdm_dp_gidinfo_t *, int *); 99 static void ibdm_event_hdlr(void *, ibt_hca_hdl_t, 100 ibt_async_code_t, ibt_async_event_t *); 101 static void ibdm_handle_classportinfo(ibmf_handle_t, 102 ibmf_msg_t *, ibdm_dp_gidinfo_t *, int *); 103 static void ibdm_update_ioc_port_gidlist(ibdm_ioc_info_t *, 104 ibdm_dp_gidinfo_t *); 105 106 static ibdm_hca_list_t *ibdm_dup_hca_attr(ibdm_hca_list_t *); 107 static ibdm_ioc_info_t *ibdm_dup_ioc_info(ibdm_ioc_info_t *, 108 ibdm_dp_gidinfo_t *gid_list); 109 static void ibdm_probe_ioc(ib_guid_t, ib_guid_t, int); 110 static ibdm_ioc_info_t *ibdm_is_ioc_present(ib_guid_t, 111 ibdm_dp_gidinfo_t *, int *); 112 static ibdm_port_attr_t *ibdm_get_port_attr(ibt_async_event_t *, 113 ibdm_hca_list_t **); 114 static sa_node_record_t *ibdm_get_node_records(ibmf_saa_handle_t, 115 size_t *, ib_guid_t); 116 static int ibdm_get_node_record_by_port(ibmf_saa_handle_t, 117 ib_guid_t, sa_node_record_t **, size_t *); 118 static sa_portinfo_record_t *ibdm_get_portinfo(ibmf_saa_handle_t, size_t *, 119 ib_lid_t); 120 static ibdm_dp_gidinfo_t *ibdm_create_gid_info(ibdm_port_attr_t *, 121 ib_gid_t, ib_gid_t); 122 static ibdm_dp_gidinfo_t *ibdm_find_gid(ib_guid_t, ib_guid_t); 123 static int ibdm_send_ioc_profile(ibdm_dp_gidinfo_t *, uint8_t); 124 static ibdm_ioc_info_t *ibdm_update_ioc_gidlist(ibdm_dp_gidinfo_t *, int); 125 static void ibdm_saa_event_cb(ibmf_saa_handle_t, ibmf_saa_subnet_event_t, 126 ibmf_saa_event_details_t *, void *); 127 static void ibdm_reprobe_update_port_srv(ibdm_ioc_info_t *, 128 ibdm_dp_gidinfo_t *); 129 static ibdm_dp_gidinfo_t *ibdm_handle_gid_rm(ibdm_dp_gidinfo_t *); 130 static void ibdm_rmfrom_glgid_list(ibdm_dp_gidinfo_t *, 131 ibdm_dp_gidinfo_t *); 132 static void ibdm_addto_gidlist(ibdm_gid_t **, ibdm_gid_t *); 133 static void ibdm_free_gid_list(ibdm_gid_t *); 134 static void ibdm_rescan_gidlist(ib_guid_t *ioc_guid); 135 static void ibdm_notify_newgid_iocs(ibdm_dp_gidinfo_t *); 136 static void ibdm_saa_event_taskq(void *); 137 static void ibdm_free_saa_event_arg(ibdm_saa_event_arg_t *); 138 static void ibdm_get_next_port(ibdm_hca_list_t **, 139 ibdm_port_attr_t **, int); 140 static void ibdm_add_to_gl_gid(ibdm_dp_gidinfo_t *, 141 ibdm_dp_gidinfo_t *); 142 static void ibdm_addto_glhcalist(ibdm_dp_gidinfo_t *, 143 ibdm_hca_list_t *); 144 static void ibdm_delete_glhca_list(ibdm_dp_gidinfo_t *); 145 static void ibdm_saa_handle_new_gid(void *); 146 static void ibdm_reset_all_dgids(ibmf_saa_handle_t); 147 static void ibdm_reset_gidinfo(ibdm_dp_gidinfo_t *); 148 static void ibdm_delete_gidinfo(ibdm_dp_gidinfo_t *); 149 static void ibdm_fill_srv_attr_mod(ib_mad_hdr_t *, ibdm_timeout_cb_args_t *); 150 static void ibdm_bump_transactionID(ibdm_dp_gidinfo_t *); 151 static ibdm_ioc_info_t *ibdm_handle_prev_iou(); 152 static int ibdm_serv_cmp(ibdm_srvents_info_t *, ibdm_srvents_info_t *, 153 int); 154 static ibdm_ioc_info_t *ibdm_get_ioc_info_with_gid(ib_guid_t, 155 ibdm_dp_gidinfo_t **); 156 157 int ibdm_dft_timeout = IBDM_DFT_TIMEOUT; 158 int ibdm_dft_retry_cnt = IBDM_DFT_NRETRIES; 159 #ifdef DEBUG 160 int ibdm_ignore_saa_event = 0; 161 #endif 162 int ibdm_enumerate_iocs = 0; 163 164 /* Modload support */ 165 static struct modlmisc ibdm_modlmisc = { 166 &mod_miscops, 167 "InfiniBand Device Manager" 168 }; 169 170 struct modlinkage ibdm_modlinkage = { 171 MODREV_1, 172 (void *)&ibdm_modlmisc, 173 NULL 174 }; 175 176 static ibt_clnt_modinfo_t ibdm_ibt_modinfo = { 177 IBTI_V_CURR, 178 IBT_DM, 179 ibdm_event_hdlr, 180 NULL, 181 "ibdm" 182 }; 183 184 /* Global variables */ 185 ibdm_t ibdm; 186 int ibdm_taskq_enable = IBDM_ENABLE_TASKQ_HANDLING; 187 char *ibdm_string = "ibdm"; 188 189 _NOTE(SCHEME_PROTECTS_DATA("Serialized access by cv", 190 ibdm.ibdm_dp_gidlist_head)) 191 192 /* 193 * _init 194 * Loadable module init, called before any other module. 195 * Initialize mutex 196 * Register with IBTF 197 */ 198 int 199 _init(void) 200 { 201 int err; 202 203 IBTF_DPRINTF_L4("ibdm", "\t_init: addr of ibdm %p", &ibdm); 204 205 if ((err = ibdm_init()) != IBDM_SUCCESS) { 206 IBTF_DPRINTF_L2("ibdm", "_init: ibdm_init failed 0x%x", err); 207 (void) ibdm_fini(); 208 return (DDI_FAILURE); 209 } 210 211 if ((err = mod_install(&ibdm_modlinkage)) != 0) { 212 IBTF_DPRINTF_L2("ibdm", "_init: mod_install failed 0x%x", err); 213 (void) ibdm_fini(); 214 } 215 return (err); 216 } 217 218 219 int 220 _fini(void) 221 { 222 int err; 223 224 if ((err = ibdm_fini()) != IBDM_SUCCESS) { 225 IBTF_DPRINTF_L2("ibdm", "_fini: ibdm_fini failed 0x%x", err); 226 (void) ibdm_init(); 227 return (EBUSY); 228 } 229 230 if ((err = mod_remove(&ibdm_modlinkage)) != 0) { 231 IBTF_DPRINTF_L2("ibdm", "_fini: mod_remove failed 0x%x", err); 232 (void) ibdm_init(); 233 } 234 return (err); 235 } 236 237 238 int 239 _info(struct modinfo *modinfop) 240 { 241 return (mod_info(&ibdm_modlinkage, modinfop)); 242 } 243 244 245 /* 246 * ibdm_init(): 247 * Register with IBTF 248 * Allocate memory for the HCAs 249 * Allocate minor-nodes for the HCAs 250 */ 251 static int 252 ibdm_init(void) 253 { 254 int i, hca_count; 255 ib_guid_t *hca_guids; 256 ibt_status_t status; 257 258 IBTF_DPRINTF_L4("ibdm", "\tibdm_init:"); 259 if (!(ibdm.ibdm_state & IBDM_LOCKS_ALLOCED)) { 260 mutex_init(&ibdm.ibdm_mutex, NULL, MUTEX_DEFAULT, NULL); 261 mutex_init(&ibdm.ibdm_hl_mutex, NULL, MUTEX_DEFAULT, NULL); 262 mutex_init(&ibdm.ibdm_ibnex_mutex, NULL, MUTEX_DEFAULT, NULL); 263 cv_init(&ibdm.ibdm_port_settle_cv, NULL, CV_DRIVER, NULL); 264 mutex_enter(&ibdm.ibdm_mutex); 265 ibdm.ibdm_state |= IBDM_LOCKS_ALLOCED; 266 } 267 268 if (!(ibdm.ibdm_state & IBDM_IBT_ATTACHED)) { 269 if ((status = ibt_attach(&ibdm_ibt_modinfo, NULL, NULL, 270 (void *)&ibdm.ibdm_ibt_clnt_hdl)) != IBT_SUCCESS) { 271 IBTF_DPRINTF_L2("ibdm", "ibdm_init: ibt_attach " 272 "failed %x", status); 273 mutex_exit(&ibdm.ibdm_mutex); 274 return (IBDM_FAILURE); 275 } 276 277 ibdm.ibdm_state |= IBDM_IBT_ATTACHED; 278 mutex_exit(&ibdm.ibdm_mutex); 279 } 280 281 282 if (!(ibdm.ibdm_state & IBDM_HCA_ATTACHED)) { 283 hca_count = ibt_get_hca_list(&hca_guids); 284 IBTF_DPRINTF_L4("ibdm", "ibdm_init: num_hcas = %d", hca_count); 285 for (i = 0; i < hca_count; i++) 286 (void) ibdm_handle_hca_attach(hca_guids[i]); 287 if (hca_count) 288 ibt_free_hca_list(hca_guids, hca_count); 289 290 mutex_enter(&ibdm.ibdm_mutex); 291 ibdm.ibdm_state |= IBDM_HCA_ATTACHED; 292 mutex_exit(&ibdm.ibdm_mutex); 293 } 294 295 if (!(ibdm.ibdm_state & IBDM_CVS_ALLOCED)) { 296 cv_init(&ibdm.ibdm_probe_cv, NULL, CV_DRIVER, NULL); 297 cv_init(&ibdm.ibdm_busy_cv, NULL, CV_DRIVER, NULL); 298 mutex_enter(&ibdm.ibdm_mutex); 299 ibdm.ibdm_state |= IBDM_CVS_ALLOCED; 300 mutex_exit(&ibdm.ibdm_mutex); 301 } 302 return (IBDM_SUCCESS); 303 } 304 305 306 static int 307 ibdm_free_iou_info(ibdm_dp_gidinfo_t *gid_info, ibdm_iou_info_t **ioup) 308 { 309 int ii, k, niocs; 310 size_t size; 311 ibdm_gid_t *delete, *head; 312 timeout_id_t timeout_id; 313 ibdm_ioc_info_t *ioc; 314 ibdm_iou_info_t *gl_iou = *ioup; 315 316 ASSERT(mutex_owned(&gid_info->gl_mutex)); 317 if (gl_iou == NULL) { 318 IBTF_DPRINTF_L4("ibdm", "\tibdm_free_iou_info: No IOU"); 319 return (0); 320 } 321 322 niocs = gl_iou->iou_info.iou_num_ctrl_slots; 323 IBTF_DPRINTF_L4("ibdm", "\tfree_iou_info: gid_info = %p, niocs %d", 324 gid_info, niocs); 325 326 for (ii = 0; ii < niocs; ii++) { 327 ioc = (ibdm_ioc_info_t *)&gl_iou->iou_ioc_info[ii]; 328 329 /* handle the case where an ioc_timeout_id is scheduled */ 330 if (ioc->ioc_timeout_id) { 331 timeout_id = ioc->ioc_timeout_id; 332 ioc->ioc_timeout_id = 0; 333 mutex_exit(&gid_info->gl_mutex); 334 IBTF_DPRINTF_L5("ibdm", "free_iou_info: " 335 "ioc_timeout_id = 0x%x", timeout_id); 336 if (untimeout(timeout_id) == -1) { 337 IBTF_DPRINTF_L2("ibdm", "free_iou_info: " 338 "untimeout ioc_timeout_id failed"); 339 mutex_enter(&gid_info->gl_mutex); 340 return (-1); 341 } 342 mutex_enter(&gid_info->gl_mutex); 343 } 344 345 /* handle the case where an ioc_dc_timeout_id is scheduled */ 346 if (ioc->ioc_dc_timeout_id) { 347 timeout_id = ioc->ioc_dc_timeout_id; 348 ioc->ioc_dc_timeout_id = 0; 349 mutex_exit(&gid_info->gl_mutex); 350 IBTF_DPRINTF_L5("ibdm", "free_iou_info: " 351 "ioc_dc_timeout_id = 0x%x", timeout_id); 352 if (untimeout(timeout_id) == -1) { 353 IBTF_DPRINTF_L2("ibdm", "free_iou_info: " 354 "untimeout ioc_dc_timeout_id failed"); 355 mutex_enter(&gid_info->gl_mutex); 356 return (-1); 357 } 358 mutex_enter(&gid_info->gl_mutex); 359 } 360 361 /* handle the case where serv[k].se_timeout_id is scheduled */ 362 for (k = 0; k < ioc->ioc_profile.ioc_service_entries; k++) { 363 if (ioc->ioc_serv[k].se_timeout_id) { 364 timeout_id = ioc->ioc_serv[k].se_timeout_id; 365 ioc->ioc_serv[k].se_timeout_id = 0; 366 mutex_exit(&gid_info->gl_mutex); 367 IBTF_DPRINTF_L5("ibdm", "free_iou_info: " 368 "ioc->ioc_serv[%d].se_timeout_id = 0x%x", 369 k, timeout_id); 370 if (untimeout(timeout_id) == -1) { 371 IBTF_DPRINTF_L2("ibdm", "free_iou_info:" 372 " untimeout se_timeout_id failed"); 373 mutex_enter(&gid_info->gl_mutex); 374 return (-1); 375 } 376 mutex_enter(&gid_info->gl_mutex); 377 } 378 } 379 380 /* delete GID list in IOC */ 381 head = ioc->ioc_gid_list; 382 while (head) { 383 IBTF_DPRINTF_L4("ibdm", "\tibdm_free_iou_info: " 384 "Deleting gid_list struct %p", head); 385 delete = head; 386 head = head->gid_next; 387 kmem_free(delete, sizeof (ibdm_gid_t)); 388 } 389 ioc->ioc_gid_list = NULL; 390 391 /* delete ioc_serv */ 392 size = ioc->ioc_profile.ioc_service_entries * 393 sizeof (ibdm_srvents_info_t); 394 if (ioc->ioc_serv && size) { 395 kmem_free(ioc->ioc_serv, size); 396 ioc->ioc_serv = NULL; 397 } 398 } 399 /* 400 * Clear the IBDM_CISCO_PROBE_DONE flag to get the IO Unit information 401 * via the switch during the probe process. 402 */ 403 gid_info->gl_flag &= ~IBDM_CISCO_PROBE_DONE; 404 405 IBTF_DPRINTF_L4("ibdm", "\tibdm_free_iou_info: deleting IOU & IOC"); 406 size = sizeof (ibdm_iou_info_t) + niocs * sizeof (ibdm_ioc_info_t); 407 kmem_free(gl_iou, size); 408 *ioup = NULL; 409 return (0); 410 } 411 412 413 /* 414 * ibdm_fini(): 415 * Un-register with IBTF 416 * De allocate memory for the GID info 417 */ 418 static int 419 ibdm_fini() 420 { 421 int ii; 422 ibdm_hca_list_t *hca_list, *temp; 423 ibdm_dp_gidinfo_t *gid_info, *tmp; 424 ibdm_gid_t *head, *delete; 425 426 IBTF_DPRINTF_L4("ibdm", "\tibdm_fini"); 427 428 mutex_enter(&ibdm.ibdm_hl_mutex); 429 if (ibdm.ibdm_state & IBDM_IBT_ATTACHED) { 430 if (ibt_detach(ibdm.ibdm_ibt_clnt_hdl) != IBT_SUCCESS) { 431 IBTF_DPRINTF_L2("ibdm", "\t_fini: ibt_detach failed"); 432 mutex_exit(&ibdm.ibdm_hl_mutex); 433 return (IBDM_FAILURE); 434 } 435 ibdm.ibdm_state &= ~IBDM_IBT_ATTACHED; 436 ibdm.ibdm_ibt_clnt_hdl = NULL; 437 } 438 439 hca_list = ibdm.ibdm_hca_list_head; 440 IBTF_DPRINTF_L4("ibdm", "\tibdm_fini: nhcas %d", ibdm.ibdm_hca_count); 441 for (ii = 0; ii < ibdm.ibdm_hca_count; ii++) { 442 temp = hca_list; 443 hca_list = hca_list->hl_next; 444 IBTF_DPRINTF_L4("ibdm", "\tibdm_fini: hca %p", temp); 445 if (ibdm_uninit_hca(temp) != IBDM_SUCCESS) { 446 IBTF_DPRINTF_L2("ibdm", "\tibdm_fini: " 447 "uninit_hca %p failed", temp); 448 mutex_exit(&ibdm.ibdm_hl_mutex); 449 return (IBDM_FAILURE); 450 } 451 } 452 mutex_exit(&ibdm.ibdm_hl_mutex); 453 454 mutex_enter(&ibdm.ibdm_mutex); 455 if (ibdm.ibdm_state & IBDM_HCA_ATTACHED) 456 ibdm.ibdm_state &= ~IBDM_HCA_ATTACHED; 457 458 gid_info = ibdm.ibdm_dp_gidlist_head; 459 while (gid_info) { 460 mutex_enter(&gid_info->gl_mutex); 461 (void) ibdm_free_iou_info(gid_info, &gid_info->gl_iou); 462 mutex_exit(&gid_info->gl_mutex); 463 ibdm_delete_glhca_list(gid_info); 464 465 tmp = gid_info; 466 gid_info = gid_info->gl_next; 467 mutex_destroy(&tmp->gl_mutex); 468 head = tmp->gl_gid; 469 while (head) { 470 IBTF_DPRINTF_L4("ibdm", 471 "\tibdm_fini: Deleting gid structs"); 472 delete = head; 473 head = head->gid_next; 474 kmem_free(delete, sizeof (ibdm_gid_t)); 475 } 476 kmem_free(tmp, sizeof (ibdm_dp_gidinfo_t)); 477 } 478 mutex_exit(&ibdm.ibdm_mutex); 479 480 if (ibdm.ibdm_state & IBDM_LOCKS_ALLOCED) { 481 ibdm.ibdm_state &= ~IBDM_LOCKS_ALLOCED; 482 mutex_destroy(&ibdm.ibdm_mutex); 483 mutex_destroy(&ibdm.ibdm_hl_mutex); 484 mutex_destroy(&ibdm.ibdm_ibnex_mutex); 485 cv_destroy(&ibdm.ibdm_port_settle_cv); 486 } 487 if (ibdm.ibdm_state & IBDM_CVS_ALLOCED) { 488 ibdm.ibdm_state &= ~IBDM_CVS_ALLOCED; 489 cv_destroy(&ibdm.ibdm_probe_cv); 490 cv_destroy(&ibdm.ibdm_busy_cv); 491 } 492 return (IBDM_SUCCESS); 493 } 494 495 496 /* 497 * ibdm_event_hdlr() 498 * 499 * IBDM registers this asynchronous event handler at the time of 500 * ibt_attach. IBDM support the following async events. For other 501 * event, simply returns success. 502 * IBT_HCA_ATTACH_EVENT: 503 * Retrieves the information about all the port that are 504 * present on this HCA, allocates the port attributes 505 * structure and calls IB nexus callback routine with 506 * the port attributes structure as an input argument. 507 * IBT_HCA_DETACH_EVENT: 508 * Retrieves the information about all the ports that are 509 * present on this HCA and calls IB nexus callback with 510 * port guid as an argument 511 * IBT_EVENT_PORT_UP: 512 * Register with IBMF and SA access 513 * Setup IBMF receive callback routine 514 * IBT_EVENT_PORT_DOWN: 515 * Un-Register with IBMF and SA access 516 * Teardown IBMF receive callback routine 517 */ 518 /*ARGSUSED*/ 519 static void 520 ibdm_event_hdlr(void *clnt_hdl, 521 ibt_hca_hdl_t hca_hdl, ibt_async_code_t code, ibt_async_event_t *event) 522 { 523 ibdm_hca_list_t *hca_list; 524 ibdm_port_attr_t *port; 525 ibmf_saa_handle_t port_sa_hdl; 526 527 IBTF_DPRINTF_L4("ibdm", "\tevent_hdlr: async code 0x%x", code); 528 529 switch (code) { 530 case IBT_HCA_ATTACH_EVENT: /* New HCA registered with IBTF */ 531 ibdm_handle_hca_attach(event->ev_hca_guid); 532 break; 533 534 case IBT_HCA_DETACH_EVENT: /* HCA unregistered with IBTF */ 535 ibdm_handle_hca_detach(event->ev_hca_guid); 536 mutex_enter(&ibdm.ibdm_ibnex_mutex); 537 if (ibdm.ibdm_ibnex_callback != NULL) { 538 (*ibdm.ibdm_ibnex_callback)((void *) 539 &event->ev_hca_guid, IBDM_EVENT_HCA_REMOVED); 540 } 541 mutex_exit(&ibdm.ibdm_ibnex_mutex); 542 break; 543 544 case IBT_EVENT_PORT_UP: 545 IBTF_DPRINTF_L4("ibdm", "\tevent_hdlr: PORT_UP"); 546 mutex_enter(&ibdm.ibdm_hl_mutex); 547 port = ibdm_get_port_attr(event, &hca_list); 548 if (port == NULL) { 549 IBTF_DPRINTF_L2("ibdm", 550 "\tevent_hdlr: HCA not present"); 551 mutex_exit(&ibdm.ibdm_hl_mutex); 552 break; 553 } 554 ibdm_initialize_port(port); 555 hca_list->hl_nports_active++; 556 cv_broadcast(&ibdm.ibdm_port_settle_cv); 557 mutex_exit(&ibdm.ibdm_hl_mutex); 558 559 /* Inform IB nexus driver */ 560 mutex_enter(&ibdm.ibdm_ibnex_mutex); 561 if (ibdm.ibdm_ibnex_callback != NULL) { 562 (*ibdm.ibdm_ibnex_callback)((void *) 563 &event->ev_hca_guid, IBDM_EVENT_PORT_UP); 564 } 565 mutex_exit(&ibdm.ibdm_ibnex_mutex); 566 break; 567 568 case IBT_ERROR_PORT_DOWN: 569 IBTF_DPRINTF_L4("ibdm", "\tevent_hdlr: PORT_DOWN"); 570 mutex_enter(&ibdm.ibdm_hl_mutex); 571 port = ibdm_get_port_attr(event, &hca_list); 572 if (port == NULL) { 573 IBTF_DPRINTF_L2("ibdm", 574 "\tevent_hdlr: HCA not present"); 575 mutex_exit(&ibdm.ibdm_hl_mutex); 576 break; 577 } 578 hca_list->hl_nports_active--; 579 port_sa_hdl = port->pa_sa_hdl; 580 (void) ibdm_fini_port(port); 581 port->pa_state = IBT_PORT_DOWN; 582 cv_broadcast(&ibdm.ibdm_port_settle_cv); 583 mutex_exit(&ibdm.ibdm_hl_mutex); 584 ibdm_reset_all_dgids(port_sa_hdl); 585 break; 586 587 case IBT_PORT_CHANGE_EVENT: 588 IBTF_DPRINTF_L4("ibdm", "\tevent_hdlr: PORT_CHANGE"); 589 if (event->ev_port_flags & IBT_PORT_CHANGE_PKEY) 590 ibdm_handle_port_change_event(event); 591 break; 592 593 default: /* Ignore all other events/errors */ 594 break; 595 } 596 } 597 598 static void 599 ibdm_handle_port_change_event(ibt_async_event_t *event) 600 { 601 ibdm_port_attr_t *port; 602 ibdm_hca_list_t *hca_list; 603 604 IBTF_DPRINTF_L2("ibdm", "\tibdm_handle_port_change_event:" 605 " HCA guid %llx", event->ev_hca_guid); 606 mutex_enter(&ibdm.ibdm_hl_mutex); 607 port = ibdm_get_port_attr(event, &hca_list); 608 if (port == NULL) { 609 IBTF_DPRINTF_L2("ibdm", "\tevent_hdlr: HCA not present"); 610 mutex_exit(&ibdm.ibdm_hl_mutex); 611 return; 612 } 613 ibdm_update_port_pkeys(port); 614 cv_broadcast(&ibdm.ibdm_port_settle_cv); 615 mutex_exit(&ibdm.ibdm_hl_mutex); 616 617 /* Inform IB nexus driver */ 618 mutex_enter(&ibdm.ibdm_ibnex_mutex); 619 if (ibdm.ibdm_ibnex_callback != NULL) { 620 (*ibdm.ibdm_ibnex_callback)((void *) 621 &event->ev_hca_guid, IBDM_EVENT_PORT_PKEY_CHANGE); 622 } 623 mutex_exit(&ibdm.ibdm_ibnex_mutex); 624 } 625 626 /* 627 * ibdm_update_port_pkeys() 628 * Update the pkey table 629 * Update the port attributes 630 */ 631 static void 632 ibdm_update_port_pkeys(ibdm_port_attr_t *port) 633 { 634 uint_t nports, size; 635 uint_t pkey_idx, opkey_idx; 636 uint16_t npkeys; 637 ibt_hca_portinfo_t *pinfop; 638 ib_pkey_t pkey; 639 ibdm_pkey_tbl_t *pkey_tbl; 640 ibdm_port_attr_t newport; 641 642 IBTF_DPRINTF_L4("ibdm", "\tupdate_port_pkeys:"); 643 ASSERT(MUTEX_HELD(&ibdm.ibdm_hl_mutex)); 644 645 /* Check whether the port is active */ 646 if (ibt_get_port_state(port->pa_hca_hdl, port->pa_port_num, NULL, 647 NULL) != IBT_SUCCESS) 648 return; 649 650 if (ibt_query_hca_ports(port->pa_hca_hdl, port->pa_port_num, 651 &pinfop, &nports, &size) != IBT_SUCCESS) { 652 /* This should not occur */ 653 port->pa_npkeys = 0; 654 port->pa_pkey_tbl = NULL; 655 return; 656 } 657 658 npkeys = pinfop->p_pkey_tbl_sz; 659 pkey_tbl = kmem_zalloc(npkeys * sizeof (ibdm_pkey_tbl_t), KM_SLEEP); 660 newport.pa_pkey_tbl = pkey_tbl; 661 newport.pa_ibmf_hdl = port->pa_ibmf_hdl; 662 663 for (pkey_idx = 0; pkey_idx < npkeys; pkey_idx++) { 664 pkey = pkey_tbl[pkey_idx].pt_pkey = 665 pinfop->p_pkey_tbl[pkey_idx]; 666 /* 667 * Is this pkey present in the current table ? 668 */ 669 for (opkey_idx = 0; opkey_idx < port->pa_npkeys; opkey_idx++) { 670 if (pkey == port->pa_pkey_tbl[opkey_idx].pt_pkey) { 671 pkey_tbl[pkey_idx].pt_qp_hdl = 672 port->pa_pkey_tbl[opkey_idx].pt_qp_hdl; 673 port->pa_pkey_tbl[opkey_idx].pt_qp_hdl = NULL; 674 break; 675 } 676 } 677 678 if (opkey_idx == port->pa_npkeys) { 679 pkey = pkey_tbl[pkey_idx].pt_pkey; 680 if (IBDM_INVALID_PKEY(pkey)) { 681 pkey_tbl[pkey_idx].pt_qp_hdl = NULL; 682 continue; 683 } 684 ibdm_port_attr_ibmf_init(&newport, pkey, pkey_idx); 685 } 686 } 687 688 for (opkey_idx = 0; opkey_idx < port->pa_npkeys; opkey_idx++) { 689 if (port->pa_pkey_tbl[opkey_idx].pt_qp_hdl != NULL) { 690 if (ibdm_port_attr_ibmf_fini(port, opkey_idx) != 691 IBDM_SUCCESS) { 692 IBTF_DPRINTF_L2("ibdm", "\tupdate_port_pkeys: " 693 "ibdm_port_attr_ibmf_fini failed for " 694 "port pkey 0x%x", 695 port->pa_pkey_tbl[opkey_idx].pt_pkey); 696 } 697 } 698 } 699 700 if (port->pa_pkey_tbl != NULL) { 701 kmem_free(port->pa_pkey_tbl, 702 port->pa_npkeys * sizeof (ibdm_pkey_tbl_t)); 703 } 704 705 port->pa_npkeys = npkeys; 706 port->pa_pkey_tbl = pkey_tbl; 707 port->pa_sn_prefix = pinfop->p_sgid_tbl[0].gid_prefix; 708 port->pa_state = pinfop->p_linkstate; 709 ibt_free_portinfo(pinfop, size); 710 } 711 712 /* 713 * ibdm_initialize_port() 714 * Register with IBMF 715 * Register with SA access 716 * Register a receive callback routine with IBMF. IBMF invokes 717 * this routine whenever a MAD arrives at this port. 718 * Update the port attributes 719 */ 720 static void 721 ibdm_initialize_port(ibdm_port_attr_t *port) 722 { 723 int ii; 724 uint_t nports, size; 725 uint_t pkey_idx; 726 ib_pkey_t pkey; 727 ibt_hca_portinfo_t *pinfop; 728 ibmf_register_info_t ibmf_reg; 729 ibmf_saa_subnet_event_args_t event_args; 730 731 IBTF_DPRINTF_L4("ibdm", "\tinitialize_port:"); 732 ASSERT(MUTEX_HELD(&ibdm.ibdm_hl_mutex)); 733 734 /* Check whether the port is active */ 735 if (ibt_get_port_state(port->pa_hca_hdl, port->pa_port_num, NULL, 736 NULL) != IBT_SUCCESS) 737 return; 738 739 if (port->pa_sa_hdl != NULL || port->pa_pkey_tbl != NULL) 740 return; 741 742 if (ibt_query_hca_ports(port->pa_hca_hdl, port->pa_port_num, 743 &pinfop, &nports, &size) != IBT_SUCCESS) { 744 /* This should not occur */ 745 port->pa_npkeys = 0; 746 port->pa_pkey_tbl = NULL; 747 return; 748 } 749 port->pa_sn_prefix = pinfop->p_sgid_tbl[0].gid_prefix; 750 751 port->pa_state = pinfop->p_linkstate; 752 port->pa_npkeys = pinfop->p_pkey_tbl_sz; 753 port->pa_pkey_tbl = (ibdm_pkey_tbl_t *)kmem_zalloc( 754 port->pa_npkeys * sizeof (ibdm_pkey_tbl_t), KM_SLEEP); 755 756 for (pkey_idx = 0; pkey_idx < port->pa_npkeys; pkey_idx++) 757 port->pa_pkey_tbl[pkey_idx].pt_pkey = 758 pinfop->p_pkey_tbl[pkey_idx]; 759 760 ibt_free_portinfo(pinfop, size); 761 762 if (ibdm_enumerate_iocs) { 763 event_args.is_event_callback = ibdm_saa_event_cb; 764 event_args.is_event_callback_arg = port; 765 if (ibmf_sa_session_open(port->pa_port_guid, 0, &event_args, 766 IBMF_VERSION, 0, &port->pa_sa_hdl) != IBMF_SUCCESS) { 767 IBTF_DPRINTF_L2("ibdm", "\tinitialize_port: " 768 "sa access registration failed"); 769 (void) ibdm_fini_port(port); 770 return; 771 } 772 773 ibmf_reg.ir_ci_guid = port->pa_hca_guid; 774 ibmf_reg.ir_port_num = port->pa_port_num; 775 ibmf_reg.ir_client_class = DEV_MGT_MANAGER; 776 777 if (ibmf_register(&ibmf_reg, IBMF_VERSION, 0, NULL, NULL, 778 &port->pa_ibmf_hdl, &port->pa_ibmf_caps) != IBMF_SUCCESS) { 779 IBTF_DPRINTF_L2("ibdm", "\tinitialize_port: " 780 "IBMF registration failed"); 781 (void) ibdm_fini_port(port); 782 return; 783 } 784 785 if (ibmf_setup_async_cb(port->pa_ibmf_hdl, 786 IBMF_QP_HANDLE_DEFAULT, 787 ibdm_ibmf_recv_cb, 0, 0) != IBMF_SUCCESS) { 788 IBTF_DPRINTF_L2("ibdm", "\tinitialize_port: " 789 "IBMF setup recv cb failed"); 790 (void) ibdm_fini_port(port); 791 return; 792 } 793 } else { 794 port->pa_sa_hdl = NULL; 795 port->pa_ibmf_hdl = NULL; 796 } 797 798 for (ii = 0; ii < port->pa_npkeys; ii++) { 799 pkey = port->pa_pkey_tbl[ii].pt_pkey; 800 if (IBDM_INVALID_PKEY(pkey)) { 801 port->pa_pkey_tbl[ii].pt_qp_hdl = NULL; 802 continue; 803 } 804 ibdm_port_attr_ibmf_init(port, pkey, ii); 805 } 806 } 807 808 809 /* 810 * ibdm_port_attr_ibmf_init: 811 * With IBMF - Alloc QP Handle and Setup Async callback 812 */ 813 static void 814 ibdm_port_attr_ibmf_init(ibdm_port_attr_t *port, ib_pkey_t pkey, int ii) 815 { 816 int ret; 817 818 if (ibdm_enumerate_iocs == 0) { 819 port->pa_pkey_tbl[ii].pt_qp_hdl = NULL; 820 return; 821 } 822 823 if ((ret = ibmf_alloc_qp(port->pa_ibmf_hdl, pkey, IB_GSI_QKEY, 824 IBMF_ALT_QP_MAD_NO_RMPP, &port->pa_pkey_tbl[ii].pt_qp_hdl)) != 825 IBMF_SUCCESS) { 826 IBTF_DPRINTF_L2("ibdm", "\tport_attr_ibmf_init: " 827 "IBMF failed to alloc qp %d", ret); 828 port->pa_pkey_tbl[ii].pt_qp_hdl = NULL; 829 return; 830 } 831 832 IBTF_DPRINTF_L4("ibdm", "\tport_attr_ibmf_init: QP handle is %p", 833 port->pa_ibmf_hdl); 834 835 if ((ret = ibmf_setup_async_cb(port->pa_ibmf_hdl, 836 port->pa_pkey_tbl[ii].pt_qp_hdl, ibdm_ibmf_recv_cb, 0, 0)) != 837 IBMF_SUCCESS) { 838 IBTF_DPRINTF_L2("ibdm", "\tport_attr_ibmf_init: " 839 "IBMF setup recv cb failed %d", ret); 840 (void) ibmf_free_qp(port->pa_ibmf_hdl, 841 &port->pa_pkey_tbl[ii].pt_qp_hdl, 0); 842 port->pa_pkey_tbl[ii].pt_qp_hdl = NULL; 843 } 844 } 845 846 847 /* 848 * ibdm_get_port_attr() 849 * Get port attributes from HCA guid and port number 850 * Return pointer to ibdm_port_attr_t on Success 851 * and NULL on failure 852 */ 853 static ibdm_port_attr_t * 854 ibdm_get_port_attr(ibt_async_event_t *event, ibdm_hca_list_t **retval) 855 { 856 ibdm_hca_list_t *hca_list; 857 ibdm_port_attr_t *port_attr; 858 int ii; 859 860 IBTF_DPRINTF_L4("ibdm", "\tget_port_attr: port# %d", event->ev_port); 861 ASSERT(MUTEX_HELD(&ibdm.ibdm_hl_mutex)); 862 hca_list = ibdm.ibdm_hca_list_head; 863 while (hca_list) { 864 if (hca_list->hl_hca_guid == event->ev_hca_guid) { 865 for (ii = 0; ii < hca_list->hl_nports; ii++) { 866 port_attr = &hca_list->hl_port_attr[ii]; 867 if (port_attr->pa_port_num == event->ev_port) { 868 *retval = hca_list; 869 return (port_attr); 870 } 871 } 872 } 873 hca_list = hca_list->hl_next; 874 } 875 return (NULL); 876 } 877 878 879 /* 880 * ibdm_update_port_attr() 881 * Update the port attributes 882 */ 883 static void 884 ibdm_update_port_attr(ibdm_port_attr_t *port) 885 { 886 uint_t nports, size; 887 uint_t pkey_idx; 888 ibt_hca_portinfo_t *portinfop; 889 890 IBTF_DPRINTF_L4("ibdm", "\tupdate_port_attr: Begin"); 891 if (ibt_query_hca_ports(port->pa_hca_hdl, 892 port->pa_port_num, &portinfop, &nports, &size) != IBT_SUCCESS) { 893 /* This should not occur */ 894 port->pa_npkeys = 0; 895 port->pa_pkey_tbl = NULL; 896 return; 897 } 898 port->pa_sn_prefix = portinfop->p_sgid_tbl[0].gid_prefix; 899 900 port->pa_state = portinfop->p_linkstate; 901 902 /* 903 * PKey information in portinfo valid only if port is 904 * ACTIVE. Bail out if not. 905 */ 906 if (port->pa_state != IBT_PORT_ACTIVE) { 907 port->pa_npkeys = 0; 908 port->pa_pkey_tbl = NULL; 909 ibt_free_portinfo(portinfop, size); 910 return; 911 } 912 913 port->pa_npkeys = portinfop->p_pkey_tbl_sz; 914 port->pa_pkey_tbl = (ibdm_pkey_tbl_t *)kmem_zalloc( 915 port->pa_npkeys * sizeof (ibdm_pkey_tbl_t), KM_SLEEP); 916 917 for (pkey_idx = 0; pkey_idx < port->pa_npkeys; pkey_idx++) { 918 port->pa_pkey_tbl[pkey_idx].pt_pkey = 919 portinfop->p_pkey_tbl[pkey_idx]; 920 } 921 ibt_free_portinfo(portinfop, size); 922 } 923 924 925 /* 926 * ibdm_handle_hca_attach() 927 */ 928 static void 929 ibdm_handle_hca_attach(ib_guid_t hca_guid) 930 { 931 uint_t size; 932 uint_t ii, nports; 933 ibt_status_t status; 934 ibt_hca_hdl_t hca_hdl; 935 ibt_hca_attr_t *hca_attr; 936 ibdm_hca_list_t *hca_list, *temp; 937 ibdm_port_attr_t *port_attr; 938 ibt_hca_portinfo_t *portinfop; 939 940 IBTF_DPRINTF_L4("ibdm", 941 "\thandle_hca_attach: hca_guid = 0x%llX", hca_guid); 942 943 /* open the HCA first */ 944 if ((status = ibt_open_hca(ibdm.ibdm_ibt_clnt_hdl, hca_guid, 945 &hca_hdl)) != IBT_SUCCESS) { 946 IBTF_DPRINTF_L2("ibdm", "\thandle_hca_attach: " 947 "open_hca failed, status 0x%x", status); 948 return; 949 } 950 951 hca_attr = (ibt_hca_attr_t *) 952 kmem_alloc(sizeof (ibt_hca_attr_t), KM_SLEEP); 953 /* ibt_query_hca always returns IBT_SUCCESS */ 954 (void) ibt_query_hca(hca_hdl, hca_attr); 955 956 IBTF_DPRINTF_L4("ibdm", "\tvid: 0x%x, pid: 0x%x, ver: 0x%x," 957 " #ports: %d", hca_attr->hca_vendor_id, hca_attr->hca_device_id, 958 hca_attr->hca_version_id, hca_attr->hca_nports); 959 960 if ((status = ibt_query_hca_ports(hca_hdl, 0, &portinfop, &nports, 961 &size)) != IBT_SUCCESS) { 962 IBTF_DPRINTF_L2("ibdm", "\thandle_hca_attach: " 963 "ibt_query_hca_ports failed, status 0x%x", status); 964 kmem_free(hca_attr, sizeof (ibt_hca_attr_t)); 965 (void) ibt_close_hca(hca_hdl); 966 return; 967 } 968 hca_list = (ibdm_hca_list_t *) 969 kmem_zalloc((sizeof (ibdm_hca_list_t)), KM_SLEEP); 970 hca_list->hl_port_attr = (ibdm_port_attr_t *)kmem_zalloc( 971 (sizeof (ibdm_port_attr_t) * hca_attr->hca_nports), KM_SLEEP); 972 hca_list->hl_hca_guid = hca_attr->hca_node_guid; 973 hca_list->hl_nports = hca_attr->hca_nports; 974 hca_list->hl_attach_time = ddi_get_time(); 975 hca_list->hl_hca_hdl = hca_hdl; 976 977 /* 978 * Init a dummy port attribute for the HCA node 979 * This is for Per-HCA Node. Initialize port_attr : 980 * hca_guid & port_guid -> hca_guid 981 * npkeys, pkey_tbl is NULL 982 * port_num, sn_prefix is 0 983 * vendorid, product_id, dev_version from HCA 984 * pa_state is IBT_PORT_ACTIVE 985 */ 986 hca_list->hl_hca_port_attr = (ibdm_port_attr_t *)kmem_zalloc( 987 sizeof (ibdm_port_attr_t), KM_SLEEP); 988 port_attr = hca_list->hl_hca_port_attr; 989 port_attr->pa_vendorid = hca_attr->hca_vendor_id; 990 port_attr->pa_productid = hca_attr->hca_device_id; 991 port_attr->pa_dev_version = hca_attr->hca_version_id; 992 port_attr->pa_hca_guid = hca_attr->hca_node_guid; 993 port_attr->pa_hca_hdl = hca_list->hl_hca_hdl; 994 port_attr->pa_port_guid = hca_attr->hca_node_guid; 995 port_attr->pa_state = IBT_PORT_ACTIVE; 996 997 998 for (ii = 0; ii < nports; ii++) { 999 port_attr = &hca_list->hl_port_attr[ii]; 1000 port_attr->pa_vendorid = hca_attr->hca_vendor_id; 1001 port_attr->pa_productid = hca_attr->hca_device_id; 1002 port_attr->pa_dev_version = hca_attr->hca_version_id; 1003 port_attr->pa_hca_guid = hca_attr->hca_node_guid; 1004 port_attr->pa_hca_hdl = hca_list->hl_hca_hdl; 1005 port_attr->pa_port_guid = portinfop[ii].p_sgid_tbl->gid_guid; 1006 port_attr->pa_sn_prefix = portinfop[ii].p_sgid_tbl->gid_prefix; 1007 port_attr->pa_port_num = portinfop[ii].p_port_num; 1008 port_attr->pa_state = portinfop[ii].p_linkstate; 1009 1010 /* 1011 * Register with IBMF, SA access when the port is in 1012 * ACTIVE state. Also register a callback routine 1013 * with IBMF to receive incoming DM MAD's. 1014 * The IBDM event handler takes care of registration of 1015 * port which are not active. 1016 */ 1017 IBTF_DPRINTF_L4("ibdm", 1018 "\thandle_hca_attach: port guid %llx Port state 0x%x", 1019 port_attr->pa_port_guid, portinfop[ii].p_linkstate); 1020 1021 if (portinfop[ii].p_linkstate == IBT_PORT_ACTIVE) { 1022 mutex_enter(&ibdm.ibdm_hl_mutex); 1023 hca_list->hl_nports_active++; 1024 ibdm_initialize_port(port_attr); 1025 cv_broadcast(&ibdm.ibdm_port_settle_cv); 1026 mutex_exit(&ibdm.ibdm_hl_mutex); 1027 } 1028 } 1029 mutex_enter(&ibdm.ibdm_hl_mutex); 1030 for (temp = ibdm.ibdm_hca_list_head; temp; temp = temp->hl_next) { 1031 if (temp->hl_hca_guid == hca_guid) { 1032 IBTF_DPRINTF_L2("ibdm", "hca_attach: HCA %llX " 1033 "already seen by IBDM", hca_guid); 1034 mutex_exit(&ibdm.ibdm_hl_mutex); 1035 (void) ibdm_uninit_hca(hca_list); 1036 return; 1037 } 1038 } 1039 ibdm.ibdm_hca_count++; 1040 if (ibdm.ibdm_hca_list_head == NULL) { 1041 ibdm.ibdm_hca_list_head = hca_list; 1042 ibdm.ibdm_hca_list_tail = hca_list; 1043 } else { 1044 ibdm.ibdm_hca_list_tail->hl_next = hca_list; 1045 ibdm.ibdm_hca_list_tail = hca_list; 1046 } 1047 mutex_exit(&ibdm.ibdm_hl_mutex); 1048 mutex_enter(&ibdm.ibdm_ibnex_mutex); 1049 if (ibdm.ibdm_ibnex_callback != NULL) { 1050 (*ibdm.ibdm_ibnex_callback)((void *) 1051 &hca_guid, IBDM_EVENT_HCA_ADDED); 1052 } 1053 mutex_exit(&ibdm.ibdm_ibnex_mutex); 1054 1055 kmem_free(hca_attr, sizeof (ibt_hca_attr_t)); 1056 ibt_free_portinfo(portinfop, size); 1057 } 1058 1059 1060 /* 1061 * ibdm_handle_hca_detach() 1062 */ 1063 static void 1064 ibdm_handle_hca_detach(ib_guid_t hca_guid) 1065 { 1066 ibdm_hca_list_t *head, *prev = NULL; 1067 size_t len; 1068 ibdm_dp_gidinfo_t *gidinfo; 1069 ibdm_port_attr_t *port_attr; 1070 int i; 1071 1072 IBTF_DPRINTF_L4("ibdm", 1073 "\thandle_hca_detach: hca_guid = 0x%llx", hca_guid); 1074 1075 /* Make sure no probes are running */ 1076 mutex_enter(&ibdm.ibdm_mutex); 1077 while (ibdm.ibdm_busy & IBDM_BUSY) 1078 cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex); 1079 ibdm.ibdm_busy |= IBDM_BUSY; 1080 mutex_exit(&ibdm.ibdm_mutex); 1081 1082 mutex_enter(&ibdm.ibdm_hl_mutex); 1083 head = ibdm.ibdm_hca_list_head; 1084 while (head) { 1085 if (head->hl_hca_guid == hca_guid) { 1086 if (prev == NULL) 1087 ibdm.ibdm_hca_list_head = head->hl_next; 1088 else 1089 prev->hl_next = head->hl_next; 1090 if (ibdm.ibdm_hca_list_tail == head) 1091 ibdm.ibdm_hca_list_tail = prev; 1092 ibdm.ibdm_hca_count--; 1093 break; 1094 } 1095 prev = head; 1096 head = head->hl_next; 1097 } 1098 mutex_exit(&ibdm.ibdm_hl_mutex); 1099 if (ibdm_uninit_hca(head) != IBDM_SUCCESS) 1100 (void) ibdm_handle_hca_attach(hca_guid); 1101 1102 #ifdef DEBUG 1103 if (ibdm_enumerate_iocs == 0) { 1104 ASSERT(ibdm.ibdm_dp_gidlist_head == NULL); 1105 } 1106 #endif 1107 1108 /* 1109 * Now clean up the HCA lists in the gidlist. 1110 */ 1111 for (gidinfo = ibdm.ibdm_dp_gidlist_head; gidinfo; gidinfo = 1112 gidinfo->gl_next) { 1113 prev = NULL; 1114 head = gidinfo->gl_hca_list; 1115 while (head) { 1116 if (head->hl_hca_guid == hca_guid) { 1117 if (prev == NULL) 1118 gidinfo->gl_hca_list = 1119 head->hl_next; 1120 else 1121 prev->hl_next = head->hl_next; 1122 for (i = 0; i < head->hl_nports; i++) { 1123 port_attr = &head->hl_port_attr[i]; 1124 if (port_attr->pa_pkey_tbl != NULL) 1125 kmem_free( 1126 port_attr->pa_pkey_tbl, 1127 port_attr->pa_npkeys * 1128 sizeof (ibdm_pkey_tbl_t)); 1129 } 1130 len = sizeof (ibdm_hca_list_t) + 1131 (head->hl_nports * 1132 sizeof (ibdm_port_attr_t)); 1133 kmem_free(head, len); 1134 1135 break; 1136 } 1137 prev = head; 1138 head = head->hl_next; 1139 } 1140 } 1141 1142 mutex_enter(&ibdm.ibdm_mutex); 1143 ibdm.ibdm_busy &= ~IBDM_BUSY; 1144 cv_broadcast(&ibdm.ibdm_busy_cv); 1145 mutex_exit(&ibdm.ibdm_mutex); 1146 } 1147 1148 1149 static int 1150 ibdm_uninit_hca(ibdm_hca_list_t *head) 1151 { 1152 int ii; 1153 ibdm_port_attr_t *port_attr; 1154 1155 for (ii = 0; ii < head->hl_nports; ii++) { 1156 port_attr = &head->hl_port_attr[ii]; 1157 if (ibdm_fini_port(port_attr) != IBDM_SUCCESS) { 1158 IBTF_DPRINTF_L2("ibdm", "uninit_hca: HCA %p port 0x%x " 1159 "ibdm_fini_port() failed", head, ii); 1160 return (IBDM_FAILURE); 1161 } 1162 } 1163 if (head->hl_hca_hdl) 1164 if (ibt_close_hca(head->hl_hca_hdl) != IBT_SUCCESS) { 1165 IBTF_DPRINTF_L2("ibdm", "uninit_hca: " 1166 "ibt_close_hca() failed"); 1167 return (IBDM_FAILURE); 1168 } 1169 kmem_free(head->hl_port_attr, 1170 head->hl_nports * sizeof (ibdm_port_attr_t)); 1171 kmem_free(head->hl_hca_port_attr, sizeof (ibdm_port_attr_t)); 1172 kmem_free(head, sizeof (ibdm_hca_list_t)); 1173 return (IBDM_SUCCESS); 1174 } 1175 1176 1177 /* 1178 * For each port on the HCA, 1179 * 1) Teardown IBMF receive callback function 1180 * 2) Unregister with IBMF 1181 * 3) Unregister with SA access 1182 */ 1183 static int 1184 ibdm_fini_port(ibdm_port_attr_t *port_attr) 1185 { 1186 int ii, ibmf_status; 1187 1188 for (ii = 0; ii < port_attr->pa_npkeys; ii++) { 1189 if (port_attr->pa_pkey_tbl == NULL) 1190 break; 1191 if (!port_attr->pa_pkey_tbl[ii].pt_qp_hdl) 1192 continue; 1193 if (ibdm_port_attr_ibmf_fini(port_attr, ii) != IBDM_SUCCESS) { 1194 IBTF_DPRINTF_L4("ibdm", "\tfini_port: " 1195 "ibdm_port_attr_ibmf_fini failed for " 1196 "port pkey 0x%x", ii); 1197 return (IBDM_FAILURE); 1198 } 1199 } 1200 1201 if (port_attr->pa_ibmf_hdl) { 1202 ibmf_status = ibmf_tear_down_async_cb(port_attr->pa_ibmf_hdl, 1203 IBMF_QP_HANDLE_DEFAULT, 0); 1204 if (ibmf_status != IBMF_SUCCESS) { 1205 IBTF_DPRINTF_L4("ibdm", "\tfini_port: " 1206 "ibmf_tear_down_async_cb failed %d", ibmf_status); 1207 return (IBDM_FAILURE); 1208 } 1209 1210 ibmf_status = ibmf_unregister(&port_attr->pa_ibmf_hdl, 0); 1211 if (ibmf_status != IBMF_SUCCESS) { 1212 IBTF_DPRINTF_L2("ibdm", "\tfini_port: " 1213 "ibmf_unregister failed %d", ibmf_status); 1214 return (IBDM_FAILURE); 1215 } 1216 1217 port_attr->pa_ibmf_hdl = NULL; 1218 } 1219 1220 if (port_attr->pa_sa_hdl) { 1221 ibmf_status = ibmf_sa_session_close(&port_attr->pa_sa_hdl, 0); 1222 if (ibmf_status != IBMF_SUCCESS) { 1223 IBTF_DPRINTF_L2("ibdm", "\tfini_port: " 1224 "ibmf_sa_session_close failed %d", ibmf_status); 1225 return (IBDM_FAILURE); 1226 } 1227 port_attr->pa_sa_hdl = NULL; 1228 } 1229 1230 if (port_attr->pa_pkey_tbl != NULL) { 1231 kmem_free(port_attr->pa_pkey_tbl, 1232 port_attr->pa_npkeys * sizeof (ibdm_pkey_tbl_t)); 1233 port_attr->pa_pkey_tbl = NULL; 1234 port_attr->pa_npkeys = 0; 1235 } 1236 1237 return (IBDM_SUCCESS); 1238 } 1239 1240 1241 /* 1242 * ibdm_port_attr_ibmf_fini: 1243 * With IBMF - Tear down Async callback and free QP Handle 1244 */ 1245 static int 1246 ibdm_port_attr_ibmf_fini(ibdm_port_attr_t *port_attr, int ii) 1247 { 1248 int ibmf_status; 1249 1250 IBTF_DPRINTF_L5("ibdm", "\tport_attr_ibmf_fini:"); 1251 1252 if (ibdm_enumerate_iocs == 0) { 1253 ASSERT(port_attr->pa_pkey_tbl[ii].pt_qp_hdl == NULL); 1254 return (IBDM_SUCCESS); 1255 } 1256 1257 if (port_attr->pa_pkey_tbl[ii].pt_qp_hdl) { 1258 ibmf_status = ibmf_tear_down_async_cb(port_attr->pa_ibmf_hdl, 1259 port_attr->pa_pkey_tbl[ii].pt_qp_hdl, 0); 1260 if (ibmf_status != IBMF_SUCCESS) { 1261 IBTF_DPRINTF_L4("ibdm", "\tport_attr_ibmf_fini: " 1262 "ibmf_tear_down_async_cb failed %d", ibmf_status); 1263 return (IBDM_FAILURE); 1264 } 1265 ibmf_status = ibmf_free_qp(port_attr->pa_ibmf_hdl, 1266 &port_attr->pa_pkey_tbl[ii].pt_qp_hdl, 0); 1267 if (ibmf_status != IBMF_SUCCESS) { 1268 IBTF_DPRINTF_L4("ibdm", "\tport_attr_ibmf_fini: " 1269 "ibmf_free_qp failed %d", ibmf_status); 1270 return (IBDM_FAILURE); 1271 } 1272 port_attr->pa_pkey_tbl[ii].pt_qp_hdl = NULL; 1273 } 1274 return (IBDM_SUCCESS); 1275 } 1276 1277 1278 /* 1279 * ibdm_gid_decr_pending: 1280 * decrement gl_pending_cmds. If zero wakeup sleeping threads 1281 */ 1282 static void 1283 ibdm_gid_decr_pending(ibdm_dp_gidinfo_t *gidinfo) 1284 { 1285 mutex_enter(&ibdm.ibdm_mutex); 1286 mutex_enter(&gidinfo->gl_mutex); 1287 if (--gidinfo->gl_pending_cmds == 0) { 1288 /* 1289 * Handle DGID getting removed. 1290 */ 1291 if (gidinfo->gl_disconnected) { 1292 mutex_exit(&gidinfo->gl_mutex); 1293 mutex_exit(&ibdm.ibdm_mutex); 1294 1295 IBTF_DPRINTF_L3(ibdm_string, "\tgid_decr_pending: " 1296 "gidinfo %p hot removal", gidinfo); 1297 ibdm_delete_gidinfo(gidinfo); 1298 1299 mutex_enter(&ibdm.ibdm_mutex); 1300 ibdm.ibdm_ngid_probes_in_progress--; 1301 ibdm_wait_probe_completion(); 1302 mutex_exit(&ibdm.ibdm_mutex); 1303 return; 1304 } 1305 mutex_exit(&gidinfo->gl_mutex); 1306 mutex_exit(&ibdm.ibdm_mutex); 1307 ibdm_notify_newgid_iocs(gidinfo); 1308 mutex_enter(&ibdm.ibdm_mutex); 1309 mutex_enter(&gidinfo->gl_mutex); 1310 1311 ibdm.ibdm_ngid_probes_in_progress--; 1312 ibdm_wait_probe_completion(); 1313 } 1314 mutex_exit(&gidinfo->gl_mutex); 1315 mutex_exit(&ibdm.ibdm_mutex); 1316 } 1317 1318 1319 /* 1320 * ibdm_wait_probe_completion: 1321 * wait for probing to complete 1322 */ 1323 static void 1324 ibdm_wait_probe_completion(void) 1325 { 1326 ASSERT(MUTEX_HELD(&ibdm.ibdm_mutex)); 1327 if (ibdm.ibdm_ngid_probes_in_progress) { 1328 IBTF_DPRINTF_L4("ibdm", "\twait for probe complete"); 1329 ibdm.ibdm_busy |= IBDM_PROBE_IN_PROGRESS; 1330 while (ibdm.ibdm_busy & IBDM_PROBE_IN_PROGRESS) 1331 cv_wait(&ibdm.ibdm_probe_cv, &ibdm.ibdm_mutex); 1332 } 1333 } 1334 1335 1336 /* 1337 * ibdm_wait_cisco_probe_completion: 1338 * wait for the reply from the Cisco FC GW switch after a setclassportinfo 1339 * request is sent. This wait can be achieved on each gid. 1340 */ 1341 static void 1342 ibdm_wait_cisco_probe_completion(ibdm_dp_gidinfo_t *gidinfo) 1343 { 1344 ASSERT(MUTEX_HELD(&gidinfo->gl_mutex)); 1345 IBTF_DPRINTF_L4("ibdm", "\twait for cisco probe complete"); 1346 gidinfo->gl_flag |= IBDM_CISCO_PROBE; 1347 while (gidinfo->gl_flag & IBDM_CISCO_PROBE) 1348 cv_wait(&gidinfo->gl_probe_cv, &gidinfo->gl_mutex); 1349 } 1350 1351 1352 /* 1353 * ibdm_wakeup_probe_gid_cv: 1354 * wakeup waiting threads (based on ibdm_ngid_probes_in_progress) 1355 */ 1356 static void 1357 ibdm_wakeup_probe_gid_cv(void) 1358 { 1359 ASSERT(MUTEX_HELD(&ibdm.ibdm_mutex)); 1360 if (!ibdm.ibdm_ngid_probes_in_progress) { 1361 IBTF_DPRINTF_L4("ibdm", "wakeup_probe_gid_thread: Wakeup"); 1362 ibdm.ibdm_busy &= ~IBDM_PROBE_IN_PROGRESS; 1363 cv_broadcast(&ibdm.ibdm_probe_cv); 1364 } 1365 1366 } 1367 1368 1369 /* 1370 * ibdm_sweep_fabric(reprobe_flag) 1371 * Find all possible Managed IOU's and their IOC's that are visible 1372 * to the host. The algorithm used is as follows 1373 * 1374 * Send a "bus walk" request for each port on the host HCA to SA access 1375 * SA returns complete set of GID's that are reachable from 1376 * source port. This is done in parallel. 1377 * 1378 * Initialize GID state to IBDM_GID_PROBE_NOT_DONE 1379 * 1380 * Sort the GID list and eliminate duplicate GID's 1381 * 1) Use DGID for sorting 1382 * 2) use PortGuid for sorting 1383 * Send SA query to retrieve NodeRecord and 1384 * extract PortGuid from that. 1385 * 1386 * Set GID state to IBDM_GID_PROBE_FAILED to all the ports that dont 1387 * support DM MAD's 1388 * Send a "Portinfo" query to get the port capabilities and 1389 * then check for DM MAD's support 1390 * 1391 * Send "ClassPortInfo" request for all the GID's in parallel, 1392 * set the GID state to IBDM_GET_CLASSPORTINFO and wait on the 1393 * cv_signal to complete. 1394 * 1395 * When DM agent on the remote GID sends back the response, IBMF 1396 * invokes DM callback routine. 1397 * 1398 * If the response is proper, send "IOUnitInfo" request and set 1399 * GID state to IBDM_GET_IOUNITINFO. 1400 * 1401 * If the response is proper, send "IocProfileInfo" request to 1402 * all the IOC simultaneously and set GID state to IBDM_GET_IOC_DETAILS. 1403 * 1404 * Send request to get Service entries simultaneously 1405 * 1406 * Signal the waiting thread when received response for all the commands. 1407 * 1408 * Set the GID state to IBDM_GID_PROBE_FAILED when received a error 1409 * response during the probing period. 1410 * 1411 * Note: 1412 * ibdm.ibdm_ngid_probes_in_progress and ibdm_gid_list_t:gl_pending_cmds 1413 * keep track of number commands in progress at any point of time. 1414 * MAD transaction ID is used to identify a particular GID 1415 * TBD: Consider registering the IBMF receive callback on demand 1416 * 1417 * Note: This routine must be called with ibdm.ibdm_mutex held 1418 * TBD: Re probe the failure GID (for certain failures) when requested 1419 * for fabric sweep next time 1420 * 1421 * Parameters : If reprobe_flag is set, All IOCs will be reprobed. 1422 */ 1423 static void 1424 ibdm_sweep_fabric(int reprobe_flag) 1425 { 1426 int ii; 1427 int new_paths = 0; 1428 uint8_t niocs; 1429 taskqid_t tid; 1430 ibdm_ioc_info_t *ioc; 1431 ibdm_hca_list_t *hca_list = NULL; 1432 ibdm_port_attr_t *port = NULL; 1433 ibdm_dp_gidinfo_t *gid_info; 1434 1435 IBTF_DPRINTF_L4("ibdm", "\tsweep_fabric: Enter"); 1436 ASSERT(MUTEX_HELD(&ibdm.ibdm_mutex)); 1437 1438 /* 1439 * Check whether a sweep already in progress. If so, just 1440 * wait for the fabric sweep to complete 1441 */ 1442 while (ibdm.ibdm_busy & IBDM_BUSY) 1443 cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex); 1444 ibdm.ibdm_busy |= IBDM_BUSY; 1445 mutex_exit(&ibdm.ibdm_mutex); 1446 1447 ibdm_dump_sweep_fabric_timestamp(0); 1448 1449 /* Rescan the GID list for any removed GIDs for reprobe */ 1450 if (reprobe_flag) 1451 ibdm_rescan_gidlist(NULL); 1452 1453 /* 1454 * Get list of all the ports reachable from the local known HCA 1455 * ports which are active 1456 */ 1457 mutex_enter(&ibdm.ibdm_hl_mutex); 1458 for (ibdm_get_next_port(&hca_list, &port, 1); port; 1459 ibdm_get_next_port(&hca_list, &port, 1)) { 1460 /* 1461 * Get PATHS to all the reachable ports from 1462 * SGID and update the global ibdm structure. 1463 */ 1464 new_paths = ibdm_get_reachable_ports(port, hca_list); 1465 ibdm.ibdm_ngids += new_paths; 1466 } 1467 mutex_exit(&ibdm.ibdm_hl_mutex); 1468 1469 mutex_enter(&ibdm.ibdm_mutex); 1470 ibdm.ibdm_ngid_probes_in_progress += ibdm.ibdm_ngids; 1471 mutex_exit(&ibdm.ibdm_mutex); 1472 1473 /* Send a request to probe GIDs asynchronously. */ 1474 for (gid_info = ibdm.ibdm_dp_gidlist_head; gid_info; 1475 gid_info = gid_info->gl_next) { 1476 mutex_enter(&gid_info->gl_mutex); 1477 gid_info->gl_reprobe_flag = reprobe_flag; 1478 mutex_exit(&gid_info->gl_mutex); 1479 1480 /* process newly encountered GIDs */ 1481 tid = taskq_dispatch(system_taskq, ibdm_probe_gid_thread, 1482 (void *)gid_info, TQ_NOSLEEP); 1483 IBTF_DPRINTF_L4("ibdm", "\tsweep_fabric: gid_info = %p" 1484 " taskq_id = %x", gid_info, tid); 1485 /* taskq failed to dispatch call it directly */ 1486 if (tid == NULL) 1487 ibdm_probe_gid_thread((void *)gid_info); 1488 } 1489 1490 mutex_enter(&ibdm.ibdm_mutex); 1491 ibdm_wait_probe_completion(); 1492 1493 /* 1494 * Update the properties, if reprobe_flag is set 1495 * Skip if gl_reprobe_flag is set, this will be 1496 * a re-inserted / new GID, for which notifications 1497 * have already been send. 1498 */ 1499 if (reprobe_flag) { 1500 for (gid_info = ibdm.ibdm_dp_gidlist_head; gid_info; 1501 gid_info = gid_info->gl_next) { 1502 if (gid_info->gl_iou == NULL) 1503 continue; 1504 if (gid_info->gl_reprobe_flag) { 1505 gid_info->gl_reprobe_flag = 0; 1506 continue; 1507 } 1508 1509 niocs = gid_info->gl_iou->iou_info.iou_num_ctrl_slots; 1510 for (ii = 0; ii < niocs; ii++) { 1511 ioc = IBDM_GIDINFO2IOCINFO(gid_info, ii); 1512 if (ioc) 1513 ibdm_reprobe_update_port_srv(ioc, 1514 gid_info); 1515 } 1516 } 1517 } else if (ibdm.ibdm_prev_iou) { 1518 ibdm_ioc_info_t *ioc_list; 1519 1520 /* 1521 * Get the list of IOCs which have changed. 1522 * If any IOCs have changed, Notify IBNexus 1523 */ 1524 ibdm.ibdm_prev_iou = 0; 1525 ioc_list = ibdm_handle_prev_iou(); 1526 if (ioc_list) { 1527 if (ibdm.ibdm_ibnex_callback != NULL) { 1528 (*ibdm.ibdm_ibnex_callback)( 1529 (void *)ioc_list, 1530 IBDM_EVENT_IOC_PROP_UPDATE); 1531 } 1532 } 1533 } 1534 1535 ibdm_dump_sweep_fabric_timestamp(1); 1536 1537 ibdm.ibdm_busy &= ~IBDM_BUSY; 1538 cv_broadcast(&ibdm.ibdm_busy_cv); 1539 IBTF_DPRINTF_L5("ibdm", "\tsweep_fabric: EXIT"); 1540 } 1541 1542 1543 /* 1544 * ibdm_is_cisco: 1545 * Check if this is a Cisco device or not. 1546 */ 1547 static boolean_t 1548 ibdm_is_cisco(ib_guid_t guid) 1549 { 1550 if ((guid >> IBDM_OUI_GUID_SHIFT) == IBDM_CISCO_COMPANY_ID) 1551 return (B_TRUE); 1552 return (B_FALSE); 1553 } 1554 1555 1556 /* 1557 * ibdm_is_cisco_switch: 1558 * Check if this switch is a CISCO switch or not. 1559 * Note that if this switch is already activated, ibdm_is_cisco_switch() 1560 * returns B_FALSE not to re-activate it again. 1561 */ 1562 static boolean_t 1563 ibdm_is_cisco_switch(ibdm_dp_gidinfo_t *gid_info) 1564 { 1565 int company_id, device_id; 1566 ASSERT(gid_info != 0); 1567 ASSERT(MUTEX_HELD(&gid_info->gl_mutex)); 1568 1569 /* 1570 * If this switch is already activated, don't re-activate it. 1571 */ 1572 if (gid_info->gl_flag & IBDM_CISCO_PROBE_DONE) 1573 return (B_FALSE); 1574 1575 /* 1576 * Check if this switch is a Cisco FC GW or not. 1577 * Use the node guid (the OUI part) instead of the vendor id 1578 * since the vendor id is zero in practice. 1579 */ 1580 company_id = gid_info->gl_nodeguid >> IBDM_OUI_GUID_SHIFT; 1581 device_id = gid_info->gl_devid; 1582 1583 if (company_id == IBDM_CISCO_COMPANY_ID && 1584 device_id == IBDM_CISCO_DEVICE_ID) 1585 return (B_TRUE); 1586 return (B_FALSE); 1587 } 1588 1589 1590 /* 1591 * ibdm_probe_gid_thread: 1592 * thread that does the actual work for sweeping the fabric 1593 * for a given GID 1594 */ 1595 static void 1596 ibdm_probe_gid_thread(void *args) 1597 { 1598 int reprobe_flag; 1599 ib_guid_t node_guid; 1600 ib_guid_t port_guid; 1601 ibdm_dp_gidinfo_t *gid_info; 1602 1603 gid_info = (ibdm_dp_gidinfo_t *)args; 1604 reprobe_flag = gid_info->gl_reprobe_flag; 1605 IBTF_DPRINTF_L4("ibdm", "\tprobe_gid_thread: gid_info = %p, flag = %d", 1606 gid_info, reprobe_flag); 1607 ASSERT(gid_info != NULL); 1608 ASSERT(gid_info->gl_pending_cmds == 0); 1609 1610 if (gid_info->gl_state != IBDM_GID_PROBE_NOT_DONE && 1611 reprobe_flag == 0) { 1612 /* 1613 * This GID may have been already probed. Send 1614 * in a CLP to check if IOUnitInfo changed? 1615 * Explicitly set gl_reprobe_flag to 0 so that 1616 * IBnex is not notified on completion 1617 */ 1618 if (gid_info->gl_state == IBDM_GID_PROBING_COMPLETE) { 1619 IBTF_DPRINTF_L4("ibdm", "\tprobe_gid_thread: " 1620 "get new IOCs information"); 1621 mutex_enter(&gid_info->gl_mutex); 1622 gid_info->gl_pending_cmds++; 1623 gid_info->gl_state = IBDM_GET_IOUNITINFO; 1624 gid_info->gl_reprobe_flag = 0; 1625 mutex_exit(&gid_info->gl_mutex); 1626 if (ibdm_send_iounitinfo(gid_info) != IBDM_SUCCESS) { 1627 mutex_enter(&gid_info->gl_mutex); 1628 --gid_info->gl_pending_cmds; 1629 mutex_exit(&gid_info->gl_mutex); 1630 mutex_enter(&ibdm.ibdm_mutex); 1631 --ibdm.ibdm_ngid_probes_in_progress; 1632 ibdm_wakeup_probe_gid_cv(); 1633 mutex_exit(&ibdm.ibdm_mutex); 1634 } 1635 } else { 1636 mutex_enter(&ibdm.ibdm_mutex); 1637 --ibdm.ibdm_ngid_probes_in_progress; 1638 ibdm_wakeup_probe_gid_cv(); 1639 mutex_exit(&ibdm.ibdm_mutex); 1640 } 1641 return; 1642 } else if (reprobe_flag && gid_info->gl_state == 1643 IBDM_GID_PROBING_COMPLETE) { 1644 /* 1645 * Reprobe all IOCs for the GID which has completed 1646 * probe. Skip other port GIDs to same IOU. 1647 * Explicitly set gl_reprobe_flag to 0 so that 1648 * IBnex is not notified on completion 1649 */ 1650 ibdm_ioc_info_t *ioc_info; 1651 uint8_t niocs, ii; 1652 1653 ASSERT(gid_info->gl_iou); 1654 mutex_enter(&gid_info->gl_mutex); 1655 niocs = gid_info->gl_iou->iou_info.iou_num_ctrl_slots; 1656 gid_info->gl_state = IBDM_GET_IOC_DETAILS; 1657 gid_info->gl_pending_cmds += niocs; 1658 gid_info->gl_reprobe_flag = 0; 1659 mutex_exit(&gid_info->gl_mutex); 1660 for (ii = 0; ii < niocs; ii++) { 1661 uchar_t slot_info; 1662 ib_dm_io_unitinfo_t *giou_info; 1663 1664 /* 1665 * Check whether IOC is present in the slot 1666 * Series of nibbles (in the field 1667 * iou_ctrl_list) represents a slot in the 1668 * IOU. 1669 * Byte format: 76543210 1670 * Bits 0-3 of first byte represent Slot 2 1671 * bits 4-7 of first byte represent slot 1, 1672 * bits 0-3 of second byte represent slot 4 1673 * and so on 1674 * Each 4-bit nibble has the following meaning 1675 * 0x0 : IOC not installed 1676 * 0x1 : IOC is present 1677 * 0xf : Slot does not exist 1678 * and all other values are reserved. 1679 */ 1680 ioc_info = IBDM_GIDINFO2IOCINFO(gid_info, ii); 1681 giou_info = &gid_info->gl_iou->iou_info; 1682 slot_info = giou_info->iou_ctrl_list[(ii/2)]; 1683 if ((ii % 2) == 0) 1684 slot_info = (slot_info >> 4); 1685 1686 if ((slot_info & 0xf) != 1) { 1687 ioc_info->ioc_state = 1688 IBDM_IOC_STATE_PROBE_FAILED; 1689 ibdm_gid_decr_pending(gid_info); 1690 continue; 1691 } 1692 1693 if (ibdm_send_ioc_profile(gid_info, ii) != 1694 IBDM_SUCCESS) { 1695 ibdm_gid_decr_pending(gid_info); 1696 } 1697 } 1698 1699 return; 1700 } else if (gid_info->gl_state != IBDM_GID_PROBE_NOT_DONE) { 1701 mutex_enter(&ibdm.ibdm_mutex); 1702 --ibdm.ibdm_ngid_probes_in_progress; 1703 ibdm_wakeup_probe_gid_cv(); 1704 mutex_exit(&ibdm.ibdm_mutex); 1705 return; 1706 } 1707 1708 /* 1709 * Check whether the destination GID supports DM agents. If 1710 * not, stop probing the GID and continue with the next GID 1711 * in the list. 1712 */ 1713 if (ibdm_is_dev_mgt_supported(gid_info) != IBDM_SUCCESS) { 1714 mutex_enter(&gid_info->gl_mutex); 1715 gid_info->gl_state = IBDM_GID_PROBING_FAILED; 1716 gid_info->gl_is_dm_capable = B_FALSE; 1717 mutex_exit(&gid_info->gl_mutex); 1718 ibdm_delete_glhca_list(gid_info); 1719 mutex_enter(&ibdm.ibdm_mutex); 1720 --ibdm.ibdm_ngid_probes_in_progress; 1721 ibdm_wakeup_probe_gid_cv(); 1722 mutex_exit(&ibdm.ibdm_mutex); 1723 return; 1724 } 1725 1726 /* 1727 * This GID is Device management capable 1728 */ 1729 mutex_enter(&gid_info->gl_mutex); 1730 gid_info->gl_is_dm_capable = B_TRUE; 1731 mutex_exit(&gid_info->gl_mutex); 1732 1733 /* Get the nodeguid and portguid of the port */ 1734 if (ibdm_get_node_port_guids(gid_info->gl_sa_hdl, gid_info->gl_dlid, 1735 &node_guid, &port_guid) != IBDM_SUCCESS) { 1736 mutex_enter(&gid_info->gl_mutex); 1737 gid_info->gl_state = IBDM_GID_PROBING_FAILED; 1738 mutex_exit(&gid_info->gl_mutex); 1739 ibdm_delete_glhca_list(gid_info); 1740 mutex_enter(&ibdm.ibdm_mutex); 1741 --ibdm.ibdm_ngid_probes_in_progress; 1742 ibdm_wakeup_probe_gid_cv(); 1743 mutex_exit(&ibdm.ibdm_mutex); 1744 return; 1745 } 1746 1747 /* 1748 * Check whether we already knew about this NodeGuid 1749 * If so, do not probe the GID and continue with the 1750 * next GID in the gid list. Set the GID state to 1751 * probing done. 1752 */ 1753 mutex_enter(&ibdm.ibdm_mutex); 1754 gid_info->gl_nodeguid = node_guid; 1755 gid_info->gl_portguid = port_guid; 1756 if (ibdm_check_dest_nodeguid(gid_info) != NULL) { 1757 mutex_exit(&ibdm.ibdm_mutex); 1758 mutex_enter(&gid_info->gl_mutex); 1759 gid_info->gl_state = IBDM_GID_PROBING_SKIPPED; 1760 mutex_exit(&gid_info->gl_mutex); 1761 ibdm_delete_glhca_list(gid_info); 1762 mutex_enter(&ibdm.ibdm_mutex); 1763 --ibdm.ibdm_ngid_probes_in_progress; 1764 ibdm_wakeup_probe_gid_cv(); 1765 mutex_exit(&ibdm.ibdm_mutex); 1766 return; 1767 } 1768 ibdm_add_to_gl_gid(gid_info, gid_info); 1769 mutex_exit(&ibdm.ibdm_mutex); 1770 1771 /* 1772 * New or reinserted GID : Enable notification to IBnex 1773 */ 1774 mutex_enter(&gid_info->gl_mutex); 1775 gid_info->gl_reprobe_flag = 1; 1776 1777 /* 1778 * A Cisco FC GW needs the special handling to get IOUnitInfo. 1779 */ 1780 if (ibdm_is_cisco_switch(gid_info)) { 1781 gid_info->gl_pending_cmds++; 1782 gid_info->gl_state = IBDM_SET_CLASSPORTINFO; 1783 mutex_exit(&gid_info->gl_mutex); 1784 1785 if (ibdm_set_classportinfo(gid_info) != IBDM_SUCCESS) { 1786 mutex_enter(&gid_info->gl_mutex); 1787 gid_info->gl_state = IBDM_GID_PROBING_FAILED; 1788 --gid_info->gl_pending_cmds; 1789 mutex_exit(&gid_info->gl_mutex); 1790 1791 /* free the hca_list on this gid_info */ 1792 ibdm_delete_glhca_list(gid_info); 1793 1794 mutex_enter(&ibdm.ibdm_mutex); 1795 --ibdm.ibdm_ngid_probes_in_progress; 1796 ibdm_wakeup_probe_gid_cv(); 1797 mutex_exit(&ibdm.ibdm_mutex); 1798 1799 return; 1800 } 1801 1802 mutex_enter(&gid_info->gl_mutex); 1803 ibdm_wait_cisco_probe_completion(gid_info); 1804 1805 IBTF_DPRINTF_L4("ibdm", "\tibdm_probe_gid_thread: " 1806 "CISCO Wakeup signal received"); 1807 } 1808 1809 /* move on to the 'GET_CLASSPORTINFO' stage */ 1810 gid_info->gl_pending_cmds++; 1811 gid_info->gl_state = IBDM_GET_CLASSPORTINFO; 1812 mutex_exit(&gid_info->gl_mutex); 1813 1814 IBTF_DPRINTF_L3(ibdm_string, "\tibdm_probe_gid_thread: " 1815 "%d: gid_info %p gl_state %d pending_cmds %d", 1816 __LINE__, gid_info, gid_info->gl_state, 1817 gid_info->gl_pending_cmds); 1818 1819 /* 1820 * Send ClassPortInfo request to the GID asynchronously. 1821 */ 1822 if (ibdm_send_classportinfo(gid_info) != IBDM_SUCCESS) { 1823 1824 mutex_enter(&gid_info->gl_mutex); 1825 gid_info->gl_state = IBDM_GID_PROBING_FAILED; 1826 --gid_info->gl_pending_cmds; 1827 mutex_exit(&gid_info->gl_mutex); 1828 1829 /* free the hca_list on this gid_info */ 1830 ibdm_delete_glhca_list(gid_info); 1831 1832 mutex_enter(&ibdm.ibdm_mutex); 1833 --ibdm.ibdm_ngid_probes_in_progress; 1834 ibdm_wakeup_probe_gid_cv(); 1835 mutex_exit(&ibdm.ibdm_mutex); 1836 1837 return; 1838 } 1839 } 1840 1841 1842 /* 1843 * ibdm_check_dest_nodeguid 1844 * Searches for the NodeGuid in the GID list 1845 * Returns matching gid_info if found and otherwise NULL 1846 * 1847 * This function is called to handle new GIDs discovered 1848 * during device sweep / probe or for GID_AVAILABLE event. 1849 * 1850 * Parameter : 1851 * gid_info GID to check 1852 */ 1853 static ibdm_dp_gidinfo_t * 1854 ibdm_check_dest_nodeguid(ibdm_dp_gidinfo_t *gid_info) 1855 { 1856 ibdm_dp_gidinfo_t *gid_list; 1857 ibdm_gid_t *tmp; 1858 1859 IBTF_DPRINTF_L4("ibdm", "\tcheck_dest_nodeguid"); 1860 1861 gid_list = ibdm.ibdm_dp_gidlist_head; 1862 while (gid_list) { 1863 if ((gid_list != gid_info) && 1864 (gid_info->gl_nodeguid == gid_list->gl_nodeguid)) { 1865 IBTF_DPRINTF_L4("ibdm", 1866 "\tcheck_dest_nodeguid: NodeGuid is present"); 1867 1868 /* Add to gid_list */ 1869 tmp = kmem_zalloc(sizeof (ibdm_gid_t), 1870 KM_SLEEP); 1871 tmp->gid_dgid_hi = gid_info->gl_dgid_hi; 1872 tmp->gid_dgid_lo = gid_info->gl_dgid_lo; 1873 tmp->gid_next = gid_list->gl_gid; 1874 gid_list->gl_gid = tmp; 1875 gid_list->gl_ngids++; 1876 return (gid_list); 1877 } 1878 1879 gid_list = gid_list->gl_next; 1880 } 1881 1882 return (NULL); 1883 } 1884 1885 1886 /* 1887 * ibdm_is_dev_mgt_supported 1888 * Get the PortInfo attribute (SA Query) 1889 * Check "CompatabilityMask" field in the Portinfo. 1890 * Return IBDM_SUCCESS if DM MAD's supported (if bit 19 set) 1891 * by the port, otherwise IBDM_FAILURE 1892 */ 1893 static int 1894 ibdm_is_dev_mgt_supported(ibdm_dp_gidinfo_t *gid_info) 1895 { 1896 int ret; 1897 size_t length = 0; 1898 sa_portinfo_record_t req, *resp = NULL; 1899 ibmf_saa_access_args_t qargs; 1900 1901 bzero(&req, sizeof (sa_portinfo_record_t)); 1902 req.EndportLID = gid_info->gl_dlid; 1903 1904 qargs.sq_attr_id = SA_PORTINFORECORD_ATTRID; 1905 qargs.sq_access_type = IBMF_SAA_RETRIEVE; 1906 qargs.sq_component_mask = SA_PORTINFO_COMPMASK_PORTLID; 1907 qargs.sq_template = &req; 1908 qargs.sq_callback = NULL; 1909 qargs.sq_callback_arg = NULL; 1910 1911 ret = ibmf_sa_access(gid_info->gl_sa_hdl, 1912 &qargs, 0, &length, (void **)&resp); 1913 1914 if ((ret != IBMF_SUCCESS) || (length == 0) || (resp == NULL)) { 1915 IBTF_DPRINTF_L2("ibdm", "\tis_dev_mgt_supported:" 1916 "failed to get PORTINFO attribute %d", ret); 1917 return (IBDM_FAILURE); 1918 } 1919 1920 if (resp->PortInfo.CapabilityMask & SM_CAP_MASK_IS_DM_SUPPD) { 1921 IBTF_DPRINTF_L4("ibdm", "\tis_dev_mgt_supported: SUPPD !!"); 1922 ret = IBDM_SUCCESS; 1923 } else { 1924 IBTF_DPRINTF_L4("ibdm", "\tis_dev_mgt_supported: " 1925 "Not SUPPD !!, cap 0x%x", resp->PortInfo.CapabilityMask); 1926 ret = IBDM_FAILURE; 1927 } 1928 kmem_free(resp, length); 1929 return (ret); 1930 } 1931 1932 1933 /* 1934 * ibdm_get_node_port_guids() 1935 * Get the NodeInfoRecord of the port 1936 * Save NodeGuid and PortGUID values in the GID list structure. 1937 * Return IBDM_SUCCESS/IBDM_FAILURE 1938 */ 1939 static int 1940 ibdm_get_node_port_guids(ibmf_saa_handle_t sa_hdl, ib_lid_t dlid, 1941 ib_guid_t *node_guid, ib_guid_t *port_guid) 1942 { 1943 int ret; 1944 size_t length = 0; 1945 sa_node_record_t req, *resp = NULL; 1946 ibmf_saa_access_args_t qargs; 1947 1948 IBTF_DPRINTF_L4("ibdm", "\tget_node_port_guids"); 1949 1950 bzero(&req, sizeof (sa_node_record_t)); 1951 req.LID = dlid; 1952 1953 qargs.sq_attr_id = SA_NODERECORD_ATTRID; 1954 qargs.sq_access_type = IBMF_SAA_RETRIEVE; 1955 qargs.sq_component_mask = SA_NODEINFO_COMPMASK_NODELID; 1956 qargs.sq_template = &req; 1957 qargs.sq_callback = NULL; 1958 qargs.sq_callback_arg = NULL; 1959 1960 ret = ibmf_sa_access(sa_hdl, &qargs, 0, &length, (void **)&resp); 1961 if ((ret != IBMF_SUCCESS) || (length == 0) || (resp == NULL)) { 1962 IBTF_DPRINTF_L2("ibdm", "\tget_node_port_guids:" 1963 " SA Retrieve Failed: %d", ret); 1964 return (IBDM_FAILURE); 1965 } 1966 IBTF_DPRINTF_L4("ibdm", "\tget_node_port_guids: NodeGuid %llx Port" 1967 "GUID %llx", resp->NodeInfo.NodeGUID, resp->NodeInfo.NodeGUID); 1968 1969 *node_guid = resp->NodeInfo.NodeGUID; 1970 *port_guid = resp->NodeInfo.PortGUID; 1971 kmem_free(resp, length); 1972 return (IBDM_SUCCESS); 1973 } 1974 1975 1976 /* 1977 * ibdm_get_reachable_ports() 1978 * Get list of the destination GID (and its path records) by 1979 * querying the SA access. 1980 * 1981 * Returns Number paths 1982 */ 1983 static int 1984 ibdm_get_reachable_ports(ibdm_port_attr_t *portinfo, ibdm_hca_list_t *hca) 1985 { 1986 uint_t ii, jj, nrecs; 1987 uint_t npaths = 0; 1988 size_t length; 1989 ib_gid_t sgid; 1990 ibdm_pkey_tbl_t *pkey_tbl; 1991 sa_path_record_t *result; 1992 sa_path_record_t *precp; 1993 ibdm_dp_gidinfo_t *gid_info; 1994 1995 ASSERT(MUTEX_HELD(&ibdm.ibdm_hl_mutex)); 1996 IBTF_DPRINTF_L4("ibdm", "\tget_reachable_ports: portinfo %p", portinfo); 1997 1998 sgid.gid_prefix = portinfo->pa_sn_prefix; 1999 sgid.gid_guid = portinfo->pa_port_guid; 2000 2001 /* get reversible paths */ 2002 if (portinfo->pa_sa_hdl && ibmf_saa_paths_from_gid(portinfo->pa_sa_hdl, 2003 sgid, IBMF_SAA_PKEY_WC, B_TRUE, 0, &nrecs, &length, &result) 2004 != IBMF_SUCCESS) { 2005 IBTF_DPRINTF_L2("ibdm", 2006 "\tget_reachable_ports: Getting path records failed"); 2007 return (0); 2008 } 2009 2010 for (ii = 0; ii < nrecs; ii++) { 2011 sa_node_record_t *nrec; 2012 size_t length; 2013 2014 precp = &result[ii]; 2015 if ((gid_info = ibdm_check_dgid(precp->DGID.gid_guid, 2016 precp->DGID.gid_prefix)) != NULL) { 2017 IBTF_DPRINTF_L5("ibdm", "\tget_reachable_ports: " 2018 "Already exists nrecs %d, ii %d", nrecs, ii); 2019 ibdm_addto_glhcalist(gid_info, hca); 2020 continue; 2021 } 2022 /* 2023 * This is a new GID. Allocate a GID structure and 2024 * initialize the structure 2025 * gl_state is initialized to IBDM_GID_PROBE_NOT_DONE (0) 2026 * by kmem_zalloc call 2027 */ 2028 gid_info = kmem_zalloc(sizeof (ibdm_dp_gidinfo_t), KM_SLEEP); 2029 mutex_init(&gid_info->gl_mutex, NULL, MUTEX_DEFAULT, NULL); 2030 cv_init(&gid_info->gl_probe_cv, NULL, CV_DRIVER, NULL); 2031 gid_info->gl_dgid_hi = precp->DGID.gid_prefix; 2032 gid_info->gl_dgid_lo = precp->DGID.gid_guid; 2033 gid_info->gl_sgid_hi = precp->SGID.gid_prefix; 2034 gid_info->gl_sgid_lo = precp->SGID.gid_guid; 2035 gid_info->gl_p_key = precp->P_Key; 2036 gid_info->gl_sa_hdl = portinfo->pa_sa_hdl; 2037 gid_info->gl_ibmf_hdl = portinfo->pa_ibmf_hdl; 2038 gid_info->gl_slid = precp->SLID; 2039 gid_info->gl_dlid = precp->DLID; 2040 gid_info->gl_transactionID = (++ibdm.ibdm_transactionID) 2041 << IBDM_GID_TRANSACTIONID_SHIFT; 2042 gid_info->gl_min_transactionID = gid_info->gl_transactionID; 2043 gid_info->gl_max_transactionID = (ibdm.ibdm_transactionID +1) 2044 << IBDM_GID_TRANSACTIONID_SHIFT; 2045 gid_info->gl_SL = precp->SL; 2046 2047 /* 2048 * get the node record with this guid if the destination 2049 * device is a Cisco one. 2050 */ 2051 if (ibdm_is_cisco(precp->DGID.gid_guid) && 2052 (gid_info->gl_nodeguid == 0 || gid_info->gl_devid == 0) && 2053 ibdm_get_node_record_by_port(portinfo->pa_sa_hdl, 2054 precp->DGID.gid_guid, &nrec, &length) == IBDM_SUCCESS) { 2055 gid_info->gl_nodeguid = nrec->NodeInfo.NodeGUID; 2056 gid_info->gl_devid = nrec->NodeInfo.DeviceID; 2057 kmem_free(nrec, length); 2058 } 2059 2060 ibdm_addto_glhcalist(gid_info, hca); 2061 2062 ibdm_dump_path_info(precp); 2063 2064 gid_info->gl_qp_hdl = NULL; 2065 ASSERT(portinfo->pa_pkey_tbl != NULL && 2066 portinfo->pa_npkeys != 0); 2067 2068 for (jj = 0; jj < portinfo->pa_npkeys; jj++) { 2069 pkey_tbl = &portinfo->pa_pkey_tbl[jj]; 2070 if ((gid_info->gl_p_key == pkey_tbl->pt_pkey) && 2071 (pkey_tbl->pt_qp_hdl != NULL)) { 2072 gid_info->gl_qp_hdl = pkey_tbl->pt_qp_hdl; 2073 break; 2074 } 2075 } 2076 2077 /* 2078 * QP handle for GID not initialized. No matching Pkey 2079 * was found!! ibdm should *not* hit this case. Flag an 2080 * error and drop the GID if ibdm does encounter this. 2081 */ 2082 if (gid_info->gl_qp_hdl == NULL) { 2083 IBTF_DPRINTF_L2(ibdm_string, 2084 "\tget_reachable_ports: No matching Pkey"); 2085 ibdm_delete_gidinfo(gid_info); 2086 continue; 2087 } 2088 if (ibdm.ibdm_dp_gidlist_head == NULL) { 2089 ibdm.ibdm_dp_gidlist_head = gid_info; 2090 ibdm.ibdm_dp_gidlist_tail = gid_info; 2091 } else { 2092 ibdm.ibdm_dp_gidlist_tail->gl_next = gid_info; 2093 gid_info->gl_prev = ibdm.ibdm_dp_gidlist_tail; 2094 ibdm.ibdm_dp_gidlist_tail = gid_info; 2095 } 2096 npaths++; 2097 } 2098 kmem_free(result, length); 2099 IBTF_DPRINTF_L4("ibdm", "\tget_reachable_ports: npaths = %d", npaths); 2100 return (npaths); 2101 } 2102 2103 2104 /* 2105 * ibdm_check_dgid() 2106 * Look in the global list to check whether we know this DGID already 2107 * Return IBDM_GID_PRESENT/IBDM_GID_NOT_PRESENT 2108 */ 2109 static ibdm_dp_gidinfo_t * 2110 ibdm_check_dgid(ib_guid_t guid, ib_sn_prefix_t prefix) 2111 { 2112 ibdm_dp_gidinfo_t *gid_list; 2113 2114 for (gid_list = ibdm.ibdm_dp_gidlist_head; gid_list; 2115 gid_list = gid_list->gl_next) { 2116 if ((guid == gid_list->gl_dgid_lo) && 2117 (prefix == gid_list->gl_dgid_hi)) { 2118 break; 2119 } 2120 } 2121 return (gid_list); 2122 } 2123 2124 2125 /* 2126 * ibdm_find_gid() 2127 * Look in the global list to find a GID entry with matching 2128 * port & node GUID. 2129 * Return pointer to gidinfo if found, else return NULL 2130 */ 2131 static ibdm_dp_gidinfo_t * 2132 ibdm_find_gid(ib_guid_t nodeguid, ib_guid_t portguid) 2133 { 2134 ibdm_dp_gidinfo_t *gid_list; 2135 2136 IBTF_DPRINTF_L4("ibdm", "ibdm_find_gid(%llx, %llx)\n", 2137 nodeguid, portguid); 2138 2139 for (gid_list = ibdm.ibdm_dp_gidlist_head; gid_list; 2140 gid_list = gid_list->gl_next) { 2141 if ((portguid == gid_list->gl_portguid) && 2142 (nodeguid == gid_list->gl_nodeguid)) { 2143 break; 2144 } 2145 } 2146 2147 IBTF_DPRINTF_L4("ibdm", "ibdm_find_gid : returned %p\n", 2148 gid_list); 2149 return (gid_list); 2150 } 2151 2152 2153 /* 2154 * ibdm_set_classportinfo() 2155 * ibdm_set_classportinfo() is a function to activate a Cisco FC GW 2156 * by sending the setClassPortInfo request with the trapLID, trapGID 2157 * and etc. to the gateway since the gateway doesn't provide the IO 2158 * Unit Information othewise. This behavior is the Cisco specific one, 2159 * and this function is called to a Cisco FC GW only. 2160 * Returns IBDM_SUCCESS/IBDM_FAILURE 2161 */ 2162 static int 2163 ibdm_set_classportinfo(ibdm_dp_gidinfo_t *gid_info) 2164 { 2165 ibmf_msg_t *msg; 2166 ib_mad_hdr_t *hdr; 2167 ibdm_timeout_cb_args_t *cb_args; 2168 void *data; 2169 ib_mad_classportinfo_t *cpi; 2170 2171 IBTF_DPRINTF_L4("ibdm", 2172 "\tset_classportinfo: gid info 0x%p", gid_info); 2173 2174 /* 2175 * Send command to set classportinfo attribute. Allocate a IBMF 2176 * packet and initialize the packet. 2177 */ 2178 if (ibmf_alloc_msg(gid_info->gl_ibmf_hdl, IBMF_ALLOC_SLEEP, 2179 &msg) != IBMF_SUCCESS) { 2180 IBTF_DPRINTF_L4("ibdm", "\tset_classportinfo: pkt alloc fail"); 2181 return (IBDM_FAILURE); 2182 } 2183 2184 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*msg)) 2185 ibdm_alloc_send_buffers(msg); 2186 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*msg)) 2187 2188 msg->im_local_addr.ia_local_lid = gid_info->gl_slid; 2189 msg->im_local_addr.ia_remote_lid = gid_info->gl_dlid; 2190 msg->im_local_addr.ia_remote_qno = 1; 2191 msg->im_local_addr.ia_p_key = gid_info->gl_p_key; 2192 msg->im_local_addr.ia_q_key = IB_GSI_QKEY; 2193 msg->im_local_addr.ia_service_level = gid_info->gl_SL; 2194 2195 hdr = IBDM_OUT_IBMFMSG_MADHDR(msg); 2196 hdr->BaseVersion = MAD_CLASS_BASE_VERS_1; 2197 hdr->MgmtClass = MAD_MGMT_CLASS_DEV_MGT; 2198 hdr->ClassVersion = IB_DM_CLASS_VERSION_1; 2199 hdr->R_Method = IB_DM_DEVMGT_METHOD_SET; 2200 hdr->Status = 0; 2201 hdr->TransactionID = h2b64(gid_info->gl_transactionID); 2202 hdr->AttributeID = h2b16(IB_DM_ATTR_CLASSPORTINFO); 2203 hdr->AttributeModifier = 0; 2204 2205 data = msg->im_msgbufs_send.im_bufs_cl_data; 2206 cpi = (ib_mad_classportinfo_t *)data; 2207 2208 /* 2209 * Set the classportinfo values to activate this Cisco FC GW. 2210 */ 2211 cpi->TrapGID_hi = h2b64(gid_info->gl_sgid_hi); 2212 cpi->TrapGID_lo = h2b64(gid_info->gl_sgid_lo); 2213 cpi->TrapLID = h2b16(gid_info->gl_slid); 2214 cpi->TrapSL = gid_info->gl_SL; 2215 cpi->TrapP_Key = h2b16(gid_info->gl_p_key); 2216 cpi->TrapQP = h2b32((((ibmf_alt_qp_t *)gid_info->gl_qp_hdl)->isq_qpn)); 2217 cpi->TrapQ_Key = h2b32((((ibmf_alt_qp_t *) 2218 gid_info->gl_qp_hdl)->isq_qkey)); 2219 2220 cb_args = &gid_info->gl_cpi_cb_args; 2221 cb_args->cb_gid_info = gid_info; 2222 cb_args->cb_retry_count = ibdm_dft_retry_cnt; 2223 cb_args->cb_req_type = IBDM_REQ_TYPE_CLASSPORTINFO; 2224 2225 mutex_enter(&gid_info->gl_mutex); 2226 gid_info->gl_timeout_id = timeout(ibdm_pkt_timeout_hdlr, 2227 cb_args, IBDM_TIMEOUT_VALUE(ibdm_dft_timeout)); 2228 mutex_exit(&gid_info->gl_mutex); 2229 2230 IBTF_DPRINTF_L5("ibdm", "\tset_classportinfo: " 2231 "timeout id %x", gid_info->gl_timeout_id); 2232 2233 if (ibmf_msg_transport(gid_info->gl_ibmf_hdl, gid_info->gl_qp_hdl, 2234 msg, NULL, ibdm_ibmf_send_cb, cb_args, 0) != IBMF_SUCCESS) { 2235 IBTF_DPRINTF_L2("ibdm", 2236 "\tset_classportinfo: ibmf send failed"); 2237 ibdm_ibmf_send_cb(gid_info->gl_ibmf_hdl, msg, cb_args); 2238 } 2239 2240 return (IBDM_SUCCESS); 2241 } 2242 2243 2244 /* 2245 * ibdm_send_classportinfo() 2246 * Send classportinfo request. When the request is completed 2247 * IBMF calls ibdm_classportinfo_cb routine to inform about 2248 * the completion. 2249 * Returns IBDM_SUCCESS/IBDM_FAILURE 2250 */ 2251 static int 2252 ibdm_send_classportinfo(ibdm_dp_gidinfo_t *gid_info) 2253 { 2254 ibmf_msg_t *msg; 2255 ib_mad_hdr_t *hdr; 2256 ibdm_timeout_cb_args_t *cb_args; 2257 2258 IBTF_DPRINTF_L4("ibdm", 2259 "\tsend_classportinfo: gid info 0x%p", gid_info); 2260 2261 /* 2262 * Send command to get classportinfo attribute. Allocate a IBMF 2263 * packet and initialize the packet. 2264 */ 2265 if (ibmf_alloc_msg(gid_info->gl_ibmf_hdl, IBMF_ALLOC_SLEEP, 2266 &msg) != IBMF_SUCCESS) { 2267 IBTF_DPRINTF_L4("ibdm", "\tsend_classportinfo: pkt alloc fail"); 2268 return (IBDM_FAILURE); 2269 } 2270 2271 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*msg)) 2272 ibdm_alloc_send_buffers(msg); 2273 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*msg)) 2274 2275 msg->im_local_addr.ia_local_lid = gid_info->gl_slid; 2276 msg->im_local_addr.ia_remote_lid = gid_info->gl_dlid; 2277 msg->im_local_addr.ia_remote_qno = 1; 2278 msg->im_local_addr.ia_p_key = gid_info->gl_p_key; 2279 msg->im_local_addr.ia_q_key = IB_GSI_QKEY; 2280 msg->im_local_addr.ia_service_level = gid_info->gl_SL; 2281 2282 hdr = IBDM_OUT_IBMFMSG_MADHDR(msg); 2283 hdr->BaseVersion = MAD_CLASS_BASE_VERS_1; 2284 hdr->MgmtClass = MAD_MGMT_CLASS_DEV_MGT; 2285 hdr->ClassVersion = IB_DM_CLASS_VERSION_1; 2286 hdr->R_Method = IB_DM_DEVMGT_METHOD_GET; 2287 hdr->Status = 0; 2288 hdr->TransactionID = h2b64(gid_info->gl_transactionID); 2289 hdr->AttributeID = h2b16(IB_DM_ATTR_CLASSPORTINFO); 2290 hdr->AttributeModifier = 0; 2291 2292 cb_args = &gid_info->gl_cpi_cb_args; 2293 cb_args->cb_gid_info = gid_info; 2294 cb_args->cb_retry_count = ibdm_dft_retry_cnt; 2295 cb_args->cb_req_type = IBDM_REQ_TYPE_CLASSPORTINFO; 2296 2297 mutex_enter(&gid_info->gl_mutex); 2298 gid_info->gl_timeout_id = timeout(ibdm_pkt_timeout_hdlr, 2299 cb_args, IBDM_TIMEOUT_VALUE(ibdm_dft_timeout)); 2300 mutex_exit(&gid_info->gl_mutex); 2301 2302 IBTF_DPRINTF_L5("ibdm", "\tsend_classportinfo: " 2303 "timeout id %x", gid_info->gl_timeout_id); 2304 2305 if (ibmf_msg_transport(gid_info->gl_ibmf_hdl, gid_info->gl_qp_hdl, 2306 msg, NULL, ibdm_ibmf_send_cb, cb_args, 0) != IBMF_SUCCESS) { 2307 IBTF_DPRINTF_L2("ibdm", 2308 "\tsend_classportinfo: ibmf send failed"); 2309 ibdm_ibmf_send_cb(gid_info->gl_ibmf_hdl, msg, cb_args); 2310 } 2311 2312 return (IBDM_SUCCESS); 2313 } 2314 2315 2316 /* 2317 * ibdm_handle_setclassportinfo() 2318 * Invoked by the IBMF when setClassPortInfo request is completed. 2319 */ 2320 static void 2321 ibdm_handle_setclassportinfo(ibmf_handle_t ibmf_hdl, 2322 ibmf_msg_t *msg, ibdm_dp_gidinfo_t *gid_info, int *flag) 2323 { 2324 void *data; 2325 timeout_id_t timeout_id; 2326 ib_mad_classportinfo_t *cpi; 2327 2328 IBTF_DPRINTF_L4("ibdm", "\thandle_setclassportinfo:ibmf hdl " 2329 "%p msg %p gid info %p", ibmf_hdl, msg, gid_info); 2330 2331 if (IBDM_IN_IBMFMSG_ATTR(msg) != IB_DM_ATTR_CLASSPORTINFO) { 2332 IBTF_DPRINTF_L4("ibdm", "\thandle_setclassportinfo: " 2333 "Not a ClassPortInfo resp"); 2334 *flag |= IBDM_IBMF_PKT_UNEXP_RESP; 2335 return; 2336 } 2337 2338 /* 2339 * Verify whether timeout handler is created/active. 2340 * If created/ active, cancel the timeout handler 2341 */ 2342 mutex_enter(&gid_info->gl_mutex); 2343 if (gid_info->gl_state != IBDM_SET_CLASSPORTINFO) { 2344 IBTF_DPRINTF_L2("ibdm", "\thandle_setclassportinfo:DUP resp"); 2345 *flag |= IBDM_IBMF_PKT_DUP_RESP; 2346 mutex_exit(&gid_info->gl_mutex); 2347 return; 2348 } 2349 ibdm_bump_transactionID(gid_info); 2350 2351 gid_info->gl_iou_cb_args.cb_req_type = 0; 2352 if (gid_info->gl_timeout_id) { 2353 timeout_id = gid_info->gl_timeout_id; 2354 mutex_exit(&gid_info->gl_mutex); 2355 IBTF_DPRINTF_L5("ibdm", "handle_setlassportinfo: " 2356 "gl_timeout_id = 0x%x", timeout_id); 2357 if (untimeout(timeout_id) == -1) { 2358 IBTF_DPRINTF_L2("ibdm", "handle_setclassportinfo: " 2359 "untimeout gl_timeout_id failed"); 2360 } 2361 mutex_enter(&gid_info->gl_mutex); 2362 gid_info->gl_timeout_id = 0; 2363 } 2364 mutex_exit(&gid_info->gl_mutex); 2365 2366 data = msg->im_msgbufs_recv.im_bufs_cl_data; 2367 cpi = (ib_mad_classportinfo_t *)data; 2368 2369 ibdm_dump_classportinfo(cpi); 2370 } 2371 2372 2373 /* 2374 * ibdm_handle_classportinfo() 2375 * Invoked by the IBMF when the classportinfo request is completed. 2376 */ 2377 static void 2378 ibdm_handle_classportinfo(ibmf_handle_t ibmf_hdl, 2379 ibmf_msg_t *msg, ibdm_dp_gidinfo_t *gid_info, int *flag) 2380 { 2381 void *data; 2382 timeout_id_t timeout_id; 2383 ib_mad_hdr_t *hdr; 2384 ib_mad_classportinfo_t *cpi; 2385 2386 IBTF_DPRINTF_L4("ibdm", "\thandle_classportinfo:ibmf hdl " 2387 "%p msg %p gid info %p", ibmf_hdl, msg, gid_info); 2388 2389 if (IBDM_IN_IBMFMSG_ATTR(msg) != IB_DM_ATTR_CLASSPORTINFO) { 2390 IBTF_DPRINTF_L4("ibdm", "\thandle_classportinfo: " 2391 "Not a ClassPortInfo resp"); 2392 *flag |= IBDM_IBMF_PKT_UNEXP_RESP; 2393 return; 2394 } 2395 2396 /* 2397 * Verify whether timeout handler is created/active. 2398 * If created/ active, cancel the timeout handler 2399 */ 2400 mutex_enter(&gid_info->gl_mutex); 2401 ibdm_bump_transactionID(gid_info); 2402 if (gid_info->gl_state != IBDM_GET_CLASSPORTINFO) { 2403 IBTF_DPRINTF_L2("ibdm", "\thandle_classportinfo:DUP resp"); 2404 *flag |= IBDM_IBMF_PKT_DUP_RESP; 2405 mutex_exit(&gid_info->gl_mutex); 2406 return; 2407 } 2408 gid_info->gl_iou_cb_args.cb_req_type = 0; 2409 if (gid_info->gl_timeout_id) { 2410 timeout_id = gid_info->gl_timeout_id; 2411 mutex_exit(&gid_info->gl_mutex); 2412 IBTF_DPRINTF_L5("ibdm", "handle_ioclassportinfo: " 2413 "gl_timeout_id = 0x%x", timeout_id); 2414 if (untimeout(timeout_id) == -1) { 2415 IBTF_DPRINTF_L2("ibdm", "handle_classportinfo: " 2416 "untimeout gl_timeout_id failed"); 2417 } 2418 mutex_enter(&gid_info->gl_mutex); 2419 gid_info->gl_timeout_id = 0; 2420 } 2421 gid_info->gl_state = IBDM_GET_IOUNITINFO; 2422 gid_info->gl_pending_cmds++; 2423 mutex_exit(&gid_info->gl_mutex); 2424 2425 data = msg->im_msgbufs_recv.im_bufs_cl_data; 2426 cpi = (ib_mad_classportinfo_t *)data; 2427 2428 /* 2429 * Cache the "RespTimeValue" and redirection information in the 2430 * global gid list data structure. This cached information will 2431 * be used to send any further requests to the GID. 2432 */ 2433 gid_info->gl_resp_timeout = 2434 (b2h32(cpi->RespTimeValue) & 0x1F); 2435 2436 gid_info->gl_redirected = ((IBDM_IN_IBMFMSG_STATUS(msg) & 2437 MAD_STATUS_REDIRECT_REQUIRED) ? B_TRUE: B_FALSE); 2438 gid_info->gl_redirect_dlid = b2h16(cpi->RedirectLID); 2439 gid_info->gl_redirect_QP = (b2h32(cpi->RedirectQP) & 0xffffff); 2440 gid_info->gl_redirect_pkey = b2h16(cpi->RedirectP_Key); 2441 gid_info->gl_redirect_qkey = b2h32(cpi->RedirectQ_Key); 2442 gid_info->gl_redirectGID_hi = b2h64(cpi->RedirectGID_hi); 2443 gid_info->gl_redirectGID_lo = b2h64(cpi->RedirectGID_lo); 2444 gid_info->gl_redirectSL = cpi->RedirectSL; 2445 2446 ibdm_dump_classportinfo(cpi); 2447 2448 /* 2449 * Send IOUnitInfo request 2450 * Reuse previously allocated IBMF packet for sending ClassPortInfo 2451 * Check whether DM agent on the remote node requested redirection 2452 * If so, send the request to the redirect DGID/DLID/PKEY/QP. 2453 */ 2454 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*msg)) 2455 ibdm_alloc_send_buffers(msg); 2456 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*msg)) 2457 msg->im_local_addr.ia_local_lid = gid_info->gl_slid; 2458 msg->im_local_addr.ia_remote_lid = gid_info->gl_dlid; 2459 2460 if (gid_info->gl_redirected == B_TRUE) { 2461 if (gid_info->gl_redirect_dlid != 0) { 2462 msg->im_local_addr.ia_remote_lid = 2463 gid_info->gl_redirect_dlid; 2464 } 2465 msg->im_local_addr.ia_remote_qno = gid_info->gl_redirect_QP; 2466 msg->im_local_addr.ia_p_key = gid_info->gl_redirect_pkey; 2467 msg->im_local_addr.ia_q_key = gid_info->gl_redirect_qkey; 2468 msg->im_local_addr.ia_service_level = gid_info->gl_redirectSL; 2469 } else { 2470 msg->im_local_addr.ia_remote_qno = 1; 2471 msg->im_local_addr.ia_p_key = gid_info->gl_p_key; 2472 msg->im_local_addr.ia_q_key = IB_GSI_QKEY; 2473 msg->im_local_addr.ia_service_level = gid_info->gl_SL; 2474 } 2475 2476 hdr = IBDM_OUT_IBMFMSG_MADHDR(msg); 2477 hdr->BaseVersion = MAD_CLASS_BASE_VERS_1; 2478 hdr->MgmtClass = MAD_MGMT_CLASS_DEV_MGT; 2479 hdr->ClassVersion = IB_DM_CLASS_VERSION_1; 2480 hdr->R_Method = IB_DM_DEVMGT_METHOD_GET; 2481 hdr->Status = 0; 2482 hdr->TransactionID = h2b64(gid_info->gl_transactionID); 2483 hdr->AttributeID = h2b16(IB_DM_ATTR_IO_UNITINFO); 2484 hdr->AttributeModifier = 0; 2485 2486 gid_info->gl_iou_cb_args.cb_req_type = IBDM_REQ_TYPE_IOUINFO; 2487 gid_info->gl_iou_cb_args.cb_gid_info = gid_info; 2488 gid_info->gl_iou_cb_args.cb_retry_count = ibdm_dft_retry_cnt; 2489 2490 mutex_enter(&gid_info->gl_mutex); 2491 gid_info->gl_timeout_id = timeout(ibdm_pkt_timeout_hdlr, 2492 &gid_info->gl_iou_cb_args, IBDM_TIMEOUT_VALUE(ibdm_dft_timeout)); 2493 mutex_exit(&gid_info->gl_mutex); 2494 2495 IBTF_DPRINTF_L5("ibdm", "handle_classportinfo:" 2496 "timeout %x", gid_info->gl_timeout_id); 2497 2498 if (ibmf_msg_transport(ibmf_hdl, gid_info->gl_qp_hdl, msg, NULL, 2499 ibdm_ibmf_send_cb, &gid_info->gl_iou_cb_args, 0) != IBMF_SUCCESS) { 2500 IBTF_DPRINTF_L2("ibdm", 2501 "\thandle_classportinfo: msg transport failed"); 2502 ibdm_ibmf_send_cb(ibmf_hdl, msg, &gid_info->gl_iou_cb_args); 2503 } 2504 (*flag) |= IBDM_IBMF_PKT_REUSED; 2505 } 2506 2507 2508 /* 2509 * ibdm_send_iounitinfo: 2510 * Sends a DM request to get IOU unitinfo. 2511 */ 2512 static int 2513 ibdm_send_iounitinfo(ibdm_dp_gidinfo_t *gid_info) 2514 { 2515 ibmf_msg_t *msg; 2516 ib_mad_hdr_t *hdr; 2517 2518 IBTF_DPRINTF_L4("ibdm", "\tsend_iounitinfo: gid info 0x%p", gid_info); 2519 2520 /* 2521 * Send command to get iounitinfo attribute. Allocate a IBMF 2522 * packet and initialize the packet. 2523 */ 2524 if (ibmf_alloc_msg(gid_info->gl_ibmf_hdl, IBMF_ALLOC_SLEEP, &msg) != 2525 IBMF_SUCCESS) { 2526 IBTF_DPRINTF_L4("ibdm", "\tsend_iounitinfo: pkt alloc fail"); 2527 return (IBDM_FAILURE); 2528 } 2529 2530 mutex_enter(&gid_info->gl_mutex); 2531 ibdm_bump_transactionID(gid_info); 2532 mutex_exit(&gid_info->gl_mutex); 2533 2534 2535 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*msg)) 2536 ibdm_alloc_send_buffers(msg); 2537 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*msg)) 2538 msg->im_local_addr.ia_local_lid = gid_info->gl_slid; 2539 msg->im_local_addr.ia_remote_lid = gid_info->gl_dlid; 2540 msg->im_local_addr.ia_remote_qno = 1; 2541 msg->im_local_addr.ia_p_key = gid_info->gl_p_key; 2542 msg->im_local_addr.ia_q_key = IB_GSI_QKEY; 2543 msg->im_local_addr.ia_service_level = gid_info->gl_SL; 2544 2545 hdr = IBDM_OUT_IBMFMSG_MADHDR(msg); 2546 hdr->BaseVersion = MAD_CLASS_BASE_VERS_1; 2547 hdr->MgmtClass = MAD_MGMT_CLASS_DEV_MGT; 2548 hdr->ClassVersion = IB_DM_CLASS_VERSION_1; 2549 hdr->R_Method = IB_DM_DEVMGT_METHOD_GET; 2550 hdr->Status = 0; 2551 hdr->TransactionID = h2b64(gid_info->gl_transactionID); 2552 hdr->AttributeID = h2b16(IB_DM_ATTR_IO_UNITINFO); 2553 hdr->AttributeModifier = 0; 2554 2555 gid_info->gl_iou_cb_args.cb_gid_info = gid_info; 2556 gid_info->gl_iou_cb_args.cb_retry_count = ibdm_dft_retry_cnt; 2557 gid_info->gl_iou_cb_args.cb_req_type = IBDM_REQ_TYPE_IOUINFO; 2558 2559 mutex_enter(&gid_info->gl_mutex); 2560 gid_info->gl_timeout_id = timeout(ibdm_pkt_timeout_hdlr, 2561 &gid_info->gl_iou_cb_args, IBDM_TIMEOUT_VALUE(ibdm_dft_timeout)); 2562 mutex_exit(&gid_info->gl_mutex); 2563 2564 IBTF_DPRINTF_L5("ibdm", "send_iouunitinfo:" 2565 "timeout %x", gid_info->gl_timeout_id); 2566 2567 if (ibmf_msg_transport(gid_info->gl_ibmf_hdl, gid_info->gl_qp_hdl, msg, 2568 NULL, ibdm_ibmf_send_cb, &gid_info->gl_iou_cb_args, 0) != 2569 IBMF_SUCCESS) { 2570 IBTF_DPRINTF_L2("ibdm", "\tsend_iounitinfo: ibmf send failed"); 2571 ibdm_ibmf_send_cb(gid_info->gl_ibmf_hdl, 2572 msg, &gid_info->gl_iou_cb_args); 2573 } 2574 return (IBDM_SUCCESS); 2575 } 2576 2577 /* 2578 * ibdm_handle_iounitinfo() 2579 * Invoked by the IBMF when IO Unitinfo request is completed. 2580 */ 2581 static void 2582 ibdm_handle_iounitinfo(ibmf_handle_t ibmf_hdl, 2583 ibmf_msg_t *msg, ibdm_dp_gidinfo_t *gid_info, int *flag) 2584 { 2585 int ii, first = B_TRUE; 2586 int num_iocs; 2587 size_t size; 2588 uchar_t slot_info; 2589 timeout_id_t timeout_id; 2590 ib_mad_hdr_t *hdr; 2591 ibdm_ioc_info_t *ioc_info; 2592 ib_dm_io_unitinfo_t *iou_info; 2593 ib_dm_io_unitinfo_t *giou_info; 2594 ibdm_timeout_cb_args_t *cb_args; 2595 2596 IBTF_DPRINTF_L4("ibdm", "\thandle_iouintinfo:" 2597 " ibmf hdl %p pkt %p gid info %p", ibmf_hdl, msg, gid_info); 2598 2599 if (IBDM_IN_IBMFMSG_ATTR(msg) != IB_DM_ATTR_IO_UNITINFO) { 2600 IBTF_DPRINTF_L4("ibdm", "\thandle_iounitinfo: " 2601 "Unexpected response"); 2602 (*flag) |= IBDM_IBMF_PKT_UNEXP_RESP; 2603 return; 2604 } 2605 2606 mutex_enter(&gid_info->gl_mutex); 2607 if (gid_info->gl_state != IBDM_GET_IOUNITINFO) { 2608 IBTF_DPRINTF_L4("ibdm", 2609 "\thandle_iounitinfo: DUP resp"); 2610 mutex_exit(&gid_info->gl_mutex); 2611 (*flag) = IBDM_IBMF_PKT_DUP_RESP; 2612 return; 2613 } 2614 gid_info->gl_iou_cb_args.cb_req_type = 0; 2615 if (gid_info->gl_timeout_id) { 2616 timeout_id = gid_info->gl_timeout_id; 2617 mutex_exit(&gid_info->gl_mutex); 2618 IBTF_DPRINTF_L5("ibdm", "handle_iounitinfo: " 2619 "gl_timeout_id = 0x%x", timeout_id); 2620 if (untimeout(timeout_id) == -1) { 2621 IBTF_DPRINTF_L2("ibdm", "handle_iounitinfo: " 2622 "untimeout gl_timeout_id failed"); 2623 } 2624 mutex_enter(&gid_info->gl_mutex); 2625 gid_info->gl_timeout_id = 0; 2626 } 2627 gid_info->gl_state = IBDM_GET_IOC_DETAILS; 2628 2629 iou_info = IBDM_IN_IBMFMSG2IOU(msg); 2630 ibdm_dump_iounitinfo(iou_info); 2631 num_iocs = iou_info->iou_num_ctrl_slots; 2632 /* 2633 * check if number of IOCs reported is zero? if yes, return. 2634 * when num_iocs are reported zero internal IOC database needs 2635 * to be updated. To ensure that save the number of IOCs in 2636 * the new field "gl_num_iocs". Use a new field instead of 2637 * "giou_info->iou_num_ctrl_slots" as that would prevent 2638 * an unnecessary kmem_alloc/kmem_free when num_iocs is 0. 2639 */ 2640 if (num_iocs == 0 && gid_info->gl_num_iocs == 0) { 2641 IBTF_DPRINTF_L4("ibdm", "\thandle_iounitinfo: no IOC's"); 2642 mutex_exit(&gid_info->gl_mutex); 2643 return; 2644 } 2645 IBTF_DPRINTF_L4("ibdm", "\thandle_iounitinfo: num_iocs = %d", num_iocs); 2646 2647 /* 2648 * if there is an existing gl_iou (IOU has been probed before) 2649 * check if the "iou_changeid" is same as saved entry in 2650 * "giou_info->iou_changeid". 2651 * (note: this logic can prevent IOC enumeration if a given 2652 * vendor doesn't support setting iou_changeid field for its IOU) 2653 * 2654 * if there is an existing gl_iou and iou_changeid has changed : 2655 * free up existing gl_iou info and its related structures. 2656 * reallocate gl_iou info all over again. 2657 * if we donot free this up; then this leads to memory leaks 2658 */ 2659 if (gid_info->gl_iou) { 2660 giou_info = &gid_info->gl_iou->iou_info; 2661 if (b2h16(iou_info->iou_changeid) == 2662 giou_info->iou_changeid) { 2663 IBTF_DPRINTF_L3("ibdm", 2664 "\thandle_iounitinfo: no IOCs changed"); 2665 gid_info->gl_state = IBDM_GID_PROBING_COMPLETE; 2666 mutex_exit(&gid_info->gl_mutex); 2667 return; 2668 } 2669 2670 /* 2671 * Store the iou info as prev_iou to be used after 2672 * sweep is done. 2673 */ 2674 ASSERT(gid_info->gl_prev_iou == NULL); 2675 IBTF_DPRINTF_L4(ibdm_string, 2676 "\thandle_iounitinfo: setting gl_prev_iou %p", 2677 gid_info->gl_prev_iou); 2678 gid_info->gl_prev_iou = gid_info->gl_iou; 2679 ibdm.ibdm_prev_iou = 1; 2680 gid_info->gl_iou = NULL; 2681 } 2682 2683 size = sizeof (ibdm_iou_info_t) + num_iocs * sizeof (ibdm_ioc_info_t); 2684 gid_info->gl_iou = (ibdm_iou_info_t *)kmem_zalloc(size, KM_SLEEP); 2685 giou_info = &gid_info->gl_iou->iou_info; 2686 gid_info->gl_iou->iou_ioc_info = (ibdm_ioc_info_t *) 2687 ((char *)gid_info->gl_iou + sizeof (ibdm_iou_info_t)); 2688 2689 giou_info->iou_num_ctrl_slots = gid_info->gl_num_iocs = num_iocs; 2690 giou_info->iou_flag = iou_info->iou_flag; 2691 bcopy(iou_info->iou_ctrl_list, giou_info->iou_ctrl_list, 128); 2692 giou_info->iou_changeid = b2h16(iou_info->iou_changeid); 2693 gid_info->gl_pending_cmds++; /* for diag code */ 2694 mutex_exit(&gid_info->gl_mutex); 2695 2696 if (ibdm_get_diagcode(gid_info, 0) != IBDM_SUCCESS) { 2697 mutex_enter(&gid_info->gl_mutex); 2698 gid_info->gl_pending_cmds--; 2699 mutex_exit(&gid_info->gl_mutex); 2700 } 2701 /* 2702 * Parallelize getting IOC controller profiles from here. 2703 * Allocate IBMF packets and send commands to get IOC profile for 2704 * each IOC present on the IOU. 2705 */ 2706 for (ii = 0; ii < num_iocs; ii++) { 2707 /* 2708 * Check whether IOC is present in the slot 2709 * Series of nibbles (in the field iou_ctrl_list) represents 2710 * a slot in the IOU. 2711 * Byte format: 76543210 2712 * Bits 0-3 of first byte represent Slot 2 2713 * bits 4-7 of first byte represent slot 1, 2714 * bits 0-3 of second byte represent slot 4 and so on 2715 * Each 4-bit nibble has the following meaning 2716 * 0x0 : IOC not installed 2717 * 0x1 : IOC is present 2718 * 0xf : Slot does not exist 2719 * and all other values are reserved. 2720 */ 2721 ioc_info = IBDM_GIDINFO2IOCINFO(gid_info, ii); 2722 slot_info = giou_info->iou_ctrl_list[(ii/2)]; 2723 if ((ii % 2) == 0) 2724 slot_info = (slot_info >> 4); 2725 2726 if ((slot_info & 0xf) != 1) { 2727 IBTF_DPRINTF_L4("ibdm", "\thandle_iouintinfo: " 2728 "No IOC is present in the slot = %d", ii); 2729 ioc_info->ioc_state = IBDM_IOC_STATE_PROBE_FAILED; 2730 continue; 2731 } 2732 2733 mutex_enter(&gid_info->gl_mutex); 2734 ibdm_bump_transactionID(gid_info); 2735 mutex_exit(&gid_info->gl_mutex); 2736 2737 /* 2738 * Re use the already allocated packet (for IOUnitinfo) to 2739 * send the first IOC controller attribute. Allocate new 2740 * IBMF packets for the rest of the IOC's 2741 */ 2742 if (first != B_TRUE) { 2743 msg = NULL; 2744 if (ibmf_alloc_msg(ibmf_hdl, IBMF_ALLOC_SLEEP, 2745 &msg) != IBMF_SUCCESS) { 2746 IBTF_DPRINTF_L4("ibdm", "\thandle_iouintinfo: " 2747 "IBMF packet allocation failed"); 2748 continue; 2749 } 2750 2751 } 2752 2753 /* allocate send buffers for all messages */ 2754 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*msg)) 2755 ibdm_alloc_send_buffers(msg); 2756 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*msg)) 2757 2758 msg->im_local_addr.ia_local_lid = gid_info->gl_slid; 2759 msg->im_local_addr.ia_remote_lid = gid_info->gl_dlid; 2760 if (gid_info->gl_redirected == B_TRUE) { 2761 if (gid_info->gl_redirect_dlid != 0) { 2762 msg->im_local_addr.ia_remote_lid = 2763 gid_info->gl_redirect_dlid; 2764 } 2765 msg->im_local_addr.ia_remote_qno = 2766 gid_info->gl_redirect_QP; 2767 msg->im_local_addr.ia_p_key = 2768 gid_info->gl_redirect_pkey; 2769 msg->im_local_addr.ia_q_key = 2770 gid_info->gl_redirect_qkey; 2771 msg->im_local_addr.ia_service_level = 2772 gid_info->gl_redirectSL; 2773 } else { 2774 msg->im_local_addr.ia_remote_qno = 1; 2775 msg->im_local_addr.ia_p_key = gid_info->gl_p_key; 2776 msg->im_local_addr.ia_q_key = IB_GSI_QKEY; 2777 msg->im_local_addr.ia_service_level = gid_info->gl_SL; 2778 } 2779 2780 hdr = IBDM_OUT_IBMFMSG_MADHDR(msg); 2781 hdr->BaseVersion = MAD_CLASS_BASE_VERS_1; 2782 hdr->MgmtClass = MAD_MGMT_CLASS_DEV_MGT; 2783 hdr->ClassVersion = IB_DM_CLASS_VERSION_1; 2784 hdr->R_Method = IB_DM_DEVMGT_METHOD_GET; 2785 hdr->Status = 0; 2786 hdr->TransactionID = h2b64(gid_info->gl_transactionID); 2787 hdr->AttributeID = h2b16(IB_DM_ATTR_IOC_CTRL_PROFILE); 2788 hdr->AttributeModifier = h2b32(ii + 1); 2789 2790 ioc_info->ioc_state = IBDM_IOC_STATE_PROBE_INVALID; 2791 cb_args = &ioc_info->ioc_cb_args; 2792 cb_args->cb_gid_info = gid_info; 2793 cb_args->cb_retry_count = ibdm_dft_retry_cnt; 2794 cb_args->cb_req_type = IBDM_REQ_TYPE_IOCINFO; 2795 cb_args->cb_ioc_num = ii; 2796 2797 mutex_enter(&gid_info->gl_mutex); 2798 gid_info->gl_pending_cmds++; /* for diag code */ 2799 2800 ioc_info->ioc_timeout_id = timeout(ibdm_pkt_timeout_hdlr, 2801 cb_args, IBDM_TIMEOUT_VALUE(ibdm_dft_timeout)); 2802 mutex_exit(&gid_info->gl_mutex); 2803 2804 IBTF_DPRINTF_L5("ibdm", "\thandle_iounitinfo:" 2805 "timeout 0x%x, ioc_num %d", ioc_info->ioc_timeout_id, ii); 2806 2807 if (ibmf_msg_transport(ibmf_hdl, gid_info->gl_qp_hdl, msg, 2808 NULL, ibdm_ibmf_send_cb, cb_args, 0) != IBMF_SUCCESS) { 2809 IBTF_DPRINTF_L2("ibdm", 2810 "\thandle_iounitinfo: msg transport failed"); 2811 ibdm_ibmf_send_cb(ibmf_hdl, msg, cb_args); 2812 } 2813 (*flag) |= IBDM_IBMF_PKT_REUSED; 2814 first = B_FALSE; 2815 gid_info->gl_iou->iou_niocs_probe_in_progress++; 2816 } 2817 } 2818 2819 2820 /* 2821 * ibdm_handle_ioc_profile() 2822 * Invoked by the IBMF when the IOCControllerProfile request 2823 * gets completed 2824 */ 2825 static void 2826 ibdm_handle_ioc_profile(ibmf_handle_t ibmf_hdl, 2827 ibmf_msg_t *msg, ibdm_dp_gidinfo_t *gid_info, int *flag) 2828 { 2829 int first = B_TRUE, reprobe = 0; 2830 uint_t ii, ioc_no, srv_start; 2831 uint_t nserv_entries; 2832 timeout_id_t timeout_id; 2833 ib_mad_hdr_t *hdr; 2834 ibdm_ioc_info_t *ioc_info; 2835 ibdm_timeout_cb_args_t *cb_args; 2836 ib_dm_ioc_ctrl_profile_t *ioc, *gioc; 2837 2838 IBTF_DPRINTF_L4("ibdm", "\thandle_ioc_profile:" 2839 " ibmf hdl %p msg %p gid info %p", ibmf_hdl, msg, gid_info); 2840 2841 ioc = IBDM_IN_IBMFMSG2IOC(msg); 2842 /* 2843 * Check whether we know this IOC already 2844 * This will return NULL if reprobe is in progress 2845 * IBDM_IOC_STATE_REPROBE_PROGRESS will be set. 2846 * Do not hold mutexes here. 2847 */ 2848 if (ibdm_is_ioc_present(ioc->ioc_guid, gid_info, flag) != NULL) { 2849 IBTF_DPRINTF_L4("ibdm", "\thandle_ioc_profile:" 2850 "IOC guid %llx is present", ioc->ioc_guid); 2851 return; 2852 } 2853 ioc_no = IBDM_IN_IBMFMSG_ATTRMOD(msg); 2854 IBTF_DPRINTF_L4("ibdm", "\thandle_ioc_profile: ioc_no = %d", ioc_no-1); 2855 2856 /* Make sure that IOC index is with the valid range */ 2857 if (IBDM_IS_IOC_NUM_INVALID(ioc_no, gid_info)) { 2858 IBTF_DPRINTF_L2("ibdm", "\thandle_ioc_profile: " 2859 "IOC index Out of range, index %d", ioc); 2860 (*flag) |= IBDM_IBMF_PKT_UNEXP_RESP; 2861 return; 2862 } 2863 ioc_info = &gid_info->gl_iou->iou_ioc_info[ioc_no - 1]; 2864 ioc_info->ioc_iou_info = gid_info->gl_iou; 2865 2866 mutex_enter(&gid_info->gl_mutex); 2867 if (ioc_info->ioc_state == IBDM_IOC_STATE_REPROBE_PROGRESS) { 2868 reprobe = 1; 2869 ioc_info->ioc_prev_serv = ioc_info->ioc_serv; 2870 ioc_info->ioc_serv = NULL; 2871 ioc_info->ioc_prev_serv_cnt = 2872 ioc_info->ioc_profile.ioc_service_entries; 2873 } else if (ioc_info->ioc_state != IBDM_IOC_STATE_PROBE_INVALID) { 2874 IBTF_DPRINTF_L2("ibdm", "\thandle_ioc_profile: DUP response" 2875 "ioc %d, ioc_state %x", ioc_no - 1, ioc_info->ioc_state); 2876 mutex_exit(&gid_info->gl_mutex); 2877 (*flag) |= IBDM_IBMF_PKT_DUP_RESP; 2878 return; 2879 } 2880 ioc_info->ioc_cb_args.cb_req_type = 0; 2881 if (ioc_info->ioc_timeout_id) { 2882 timeout_id = ioc_info->ioc_timeout_id; 2883 ioc_info->ioc_timeout_id = 0; 2884 mutex_exit(&gid_info->gl_mutex); 2885 IBTF_DPRINTF_L5("ibdm", "handle_ioc_profile: " 2886 "ioc_timeout_id = 0x%x", timeout_id); 2887 if (untimeout(timeout_id) == -1) { 2888 IBTF_DPRINTF_L2("ibdm", "handle_ioc_profile: " 2889 "untimeout ioc_timeout_id failed"); 2890 } 2891 mutex_enter(&gid_info->gl_mutex); 2892 } 2893 2894 ioc_info->ioc_state = IBDM_IOC_STATE_PROBE_SUCCESS; 2895 if (reprobe == 0) { 2896 ioc_info->ioc_iou_guid = gid_info->gl_nodeguid; 2897 ioc_info->ioc_nodeguid = gid_info->gl_nodeguid; 2898 } 2899 2900 /* 2901 * Save all the IOC information in the global structures. 2902 * Note the wire format is Big Endian and the Sparc process also 2903 * big endian. So, there is no need to convert the data fields 2904 * The conversion routines used below are ineffective on Sparc 2905 * machines where as they will be effective on little endian 2906 * machines such as Intel processors. 2907 */ 2908 gioc = (ib_dm_ioc_ctrl_profile_t *)&ioc_info->ioc_profile; 2909 2910 /* 2911 * Restrict updates to onlyport GIDs and service entries during reprobe 2912 */ 2913 if (reprobe == 0) { 2914 gioc->ioc_guid = b2h64(ioc->ioc_guid); 2915 gioc->ioc_vendorid = 2916 ((b2h32(ioc->ioc_vendorid) & IB_DM_VENDORID_MASK) 2917 >> IB_DM_VENDORID_SHIFT); 2918 gioc->ioc_deviceid = b2h32(ioc->ioc_deviceid); 2919 gioc->ioc_device_ver = b2h16(ioc->ioc_device_ver); 2920 gioc->ioc_subsys_vendorid = 2921 ((b2h32(ioc->ioc_subsys_vendorid) & IB_DM_VENDORID_MASK) 2922 >> IB_DM_VENDORID_SHIFT); 2923 gioc->ioc_subsys_id = b2h32(ioc->ioc_subsys_id); 2924 gioc->ioc_io_class = b2h16(ioc->ioc_io_class); 2925 gioc->ioc_io_subclass = b2h16(ioc->ioc_io_subclass); 2926 gioc->ioc_protocol = b2h16(ioc->ioc_protocol); 2927 gioc->ioc_protocol_ver = b2h16(ioc->ioc_protocol_ver); 2928 gioc->ioc_send_msg_qdepth = 2929 b2h16(ioc->ioc_send_msg_qdepth); 2930 gioc->ioc_rdma_read_qdepth = 2931 b2h16(ioc->ioc_rdma_read_qdepth); 2932 gioc->ioc_send_msg_sz = b2h32(ioc->ioc_send_msg_sz); 2933 gioc->ioc_rdma_xfer_sz = b2h32(ioc->ioc_rdma_xfer_sz); 2934 gioc->ioc_ctrl_opcap_mask = ioc->ioc_ctrl_opcap_mask; 2935 bcopy(ioc->ioc_id_string, gioc->ioc_id_string, 2936 IB_DM_IOC_ID_STRING_LEN); 2937 2938 ioc_info->ioc_iou_diagcode = gid_info->gl_iou->iou_diagcode; 2939 ioc_info->ioc_iou_dc_valid = gid_info->gl_iou->iou_dc_valid; 2940 ioc_info->ioc_diagdeviceid = (IB_DM_IOU_DEVICEID_MASK & 2941 gid_info->gl_iou->iou_info.iou_flag) ? B_TRUE : B_FALSE; 2942 2943 if (ioc_info->ioc_diagdeviceid == B_TRUE) { 2944 gid_info->gl_pending_cmds++; 2945 IBTF_DPRINTF_L3(ibdm_string, 2946 "\tibdm_handle_ioc_profile: " 2947 "%d: gid_info %p gl_state %d pending_cmds %d", 2948 __LINE__, gid_info, gid_info->gl_state, 2949 gid_info->gl_pending_cmds); 2950 } 2951 } 2952 gioc->ioc_service_entries = ioc->ioc_service_entries; 2953 mutex_exit(&gid_info->gl_mutex); 2954 2955 ibdm_dump_ioc_profile(gioc); 2956 2957 if ((ioc_info->ioc_diagdeviceid == B_TRUE) && (reprobe == 0)) { 2958 if (ibdm_get_diagcode(gid_info, ioc_no) != IBDM_SUCCESS) { 2959 mutex_enter(&gid_info->gl_mutex); 2960 gid_info->gl_pending_cmds--; 2961 mutex_exit(&gid_info->gl_mutex); 2962 } 2963 } 2964 ioc_info->ioc_serv = (ibdm_srvents_info_t *)kmem_zalloc( 2965 (gioc->ioc_service_entries * sizeof (ibdm_srvents_info_t)), 2966 KM_SLEEP); 2967 2968 /* 2969 * In one single request, maximum number of requests that can be 2970 * obtained is 4. If number of service entries are more than four, 2971 * calculate number requests needed and send them parallelly. 2972 */ 2973 nserv_entries = ioc->ioc_service_entries; 2974 ii = 0; 2975 while (nserv_entries) { 2976 mutex_enter(&gid_info->gl_mutex); 2977 gid_info->gl_pending_cmds++; 2978 ibdm_bump_transactionID(gid_info); 2979 mutex_exit(&gid_info->gl_mutex); 2980 2981 if (first != B_TRUE) { 2982 if (ibmf_alloc_msg(ibmf_hdl, IBMF_ALLOC_SLEEP, 2983 &msg) != IBMF_SUCCESS) { 2984 continue; 2985 } 2986 2987 } 2988 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*msg)) 2989 ibdm_alloc_send_buffers(msg); 2990 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*msg)) 2991 msg->im_local_addr.ia_local_lid = gid_info->gl_slid; 2992 msg->im_local_addr.ia_remote_lid = gid_info->gl_dlid; 2993 if (gid_info->gl_redirected == B_TRUE) { 2994 if (gid_info->gl_redirect_dlid != 0) { 2995 msg->im_local_addr.ia_remote_lid = 2996 gid_info->gl_redirect_dlid; 2997 } 2998 msg->im_local_addr.ia_remote_qno = 2999 gid_info->gl_redirect_QP; 3000 msg->im_local_addr.ia_p_key = 3001 gid_info->gl_redirect_pkey; 3002 msg->im_local_addr.ia_q_key = 3003 gid_info->gl_redirect_qkey; 3004 msg->im_local_addr.ia_service_level = 3005 gid_info->gl_redirectSL; 3006 } else { 3007 msg->im_local_addr.ia_remote_qno = 1; 3008 msg->im_local_addr.ia_p_key = gid_info->gl_p_key; 3009 msg->im_local_addr.ia_q_key = IB_GSI_QKEY; 3010 msg->im_local_addr.ia_service_level = gid_info->gl_SL; 3011 } 3012 3013 hdr = IBDM_OUT_IBMFMSG_MADHDR(msg); 3014 hdr->BaseVersion = MAD_CLASS_BASE_VERS_1; 3015 hdr->MgmtClass = MAD_MGMT_CLASS_DEV_MGT; 3016 hdr->ClassVersion = IB_DM_CLASS_VERSION_1; 3017 hdr->R_Method = IB_DM_DEVMGT_METHOD_GET; 3018 hdr->Status = 0; 3019 hdr->TransactionID = h2b64(gid_info->gl_transactionID); 3020 hdr->AttributeID = h2b16(IB_DM_ATTR_SERVICE_ENTRIES); 3021 3022 srv_start = ii * 4; 3023 cb_args = &ioc_info->ioc_serv[srv_start].se_cb_args; 3024 cb_args->cb_gid_info = gid_info; 3025 cb_args->cb_retry_count = ibdm_dft_retry_cnt; 3026 cb_args->cb_req_type = IBDM_REQ_TYPE_SRVENTS; 3027 cb_args->cb_srvents_start = srv_start; 3028 cb_args->cb_ioc_num = ioc_no - 1; 3029 3030 if (nserv_entries >= IBDM_MAX_SERV_ENTRIES_PER_REQ) { 3031 nserv_entries -= IBDM_MAX_SERV_ENTRIES_PER_REQ; 3032 cb_args->cb_srvents_end = (cb_args->cb_srvents_start + 3033 IBDM_MAX_SERV_ENTRIES_PER_REQ - 1); 3034 } else { 3035 cb_args->cb_srvents_end = 3036 (cb_args->cb_srvents_start + nserv_entries - 1); 3037 nserv_entries = 0; 3038 } 3039 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*hdr)) 3040 ibdm_fill_srv_attr_mod(hdr, cb_args); 3041 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*hdr)) 3042 3043 mutex_enter(&gid_info->gl_mutex); 3044 ioc_info->ioc_serv[srv_start].se_timeout_id = timeout( 3045 ibdm_pkt_timeout_hdlr, cb_args, 3046 IBDM_TIMEOUT_VALUE(ibdm_dft_timeout)); 3047 mutex_exit(&gid_info->gl_mutex); 3048 3049 IBTF_DPRINTF_L5("ibdm", "\thandle_ioc_profile:" 3050 "timeout %x, ioc %d srv %d", 3051 ioc_info->ioc_serv[srv_start].se_timeout_id, 3052 ioc_no - 1, srv_start); 3053 3054 if (ibmf_msg_transport(ibmf_hdl, gid_info->gl_qp_hdl, msg, 3055 NULL, ibdm_ibmf_send_cb, cb_args, 0) != IBMF_SUCCESS) { 3056 IBTF_DPRINTF_L2("ibdm", 3057 "\thandle_ioc_profile: msg send failed"); 3058 ibdm_ibmf_send_cb(ibmf_hdl, msg, cb_args); 3059 } 3060 (*flag) |= IBDM_IBMF_PKT_REUSED; 3061 first = B_FALSE; 3062 ii++; 3063 } 3064 } 3065 3066 3067 /* 3068 * ibdm_handle_srventry_mad() 3069 */ 3070 static void 3071 ibdm_handle_srventry_mad(ibmf_msg_t *msg, 3072 ibdm_dp_gidinfo_t *gid_info, int *flag) 3073 { 3074 uint_t ii, ioc_no, attrmod; 3075 uint_t nentries, start, end; 3076 timeout_id_t timeout_id; 3077 ib_dm_srv_t *srv_ents; 3078 ibdm_ioc_info_t *ioc_info; 3079 ibdm_srvents_info_t *gsrv_ents; 3080 3081 IBTF_DPRINTF_L4("ibdm", "\thandle_srventry_mad:" 3082 " IBMF msg %p gid info %p", msg, gid_info); 3083 3084 srv_ents = IBDM_IN_IBMFMSG2SRVENT(msg); 3085 /* 3086 * Get the start and end index of the service entries 3087 * Upper 16 bits identify the IOC 3088 * Lower 16 bits specify the range of service entries 3089 * LSB specifies (Big endian) end of the range 3090 * MSB specifies (Big endian) start of the range 3091 */ 3092 attrmod = IBDM_IN_IBMFMSG_ATTRMOD(msg); 3093 ioc_no = ((attrmod >> 16) & IBDM_16_BIT_MASK); 3094 end = ((attrmod >> 8) & IBDM_8_BIT_MASK); 3095 start = (attrmod & IBDM_8_BIT_MASK); 3096 3097 /* Make sure that IOC index is with the valid range */ 3098 if ((ioc_no < 1) | 3099 (ioc_no > gid_info->gl_iou->iou_info.iou_num_ctrl_slots)) { 3100 IBTF_DPRINTF_L2("ibdm", "\thandle_srventry_mad: " 3101 "IOC index Out of range, index %d", ioc_no); 3102 (*flag) |= IBDM_IBMF_PKT_UNEXP_RESP; 3103 return; 3104 } 3105 ioc_info = IBDM_GIDINFO2IOCINFO(gid_info, (ioc_no -1)); 3106 3107 /* 3108 * Make sure that the "start" and "end" service indexes are 3109 * with in the valid range 3110 */ 3111 nentries = ioc_info->ioc_profile.ioc_service_entries; 3112 if ((start > end) | (start >= nentries) | (end >= nentries)) { 3113 IBTF_DPRINTF_L2("ibdm", "\thandle_srventry_mad: " 3114 "Attr modifier 0x%x, #Serv entries %d", attrmod, nentries); 3115 (*flag) |= IBDM_IBMF_PKT_UNEXP_RESP; 3116 return; 3117 } 3118 gsrv_ents = &ioc_info->ioc_serv[start]; 3119 mutex_enter(&gid_info->gl_mutex); 3120 if (gsrv_ents->se_state != IBDM_SE_INVALID) { 3121 IBTF_DPRINTF_L2("ibdm", "\thandle_srventry_mad: " 3122 "already known, ioc %d, srv %d, se_state %x", 3123 ioc_no - 1, start, gsrv_ents->se_state); 3124 mutex_exit(&gid_info->gl_mutex); 3125 (*flag) |= IBDM_IBMF_PKT_DUP_RESP; 3126 return; 3127 } 3128 ioc_info->ioc_serv[start].se_cb_args.cb_req_type = 0; 3129 if (ioc_info->ioc_serv[start].se_timeout_id) { 3130 IBTF_DPRINTF_L2("ibdm", 3131 "\thandle_srventry_mad: ioc %d start %d", ioc_no, start); 3132 timeout_id = ioc_info->ioc_serv[start].se_timeout_id; 3133 ioc_info->ioc_serv[start].se_timeout_id = 0; 3134 mutex_exit(&gid_info->gl_mutex); 3135 IBTF_DPRINTF_L5("ibdm", "handle_srverntry_mad: " 3136 "se_timeout_id = 0x%x", timeout_id); 3137 if (untimeout(timeout_id) == -1) { 3138 IBTF_DPRINTF_L2("ibdm", "handle_srventry_mad: " 3139 "untimeout se_timeout_id failed"); 3140 } 3141 mutex_enter(&gid_info->gl_mutex); 3142 } 3143 3144 gsrv_ents->se_state = IBDM_SE_VALID; 3145 mutex_exit(&gid_info->gl_mutex); 3146 for (ii = start; ii <= end; ii++, srv_ents++, gsrv_ents++) { 3147 gsrv_ents->se_attr.srv_id = b2h64(srv_ents->srv_id); 3148 bcopy(srv_ents->srv_name, 3149 gsrv_ents->se_attr.srv_name, IB_DM_MAX_SVC_NAME_LEN); 3150 ibdm_dump_service_entries(&gsrv_ents->se_attr); 3151 } 3152 } 3153 3154 3155 /* 3156 * ibdm_get_diagcode: 3157 * Send request to get IOU/IOC diag code 3158 * Returns IBDM_SUCCESS/IBDM_FAILURE 3159 */ 3160 static int 3161 ibdm_get_diagcode(ibdm_dp_gidinfo_t *gid_info, int attr) 3162 { 3163 ibmf_msg_t *msg; 3164 ib_mad_hdr_t *hdr; 3165 ibdm_ioc_info_t *ioc; 3166 ibdm_timeout_cb_args_t *cb_args; 3167 timeout_id_t *timeout_id; 3168 3169 IBTF_DPRINTF_L4("ibdm", "\tget_diagcode: gid info %p, attr = %d", 3170 gid_info, attr); 3171 3172 if (ibmf_alloc_msg(gid_info->gl_ibmf_hdl, IBMF_ALLOC_SLEEP, 3173 &msg) != IBMF_SUCCESS) { 3174 IBTF_DPRINTF_L4("ibdm", "\tget_diagcode: pkt alloc fail"); 3175 return (IBDM_FAILURE); 3176 } 3177 3178 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*msg)) 3179 ibdm_alloc_send_buffers(msg); 3180 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*msg)) 3181 3182 mutex_enter(&gid_info->gl_mutex); 3183 ibdm_bump_transactionID(gid_info); 3184 mutex_exit(&gid_info->gl_mutex); 3185 3186 msg->im_local_addr.ia_local_lid = gid_info->gl_slid; 3187 msg->im_local_addr.ia_remote_lid = gid_info->gl_dlid; 3188 if (gid_info->gl_redirected == B_TRUE) { 3189 if (gid_info->gl_redirect_dlid != 0) { 3190 msg->im_local_addr.ia_remote_lid = 3191 gid_info->gl_redirect_dlid; 3192 } 3193 3194 msg->im_local_addr.ia_remote_qno = gid_info->gl_redirect_QP; 3195 msg->im_local_addr.ia_p_key = gid_info->gl_redirect_pkey; 3196 msg->im_local_addr.ia_q_key = gid_info->gl_redirect_qkey; 3197 msg->im_local_addr.ia_service_level = gid_info->gl_redirectSL; 3198 } else { 3199 msg->im_local_addr.ia_remote_qno = 1; 3200 msg->im_local_addr.ia_p_key = gid_info->gl_p_key; 3201 msg->im_local_addr.ia_q_key = IB_GSI_QKEY; 3202 msg->im_local_addr.ia_service_level = gid_info->gl_SL; 3203 } 3204 3205 hdr = IBDM_OUT_IBMFMSG_MADHDR(msg); 3206 hdr->BaseVersion = MAD_CLASS_BASE_VERS_1; 3207 hdr->MgmtClass = MAD_MGMT_CLASS_DEV_MGT; 3208 hdr->ClassVersion = IB_DM_CLASS_VERSION_1; 3209 hdr->R_Method = IB_DM_DEVMGT_METHOD_GET; 3210 hdr->Status = 0; 3211 hdr->TransactionID = h2b64(gid_info->gl_transactionID); 3212 3213 hdr->AttributeID = h2b16(IB_DM_ATTR_DIAG_CODE); 3214 hdr->AttributeModifier = h2b32(attr); 3215 3216 if (attr == 0) { 3217 cb_args = &gid_info->gl_iou_cb_args; 3218 gid_info->gl_iou->iou_dc_valid = B_FALSE; 3219 cb_args->cb_ioc_num = 0; 3220 cb_args->cb_req_type = IBDM_REQ_TYPE_IOU_DIAGCODE; 3221 timeout_id = &gid_info->gl_timeout_id; 3222 } else { 3223 ioc = IBDM_GIDINFO2IOCINFO(gid_info, (attr - 1)); 3224 ioc->ioc_dc_valid = B_FALSE; 3225 cb_args = &ioc->ioc_dc_cb_args; 3226 cb_args->cb_ioc_num = attr - 1; 3227 cb_args->cb_req_type = IBDM_REQ_TYPE_IOC_DIAGCODE; 3228 timeout_id = &ioc->ioc_dc_timeout_id; 3229 } 3230 cb_args->cb_gid_info = gid_info; 3231 cb_args->cb_retry_count = ibdm_dft_retry_cnt; 3232 cb_args->cb_srvents_start = 0; 3233 3234 mutex_enter(&gid_info->gl_mutex); 3235 *timeout_id = timeout(ibdm_pkt_timeout_hdlr, 3236 cb_args, IBDM_TIMEOUT_VALUE(ibdm_dft_timeout)); 3237 mutex_exit(&gid_info->gl_mutex); 3238 3239 IBTF_DPRINTF_L5("ibdm", "\tget_diagcode:" 3240 "timeout %x, ioc %d", *timeout_id, cb_args->cb_ioc_num); 3241 3242 if (ibmf_msg_transport(gid_info->gl_ibmf_hdl, gid_info->gl_qp_hdl, 3243 msg, NULL, ibdm_ibmf_send_cb, cb_args, 0) != IBMF_SUCCESS) { 3244 IBTF_DPRINTF_L2("ibdm", "\tget_diagcode: ibmf send failed"); 3245 ibdm_ibmf_send_cb(gid_info->gl_ibmf_hdl, msg, cb_args); 3246 } 3247 return (IBDM_SUCCESS); 3248 } 3249 3250 /* 3251 * ibdm_handle_diagcode: 3252 * Process the DiagCode MAD response and update local DM 3253 * data structure. 3254 */ 3255 static void 3256 ibdm_handle_diagcode(ibmf_msg_t *ibmf_msg, 3257 ibdm_dp_gidinfo_t *gid_info, int *flag) 3258 { 3259 uint16_t attrmod, *diagcode; 3260 ibdm_iou_info_t *iou; 3261 ibdm_ioc_info_t *ioc; 3262 timeout_id_t timeout_id; 3263 ibdm_timeout_cb_args_t *cb_args; 3264 3265 diagcode = (uint16_t *)ibmf_msg->im_msgbufs_recv.im_bufs_cl_data; 3266 3267 mutex_enter(&gid_info->gl_mutex); 3268 attrmod = IBDM_IN_IBMFMSG_ATTRMOD(ibmf_msg); 3269 iou = gid_info->gl_iou; 3270 if (attrmod == 0) { 3271 if (iou->iou_dc_valid != B_FALSE) { 3272 (*flag) |= IBDM_IBMF_PKT_DUP_RESP; 3273 IBTF_DPRINTF_L4("ibdm", 3274 "\thandle_diagcode: Duplicate IOU DiagCode"); 3275 mutex_exit(&gid_info->gl_mutex); 3276 return; 3277 } 3278 cb_args = &gid_info->gl_iou_cb_args; 3279 cb_args->cb_req_type = 0; 3280 iou->iou_diagcode = b2h16(*diagcode); 3281 iou->iou_dc_valid = B_TRUE; 3282 if (gid_info->gl_timeout_id) { 3283 timeout_id = gid_info->gl_timeout_id; 3284 mutex_exit(&gid_info->gl_mutex); 3285 IBTF_DPRINTF_L5("ibdm", "\thandle_diagcode: " 3286 "gl_timeout_id = 0x%x", timeout_id); 3287 if (untimeout(timeout_id) == -1) { 3288 IBTF_DPRINTF_L2("ibdm", "handle_diagcode: " 3289 "untimeout gl_timeout_id failed"); 3290 } 3291 mutex_enter(&gid_info->gl_mutex); 3292 gid_info->gl_timeout_id = 0; 3293 } 3294 } else { 3295 ioc = IBDM_GIDINFO2IOCINFO(gid_info, (attrmod - 1)); 3296 if (ioc->ioc_dc_valid != B_FALSE) { 3297 (*flag) |= IBDM_IBMF_PKT_DUP_RESP; 3298 IBTF_DPRINTF_L4("ibdm", 3299 "\thandle_diagcode: Duplicate IOC DiagCode"); 3300 mutex_exit(&gid_info->gl_mutex); 3301 return; 3302 } 3303 cb_args = &ioc->ioc_dc_cb_args; 3304 cb_args->cb_req_type = 0; 3305 ioc->ioc_diagcode = b2h16(*diagcode); 3306 ioc->ioc_dc_valid = B_TRUE; 3307 timeout_id = iou->iou_ioc_info[attrmod - 1].ioc_dc_timeout_id; 3308 if (timeout_id) { 3309 iou->iou_ioc_info[attrmod - 1].ioc_dc_timeout_id = 0; 3310 mutex_exit(&gid_info->gl_mutex); 3311 IBTF_DPRINTF_L5("ibdm", "handle_diagcode: " 3312 "timeout_id = 0x%x", timeout_id); 3313 if (untimeout(timeout_id) == -1) { 3314 IBTF_DPRINTF_L2("ibdm", "\thandle_diagcode: " 3315 "untimeout ioc_dc_timeout_id failed"); 3316 } 3317 mutex_enter(&gid_info->gl_mutex); 3318 } 3319 } 3320 mutex_exit(&gid_info->gl_mutex); 3321 3322 IBTF_DPRINTF_L4("ibdm", "\thandle_diagcode: DiagCode : 0x%x" 3323 "attrmod : 0x%x", b2h16(*diagcode), attrmod); 3324 } 3325 3326 3327 /* 3328 * ibdm_is_ioc_present() 3329 * Return ibdm_ioc_info_t if IOC guid is found in the global gid list 3330 */ 3331 static ibdm_ioc_info_t * 3332 ibdm_is_ioc_present(ib_guid_t ioc_guid, 3333 ibdm_dp_gidinfo_t *gid_info, int *flag) 3334 { 3335 int ii; 3336 ibdm_ioc_info_t *ioc; 3337 ibdm_dp_gidinfo_t *head; 3338 ib_dm_io_unitinfo_t *iou; 3339 3340 mutex_enter(&ibdm.ibdm_mutex); 3341 head = ibdm.ibdm_dp_gidlist_head; 3342 while (head) { 3343 mutex_enter(&head->gl_mutex); 3344 if (head->gl_iou == NULL) { 3345 mutex_exit(&head->gl_mutex); 3346 head = head->gl_next; 3347 continue; 3348 } 3349 iou = &head->gl_iou->iou_info; 3350 for (ii = 0; ii < iou->iou_num_ctrl_slots; ii++) { 3351 ioc = IBDM_GIDINFO2IOCINFO(head, ii); 3352 if ((ioc->ioc_state == IBDM_IOC_STATE_PROBE_SUCCESS) && 3353 (ioc->ioc_profile.ioc_guid == ioc_guid)) { 3354 if (gid_info == head) { 3355 *flag |= IBDM_IBMF_PKT_DUP_RESP; 3356 } else if (ibdm_check_dgid(head->gl_dgid_lo, 3357 head->gl_dgid_hi) != NULL) { 3358 IBTF_DPRINTF_L4("ibdm", "\tis_ioc_" 3359 "present: gid not present"); 3360 ibdm_add_to_gl_gid(gid_info, head); 3361 } 3362 mutex_exit(&head->gl_mutex); 3363 mutex_exit(&ibdm.ibdm_mutex); 3364 return (ioc); 3365 } 3366 } 3367 mutex_exit(&head->gl_mutex); 3368 head = head->gl_next; 3369 } 3370 mutex_exit(&ibdm.ibdm_mutex); 3371 return (NULL); 3372 } 3373 3374 3375 /* 3376 * ibdm_ibmf_send_cb() 3377 * IBMF invokes this callback routine after posting the DM MAD to 3378 * the HCA. 3379 */ 3380 /*ARGSUSED*/ 3381 static void 3382 ibdm_ibmf_send_cb(ibmf_handle_t ibmf_hdl, ibmf_msg_t *ibmf_msg, void *arg) 3383 { 3384 ibdm_dump_ibmf_msg(ibmf_msg, 1); 3385 ibdm_free_send_buffers(ibmf_msg); 3386 if (ibmf_free_msg(ibmf_hdl, &ibmf_msg) != IBMF_SUCCESS) { 3387 IBTF_DPRINTF_L4("ibdm", 3388 "\tibmf_send_cb: IBMF free msg failed"); 3389 } 3390 } 3391 3392 3393 /* 3394 * ibdm_ibmf_recv_cb() 3395 * Invoked by the IBMF when a response to the one of the DM requests 3396 * is received. 3397 */ 3398 /*ARGSUSED*/ 3399 static void 3400 ibdm_ibmf_recv_cb(ibmf_handle_t ibmf_hdl, ibmf_msg_t *msg, void *arg) 3401 { 3402 ibdm_taskq_args_t *taskq_args; 3403 3404 /* 3405 * If the taskq enable is set then dispatch a taskq to process 3406 * the MAD, otherwise just process it on this thread 3407 */ 3408 if (ibdm_taskq_enable != IBDM_ENABLE_TASKQ_HANDLING) { 3409 ibdm_process_incoming_mad(ibmf_hdl, msg, arg); 3410 return; 3411 } 3412 3413 /* 3414 * create a taskq and dispatch it to process the incoming MAD 3415 */ 3416 taskq_args = kmem_alloc(sizeof (ibdm_taskq_args_t), KM_NOSLEEP); 3417 if (taskq_args == NULL) { 3418 IBTF_DPRINTF_L2("ibdm", "ibmf_recv_cb: kmem_alloc failed for" 3419 "taskq_args"); 3420 if (ibmf_free_msg(ibmf_hdl, &msg) != IBMF_SUCCESS) { 3421 IBTF_DPRINTF_L4("ibmf_recv_cb", 3422 "\tibmf_recv_cb: IBMF free msg failed"); 3423 } 3424 return; 3425 } 3426 taskq_args->tq_ibmf_handle = ibmf_hdl; 3427 taskq_args->tq_ibmf_msg = msg; 3428 taskq_args->tq_args = arg; 3429 3430 if (taskq_dispatch(system_taskq, ibdm_recv_incoming_mad, taskq_args, 3431 TQ_NOSLEEP) == 0) { 3432 IBTF_DPRINTF_L2("ibdm", "ibmf_recv_cb: taskq_dispatch failed"); 3433 if (ibmf_free_msg(ibmf_hdl, &msg) != IBMF_SUCCESS) { 3434 IBTF_DPRINTF_L4("ibmf_recv_cb", 3435 "\tibmf_recv_cb: IBMF free msg failed"); 3436 } 3437 kmem_free(taskq_args, sizeof (ibdm_taskq_args_t)); 3438 return; 3439 } 3440 3441 /* taskq_args are deleted in ibdm_recv_incoming_mad() */ 3442 } 3443 3444 3445 void 3446 ibdm_recv_incoming_mad(void *args) 3447 { 3448 ibdm_taskq_args_t *taskq_args; 3449 3450 taskq_args = (ibdm_taskq_args_t *)args; 3451 3452 IBTF_DPRINTF_L4("ibdm", "\tibdm_recv_incoming_mad: " 3453 "Processing incoming MAD via taskq"); 3454 3455 ibdm_process_incoming_mad(taskq_args->tq_ibmf_handle, 3456 taskq_args->tq_ibmf_msg, taskq_args->tq_args); 3457 3458 kmem_free(taskq_args, sizeof (ibdm_taskq_args_t)); 3459 } 3460 3461 3462 /* 3463 * Calls ibdm_process_incoming_mad with all function arguments extracted 3464 * from args 3465 */ 3466 /*ARGSUSED*/ 3467 static void 3468 ibdm_process_incoming_mad(ibmf_handle_t ibmf_hdl, ibmf_msg_t *msg, void *arg) 3469 { 3470 int flag = 0; 3471 int ret; 3472 uint64_t transaction_id; 3473 ib_mad_hdr_t *hdr; 3474 ibdm_dp_gidinfo_t *gid_info = NULL; 3475 3476 IBTF_DPRINTF_L4("ibdm", 3477 "\tprocess_incoming_mad: ibmf hdl %p pkt %p", ibmf_hdl, msg); 3478 ibdm_dump_ibmf_msg(msg, 0); 3479 3480 /* 3481 * IBMF calls this routine for every DM MAD that arrives at this port. 3482 * But we handle only the responses for requests we sent. We drop all 3483 * the DM packets that does not have response bit set in the MAD 3484 * header(this eliminates all the requests sent to this port). 3485 * We handle only DM class version 1 MAD's 3486 */ 3487 hdr = IBDM_IN_IBMFMSG_MADHDR(msg); 3488 if (ibdm_verify_mad_status(hdr) != IBDM_SUCCESS) { 3489 if (ibmf_free_msg(ibmf_hdl, &msg) != IBMF_SUCCESS) { 3490 IBTF_DPRINTF_L2("ibdm", "\tprocess_incoming_mad: " 3491 "IBMF free msg failed DM request drop it"); 3492 } 3493 return; 3494 } 3495 3496 transaction_id = b2h64(hdr->TransactionID); 3497 3498 mutex_enter(&ibdm.ibdm_mutex); 3499 gid_info = ibdm.ibdm_dp_gidlist_head; 3500 while (gid_info) { 3501 if ((gid_info->gl_transactionID & 3502 IBDM_GID_TRANSACTIONID_MASK) == 3503 (transaction_id & IBDM_GID_TRANSACTIONID_MASK)) 3504 break; 3505 gid_info = gid_info->gl_next; 3506 } 3507 mutex_exit(&ibdm.ibdm_mutex); 3508 3509 if (gid_info == NULL) { 3510 /* Drop the packet */ 3511 IBTF_DPRINTF_L2("ibdm", "process_incoming_mad: transaction ID" 3512 " does not match: 0x%llx", transaction_id); 3513 if (ibmf_free_msg(ibmf_hdl, &msg) != IBMF_SUCCESS) { 3514 IBTF_DPRINTF_L2("ibdm", "process_incoming_mad: " 3515 "IBMF free msg failed DM request drop it"); 3516 } 3517 return; 3518 } 3519 3520 /* Handle redirection for all the MAD's, except ClassPortInfo */ 3521 if (((IBDM_IN_IBMFMSG_STATUS(msg) & MAD_STATUS_REDIRECT_REQUIRED)) && 3522 (IBDM_IN_IBMFMSG_ATTR(msg) != IB_DM_ATTR_CLASSPORTINFO)) { 3523 ret = ibdm_handle_redirection(msg, gid_info, &flag); 3524 if (ret == IBDM_SUCCESS) { 3525 return; 3526 } 3527 } else { 3528 uint_t gl_state; 3529 3530 mutex_enter(&gid_info->gl_mutex); 3531 gl_state = gid_info->gl_state; 3532 mutex_exit(&gid_info->gl_mutex); 3533 3534 switch (gl_state) { 3535 3536 case IBDM_SET_CLASSPORTINFO: 3537 ibdm_handle_setclassportinfo( 3538 ibmf_hdl, msg, gid_info, &flag); 3539 break; 3540 3541 case IBDM_GET_CLASSPORTINFO: 3542 ibdm_handle_classportinfo( 3543 ibmf_hdl, msg, gid_info, &flag); 3544 break; 3545 3546 case IBDM_GET_IOUNITINFO: 3547 ibdm_handle_iounitinfo(ibmf_hdl, msg, gid_info, &flag); 3548 break; 3549 3550 case IBDM_GET_IOC_DETAILS: 3551 switch (IBDM_IN_IBMFMSG_ATTR(msg)) { 3552 3553 case IB_DM_ATTR_SERVICE_ENTRIES: 3554 ibdm_handle_srventry_mad(msg, gid_info, &flag); 3555 break; 3556 3557 case IB_DM_ATTR_IOC_CTRL_PROFILE: 3558 ibdm_handle_ioc_profile( 3559 ibmf_hdl, msg, gid_info, &flag); 3560 break; 3561 3562 case IB_DM_ATTR_DIAG_CODE: 3563 ibdm_handle_diagcode(msg, gid_info, &flag); 3564 break; 3565 3566 default: 3567 IBTF_DPRINTF_L2("ibdm", "process_incoming_mad: " 3568 "Error state, wrong attribute :-("); 3569 (void) ibmf_free_msg(ibmf_hdl, &msg); 3570 return; 3571 } 3572 break; 3573 default: 3574 IBTF_DPRINTF_L2("ibdm", 3575 "process_incoming_mad: Dropping the packet" 3576 " gl_state %x", gl_state); 3577 if (ibmf_free_msg(ibmf_hdl, &msg) != IBMF_SUCCESS) { 3578 IBTF_DPRINTF_L2("ibdm", "process_incoming_mad: " 3579 "IBMF free msg failed DM request drop it"); 3580 } 3581 return; 3582 } 3583 } 3584 3585 if ((flag & IBDM_IBMF_PKT_DUP_RESP) || 3586 (flag & IBDM_IBMF_PKT_UNEXP_RESP)) { 3587 IBTF_DPRINTF_L2("ibdm", 3588 "\tprocess_incoming_mad:Dup/unexp resp : 0x%x", flag); 3589 if (ibmf_free_msg(ibmf_hdl, &msg) != IBMF_SUCCESS) { 3590 IBTF_DPRINTF_L2("ibdm", "process_incoming_mad: " 3591 "IBMF free msg failed DM request drop it"); 3592 } 3593 return; 3594 } 3595 3596 mutex_enter(&gid_info->gl_mutex); 3597 if (gid_info->gl_pending_cmds < 1) { 3598 IBTF_DPRINTF_L2("ibdm", 3599 "\tprocess_incoming_mad: pending commands negative"); 3600 } 3601 if (--gid_info->gl_pending_cmds) { 3602 IBTF_DPRINTF_L4("ibdm", "\tprocess_incoming_mad: " 3603 "gid_info %p pending cmds %d", 3604 gid_info, gid_info->gl_pending_cmds); 3605 mutex_exit(&gid_info->gl_mutex); 3606 } else { 3607 uint_t prev_state; 3608 IBTF_DPRINTF_L4("ibdm", "\tprocess_incoming_mad: Probing DONE"); 3609 prev_state = gid_info->gl_state; 3610 gid_info->gl_state = IBDM_GID_PROBING_COMPLETE; 3611 if (prev_state == IBDM_SET_CLASSPORTINFO) { 3612 IBTF_DPRINTF_L4("ibdm", 3613 "\tprocess_incoming_mad: " 3614 "Setclassportinfo for Cisco FC GW is done."); 3615 gid_info->gl_flag &= ~IBDM_CISCO_PROBE; 3616 gid_info->gl_flag |= IBDM_CISCO_PROBE_DONE; 3617 mutex_exit(&gid_info->gl_mutex); 3618 cv_broadcast(&gid_info->gl_probe_cv); 3619 } else { 3620 mutex_exit(&gid_info->gl_mutex); 3621 ibdm_notify_newgid_iocs(gid_info); 3622 mutex_enter(&ibdm.ibdm_mutex); 3623 if (--ibdm.ibdm_ngid_probes_in_progress == 0) { 3624 IBTF_DPRINTF_L4("ibdm", 3625 "\tprocess_incoming_mad: Wakeup"); 3626 ibdm.ibdm_busy &= ~IBDM_PROBE_IN_PROGRESS; 3627 cv_broadcast(&ibdm.ibdm_probe_cv); 3628 } 3629 mutex_exit(&ibdm.ibdm_mutex); 3630 } 3631 } 3632 3633 /* 3634 * Do not deallocate the IBMF packet if atleast one request 3635 * is posted. IBMF packet is reused. 3636 */ 3637 if (!(flag & IBDM_IBMF_PKT_REUSED)) { 3638 if (ibmf_free_msg(ibmf_hdl, &msg) != IBMF_SUCCESS) { 3639 IBTF_DPRINTF_L2("ibdm", "\tprocess_incoming_mad: " 3640 "IBMF free msg failed DM request drop it"); 3641 } 3642 } 3643 } 3644 3645 3646 /* 3647 * ibdm_verify_mad_status() 3648 * Verifies the MAD status 3649 * Returns IBDM_SUCCESS if status is correct 3650 * Returns IBDM_FAILURE for bogus MAD status 3651 */ 3652 static int 3653 ibdm_verify_mad_status(ib_mad_hdr_t *hdr) 3654 { 3655 int ret = 0; 3656 3657 if ((hdr->R_Method != IB_DM_DEVMGT_METHOD_GET_RESP) || 3658 (hdr->ClassVersion != IB_DM_CLASS_VERSION_1)) { 3659 return (IBDM_FAILURE); 3660 } 3661 3662 if (b2h16(hdr->Status) == 0) 3663 ret = IBDM_SUCCESS; 3664 else if ((b2h16(hdr->Status) & 0x1f) == MAD_STATUS_REDIRECT_REQUIRED) 3665 ret = IBDM_SUCCESS; 3666 else { 3667 IBTF_DPRINTF_L2("ibdm", 3668 "\tverify_mad_status: Status : 0x%x", b2h16(hdr->Status)); 3669 ret = IBDM_FAILURE; 3670 } 3671 return (ret); 3672 } 3673 3674 3675 3676 /* 3677 * ibdm_handle_redirection() 3678 * Returns IBDM_SUCCESS/IBDM_FAILURE 3679 */ 3680 static int 3681 ibdm_handle_redirection(ibmf_msg_t *msg, 3682 ibdm_dp_gidinfo_t *gid_info, int *flag) 3683 { 3684 int attrmod, ioc_no, start; 3685 void *data; 3686 timeout_id_t *timeout_id; 3687 ib_mad_hdr_t *hdr; 3688 ibdm_ioc_info_t *ioc = NULL; 3689 ibdm_timeout_cb_args_t *cb_args; 3690 ib_mad_classportinfo_t *cpi; 3691 3692 IBTF_DPRINTF_L4("ibdm", "\thandle_redirection: Enter"); 3693 mutex_enter(&gid_info->gl_mutex); 3694 switch (gid_info->gl_state) { 3695 case IBDM_GET_IOUNITINFO: 3696 cb_args = &gid_info->gl_iou_cb_args; 3697 timeout_id = &gid_info->gl_timeout_id; 3698 break; 3699 3700 case IBDM_GET_IOC_DETAILS: 3701 attrmod = IBDM_IN_IBMFMSG_ATTRMOD(msg); 3702 switch (IBDM_IN_IBMFMSG_ATTR(msg)) { 3703 3704 case IB_DM_ATTR_DIAG_CODE: 3705 if (attrmod == 0) { 3706 cb_args = &gid_info->gl_iou_cb_args; 3707 timeout_id = &gid_info->gl_timeout_id; 3708 break; 3709 } 3710 if (IBDM_IS_IOC_NUM_INVALID(attrmod, gid_info)) { 3711 IBTF_DPRINTF_L2("ibdm", "\thandle_redirction:" 3712 "IOC# Out of range %d", attrmod); 3713 (*flag) |= IBDM_IBMF_PKT_UNEXP_RESP; 3714 mutex_exit(&gid_info->gl_mutex); 3715 return (IBDM_FAILURE); 3716 } 3717 ioc = IBDM_GIDINFO2IOCINFO(gid_info, (attrmod -1)); 3718 cb_args = &ioc->ioc_dc_cb_args; 3719 timeout_id = &ioc->ioc_dc_timeout_id; 3720 break; 3721 3722 case IB_DM_ATTR_IOC_CTRL_PROFILE: 3723 if (IBDM_IS_IOC_NUM_INVALID(attrmod, gid_info)) { 3724 IBTF_DPRINTF_L2("ibdm", "\thandle_redirction:" 3725 "IOC# Out of range %d", attrmod); 3726 (*flag) |= IBDM_IBMF_PKT_UNEXP_RESP; 3727 mutex_exit(&gid_info->gl_mutex); 3728 return (IBDM_FAILURE); 3729 } 3730 ioc = IBDM_GIDINFO2IOCINFO(gid_info, (attrmod -1)); 3731 cb_args = &ioc->ioc_cb_args; 3732 timeout_id = &ioc->ioc_timeout_id; 3733 break; 3734 3735 case IB_DM_ATTR_SERVICE_ENTRIES: 3736 ioc_no = ((attrmod >> 16) & IBDM_16_BIT_MASK); 3737 if (IBDM_IS_IOC_NUM_INVALID(ioc_no, gid_info)) { 3738 IBTF_DPRINTF_L2("ibdm", "\thandle_redirction:" 3739 "IOC# Out of range %d", ioc_no); 3740 (*flag) |= IBDM_IBMF_PKT_UNEXP_RESP; 3741 mutex_exit(&gid_info->gl_mutex); 3742 return (IBDM_FAILURE); 3743 } 3744 start = (attrmod & IBDM_8_BIT_MASK); 3745 ioc = IBDM_GIDINFO2IOCINFO(gid_info, (ioc_no -1)); 3746 if (start > ioc->ioc_profile.ioc_service_entries) { 3747 IBTF_DPRINTF_L2("ibdm", "\thandle_redirction:" 3748 " SE index Out of range %d", start); 3749 (*flag) |= IBDM_IBMF_PKT_UNEXP_RESP; 3750 mutex_exit(&gid_info->gl_mutex); 3751 return (IBDM_FAILURE); 3752 } 3753 cb_args = &ioc->ioc_serv[start].se_cb_args; 3754 timeout_id = &ioc->ioc_serv[start].se_timeout_id; 3755 break; 3756 3757 default: 3758 /* ERROR State */ 3759 IBTF_DPRINTF_L2("ibdm", 3760 "\thandle_redirection: wrong attribute :-("); 3761 (*flag) |= IBDM_IBMF_PKT_UNEXP_RESP; 3762 mutex_exit(&gid_info->gl_mutex); 3763 return (IBDM_FAILURE); 3764 } 3765 break; 3766 default: 3767 /* ERROR State */ 3768 IBTF_DPRINTF_L2("ibdm", 3769 "\thandle_redirection: Error state :-("); 3770 (*flag) |= IBDM_IBMF_PKT_UNEXP_RESP; 3771 mutex_exit(&gid_info->gl_mutex); 3772 return (IBDM_FAILURE); 3773 } 3774 if ((*timeout_id) != 0) { 3775 mutex_exit(&gid_info->gl_mutex); 3776 if (untimeout(*timeout_id) == -1) { 3777 IBTF_DPRINTF_L2("ibdm", "\thandle_redirection: " 3778 "untimeout failed %x", *timeout_id); 3779 } else { 3780 IBTF_DPRINTF_L5("ibdm", 3781 "\thandle_redirection: timeout %x", *timeout_id); 3782 } 3783 mutex_enter(&gid_info->gl_mutex); 3784 *timeout_id = 0; 3785 } 3786 3787 data = msg->im_msgbufs_recv.im_bufs_cl_data; 3788 cpi = (ib_mad_classportinfo_t *)data; 3789 3790 gid_info->gl_resp_timeout = 3791 (b2h32(cpi->RespTimeValue) & 0x1F); 3792 3793 gid_info->gl_redirected = B_TRUE; 3794 gid_info->gl_redirect_dlid = b2h16(cpi->RedirectLID); 3795 gid_info->gl_redirect_QP = (b2h32(cpi->RedirectQP) & 0xffffff); 3796 gid_info->gl_redirect_pkey = b2h16(cpi->RedirectP_Key); 3797 gid_info->gl_redirect_qkey = b2h32(cpi->RedirectQ_Key); 3798 gid_info->gl_redirectGID_hi = b2h64(cpi->RedirectGID_hi); 3799 gid_info->gl_redirectGID_lo = b2h64(cpi->RedirectGID_lo); 3800 gid_info->gl_redirectSL = cpi->RedirectSL; 3801 3802 if (gid_info->gl_redirect_dlid != 0) { 3803 msg->im_local_addr.ia_remote_lid = 3804 gid_info->gl_redirect_dlid; 3805 } 3806 ibdm_bump_transactionID(gid_info); 3807 mutex_exit(&gid_info->gl_mutex); 3808 3809 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*msg, *hdr)) 3810 ibdm_alloc_send_buffers(msg); 3811 3812 hdr = IBDM_OUT_IBMFMSG_MADHDR(msg); 3813 hdr->BaseVersion = MAD_CLASS_BASE_VERS_1; 3814 hdr->MgmtClass = MAD_MGMT_CLASS_DEV_MGT; 3815 hdr->ClassVersion = IB_DM_CLASS_VERSION_1; 3816 hdr->R_Method = IB_DM_DEVMGT_METHOD_GET; 3817 hdr->Status = 0; 3818 hdr->TransactionID = h2b64(gid_info->gl_transactionID); 3819 hdr->AttributeID = 3820 msg->im_msgbufs_recv.im_bufs_mad_hdr->AttributeID; 3821 hdr->AttributeModifier = 3822 msg->im_msgbufs_recv.im_bufs_mad_hdr->AttributeModifier; 3823 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*msg, *hdr)) 3824 3825 msg->im_local_addr.ia_remote_qno = gid_info->gl_redirect_QP; 3826 msg->im_local_addr.ia_p_key = gid_info->gl_redirect_pkey; 3827 msg->im_local_addr.ia_q_key = gid_info->gl_redirect_qkey; 3828 msg->im_local_addr.ia_service_level = gid_info->gl_redirectSL; 3829 3830 mutex_enter(&gid_info->gl_mutex); 3831 *timeout_id = timeout(ibdm_pkt_timeout_hdlr, 3832 cb_args, IBDM_TIMEOUT_VALUE(ibdm_dft_timeout)); 3833 mutex_exit(&gid_info->gl_mutex); 3834 3835 IBTF_DPRINTF_L5("ibdm", "\thandle_redirect:" 3836 "timeout %x", *timeout_id); 3837 3838 if (ibmf_msg_transport(gid_info->gl_ibmf_hdl, gid_info->gl_qp_hdl, 3839 msg, NULL, ibdm_ibmf_send_cb, cb_args, 0) != IBMF_SUCCESS) { 3840 IBTF_DPRINTF_L4("ibdm", "\thandle_redirection:" 3841 "message transport failed"); 3842 ibdm_ibmf_send_cb(gid_info->gl_ibmf_hdl, msg, cb_args); 3843 } 3844 (*flag) |= IBDM_IBMF_PKT_REUSED; 3845 IBTF_DPRINTF_L4("ibdm", "\thandle_redirection: Exit"); 3846 return (IBDM_SUCCESS); 3847 } 3848 3849 3850 /* 3851 * ibdm_pkt_timeout_hdlr 3852 * This timeout handler is registed for every IBMF packet that is 3853 * sent through the IBMF. It gets called when no response is received 3854 * within the specified time for the packet. No retries for the failed 3855 * commands currently. Drops the failed IBMF packet and update the 3856 * pending list commands. 3857 */ 3858 static void 3859 ibdm_pkt_timeout_hdlr(void *arg) 3860 { 3861 ibdm_iou_info_t *iou; 3862 ibdm_ioc_info_t *ioc; 3863 ibdm_timeout_cb_args_t *cb_args = arg; 3864 ibdm_dp_gidinfo_t *gid_info; 3865 int srv_ent; 3866 uint_t new_gl_state; 3867 3868 IBTF_DPRINTF_L2("ibdm", "\tpkt_timeout_hdlr: gid_info: %p " 3869 "rtype 0x%x iocidx 0x%x srvidx %d", cb_args->cb_gid_info, 3870 cb_args->cb_req_type, cb_args->cb_ioc_num, 3871 cb_args->cb_srvents_start); 3872 3873 gid_info = cb_args->cb_gid_info; 3874 mutex_enter(&gid_info->gl_mutex); 3875 3876 if ((gid_info->gl_state == IBDM_GID_PROBING_COMPLETE) || 3877 (cb_args->cb_req_type == 0)) { 3878 3879 IBTF_DPRINTF_L2("ibdm", "\tpkt_timeout_hdlr: req completed" 3880 "rtype 0x%x iocidx 0x%x srvidx %d", cb_args->cb_req_type, 3881 cb_args->cb_ioc_num, cb_args->cb_srvents_start); 3882 3883 if (gid_info->gl_timeout_id) 3884 gid_info->gl_timeout_id = 0; 3885 mutex_exit(&gid_info->gl_mutex); 3886 return; 3887 } 3888 if (cb_args->cb_retry_count) { 3889 cb_args->cb_retry_count--; 3890 /* 3891 * A new timeout_id is set inside ibdm_retry_command(). 3892 * When the function returns an error, the timeout_id 3893 * is reset (to zero) in the switch statement below. 3894 */ 3895 if (ibdm_retry_command(cb_args) == IBDM_SUCCESS) { 3896 mutex_exit(&gid_info->gl_mutex); 3897 return; 3898 } 3899 cb_args->cb_retry_count = 0; 3900 } 3901 3902 IBTF_DPRINTF_L2("ibdm", "\tpkt_timeout_hdlr: command failed: gid %p" 3903 " rtype 0x%x iocidx 0x%x srvidx %d", cb_args->cb_gid_info, 3904 cb_args->cb_req_type, cb_args->cb_ioc_num, 3905 cb_args->cb_srvents_start); 3906 3907 switch (cb_args->cb_req_type) { 3908 3909 case IBDM_REQ_TYPE_CLASSPORTINFO: 3910 case IBDM_REQ_TYPE_IOUINFO: 3911 new_gl_state = IBDM_GID_PROBING_FAILED; 3912 if (gid_info->gl_timeout_id) 3913 gid_info->gl_timeout_id = 0; 3914 break; 3915 3916 case IBDM_REQ_TYPE_IOCINFO: 3917 new_gl_state = IBDM_GID_PROBING_COMPLETE; 3918 iou = gid_info->gl_iou; 3919 ioc = &iou->iou_ioc_info[cb_args->cb_ioc_num]; 3920 ioc->ioc_state = IBDM_IOC_STATE_PROBE_FAILED; 3921 if (ioc->ioc_timeout_id) 3922 ioc->ioc_timeout_id = 0; 3923 break; 3924 3925 case IBDM_REQ_TYPE_SRVENTS: 3926 new_gl_state = IBDM_GID_PROBING_COMPLETE; 3927 iou = gid_info->gl_iou; 3928 ioc = &iou->iou_ioc_info[cb_args->cb_ioc_num]; 3929 ioc->ioc_state = IBDM_IOC_STATE_PROBE_FAILED; 3930 srv_ent = cb_args->cb_srvents_start; 3931 if (ioc->ioc_serv[srv_ent].se_timeout_id) 3932 ioc->ioc_serv[srv_ent].se_timeout_id = 0; 3933 break; 3934 3935 case IBDM_REQ_TYPE_IOU_DIAGCODE: 3936 new_gl_state = IBDM_GID_PROBING_COMPLETE; 3937 iou = gid_info->gl_iou; 3938 iou->iou_dc_valid = B_FALSE; 3939 if (gid_info->gl_timeout_id) 3940 gid_info->gl_timeout_id = 0; 3941 break; 3942 3943 case IBDM_REQ_TYPE_IOC_DIAGCODE: 3944 new_gl_state = IBDM_GID_PROBING_COMPLETE; 3945 iou = gid_info->gl_iou; 3946 ioc = &iou->iou_ioc_info[cb_args->cb_ioc_num]; 3947 ioc->ioc_dc_valid = B_FALSE; 3948 if (ioc->ioc_dc_timeout_id) 3949 ioc->ioc_dc_timeout_id = 0; 3950 break; 3951 3952 default: /* ERROR State */ 3953 new_gl_state = IBDM_GID_PROBING_FAILED; 3954 if (gid_info->gl_timeout_id) 3955 gid_info->gl_timeout_id = 0; 3956 IBTF_DPRINTF_L2("ibdm", 3957 "\tpkt_timeout_hdlr: wrong request type."); 3958 break; 3959 } 3960 3961 --gid_info->gl_pending_cmds; /* decrease the counter */ 3962 3963 if (gid_info->gl_pending_cmds == 0) { 3964 gid_info->gl_state = new_gl_state; 3965 mutex_exit(&gid_info->gl_mutex); 3966 /* 3967 * Delete this gid_info if the gid probe fails. 3968 */ 3969 if (new_gl_state == IBDM_GID_PROBING_FAILED) { 3970 ibdm_delete_glhca_list(gid_info); 3971 } 3972 ibdm_notify_newgid_iocs(gid_info); 3973 mutex_enter(&ibdm.ibdm_mutex); 3974 if (--ibdm.ibdm_ngid_probes_in_progress == 0) { 3975 IBTF_DPRINTF_L4("ibdm", "\tpkt_timeout_hdlr: Wakeup"); 3976 ibdm.ibdm_busy &= ~IBDM_PROBE_IN_PROGRESS; 3977 cv_broadcast(&ibdm.ibdm_probe_cv); 3978 } 3979 mutex_exit(&ibdm.ibdm_mutex); 3980 } else { 3981 /* 3982 * Reset gl_pending_cmd if the extra timeout happens since 3983 * gl_pending_cmd becomes negative as a result. 3984 */ 3985 if (gid_info->gl_pending_cmds < 0) { 3986 gid_info->gl_pending_cmds = 0; 3987 IBTF_DPRINTF_L2("ibdm", 3988 "\tpkt_timeout_hdlr: extra timeout request." 3989 " reset gl_pending_cmds"); 3990 } 3991 mutex_exit(&gid_info->gl_mutex); 3992 /* 3993 * Delete this gid_info if the gid probe fails. 3994 */ 3995 if (new_gl_state == IBDM_GID_PROBING_FAILED) { 3996 ibdm_delete_glhca_list(gid_info); 3997 } 3998 } 3999 } 4000 4001 4002 /* 4003 * ibdm_retry_command() 4004 * Retries the failed command. 4005 * Returns IBDM_FAILURE/IBDM_SUCCESS 4006 */ 4007 static int 4008 ibdm_retry_command(ibdm_timeout_cb_args_t *cb_args) 4009 { 4010 int ret; 4011 ibmf_msg_t *msg; 4012 ib_mad_hdr_t *hdr; 4013 ibdm_dp_gidinfo_t *gid_info = cb_args->cb_gid_info; 4014 timeout_id_t *timeout_id; 4015 ibdm_ioc_info_t *ioc; 4016 int ioc_no; 4017 ASSERT(MUTEX_HELD(&gid_info->gl_mutex)); 4018 4019 IBTF_DPRINTF_L2("ibdm", "\tretry_command: gid_info: %p " 4020 "rtype 0x%x iocidx 0x%x srvidx %d", cb_args->cb_gid_info, 4021 cb_args->cb_req_type, cb_args->cb_ioc_num, 4022 cb_args->cb_srvents_start); 4023 4024 ret = ibmf_alloc_msg(gid_info->gl_ibmf_hdl, IBMF_ALLOC_NOSLEEP, &msg); 4025 4026 4027 /* 4028 * Reset the gid if alloc_msg failed with BAD_HANDLE 4029 * ibdm_reset_gidinfo reinits the gid_info 4030 */ 4031 if (ret == IBMF_BAD_HANDLE) { 4032 IBTF_DPRINTF_L3(ibdm_string, "\tretry_command: gid %p hdl bad", 4033 gid_info); 4034 4035 mutex_exit(&gid_info->gl_mutex); 4036 ibdm_reset_gidinfo(gid_info); 4037 mutex_enter(&gid_info->gl_mutex); 4038 4039 /* Retry alloc */ 4040 ret = ibmf_alloc_msg(gid_info->gl_ibmf_hdl, IBMF_ALLOC_NOSLEEP, 4041 &msg); 4042 } 4043 4044 if (ret != IBDM_SUCCESS) { 4045 IBTF_DPRINTF_L2("ibdm", "\tretry_command: alloc failed: %p " 4046 "rtype 0x%x iocidx 0x%x srvidx %d", cb_args->cb_gid_info, 4047 cb_args->cb_req_type, cb_args->cb_ioc_num, 4048 cb_args->cb_srvents_start); 4049 return (IBDM_FAILURE); 4050 } 4051 4052 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*msg)) 4053 ibdm_alloc_send_buffers(msg); 4054 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*msg)) 4055 4056 ibdm_bump_transactionID(gid_info); 4057 4058 msg->im_local_addr.ia_local_lid = gid_info->gl_slid; 4059 msg->im_local_addr.ia_remote_lid = gid_info->gl_dlid; 4060 if (gid_info->gl_redirected == B_TRUE) { 4061 if (gid_info->gl_redirect_dlid != 0) { 4062 msg->im_local_addr.ia_remote_lid = 4063 gid_info->gl_redirect_dlid; 4064 } 4065 msg->im_local_addr.ia_remote_qno = gid_info->gl_redirect_QP; 4066 msg->im_local_addr.ia_p_key = gid_info->gl_redirect_pkey; 4067 msg->im_local_addr.ia_q_key = gid_info->gl_redirect_qkey; 4068 msg->im_local_addr.ia_service_level = gid_info->gl_redirectSL; 4069 } else { 4070 msg->im_local_addr.ia_remote_qno = 1; 4071 msg->im_local_addr.ia_p_key = gid_info->gl_p_key; 4072 msg->im_local_addr.ia_q_key = IB_GSI_QKEY; 4073 msg->im_local_addr.ia_service_level = gid_info->gl_SL; 4074 } 4075 hdr = IBDM_OUT_IBMFMSG_MADHDR(msg); 4076 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*hdr)) 4077 hdr->BaseVersion = MAD_CLASS_BASE_VERS_1; 4078 hdr->MgmtClass = MAD_MGMT_CLASS_DEV_MGT; 4079 hdr->ClassVersion = IB_DM_CLASS_VERSION_1; 4080 hdr->R_Method = IB_DM_DEVMGT_METHOD_GET; 4081 hdr->Status = 0; 4082 hdr->TransactionID = h2b64(gid_info->gl_transactionID); 4083 4084 switch (cb_args->cb_req_type) { 4085 case IBDM_REQ_TYPE_CLASSPORTINFO: 4086 hdr->AttributeID = h2b16(IB_DM_ATTR_CLASSPORTINFO); 4087 hdr->AttributeModifier = 0; 4088 timeout_id = &gid_info->gl_timeout_id; 4089 break; 4090 case IBDM_REQ_TYPE_IOUINFO: 4091 hdr->AttributeID = h2b16(IB_DM_ATTR_IO_UNITINFO); 4092 hdr->AttributeModifier = 0; 4093 timeout_id = &gid_info->gl_timeout_id; 4094 break; 4095 case IBDM_REQ_TYPE_IOCINFO: 4096 hdr->AttributeID = h2b16(IB_DM_ATTR_IOC_CTRL_PROFILE); 4097 hdr->AttributeModifier = h2b32(cb_args->cb_ioc_num + 1); 4098 ioc = IBDM_GIDINFO2IOCINFO(gid_info, cb_args->cb_ioc_num); 4099 timeout_id = &ioc->ioc_timeout_id; 4100 break; 4101 case IBDM_REQ_TYPE_SRVENTS: 4102 hdr->AttributeID = h2b16(IB_DM_ATTR_SERVICE_ENTRIES); 4103 ibdm_fill_srv_attr_mod(hdr, cb_args); 4104 ioc = IBDM_GIDINFO2IOCINFO(gid_info, cb_args->cb_ioc_num); 4105 timeout_id = 4106 &ioc->ioc_serv[cb_args->cb_srvents_start].se_timeout_id; 4107 break; 4108 case IBDM_REQ_TYPE_IOU_DIAGCODE: 4109 hdr->AttributeID = h2b16(IB_DM_ATTR_DIAG_CODE); 4110 hdr->AttributeModifier = 0; 4111 timeout_id = &gid_info->gl_timeout_id; 4112 break; 4113 case IBDM_REQ_TYPE_IOC_DIAGCODE: 4114 hdr->AttributeID = h2b16(IB_DM_ATTR_DIAG_CODE); 4115 hdr->AttributeModifier = h2b32(cb_args->cb_ioc_num + 1); 4116 ioc_no = cb_args->cb_ioc_num; 4117 ioc = &gid_info->gl_iou->iou_ioc_info[ioc_no]; 4118 timeout_id = &ioc->ioc_dc_timeout_id; 4119 break; 4120 } 4121 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*hdr)) 4122 4123 *timeout_id = timeout(ibdm_pkt_timeout_hdlr, 4124 cb_args, IBDM_TIMEOUT_VALUE(ibdm_dft_timeout)); 4125 4126 mutex_exit(&gid_info->gl_mutex); 4127 4128 IBTF_DPRINTF_L5("ibdm", "\tretry_command: %p,%x,%d,%d:" 4129 "timeout %x", cb_args->cb_req_type, cb_args->cb_ioc_num, 4130 cb_args->cb_srvents_start, *timeout_id); 4131 4132 if (ibmf_msg_transport(gid_info->gl_ibmf_hdl, 4133 gid_info->gl_qp_hdl, msg, NULL, ibdm_ibmf_send_cb, 4134 cb_args, 0) != IBMF_SUCCESS) { 4135 IBTF_DPRINTF_L2("ibdm", "\tretry_command: send failed: %p " 4136 "rtype 0x%x iocidx 0x%x srvidx %d", cb_args->cb_gid_info, 4137 cb_args->cb_req_type, cb_args->cb_ioc_num, 4138 cb_args->cb_srvents_start); 4139 ibdm_ibmf_send_cb(gid_info->gl_ibmf_hdl, msg, cb_args); 4140 } 4141 mutex_enter(&gid_info->gl_mutex); 4142 return (IBDM_SUCCESS); 4143 } 4144 4145 4146 /* 4147 * ibdm_update_ioc_port_gidlist() 4148 */ 4149 static void 4150 ibdm_update_ioc_port_gidlist(ibdm_ioc_info_t *dest, 4151 ibdm_dp_gidinfo_t *gid_info) 4152 { 4153 int ii, ngid_ents; 4154 ibdm_gid_t *tmp; 4155 ibdm_hca_list_t *gid_hca_head, *temp; 4156 ibdm_hca_list_t *ioc_head = NULL; 4157 ASSERT(MUTEX_HELD(&gid_info->gl_mutex)); 4158 4159 IBTF_DPRINTF_L5("ibdm", "\tupdate_ioc_port_gidlist: Enter"); 4160 4161 ngid_ents = gid_info->gl_ngids; 4162 dest->ioc_nportgids = ngid_ents; 4163 dest->ioc_gid_list = kmem_zalloc(sizeof (ibdm_gid_t) * 4164 ngid_ents, KM_SLEEP); 4165 tmp = gid_info->gl_gid; 4166 for (ii = 0; (ii < ngid_ents) && (tmp); ii++) { 4167 dest->ioc_gid_list[ii].gid_dgid_hi = tmp->gid_dgid_hi; 4168 dest->ioc_gid_list[ii].gid_dgid_lo = tmp->gid_dgid_lo; 4169 tmp = tmp->gid_next; 4170 } 4171 4172 gid_hca_head = gid_info->gl_hca_list; 4173 while (gid_hca_head) { 4174 temp = ibdm_dup_hca_attr(gid_hca_head); 4175 temp->hl_next = ioc_head; 4176 ioc_head = temp; 4177 gid_hca_head = gid_hca_head->hl_next; 4178 } 4179 dest->ioc_hca_list = ioc_head; 4180 } 4181 4182 4183 /* 4184 * ibdm_alloc_send_buffers() 4185 * Allocates memory for the IBMF send buffer to send and/or receive 4186 * the Device Management MAD packet. 4187 */ 4188 static void 4189 ibdm_alloc_send_buffers(ibmf_msg_t *msgp) 4190 { 4191 msgp->im_msgbufs_send.im_bufs_mad_hdr = 4192 kmem_zalloc(IBDM_MAD_SIZE, KM_SLEEP); 4193 4194 msgp->im_msgbufs_send.im_bufs_cl_hdr = (uchar_t *) 4195 msgp->im_msgbufs_send.im_bufs_mad_hdr + sizeof (ib_mad_hdr_t); 4196 msgp->im_msgbufs_send.im_bufs_cl_hdr_len = IBDM_DM_MAD_HDR_SZ; 4197 4198 msgp->im_msgbufs_send.im_bufs_cl_data = 4199 ((char *)msgp->im_msgbufs_send.im_bufs_cl_hdr + IBDM_DM_MAD_HDR_SZ); 4200 msgp->im_msgbufs_send.im_bufs_cl_data_len = 4201 IBDM_MAD_SIZE - sizeof (ib_mad_hdr_t) - IBDM_DM_MAD_HDR_SZ; 4202 } 4203 4204 4205 /* 4206 * ibdm_alloc_send_buffers() 4207 * De-allocates memory for the IBMF send buffer 4208 */ 4209 static void 4210 ibdm_free_send_buffers(ibmf_msg_t *msgp) 4211 { 4212 if (msgp->im_msgbufs_send.im_bufs_mad_hdr != NULL) 4213 kmem_free(msgp->im_msgbufs_send.im_bufs_mad_hdr, IBDM_MAD_SIZE); 4214 } 4215 4216 /* 4217 * ibdm_probe_ioc() 4218 * 1. Gets the node records for the port GUID. This detects all the port 4219 * to the IOU. 4220 * 2. Selectively probes all the IOC, given it's node GUID 4221 * 3. In case of reprobe, only the IOC to be reprobed is send the IOC 4222 * Controller Profile asynchronously 4223 */ 4224 /*ARGSUSED*/ 4225 static void 4226 ibdm_probe_ioc(ib_guid_t nodeguid, ib_guid_t ioc_guid, int reprobe_flag) 4227 { 4228 int ii, nrecords; 4229 size_t nr_len = 0, pi_len = 0; 4230 ib_gid_t sgid, dgid; 4231 ibdm_hca_list_t *hca_list = NULL; 4232 sa_node_record_t *nr, *tmp; 4233 ibdm_port_attr_t *port = NULL; 4234 ibdm_dp_gidinfo_t *reprobe_gid, *new_gid, *node_gid; 4235 ibdm_dp_gidinfo_t *temp_gidinfo; 4236 ibdm_gid_t *temp_gid; 4237 sa_portinfo_record_t *pi; 4238 4239 IBTF_DPRINTF_L4("ibdm", "\tprobe_ioc(%llx, %llx, %x): Begin", 4240 nodeguid, ioc_guid, reprobe_flag); 4241 4242 /* Rescan the GID list for any removed GIDs for reprobe */ 4243 if (reprobe_flag) 4244 ibdm_rescan_gidlist(&ioc_guid); 4245 4246 mutex_enter(&ibdm.ibdm_hl_mutex); 4247 for (ibdm_get_next_port(&hca_list, &port, 1); port; 4248 ibdm_get_next_port(&hca_list, &port, 1)) { 4249 reprobe_gid = new_gid = node_gid = NULL; 4250 4251 nr = ibdm_get_node_records(port->pa_sa_hdl, &nr_len, nodeguid); 4252 if (nr == NULL) { 4253 IBTF_DPRINTF_L4("ibdm", "\tprobe_ioc: no records"); 4254 continue; 4255 } 4256 nrecords = (nr_len / sizeof (sa_node_record_t)); 4257 for (tmp = nr, ii = 0; (ii < nrecords); ii++, tmp++) { 4258 if ((pi = ibdm_get_portinfo( 4259 port->pa_sa_hdl, &pi_len, tmp->LID)) == NULL) { 4260 IBTF_DPRINTF_L4("ibdm", 4261 "\tibdm_get_portinfo: no portinfo recs"); 4262 continue; 4263 } 4264 4265 /* 4266 * If Device Management is not supported on 4267 * this port, skip the rest. 4268 */ 4269 if (!(pi->PortInfo.CapabilityMask & 4270 SM_CAP_MASK_IS_DM_SUPPD)) { 4271 kmem_free(pi, pi_len); 4272 continue; 4273 } 4274 4275 /* 4276 * For reprobes: Check if GID, already in 4277 * the list. If so, set the state to SKIPPED 4278 */ 4279 if (((temp_gidinfo = ibdm_find_gid(nodeguid, 4280 tmp->NodeInfo.PortGUID)) != NULL) && 4281 temp_gidinfo->gl_state == 4282 IBDM_GID_PROBING_COMPLETE) { 4283 ASSERT(reprobe_gid == NULL); 4284 ibdm_addto_glhcalist(temp_gidinfo, 4285 hca_list); 4286 reprobe_gid = temp_gidinfo; 4287 kmem_free(pi, pi_len); 4288 continue; 4289 } else if (temp_gidinfo != NULL) { 4290 kmem_free(pi, pi_len); 4291 ibdm_addto_glhcalist(temp_gidinfo, 4292 hca_list); 4293 continue; 4294 } 4295 4296 IBTF_DPRINTF_L4("ibdm", "\tprobe_ioc : " 4297 "create_gid : prefix %llx, guid %llx\n", 4298 pi->PortInfo.GidPrefix, 4299 tmp->NodeInfo.PortGUID); 4300 4301 sgid.gid_prefix = port->pa_sn_prefix; 4302 sgid.gid_guid = port->pa_port_guid; 4303 dgid.gid_prefix = pi->PortInfo.GidPrefix; 4304 dgid.gid_guid = tmp->NodeInfo.PortGUID; 4305 new_gid = ibdm_create_gid_info(port, sgid, 4306 dgid); 4307 if (new_gid == NULL) { 4308 IBTF_DPRINTF_L2("ibdm", "\tprobe_ioc: " 4309 "create_gid_info failed\n"); 4310 kmem_free(pi, pi_len); 4311 continue; 4312 } 4313 if (node_gid == NULL) { 4314 node_gid = new_gid; 4315 ibdm_add_to_gl_gid(node_gid, node_gid); 4316 } else { 4317 IBTF_DPRINTF_L4("ibdm", 4318 "\tprobe_ioc: new gid"); 4319 temp_gid = kmem_zalloc( 4320 sizeof (ibdm_gid_t), KM_SLEEP); 4321 temp_gid->gid_dgid_hi = 4322 new_gid->gl_dgid_hi; 4323 temp_gid->gid_dgid_lo = 4324 new_gid->gl_dgid_lo; 4325 temp_gid->gid_next = node_gid->gl_gid; 4326 node_gid->gl_gid = temp_gid; 4327 node_gid->gl_ngids++; 4328 } 4329 new_gid->gl_is_dm_capable = B_TRUE; 4330 new_gid->gl_nodeguid = nodeguid; 4331 new_gid->gl_portguid = dgid.gid_guid; 4332 ibdm_addto_glhcalist(new_gid, hca_list); 4333 4334 /* 4335 * Set the state to skipped as all these 4336 * gids point to the same node. 4337 * We (re)probe only one GID below and reset 4338 * state appropriately 4339 */ 4340 new_gid->gl_state = IBDM_GID_PROBING_SKIPPED; 4341 new_gid->gl_devid = (*tmp).NodeInfo.DeviceID; 4342 kmem_free(pi, pi_len); 4343 } 4344 kmem_free(nr, nr_len); 4345 4346 IBTF_DPRINTF_L4("ibdm", "\tprobe_ioc : reprobe_flag %d " 4347 "reprobe_gid %p new_gid %p node_gid %p", 4348 reprobe_flag, reprobe_gid, new_gid, node_gid); 4349 4350 if (reprobe_flag != 0 && reprobe_gid != NULL) { 4351 int niocs, jj; 4352 ibdm_ioc_info_t *tmp_ioc; 4353 int ioc_matched = 0; 4354 4355 mutex_exit(&ibdm.ibdm_hl_mutex); 4356 mutex_enter(&reprobe_gid->gl_mutex); 4357 reprobe_gid->gl_state = IBDM_GET_IOC_DETAILS; 4358 niocs = 4359 reprobe_gid->gl_iou->iou_info.iou_num_ctrl_slots; 4360 reprobe_gid->gl_pending_cmds++; 4361 mutex_exit(&reprobe_gid->gl_mutex); 4362 4363 for (jj = 0; jj < niocs; jj++) { 4364 tmp_ioc = 4365 IBDM_GIDINFO2IOCINFO(reprobe_gid, jj); 4366 if (tmp_ioc->ioc_profile.ioc_guid != ioc_guid) 4367 continue; 4368 4369 ioc_matched = 1; 4370 4371 /* 4372 * Explicitly set gl_reprobe_flag to 0 so that 4373 * IBnex is not notified on completion 4374 */ 4375 mutex_enter(&reprobe_gid->gl_mutex); 4376 reprobe_gid->gl_reprobe_flag = 0; 4377 mutex_exit(&reprobe_gid->gl_mutex); 4378 4379 mutex_enter(&ibdm.ibdm_mutex); 4380 ibdm.ibdm_ngid_probes_in_progress++; 4381 mutex_exit(&ibdm.ibdm_mutex); 4382 if (ibdm_send_ioc_profile(reprobe_gid, jj) != 4383 IBDM_SUCCESS) { 4384 IBTF_DPRINTF_L4("ibdm", 4385 "\tprobe_ioc: " 4386 "send_ioc_profile failed " 4387 "for ioc %d", jj); 4388 ibdm_gid_decr_pending(reprobe_gid); 4389 break; 4390 } 4391 mutex_enter(&ibdm.ibdm_mutex); 4392 ibdm_wait_probe_completion(); 4393 mutex_exit(&ibdm.ibdm_mutex); 4394 break; 4395 } 4396 if (ioc_matched == 0) 4397 ibdm_gid_decr_pending(reprobe_gid); 4398 else { 4399 mutex_enter(&ibdm.ibdm_hl_mutex); 4400 break; 4401 } 4402 } else if (new_gid != NULL) { 4403 mutex_exit(&ibdm.ibdm_hl_mutex); 4404 node_gid = node_gid ? node_gid : new_gid; 4405 4406 /* 4407 * New or reinserted GID : Enable notification 4408 * to IBnex 4409 */ 4410 mutex_enter(&node_gid->gl_mutex); 4411 node_gid->gl_reprobe_flag = 1; 4412 mutex_exit(&node_gid->gl_mutex); 4413 4414 ibdm_probe_gid(node_gid); 4415 4416 mutex_enter(&ibdm.ibdm_hl_mutex); 4417 } 4418 } 4419 mutex_exit(&ibdm.ibdm_hl_mutex); 4420 IBTF_DPRINTF_L4("ibdm", "\tprobe_ioc : End\n"); 4421 } 4422 4423 4424 /* 4425 * ibdm_probe_gid() 4426 * Selectively probes the GID 4427 */ 4428 static void 4429 ibdm_probe_gid(ibdm_dp_gidinfo_t *gid_info) 4430 { 4431 IBTF_DPRINTF_L4("ibdm", "\tprobe_gid:"); 4432 4433 /* 4434 * A Cisco FC GW needs the special handling to get IOUnitInfo. 4435 */ 4436 mutex_enter(&gid_info->gl_mutex); 4437 if (ibdm_is_cisco_switch(gid_info)) { 4438 gid_info->gl_pending_cmds++; 4439 gid_info->gl_state = IBDM_SET_CLASSPORTINFO; 4440 mutex_exit(&gid_info->gl_mutex); 4441 4442 if (ibdm_set_classportinfo(gid_info) != IBDM_SUCCESS) { 4443 4444 mutex_enter(&gid_info->gl_mutex); 4445 gid_info->gl_state = IBDM_GID_PROBING_FAILED; 4446 --gid_info->gl_pending_cmds; 4447 mutex_exit(&gid_info->gl_mutex); 4448 4449 /* free the hca_list on this gid_info */ 4450 ibdm_delete_glhca_list(gid_info); 4451 gid_info = gid_info->gl_next; 4452 return; 4453 } 4454 4455 mutex_enter(&gid_info->gl_mutex); 4456 ibdm_wait_cisco_probe_completion(gid_info); 4457 4458 IBTF_DPRINTF_L4("ibdm", 4459 "\tprobe_gid: CISCO Wakeup signal received"); 4460 } 4461 4462 /* move on to the 'GET_CLASSPORTINFO' stage */ 4463 gid_info->gl_pending_cmds++; 4464 gid_info->gl_state = IBDM_GET_CLASSPORTINFO; 4465 mutex_exit(&gid_info->gl_mutex); 4466 4467 if (ibdm_send_classportinfo(gid_info) != IBDM_SUCCESS) { 4468 4469 mutex_enter(&gid_info->gl_mutex); 4470 gid_info->gl_state = IBDM_GID_PROBING_FAILED; 4471 --gid_info->gl_pending_cmds; 4472 mutex_exit(&gid_info->gl_mutex); 4473 4474 /* free the hca_list on this gid_info */ 4475 ibdm_delete_glhca_list(gid_info); 4476 gid_info = gid_info->gl_next; 4477 return; 4478 } 4479 4480 mutex_enter(&ibdm.ibdm_mutex); 4481 ibdm.ibdm_ngid_probes_in_progress++; 4482 gid_info = gid_info->gl_next; 4483 ibdm_wait_probe_completion(); 4484 mutex_exit(&ibdm.ibdm_mutex); 4485 4486 IBTF_DPRINTF_L4("ibdm", "\tprobe_gid: Wakeup signal received"); 4487 } 4488 4489 4490 /* 4491 * ibdm_create_gid_info() 4492 * Allocates a gid_info structure and initializes 4493 * Returns pointer to the structure on success 4494 * and NULL on failure 4495 */ 4496 static ibdm_dp_gidinfo_t * 4497 ibdm_create_gid_info(ibdm_port_attr_t *port, ib_gid_t sgid, ib_gid_t dgid) 4498 { 4499 uint8_t ii, npaths; 4500 sa_path_record_t *path; 4501 size_t len; 4502 ibdm_pkey_tbl_t *pkey_tbl; 4503 ibdm_dp_gidinfo_t *gid_info = NULL; 4504 int ret; 4505 4506 IBTF_DPRINTF_L4("ibdm", "\tcreate_gid_info: Begin"); 4507 npaths = 1; 4508 4509 /* query for reversible paths */ 4510 if (port->pa_sa_hdl) 4511 ret = ibmf_saa_gid_to_pathrecords(port->pa_sa_hdl, 4512 sgid, dgid, IBMF_SAA_PKEY_WC, 0, B_TRUE, &npaths, 0, 4513 &len, &path); 4514 else 4515 return (NULL); 4516 4517 if (ret == IBMF_SUCCESS && path) { 4518 ibdm_dump_path_info(path); 4519 4520 gid_info = kmem_zalloc( 4521 sizeof (ibdm_dp_gidinfo_t), KM_SLEEP); 4522 mutex_init(&gid_info->gl_mutex, NULL, MUTEX_DEFAULT, NULL); 4523 cv_init(&gid_info->gl_probe_cv, NULL, CV_DRIVER, NULL); 4524 gid_info->gl_dgid_hi = path->DGID.gid_prefix; 4525 gid_info->gl_dgid_lo = path->DGID.gid_guid; 4526 gid_info->gl_sgid_hi = path->SGID.gid_prefix; 4527 gid_info->gl_sgid_lo = path->SGID.gid_guid; 4528 gid_info->gl_p_key = path->P_Key; 4529 gid_info->gl_sa_hdl = port->pa_sa_hdl; 4530 gid_info->gl_ibmf_hdl = port->pa_ibmf_hdl; 4531 gid_info->gl_slid = path->SLID; 4532 gid_info->gl_dlid = path->DLID; 4533 gid_info->gl_transactionID = (++ibdm.ibdm_transactionID) 4534 << IBDM_GID_TRANSACTIONID_SHIFT; 4535 gid_info->gl_min_transactionID = gid_info->gl_transactionID; 4536 gid_info->gl_max_transactionID = (ibdm.ibdm_transactionID +1) 4537 << IBDM_GID_TRANSACTIONID_SHIFT; 4538 gid_info->gl_SL = path->SL; 4539 4540 gid_info->gl_qp_hdl = IBMF_QP_HANDLE_DEFAULT; 4541 for (ii = 0; ii < port->pa_npkeys; ii++) { 4542 if (port->pa_pkey_tbl == NULL) 4543 break; 4544 4545 pkey_tbl = &port->pa_pkey_tbl[ii]; 4546 if ((gid_info->gl_p_key == pkey_tbl->pt_pkey) && 4547 (pkey_tbl->pt_qp_hdl != NULL)) { 4548 gid_info->gl_qp_hdl = pkey_tbl->pt_qp_hdl; 4549 break; 4550 } 4551 } 4552 kmem_free(path, len); 4553 4554 /* 4555 * QP handle for GID not initialized. No matching Pkey 4556 * was found!! ibdm should *not* hit this case. Flag an 4557 * error and drop the GID if ibdm does encounter this. 4558 */ 4559 if (gid_info->gl_qp_hdl == NULL) { 4560 IBTF_DPRINTF_L2(ibdm_string, 4561 "\tcreate_gid_info: No matching Pkey"); 4562 ibdm_delete_gidinfo(gid_info); 4563 return (NULL); 4564 } 4565 4566 ibdm.ibdm_ngids++; 4567 if (ibdm.ibdm_dp_gidlist_head == NULL) { 4568 ibdm.ibdm_dp_gidlist_head = gid_info; 4569 ibdm.ibdm_dp_gidlist_tail = gid_info; 4570 } else { 4571 ibdm.ibdm_dp_gidlist_tail->gl_next = gid_info; 4572 gid_info->gl_prev = ibdm.ibdm_dp_gidlist_tail; 4573 ibdm.ibdm_dp_gidlist_tail = gid_info; 4574 } 4575 } 4576 4577 return (gid_info); 4578 } 4579 4580 4581 /* 4582 * ibdm_get_node_records 4583 * Sends a SA query to get the NODE record 4584 * Returns pointer to the sa_node_record_t on success 4585 * and NULL on failure 4586 */ 4587 static sa_node_record_t * 4588 ibdm_get_node_records(ibmf_saa_handle_t sa_hdl, size_t *length, ib_guid_t guid) 4589 { 4590 sa_node_record_t req, *resp = NULL; 4591 ibmf_saa_access_args_t args; 4592 int ret; 4593 4594 IBTF_DPRINTF_L4("ibdm", "\tget_node_records: Begin"); 4595 4596 bzero(&req, sizeof (sa_node_record_t)); 4597 req.NodeInfo.NodeGUID = guid; 4598 4599 args.sq_attr_id = SA_NODERECORD_ATTRID; 4600 args.sq_access_type = IBMF_SAA_RETRIEVE; 4601 args.sq_component_mask = SA_NODEINFO_COMPMASK_NODEGUID; 4602 args.sq_template = &req; 4603 args.sq_callback = NULL; 4604 args.sq_callback_arg = NULL; 4605 4606 ret = ibmf_sa_access(sa_hdl, &args, 0, length, (void **) &resp); 4607 if (ret != IBMF_SUCCESS) { 4608 IBTF_DPRINTF_L2("ibdm", "\tget_node_records:" 4609 " SA Retrieve Failed: %d", ret); 4610 return (NULL); 4611 } 4612 if ((resp == NULL) || (*length == 0)) { 4613 IBTF_DPRINTF_L2("ibdm", "\tget_node_records: No records"); 4614 return (NULL); 4615 } 4616 4617 IBTF_DPRINTF_L4("ibdm", "\tget_node_records: NodeGuid %llx " 4618 "PortGUID %llx", resp->NodeInfo.NodeGUID, resp->NodeInfo.PortGUID); 4619 4620 return (resp); 4621 } 4622 4623 4624 /* 4625 * ibdm_get_portinfo() 4626 * Sends a SA query to get the PortInfo record 4627 * Returns pointer to the sa_portinfo_record_t on success 4628 * and NULL on failure 4629 */ 4630 static sa_portinfo_record_t * 4631 ibdm_get_portinfo(ibmf_saa_handle_t sa_hdl, size_t *length, ib_lid_t lid) 4632 { 4633 sa_portinfo_record_t req, *resp = NULL; 4634 ibmf_saa_access_args_t args; 4635 int ret; 4636 4637 IBTF_DPRINTF_L4("ibdm", "\tget_portinfo: Begin"); 4638 4639 bzero(&req, sizeof (sa_portinfo_record_t)); 4640 req.EndportLID = lid; 4641 4642 args.sq_attr_id = SA_PORTINFORECORD_ATTRID; 4643 args.sq_access_type = IBMF_SAA_RETRIEVE; 4644 args.sq_component_mask = SA_PORTINFO_COMPMASK_PORTLID; 4645 args.sq_template = &req; 4646 args.sq_callback = NULL; 4647 args.sq_callback_arg = NULL; 4648 4649 ret = ibmf_sa_access(sa_hdl, &args, 0, length, (void **) &resp); 4650 if (ret != IBMF_SUCCESS) { 4651 IBTF_DPRINTF_L2("ibdm", "\tget_portinfo:" 4652 " SA Retrieve Failed: 0x%X", ret); 4653 return (NULL); 4654 } 4655 if ((*length == 0) || (resp == NULL)) 4656 return (NULL); 4657 4658 IBTF_DPRINTF_L4("ibdm", "\tget_portinfo: GidPrefix %llx Cap 0x%x", 4659 resp->PortInfo.GidPrefix, resp->PortInfo.CapabilityMask); 4660 return (resp); 4661 } 4662 4663 4664 /* 4665 * ibdm_ibnex_register_callback 4666 * IB nexus callback routine for HCA attach and detach notification 4667 */ 4668 void 4669 ibdm_ibnex_register_callback(ibdm_callback_t ibnex_dm_callback) 4670 { 4671 IBTF_DPRINTF_L4("ibdm", "\tibnex_register_callbacks"); 4672 mutex_enter(&ibdm.ibdm_ibnex_mutex); 4673 ibdm.ibdm_ibnex_callback = ibnex_dm_callback; 4674 mutex_exit(&ibdm.ibdm_ibnex_mutex); 4675 } 4676 4677 4678 /* 4679 * ibdm_ibnex_unregister_callbacks 4680 */ 4681 void 4682 ibdm_ibnex_unregister_callback() 4683 { 4684 IBTF_DPRINTF_L4("ibdm", "\tibnex_unregister_callbacks"); 4685 mutex_enter(&ibdm.ibdm_ibnex_mutex); 4686 ibdm.ibdm_ibnex_callback = NULL; 4687 mutex_exit(&ibdm.ibdm_ibnex_mutex); 4688 } 4689 4690 /* 4691 * ibdm_get_waittime() 4692 * Calculates the wait time based on the last HCA attach time 4693 */ 4694 static time_t 4695 ibdm_get_waittime(ib_guid_t hca_guid, int dft_wait) 4696 { 4697 int ii; 4698 time_t temp, wait_time = 0; 4699 ibdm_hca_list_t *hca; 4700 4701 IBTF_DPRINTF_L4("ibdm", "\tget_waittime hcaguid:%llx" 4702 "\tport settling time %d", hca_guid, dft_wait); 4703 4704 ASSERT(mutex_owned(&ibdm.ibdm_hl_mutex)); 4705 4706 hca = ibdm.ibdm_hca_list_head; 4707 4708 if (hca_guid) { 4709 for (ii = 0; ii < ibdm.ibdm_hca_count; ii++) { 4710 if ((hca_guid == hca->hl_hca_guid) && 4711 (hca->hl_nports != hca->hl_nports_active)) { 4712 wait_time = 4713 ddi_get_time() - hca->hl_attach_time; 4714 wait_time = ((wait_time >= dft_wait) ? 4715 0 : (dft_wait - wait_time)); 4716 break; 4717 } 4718 hca = hca->hl_next; 4719 } 4720 IBTF_DPRINTF_L2("ibdm", "\tget_waittime: wait_time = %ld secs", 4721 (long)wait_time); 4722 return (wait_time); 4723 } 4724 4725 for (ii = 0; ii < ibdm.ibdm_hca_count; ii++) { 4726 if (hca->hl_nports != hca->hl_nports_active) { 4727 temp = ddi_get_time() - hca->hl_attach_time; 4728 temp = ((temp >= dft_wait) ? 0 : (dft_wait - temp)); 4729 wait_time = (temp > wait_time) ? temp : wait_time; 4730 } 4731 hca = hca->hl_next; 4732 } 4733 IBTF_DPRINTF_L2("ibdm", "\tget_waittime: wait_time = %ld secs", 4734 (long)wait_time); 4735 return (wait_time); 4736 } 4737 4738 void 4739 ibdm_ibnex_port_settle_wait(ib_guid_t hca_guid, int dft_wait) 4740 { 4741 time_t wait_time; 4742 clock_t delta; 4743 4744 mutex_enter(&ibdm.ibdm_hl_mutex); 4745 4746 while ((wait_time = ibdm_get_waittime(hca_guid, dft_wait)) > 0) { 4747 if (wait_time > dft_wait) { 4748 IBTF_DPRINTF_L1("ibdm", 4749 "\tibnex_port_settle_wait: wait_time = %ld secs; " 4750 "Resetting to %d secs", 4751 (long)wait_time, dft_wait); 4752 wait_time = dft_wait; 4753 } 4754 delta = drv_usectohz(wait_time * 1000000); 4755 (void) cv_reltimedwait(&ibdm.ibdm_port_settle_cv, 4756 &ibdm.ibdm_hl_mutex, delta, TR_CLOCK_TICK); 4757 } 4758 4759 mutex_exit(&ibdm.ibdm_hl_mutex); 4760 } 4761 4762 4763 /* 4764 * ibdm_ibnex_probe_hcaport 4765 * Probes the presence of HCA port (with HCA dip and port number) 4766 * Returns port attributes structure on SUCCESS 4767 */ 4768 ibdm_port_attr_t * 4769 ibdm_ibnex_probe_hcaport(ib_guid_t hca_guid, uint8_t port_num) 4770 { 4771 int ii, jj; 4772 ibdm_hca_list_t *hca_list; 4773 ibdm_port_attr_t *port_attr; 4774 4775 IBTF_DPRINTF_L4("ibdm", "\tibnex_probe_hcaport:"); 4776 4777 mutex_enter(&ibdm.ibdm_hl_mutex); 4778 hca_list = ibdm.ibdm_hca_list_head; 4779 for (ii = 0; ii < ibdm.ibdm_hca_count; ii++) { 4780 if (hca_list->hl_hca_guid == hca_guid) { 4781 for (jj = 0; jj < hca_list->hl_nports; jj++) { 4782 if (hca_list->hl_port_attr[jj].pa_port_num == 4783 port_num) { 4784 break; 4785 } 4786 } 4787 if (jj != hca_list->hl_nports) 4788 break; 4789 } 4790 hca_list = hca_list->hl_next; 4791 } 4792 if (ii == ibdm.ibdm_hca_count) { 4793 IBTF_DPRINTF_L2("ibdm", "\tibnex_probe_hcaport: not found"); 4794 mutex_exit(&ibdm.ibdm_hl_mutex); 4795 return (NULL); 4796 } 4797 port_attr = (ibdm_port_attr_t *)kmem_zalloc( 4798 sizeof (ibdm_port_attr_t), KM_SLEEP); 4799 bcopy((char *)&hca_list->hl_port_attr[jj], 4800 port_attr, sizeof (ibdm_port_attr_t)); 4801 ibdm_update_port_attr(port_attr); 4802 4803 mutex_exit(&ibdm.ibdm_hl_mutex); 4804 return (port_attr); 4805 } 4806 4807 4808 /* 4809 * ibdm_ibnex_get_port_attrs 4810 * Scan all HCAs for a matching port_guid. 4811 * Returns "port attributes" structure on success. 4812 */ 4813 ibdm_port_attr_t * 4814 ibdm_ibnex_get_port_attrs(ib_guid_t port_guid) 4815 { 4816 int ii, jj; 4817 ibdm_hca_list_t *hca_list; 4818 ibdm_port_attr_t *port_attr; 4819 4820 IBTF_DPRINTF_L4("ibdm", "\tibnex_get_port_attrs:"); 4821 4822 mutex_enter(&ibdm.ibdm_hl_mutex); 4823 hca_list = ibdm.ibdm_hca_list_head; 4824 4825 for (ii = 0; ii < ibdm.ibdm_hca_count; ii++) { 4826 for (jj = 0; jj < hca_list->hl_nports; jj++) { 4827 if (hca_list->hl_port_attr[jj].pa_port_guid == 4828 port_guid) { 4829 break; 4830 } 4831 } 4832 if (jj != hca_list->hl_nports) 4833 break; 4834 hca_list = hca_list->hl_next; 4835 } 4836 4837 if (ii == ibdm.ibdm_hca_count) { 4838 IBTF_DPRINTF_L2("ibdm", "\tibnex_get_port_attrs: not found"); 4839 mutex_exit(&ibdm.ibdm_hl_mutex); 4840 return (NULL); 4841 } 4842 4843 port_attr = (ibdm_port_attr_t *)kmem_alloc(sizeof (ibdm_port_attr_t), 4844 KM_SLEEP); 4845 bcopy((char *)&hca_list->hl_port_attr[jj], port_attr, 4846 sizeof (ibdm_port_attr_t)); 4847 ibdm_update_port_attr(port_attr); 4848 4849 mutex_exit(&ibdm.ibdm_hl_mutex); 4850 return (port_attr); 4851 } 4852 4853 4854 /* 4855 * ibdm_ibnex_free_port_attr() 4856 */ 4857 void 4858 ibdm_ibnex_free_port_attr(ibdm_port_attr_t *port_attr) 4859 { 4860 IBTF_DPRINTF_L4("ibdm", "\tibnex_free_port_attr:"); 4861 if (port_attr) { 4862 if (port_attr->pa_pkey_tbl != NULL) { 4863 kmem_free(port_attr->pa_pkey_tbl, 4864 (port_attr->pa_npkeys * sizeof (ibdm_pkey_tbl_t))); 4865 } 4866 kmem_free(port_attr, sizeof (ibdm_port_attr_t)); 4867 } 4868 } 4869 4870 4871 /* 4872 * ibdm_ibnex_get_hca_list() 4873 * Returns portinfo for all the port for all the HCA's 4874 */ 4875 void 4876 ibdm_ibnex_get_hca_list(ibdm_hca_list_t **hca, int *count) 4877 { 4878 ibdm_hca_list_t *head = NULL, *temp, *temp1; 4879 int ii; 4880 4881 IBTF_DPRINTF_L4("ibdm", "\tibnex_get_hca_list:"); 4882 4883 mutex_enter(&ibdm.ibdm_hl_mutex); 4884 temp = ibdm.ibdm_hca_list_head; 4885 for (ii = 0; ii < ibdm.ibdm_hca_count; ii++) { 4886 temp1 = ibdm_dup_hca_attr(temp); 4887 temp1->hl_next = head; 4888 head = temp1; 4889 temp = temp->hl_next; 4890 } 4891 *count = ibdm.ibdm_hca_count; 4892 *hca = head; 4893 mutex_exit(&ibdm.ibdm_hl_mutex); 4894 } 4895 4896 4897 /* 4898 * ibdm_ibnex_get_hca_info_by_guid() 4899 */ 4900 ibdm_hca_list_t * 4901 ibdm_ibnex_get_hca_info_by_guid(ib_guid_t hca_guid) 4902 { 4903 ibdm_hca_list_t *head = NULL, *hca = NULL; 4904 4905 IBTF_DPRINTF_L4("ibdm", "\tibnex_get_hca_info_by_dip"); 4906 4907 mutex_enter(&ibdm.ibdm_hl_mutex); 4908 head = ibdm.ibdm_hca_list_head; 4909 while (head) { 4910 if (head->hl_hca_guid == hca_guid) { 4911 hca = ibdm_dup_hca_attr(head); 4912 hca->hl_next = NULL; 4913 break; 4914 } 4915 head = head->hl_next; 4916 } 4917 mutex_exit(&ibdm.ibdm_hl_mutex); 4918 IBTF_DPRINTF_L4("ibdm", "\tibnex_get_hca_info_by_dip %p", hca); 4919 return (hca); 4920 } 4921 4922 4923 /* 4924 * ibdm_dup_hca_attr() 4925 * Allocate a new HCA attribute strucuture and initialize 4926 * hca attribute structure with the incoming HCA attributes 4927 * returned the allocated hca attributes. 4928 */ 4929 static ibdm_hca_list_t * 4930 ibdm_dup_hca_attr(ibdm_hca_list_t *in_hca) 4931 { 4932 int len; 4933 ibdm_hca_list_t *out_hca; 4934 4935 len = sizeof (ibdm_hca_list_t) + 4936 (in_hca->hl_nports * sizeof (ibdm_port_attr_t)); 4937 IBTF_DPRINTF_L4("ibdm", "\tdup_hca_attr len %d", len); 4938 out_hca = (ibdm_hca_list_t *)kmem_alloc(len, KM_SLEEP); 4939 bcopy((char *)in_hca, 4940 (char *)out_hca, sizeof (ibdm_hca_list_t)); 4941 if (in_hca->hl_nports) { 4942 out_hca->hl_port_attr = (ibdm_port_attr_t *) 4943 ((char *)out_hca + sizeof (ibdm_hca_list_t)); 4944 bcopy((char *)in_hca->hl_port_attr, 4945 (char *)out_hca->hl_port_attr, 4946 (in_hca->hl_nports * sizeof (ibdm_port_attr_t))); 4947 for (len = 0; len < out_hca->hl_nports; len++) 4948 ibdm_update_port_attr(&out_hca->hl_port_attr[len]); 4949 } 4950 return (out_hca); 4951 } 4952 4953 4954 /* 4955 * ibdm_ibnex_free_hca_list() 4956 * Free one/more HCA lists 4957 */ 4958 void 4959 ibdm_ibnex_free_hca_list(ibdm_hca_list_t *hca_list) 4960 { 4961 int ii; 4962 size_t len; 4963 ibdm_hca_list_t *temp; 4964 ibdm_port_attr_t *port; 4965 4966 IBTF_DPRINTF_L4("ibdm", "\tibnex_free_hca_list:"); 4967 ASSERT(hca_list); 4968 while (hca_list) { 4969 temp = hca_list; 4970 hca_list = hca_list->hl_next; 4971 for (ii = 0; ii < temp->hl_nports; ii++) { 4972 port = &temp->hl_port_attr[ii]; 4973 len = (port->pa_npkeys * sizeof (ibdm_pkey_tbl_t)); 4974 if (len != 0) 4975 kmem_free(port->pa_pkey_tbl, len); 4976 } 4977 len = sizeof (ibdm_hca_list_t) + (temp->hl_nports * 4978 sizeof (ibdm_port_attr_t)); 4979 kmem_free(temp, len); 4980 } 4981 } 4982 4983 4984 /* 4985 * ibdm_ibnex_probe_iocguid() 4986 * Probes the IOC on the fabric and returns the IOC information 4987 * if present. Otherwise, NULL is returned 4988 */ 4989 /* ARGSUSED */ 4990 ibdm_ioc_info_t * 4991 ibdm_ibnex_probe_ioc(ib_guid_t iou, ib_guid_t ioc_guid, int reprobe_flag) 4992 { 4993 int k; 4994 ibdm_ioc_info_t *ioc_info; 4995 ibdm_dp_gidinfo_t *gid_info; /* used as index and arg */ 4996 timeout_id_t *timeout_id; 4997 4998 IBTF_DPRINTF_L4("ibdm", "\tibnex_probe_ioc: (%llX, %llX, %d) Begin", 4999 iou, ioc_guid, reprobe_flag); 5000 5001 if (ibdm_enumerate_iocs == 0) 5002 return (NULL); 5003 5004 /* Check whether we know this already */ 5005 ioc_info = ibdm_get_ioc_info_with_gid(ioc_guid, &gid_info); 5006 if (ioc_info == NULL) { 5007 mutex_enter(&ibdm.ibdm_mutex); 5008 while (ibdm.ibdm_busy & IBDM_BUSY) 5009 cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex); 5010 ibdm.ibdm_busy |= IBDM_BUSY; 5011 mutex_exit(&ibdm.ibdm_mutex); 5012 ibdm_probe_ioc(iou, ioc_guid, 0); 5013 mutex_enter(&ibdm.ibdm_mutex); 5014 ibdm.ibdm_busy &= ~IBDM_BUSY; 5015 cv_broadcast(&ibdm.ibdm_busy_cv); 5016 mutex_exit(&ibdm.ibdm_mutex); 5017 ioc_info = ibdm_get_ioc_info_with_gid(ioc_guid, &gid_info); 5018 } else if (reprobe_flag) { /* Handle Reprobe for the IOC */ 5019 ASSERT(gid_info != NULL); 5020 /* Free the ioc_list before reprobe; and cancel any timers */ 5021 mutex_enter(&ibdm.ibdm_mutex); 5022 mutex_enter(&gid_info->gl_mutex); 5023 if (ioc_info->ioc_timeout_id) { 5024 timeout_id = ioc_info->ioc_timeout_id; 5025 ioc_info->ioc_timeout_id = 0; 5026 mutex_exit(&gid_info->gl_mutex); 5027 IBTF_DPRINTF_L5("ibdm", "\tprobe_ioc: " 5028 "ioc_timeout_id = 0x%x", timeout_id); 5029 if (untimeout(timeout_id) == -1) { 5030 IBTF_DPRINTF_L2("ibdm", "\tprobe_ioc: " 5031 "untimeout ioc_timeout_id failed"); 5032 } 5033 mutex_enter(&gid_info->gl_mutex); 5034 } 5035 if (ioc_info->ioc_dc_timeout_id) { 5036 timeout_id = ioc_info->ioc_dc_timeout_id; 5037 ioc_info->ioc_dc_timeout_id = 0; 5038 mutex_exit(&gid_info->gl_mutex); 5039 IBTF_DPRINTF_L5("ibdm", "\tprobe_ioc: " 5040 "ioc_dc_timeout_id = 0x%x", timeout_id); 5041 if (untimeout(timeout_id) == -1) { 5042 IBTF_DPRINTF_L2("ibdm", "\tprobe_ioc: " 5043 "untimeout ioc_dc_timeout_id failed"); 5044 } 5045 mutex_enter(&gid_info->gl_mutex); 5046 } 5047 for (k = 0; k < ioc_info->ioc_profile.ioc_service_entries; k++) 5048 if (ioc_info->ioc_serv[k].se_timeout_id) { 5049 timeout_id = ioc_info->ioc_serv[k]. 5050 se_timeout_id; 5051 ioc_info->ioc_serv[k].se_timeout_id = 0; 5052 mutex_exit(&gid_info->gl_mutex); 5053 IBTF_DPRINTF_L5("ibdm", "\tprobe_ioc: " 5054 "ioc_info->ioc_serv[k].se_timeout_id = %x", 5055 k, timeout_id); 5056 if (untimeout(timeout_id) == -1) { 5057 IBTF_DPRINTF_L2("ibdm", "\tprobe_ioc: " 5058 "untimeout se_timeout_id %d " 5059 "failed", k); 5060 } 5061 mutex_enter(&gid_info->gl_mutex); 5062 } 5063 mutex_exit(&gid_info->gl_mutex); 5064 mutex_exit(&ibdm.ibdm_mutex); 5065 ibdm_ibnex_free_ioc_list(ioc_info); 5066 5067 mutex_enter(&ibdm.ibdm_mutex); 5068 while (ibdm.ibdm_busy & IBDM_BUSY) 5069 cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex); 5070 ibdm.ibdm_busy |= IBDM_BUSY; 5071 mutex_exit(&ibdm.ibdm_mutex); 5072 5073 ibdm_probe_ioc(iou, ioc_guid, 1); 5074 5075 /* 5076 * Skip if gl_reprobe_flag is set, this will be 5077 * a re-inserted / new GID, for which notifications 5078 * have already been send. 5079 */ 5080 for (gid_info = ibdm.ibdm_dp_gidlist_head; gid_info; 5081 gid_info = gid_info->gl_next) { 5082 uint8_t ii, niocs; 5083 ibdm_ioc_info_t *ioc; 5084 5085 if (gid_info->gl_iou == NULL) 5086 continue; 5087 5088 if (gid_info->gl_reprobe_flag) { 5089 gid_info->gl_reprobe_flag = 0; 5090 continue; 5091 } 5092 5093 niocs = gid_info->gl_iou->iou_info.iou_num_ctrl_slots; 5094 for (ii = 0; ii < niocs; ii++) { 5095 ioc = IBDM_GIDINFO2IOCINFO(gid_info, ii); 5096 if (ioc->ioc_profile.ioc_guid == ioc_guid) { 5097 mutex_enter(&ibdm.ibdm_mutex); 5098 ibdm_reprobe_update_port_srv(ioc, 5099 gid_info); 5100 mutex_exit(&ibdm.ibdm_mutex); 5101 } 5102 } 5103 } 5104 mutex_enter(&ibdm.ibdm_mutex); 5105 ibdm.ibdm_busy &= ~IBDM_BUSY; 5106 cv_broadcast(&ibdm.ibdm_busy_cv); 5107 mutex_exit(&ibdm.ibdm_mutex); 5108 5109 ioc_info = ibdm_get_ioc_info_with_gid(ioc_guid, &gid_info); 5110 } 5111 return (ioc_info); 5112 } 5113 5114 5115 /* 5116 * ibdm_get_ioc_info_with_gid() 5117 * Returns pointer to ibdm_ioc_info_t if it finds 5118 * matching record for the ioc_guid. Otherwise NULL is returned. 5119 * The pointer to gid_info is set to the second argument in case that 5120 * the non-NULL value returns (and the second argument is not NULL). 5121 * 5122 * Note. use the same strings as "ibnex_get_ioc_info" in 5123 * IBTF_DPRINTF() to keep compatibility. 5124 */ 5125 static ibdm_ioc_info_t * 5126 ibdm_get_ioc_info_with_gid(ib_guid_t ioc_guid, 5127 ibdm_dp_gidinfo_t **gid_info) 5128 { 5129 int ii; 5130 ibdm_ioc_info_t *ioc = NULL, *tmp = NULL; 5131 ibdm_dp_gidinfo_t *gid_list; 5132 ib_dm_io_unitinfo_t *iou; 5133 5134 IBTF_DPRINTF_L4("ibdm", "\tibnex_get_ioc_info: GUID %llx", ioc_guid); 5135 5136 mutex_enter(&ibdm.ibdm_mutex); 5137 while (ibdm.ibdm_busy & IBDM_BUSY) 5138 cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex); 5139 ibdm.ibdm_busy |= IBDM_BUSY; 5140 5141 if (gid_info) 5142 *gid_info = NULL; /* clear the value of gid_info */ 5143 5144 gid_list = ibdm.ibdm_dp_gidlist_head; 5145 while (gid_list) { 5146 mutex_enter(&gid_list->gl_mutex); 5147 if (gid_list->gl_state != IBDM_GID_PROBING_COMPLETE) { 5148 mutex_exit(&gid_list->gl_mutex); 5149 gid_list = gid_list->gl_next; 5150 continue; 5151 } 5152 if (gid_list->gl_iou == NULL) { 5153 IBTF_DPRINTF_L2("ibdm", 5154 "\tget_ioc_info: No IOU info"); 5155 mutex_exit(&gid_list->gl_mutex); 5156 gid_list = gid_list->gl_next; 5157 continue; 5158 } 5159 iou = &gid_list->gl_iou->iou_info; 5160 for (ii = 0; ii < iou->iou_num_ctrl_slots; ii++) { 5161 tmp = IBDM_GIDINFO2IOCINFO(gid_list, ii); 5162 if ((tmp->ioc_profile.ioc_guid == ioc_guid) && 5163 (tmp->ioc_state == IBDM_IOC_STATE_PROBE_SUCCESS)) { 5164 ioc = ibdm_dup_ioc_info(tmp, gid_list); 5165 if (gid_info) 5166 *gid_info = gid_list; /* set this ptr */ 5167 mutex_exit(&gid_list->gl_mutex); 5168 ibdm.ibdm_busy &= ~IBDM_BUSY; 5169 cv_broadcast(&ibdm.ibdm_busy_cv); 5170 mutex_exit(&ibdm.ibdm_mutex); 5171 IBTF_DPRINTF_L4("ibdm", "\tget_ioc_info: End"); 5172 return (ioc); 5173 } 5174 } 5175 if (ii == iou->iou_num_ctrl_slots) 5176 ioc = NULL; 5177 5178 mutex_exit(&gid_list->gl_mutex); 5179 gid_list = gid_list->gl_next; 5180 } 5181 5182 ibdm.ibdm_busy &= ~IBDM_BUSY; 5183 cv_broadcast(&ibdm.ibdm_busy_cv); 5184 mutex_exit(&ibdm.ibdm_mutex); 5185 IBTF_DPRINTF_L4("ibdm", "\tget_ioc_info: failure End"); 5186 return (ioc); 5187 } 5188 5189 /* 5190 * ibdm_ibnex_get_ioc_info() 5191 * Returns pointer to ibdm_ioc_info_t if it finds 5192 * matching record for the ioc_guid, otherwise NULL 5193 * is returned 5194 * 5195 * Note. this is a wrapper function to ibdm_get_ioc_info_with_gid() now. 5196 */ 5197 ibdm_ioc_info_t * 5198 ibdm_ibnex_get_ioc_info(ib_guid_t ioc_guid) 5199 { 5200 if (ibdm_enumerate_iocs == 0) 5201 return (NULL); 5202 5203 /* will not use the gid_info pointer, so the second arg is NULL */ 5204 return (ibdm_get_ioc_info_with_gid(ioc_guid, NULL)); 5205 } 5206 5207 /* 5208 * ibdm_ibnex_get_ioc_count() 5209 * Returns number of ibdm_ioc_info_t it finds 5210 */ 5211 int 5212 ibdm_ibnex_get_ioc_count(void) 5213 { 5214 int count = 0, k; 5215 ibdm_ioc_info_t *ioc; 5216 ibdm_dp_gidinfo_t *gid_list; 5217 5218 if (ibdm_enumerate_iocs == 0) 5219 return (0); 5220 5221 mutex_enter(&ibdm.ibdm_mutex); 5222 ibdm_sweep_fabric(0); 5223 5224 while (ibdm.ibdm_busy & IBDM_BUSY) 5225 cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex); 5226 ibdm.ibdm_busy |= IBDM_BUSY; 5227 5228 for (gid_list = ibdm.ibdm_dp_gidlist_head; gid_list; 5229 gid_list = gid_list->gl_next) { 5230 mutex_enter(&gid_list->gl_mutex); 5231 if ((gid_list->gl_state != IBDM_GID_PROBING_COMPLETE) || 5232 (gid_list->gl_iou == NULL)) { 5233 mutex_exit(&gid_list->gl_mutex); 5234 continue; 5235 } 5236 for (k = 0; k < gid_list->gl_iou->iou_info.iou_num_ctrl_slots; 5237 k++) { 5238 ioc = IBDM_GIDINFO2IOCINFO(gid_list, k); 5239 if (ioc->ioc_state == IBDM_IOC_STATE_PROBE_SUCCESS) 5240 ++count; 5241 } 5242 mutex_exit(&gid_list->gl_mutex); 5243 } 5244 ibdm.ibdm_busy &= ~IBDM_BUSY; 5245 cv_broadcast(&ibdm.ibdm_busy_cv); 5246 mutex_exit(&ibdm.ibdm_mutex); 5247 5248 IBTF_DPRINTF_L4("ibdm", "\tget_ioc_count: count = %d", count); 5249 return (count); 5250 } 5251 5252 5253 /* 5254 * ibdm_ibnex_get_ioc_list() 5255 * Returns information about all the IOCs present on the fabric. 5256 * Reprobes the IOCs and the GID list if list_flag is set to REPROBE_ALL. 5257 * Does not sweep fabric if DONOT_PROBE is set 5258 */ 5259 ibdm_ioc_info_t * 5260 ibdm_ibnex_get_ioc_list(ibdm_ibnex_get_ioclist_mtd_t list_flag) 5261 { 5262 int ii; 5263 ibdm_ioc_info_t *ioc_list = NULL, *tmp, *ioc; 5264 ibdm_dp_gidinfo_t *gid_list; 5265 ib_dm_io_unitinfo_t *iou; 5266 5267 IBTF_DPRINTF_L4("ibdm", "\tget_ioc_list: Enter"); 5268 5269 if (ibdm_enumerate_iocs == 0) 5270 return (NULL); 5271 5272 mutex_enter(&ibdm.ibdm_mutex); 5273 if (list_flag != IBDM_IBNEX_DONOT_PROBE) 5274 ibdm_sweep_fabric(list_flag == IBDM_IBNEX_REPROBE_ALL); 5275 5276 while (ibdm.ibdm_busy & IBDM_BUSY) 5277 cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex); 5278 ibdm.ibdm_busy |= IBDM_BUSY; 5279 5280 gid_list = ibdm.ibdm_dp_gidlist_head; 5281 while (gid_list) { 5282 mutex_enter(&gid_list->gl_mutex); 5283 if (gid_list->gl_state != IBDM_GID_PROBING_COMPLETE) { 5284 mutex_exit(&gid_list->gl_mutex); 5285 gid_list = gid_list->gl_next; 5286 continue; 5287 } 5288 if (gid_list->gl_iou == NULL) { 5289 IBTF_DPRINTF_L2("ibdm", 5290 "\tget_ioc_list: No IOU info"); 5291 mutex_exit(&gid_list->gl_mutex); 5292 gid_list = gid_list->gl_next; 5293 continue; 5294 } 5295 iou = &gid_list->gl_iou->iou_info; 5296 for (ii = 0; ii < iou->iou_num_ctrl_slots; ii++) { 5297 ioc = IBDM_GIDINFO2IOCINFO(gid_list, ii); 5298 if (ioc->ioc_state == IBDM_IOC_STATE_PROBE_SUCCESS) { 5299 tmp = ibdm_dup_ioc_info(ioc, gid_list); 5300 tmp->ioc_next = ioc_list; 5301 ioc_list = tmp; 5302 } 5303 } 5304 mutex_exit(&gid_list->gl_mutex); 5305 gid_list = gid_list->gl_next; 5306 } 5307 ibdm.ibdm_busy &= ~IBDM_BUSY; 5308 cv_broadcast(&ibdm.ibdm_busy_cv); 5309 mutex_exit(&ibdm.ibdm_mutex); 5310 5311 IBTF_DPRINTF_L4("ibdm", "\tget_ioc_list: End"); 5312 return (ioc_list); 5313 } 5314 5315 /* 5316 * ibdm_dup_ioc_info() 5317 * Duplicate the IOC information and return the IOC 5318 * information. 5319 */ 5320 static ibdm_ioc_info_t * 5321 ibdm_dup_ioc_info(ibdm_ioc_info_t *in_ioc, ibdm_dp_gidinfo_t *gid_list) 5322 { 5323 ibdm_ioc_info_t *out_ioc; 5324 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*out_ioc)); 5325 ASSERT(MUTEX_HELD(&gid_list->gl_mutex)); 5326 5327 out_ioc = kmem_alloc(sizeof (ibdm_ioc_info_t), KM_SLEEP); 5328 bcopy(in_ioc, out_ioc, sizeof (ibdm_ioc_info_t)); 5329 ibdm_update_ioc_port_gidlist(out_ioc, gid_list); 5330 out_ioc->ioc_iou_dc_valid = gid_list->gl_iou->iou_dc_valid; 5331 out_ioc->ioc_iou_diagcode = gid_list->gl_iou->iou_diagcode; 5332 5333 return (out_ioc); 5334 } 5335 5336 5337 /* 5338 * ibdm_free_ioc_list() 5339 * Deallocate memory for IOC list structure 5340 */ 5341 void 5342 ibdm_ibnex_free_ioc_list(ibdm_ioc_info_t *ioc) 5343 { 5344 ibdm_ioc_info_t *temp; 5345 5346 IBTF_DPRINTF_L4("ibdm", "\tibnex_free_ioc_list:"); 5347 while (ioc) { 5348 temp = ioc; 5349 ioc = ioc->ioc_next; 5350 kmem_free(temp->ioc_gid_list, 5351 (sizeof (ibdm_gid_t) * temp->ioc_nportgids)); 5352 if (temp->ioc_hca_list) 5353 ibdm_ibnex_free_hca_list(temp->ioc_hca_list); 5354 kmem_free(temp, sizeof (ibdm_ioc_info_t)); 5355 } 5356 } 5357 5358 5359 /* 5360 * ibdm_ibnex_update_pkey_tbls 5361 * Updates the DM P_Key database. 5362 * NOTE: Two cases are handled here: P_Key being added or removed. 5363 * 5364 * Arguments : NONE 5365 * Return Values : NONE 5366 */ 5367 void 5368 ibdm_ibnex_update_pkey_tbls(void) 5369 { 5370 int h, pp, pidx; 5371 uint_t nports; 5372 uint_t size; 5373 ib_pkey_t new_pkey; 5374 ib_pkey_t *orig_pkey; 5375 ibdm_hca_list_t *hca_list; 5376 ibdm_port_attr_t *port; 5377 ibt_hca_portinfo_t *pinfop; 5378 5379 IBTF_DPRINTF_L4("ibdm", "\tibnex_update_pkey_tbls:"); 5380 5381 mutex_enter(&ibdm.ibdm_hl_mutex); 5382 hca_list = ibdm.ibdm_hca_list_head; 5383 5384 for (h = 0; h < ibdm.ibdm_hca_count; h++) { 5385 5386 /* This updates P_Key Tables for all ports of this HCA */ 5387 (void) ibt_query_hca_ports(hca_list->hl_hca_hdl, 0, &pinfop, 5388 &nports, &size); 5389 5390 /* number of ports shouldn't have changed */ 5391 ASSERT(nports == hca_list->hl_nports); 5392 5393 for (pp = 0; pp < hca_list->hl_nports; pp++) { 5394 port = &hca_list->hl_port_attr[pp]; 5395 5396 /* 5397 * First figure out the P_Keys from IBTL. 5398 * Three things could have happened: 5399 * New P_Keys added 5400 * Existing P_Keys removed 5401 * Both of the above two 5402 * 5403 * Loop through the P_Key Indices and check if a 5404 * give P_Key_Ix matches that of the one seen by 5405 * IBDM. If they match no action is needed. 5406 * 5407 * If they don't match: 5408 * 1. if orig_pkey is invalid and new_pkey is valid 5409 * ---> add new_pkey to DM database 5410 * 2. if orig_pkey is valid and new_pkey is invalid 5411 * ---> remove orig_pkey from DM database 5412 * 3. if orig_pkey and new_pkey are both valid: 5413 * ---> remov orig_pkey from DM database 5414 * ---> add new_pkey to DM database 5415 * 4. if orig_pkey and new_pkey are both invalid: 5416 * ---> do nothing. Updated DM database. 5417 */ 5418 5419 for (pidx = 0; pidx < port->pa_npkeys; pidx++) { 5420 new_pkey = pinfop[pp].p_pkey_tbl[pidx]; 5421 orig_pkey = &port->pa_pkey_tbl[pidx].pt_pkey; 5422 5423 /* keys match - do nothing */ 5424 if (*orig_pkey == new_pkey) 5425 continue; 5426 5427 if (IBDM_INVALID_PKEY(*orig_pkey) && 5428 !IBDM_INVALID_PKEY(new_pkey)) { 5429 /* P_Key was added */ 5430 IBTF_DPRINTF_L5("ibdm", 5431 "\tibnex_update_pkey_tbls: new " 5432 "P_Key added = 0x%x", new_pkey); 5433 *orig_pkey = new_pkey; 5434 ibdm_port_attr_ibmf_init(port, 5435 new_pkey, pp); 5436 } else if (!IBDM_INVALID_PKEY(*orig_pkey) && 5437 IBDM_INVALID_PKEY(new_pkey)) { 5438 /* P_Key was removed */ 5439 IBTF_DPRINTF_L5("ibdm", 5440 "\tibnex_update_pkey_tbls: P_Key " 5441 "removed = 0x%x", *orig_pkey); 5442 *orig_pkey = new_pkey; 5443 (void) ibdm_port_attr_ibmf_fini(port, 5444 pidx); 5445 } else if (!IBDM_INVALID_PKEY(*orig_pkey) && 5446 !IBDM_INVALID_PKEY(new_pkey)) { 5447 /* P_Key were replaced */ 5448 IBTF_DPRINTF_L5("ibdm", 5449 "\tibnex_update_pkey_tbls: P_Key " 5450 "replaced 0x%x with 0x%x", 5451 *orig_pkey, new_pkey); 5452 (void) ibdm_port_attr_ibmf_fini(port, 5453 pidx); 5454 *orig_pkey = new_pkey; 5455 ibdm_port_attr_ibmf_init(port, 5456 new_pkey, pp); 5457 } else { 5458 /* 5459 * P_Keys are invalid 5460 * set anyway to reflect if 5461 * INVALID_FULL was changed to 5462 * INVALID_LIMITED or vice-versa. 5463 */ 5464 *orig_pkey = new_pkey; 5465 } /* end of else */ 5466 5467 } /* loop of p_key index */ 5468 5469 } /* loop of #ports of HCA */ 5470 5471 ibt_free_portinfo(pinfop, size); 5472 hca_list = hca_list->hl_next; 5473 5474 } /* loop for all HCAs in the system */ 5475 5476 mutex_exit(&ibdm.ibdm_hl_mutex); 5477 } 5478 5479 5480 /* 5481 * ibdm_send_ioc_profile() 5482 * Send IOC Controller Profile request. When the request is completed 5483 * IBMF calls ibdm_process_incoming_mad routine to inform about 5484 * the completion. 5485 */ 5486 static int 5487 ibdm_send_ioc_profile(ibdm_dp_gidinfo_t *gid_info, uint8_t ioc_no) 5488 { 5489 ibmf_msg_t *msg; 5490 ib_mad_hdr_t *hdr; 5491 ibdm_ioc_info_t *ioc_info = &(gid_info->gl_iou->iou_ioc_info[ioc_no]); 5492 ibdm_timeout_cb_args_t *cb_args; 5493 5494 IBTF_DPRINTF_L4("ibdm", "\tsend_ioc_profile: " 5495 "gid info 0x%p, ioc_no = %d", gid_info, ioc_no); 5496 5497 /* 5498 * Send command to get IOC profile. 5499 * Allocate a IBMF packet and initialize the packet. 5500 */ 5501 if (ibmf_alloc_msg(gid_info->gl_ibmf_hdl, IBMF_ALLOC_SLEEP, 5502 &msg) != IBMF_SUCCESS) { 5503 IBTF_DPRINTF_L2("ibdm", "\tsend_ioc_profile: pkt alloc fail"); 5504 return (IBDM_FAILURE); 5505 } 5506 5507 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*msg)) 5508 ibdm_alloc_send_buffers(msg); 5509 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*msg)) 5510 5511 mutex_enter(&gid_info->gl_mutex); 5512 ibdm_bump_transactionID(gid_info); 5513 mutex_exit(&gid_info->gl_mutex); 5514 5515 msg->im_local_addr.ia_local_lid = gid_info->gl_slid; 5516 msg->im_local_addr.ia_remote_lid = gid_info->gl_dlid; 5517 if (gid_info->gl_redirected == B_TRUE) { 5518 if (gid_info->gl_redirect_dlid != 0) { 5519 msg->im_local_addr.ia_remote_lid = 5520 gid_info->gl_redirect_dlid; 5521 } 5522 msg->im_local_addr.ia_remote_qno = gid_info->gl_redirect_QP; 5523 msg->im_local_addr.ia_p_key = gid_info->gl_redirect_pkey; 5524 msg->im_local_addr.ia_q_key = gid_info->gl_redirect_qkey; 5525 msg->im_local_addr.ia_service_level = gid_info->gl_redirectSL; 5526 } else { 5527 msg->im_local_addr.ia_remote_qno = 1; 5528 msg->im_local_addr.ia_p_key = gid_info->gl_p_key; 5529 msg->im_local_addr.ia_q_key = IB_GSI_QKEY; 5530 msg->im_local_addr.ia_service_level = gid_info->gl_SL; 5531 } 5532 5533 hdr = IBDM_OUT_IBMFMSG_MADHDR(msg); 5534 hdr->BaseVersion = MAD_CLASS_BASE_VERS_1; 5535 hdr->MgmtClass = MAD_MGMT_CLASS_DEV_MGT; 5536 hdr->ClassVersion = IB_DM_CLASS_VERSION_1; 5537 hdr->R_Method = IB_DM_DEVMGT_METHOD_GET; 5538 hdr->Status = 0; 5539 hdr->TransactionID = h2b64(gid_info->gl_transactionID); 5540 hdr->AttributeID = h2b16(IB_DM_ATTR_IOC_CTRL_PROFILE); 5541 hdr->AttributeModifier = h2b32(ioc_no + 1); 5542 5543 ioc_info->ioc_state = IBDM_IOC_STATE_REPROBE_PROGRESS; 5544 cb_args = &ioc_info->ioc_cb_args; 5545 cb_args->cb_gid_info = gid_info; 5546 cb_args->cb_retry_count = ibdm_dft_retry_cnt; 5547 cb_args->cb_req_type = IBDM_REQ_TYPE_IOCINFO; 5548 cb_args->cb_ioc_num = ioc_no; 5549 5550 mutex_enter(&gid_info->gl_mutex); 5551 ioc_info->ioc_timeout_id = timeout(ibdm_pkt_timeout_hdlr, 5552 cb_args, IBDM_TIMEOUT_VALUE(ibdm_dft_timeout)); 5553 mutex_exit(&gid_info->gl_mutex); 5554 5555 IBTF_DPRINTF_L5("ibdm", "\tsend_ioc_profile:" 5556 "timeout %x", ioc_info->ioc_timeout_id); 5557 5558 if (ibmf_msg_transport(gid_info->gl_ibmf_hdl, gid_info->gl_qp_hdl, msg, 5559 NULL, ibdm_ibmf_send_cb, cb_args, 0) != IBMF_SUCCESS) { 5560 IBTF_DPRINTF_L2("ibdm", 5561 "\tsend_ioc_profile: msg transport failed"); 5562 ibdm_ibmf_send_cb(gid_info->gl_ibmf_hdl, msg, cb_args); 5563 } 5564 ioc_info->ioc_state = IBDM_IOC_STATE_REPROBE_PROGRESS; 5565 return (IBDM_SUCCESS); 5566 } 5567 5568 5569 /* 5570 * ibdm_port_reachable 5571 * Returns B_TRUE if the port GID is reachable by sending 5572 * a SA query to get the NODE record for this port GUID. 5573 */ 5574 static boolean_t 5575 ibdm_port_reachable(ibmf_saa_handle_t sa_hdl, ib_guid_t guid) 5576 { 5577 sa_node_record_t *resp; 5578 size_t length; 5579 5580 /* 5581 * Verify if it's reachable by getting the node record. 5582 */ 5583 if (ibdm_get_node_record_by_port(sa_hdl, guid, &resp, &length) == 5584 IBDM_SUCCESS) { 5585 kmem_free(resp, length); 5586 return (B_TRUE); 5587 } 5588 return (B_FALSE); 5589 } 5590 5591 /* 5592 * ibdm_get_node_record_by_port 5593 * Sends a SA query to get the NODE record for port GUID 5594 * Returns IBDM_SUCCESS if the port GID is reachable. 5595 * 5596 * Note: the caller must be responsible for freeing the resource 5597 * by calling kmem_free(resp, length) later. 5598 */ 5599 static int 5600 ibdm_get_node_record_by_port(ibmf_saa_handle_t sa_hdl, ib_guid_t guid, 5601 sa_node_record_t **resp, size_t *length) 5602 { 5603 sa_node_record_t req; 5604 ibmf_saa_access_args_t args; 5605 int ret; 5606 ASSERT(resp != NULL && length != NULL); 5607 5608 IBTF_DPRINTF_L4("ibdm", "\tport_reachable: port_guid %llx", 5609 guid); 5610 5611 bzero(&req, sizeof (sa_node_record_t)); 5612 req.NodeInfo.PortGUID = guid; 5613 5614 args.sq_attr_id = SA_NODERECORD_ATTRID; 5615 args.sq_access_type = IBMF_SAA_RETRIEVE; 5616 args.sq_component_mask = SA_NODEINFO_COMPMASK_PORTGUID; 5617 args.sq_template = &req; 5618 args.sq_callback = NULL; 5619 args.sq_callback_arg = NULL; 5620 5621 ret = ibmf_sa_access(sa_hdl, &args, 0, length, (void **) resp); 5622 if (ret != IBMF_SUCCESS) { 5623 IBTF_DPRINTF_L2("ibdm", "\tport_reachable:" 5624 " SA Retrieve Failed: %d", ret); 5625 return (IBDM_FAILURE); 5626 } 5627 if (*resp == NULL || *length == 0) { 5628 IBTF_DPRINTF_L2("ibdm", "\tport_reachable: No records"); 5629 return (IBDM_FAILURE); 5630 } 5631 /* 5632 * There is one NodeRecord on each endport on a subnet. 5633 */ 5634 ASSERT(*length == sizeof (sa_node_record_t)); 5635 5636 return (IBDM_SUCCESS); 5637 } 5638 5639 5640 /* 5641 * Update the gidlist for all affected IOCs when GID becomes 5642 * available/unavailable. 5643 * 5644 * Parameters : 5645 * gidinfo - Incoming / Outgoing GID. 5646 * add_flag - 1 for GID added, 0 for GID removed. 5647 * - (-1) : IOC gid list updated, ioc_list required. 5648 * 5649 * This function gets the GID for the node GUID corresponding to the 5650 * port GID. Gets the IOU info 5651 */ 5652 static ibdm_ioc_info_t * 5653 ibdm_update_ioc_gidlist(ibdm_dp_gidinfo_t *gid_info, int avail_flag) 5654 { 5655 ibdm_dp_gidinfo_t *node_gid = NULL; 5656 uint8_t niocs, ii; 5657 ibdm_ioc_info_t *ioc, *ioc_list = NULL, *tmp; 5658 5659 IBTF_DPRINTF_L4("ibdm", "\tupdate_ioc_gidlist"); 5660 5661 switch (avail_flag) { 5662 case 1 : 5663 node_gid = ibdm_check_dest_nodeguid(gid_info); 5664 break; 5665 case 0 : 5666 node_gid = ibdm_handle_gid_rm(gid_info); 5667 break; 5668 case -1 : 5669 node_gid = gid_info; 5670 break; 5671 default : 5672 break; 5673 } 5674 5675 if (node_gid == NULL) { 5676 IBTF_DPRINTF_L4("ibdm", "\tupdate_ioc_gidlist: " 5677 "No node GID found, port gid 0x%p, avail_flag %d", 5678 gid_info, avail_flag); 5679 return (NULL); 5680 } 5681 5682 mutex_enter(&node_gid->gl_mutex); 5683 if ((node_gid->gl_state != IBDM_GID_PROBING_COMPLETE && 5684 node_gid->gl_state != IBDM_GID_PROBING_SKIPPED) || 5685 node_gid->gl_iou == NULL) { 5686 IBTF_DPRINTF_L4("ibdm", "\tupdate_ioc_gidlist " 5687 "gl_state %x, gl_iou %p", node_gid->gl_state, 5688 node_gid->gl_iou); 5689 mutex_exit(&node_gid->gl_mutex); 5690 return (NULL); 5691 } 5692 5693 niocs = node_gid->gl_iou->iou_info.iou_num_ctrl_slots; 5694 IBTF_DPRINTF_L4("ibdm", "\tupdate_ioc_gidlist : niocs %x", 5695 niocs); 5696 for (ii = 0; ii < niocs; ii++) { 5697 ioc = IBDM_GIDINFO2IOCINFO(node_gid, ii); 5698 /* 5699 * Skip IOCs for which probe is not complete or 5700 * reprobe is progress 5701 */ 5702 if (ioc->ioc_state == IBDM_IOC_STATE_PROBE_SUCCESS) { 5703 tmp = ibdm_dup_ioc_info(ioc, node_gid); 5704 tmp->ioc_info_updated.ib_gid_prop_updated = 1; 5705 tmp->ioc_next = ioc_list; 5706 ioc_list = tmp; 5707 } 5708 } 5709 mutex_exit(&node_gid->gl_mutex); 5710 5711 IBTF_DPRINTF_L4("ibdm", "\tupdate_ioc_gidlist : return %p", 5712 ioc_list); 5713 return (ioc_list); 5714 } 5715 5716 /* 5717 * ibdm_saa_event_cb : 5718 * Event handling which does *not* require ibdm_hl_mutex to be 5719 * held are executed in the same thread. This is to prevent 5720 * deadlocks with HCA port down notifications which hold the 5721 * ibdm_hl_mutex. 5722 * 5723 * GID_AVAILABLE event is handled here. A taskq is spawned to 5724 * handle GID_UNAVAILABLE. 5725 * 5726 * A new mutex ibdm_ibnex_mutex has been introduced to protect 5727 * ibnex_callback. This has been done to prevent any possible 5728 * deadlock (described above) while handling GID_AVAILABLE. 5729 * 5730 * IBMF calls the event callback for a HCA port. The SA handle 5731 * for this port would be valid, till the callback returns. 5732 * IBDM calling IBDM using the above SA handle should be valid. 5733 * 5734 * IBDM will additionally check (SA handle != NULL), before 5735 * calling IBMF. 5736 */ 5737 /*ARGSUSED*/ 5738 static void 5739 ibdm_saa_event_cb(ibmf_saa_handle_t ibmf_saa_handle, 5740 ibmf_saa_subnet_event_t ibmf_saa_event, 5741 ibmf_saa_event_details_t *event_details, void *callback_arg) 5742 { 5743 ibdm_saa_event_arg_t *event_arg; 5744 ib_gid_t sgid, dgid; 5745 ibdm_port_attr_t *hca_port; 5746 ibdm_dp_gidinfo_t *gid_info, *node_gid_info = NULL; 5747 sa_node_record_t *nrec; 5748 size_t length; 5749 5750 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*event_arg)); 5751 5752 hca_port = (ibdm_port_attr_t *)callback_arg; 5753 5754 IBTF_DPRINTF_L4("ibdm", "\tsaa_event_cb(%x, %x, %x, %x)\n", 5755 ibmf_saa_handle, ibmf_saa_event, event_details, 5756 callback_arg); 5757 5758 #ifdef DEBUG 5759 if (ibdm_ignore_saa_event) 5760 return; 5761 #endif 5762 5763 if (ibmf_saa_event == IBMF_SAA_EVENT_GID_AVAILABLE) { 5764 /* 5765 * Ensure no other probe / sweep fabric is in 5766 * progress. 5767 */ 5768 mutex_enter(&ibdm.ibdm_mutex); 5769 while (ibdm.ibdm_busy & IBDM_BUSY) 5770 cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex); 5771 ibdm.ibdm_busy |= IBDM_BUSY; 5772 mutex_exit(&ibdm.ibdm_mutex); 5773 5774 /* 5775 * If we already know about this GID, return. 5776 * GID_AVAILABLE may be reported for multiple HCA 5777 * ports. 5778 */ 5779 if ((ibdm_check_dgid(event_details->ie_gid.gid_guid, 5780 event_details->ie_gid.gid_prefix)) != NULL) { 5781 mutex_enter(&ibdm.ibdm_mutex); 5782 ibdm.ibdm_busy &= ~IBDM_BUSY; 5783 cv_broadcast(&ibdm.ibdm_busy_cv); 5784 mutex_exit(&ibdm.ibdm_mutex); 5785 return; 5786 } 5787 5788 IBTF_DPRINTF_L4("ibdm", "\tGID (prefix %x, guid %llx) " 5789 "Insertion notified", 5790 event_details->ie_gid.gid_prefix, 5791 event_details->ie_gid.gid_guid); 5792 5793 /* This is a new gid, insert it to GID list */ 5794 sgid.gid_prefix = hca_port->pa_sn_prefix; 5795 sgid.gid_guid = hca_port->pa_port_guid; 5796 dgid.gid_prefix = event_details->ie_gid.gid_prefix; 5797 dgid.gid_guid = event_details->ie_gid.gid_guid; 5798 gid_info = ibdm_create_gid_info(hca_port, sgid, dgid); 5799 if (gid_info == NULL) { 5800 IBTF_DPRINTF_L4("ibdm", "\tGID_AVAILABLE: " 5801 "create_gid_info returned NULL"); 5802 mutex_enter(&ibdm.ibdm_mutex); 5803 ibdm.ibdm_busy &= ~IBDM_BUSY; 5804 cv_broadcast(&ibdm.ibdm_busy_cv); 5805 mutex_exit(&ibdm.ibdm_mutex); 5806 return; 5807 } 5808 mutex_enter(&gid_info->gl_mutex); 5809 gid_info->gl_state = IBDM_GID_PROBING_SKIPPED; 5810 mutex_exit(&gid_info->gl_mutex); 5811 5812 /* Get the node GUID */ 5813 if (ibdm_get_node_record_by_port(ibmf_saa_handle, dgid.gid_guid, 5814 &nrec, &length) != IBDM_SUCCESS) { 5815 /* 5816 * Set the state to PROBE_NOT_DONE for the 5817 * next sweep to probe it 5818 */ 5819 IBTF_DPRINTF_L2("ibdm", "\tsaa_event_taskq: " 5820 "Skipping GID : port GUID not found"); 5821 mutex_enter(&gid_info->gl_mutex); 5822 gid_info->gl_state = IBDM_GID_PROBE_NOT_DONE; 5823 mutex_exit(&gid_info->gl_mutex); 5824 mutex_enter(&ibdm.ibdm_mutex); 5825 ibdm.ibdm_busy &= ~IBDM_BUSY; 5826 cv_broadcast(&ibdm.ibdm_busy_cv); 5827 mutex_exit(&ibdm.ibdm_mutex); 5828 return; 5829 } 5830 gid_info->gl_nodeguid = nrec->NodeInfo.NodeGUID; 5831 gid_info->gl_devid = nrec->NodeInfo.DeviceID; 5832 kmem_free(nrec, length); 5833 gid_info->gl_portguid = dgid.gid_guid; 5834 5835 /* 5836 * Get the gid info with the same node GUID. 5837 */ 5838 mutex_enter(&ibdm.ibdm_mutex); 5839 node_gid_info = ibdm.ibdm_dp_gidlist_head; 5840 while (node_gid_info) { 5841 if (node_gid_info->gl_nodeguid == 5842 gid_info->gl_nodeguid && 5843 node_gid_info->gl_iou != NULL) { 5844 break; 5845 } 5846 node_gid_info = node_gid_info->gl_next; 5847 } 5848 mutex_exit(&ibdm.ibdm_mutex); 5849 5850 /* 5851 * Handling a new GID requires filling of gl_hca_list. 5852 * This require ibdm hca_list to be parsed and hence 5853 * holding the ibdm_hl_mutex. Spawning a new thread to 5854 * handle this. 5855 */ 5856 if (node_gid_info == NULL) { 5857 if (taskq_dispatch(system_taskq, 5858 ibdm_saa_handle_new_gid, (void *)gid_info, 5859 TQ_NOSLEEP) == NULL) { 5860 IBTF_DPRINTF_L2("ibdm", "\tsaa_event_cb: " 5861 "new_gid taskq_dispatch failed"); 5862 return; 5863 } 5864 } 5865 5866 mutex_enter(&ibdm.ibdm_mutex); 5867 ibdm.ibdm_busy &= ~IBDM_BUSY; 5868 cv_broadcast(&ibdm.ibdm_busy_cv); 5869 mutex_exit(&ibdm.ibdm_mutex); 5870 return; 5871 } 5872 5873 if (ibmf_saa_event != IBMF_SAA_EVENT_GID_UNAVAILABLE) 5874 return; 5875 5876 /* 5877 * GID UNAVAIL EVENT: Try to locate the GID in the GID list. 5878 * If we don't find it we just return. 5879 */ 5880 mutex_enter(&ibdm.ibdm_mutex); 5881 gid_info = ibdm.ibdm_dp_gidlist_head; 5882 while (gid_info) { 5883 if (gid_info->gl_portguid == 5884 event_details->ie_gid.gid_guid) { 5885 break; 5886 } 5887 gid_info = gid_info->gl_next; 5888 } 5889 mutex_exit(&ibdm.ibdm_mutex); 5890 if (gid_info == NULL) { 5891 IBTF_DPRINTF_L2("ibdm", "\tsaa_event_cb: " 5892 "GID for GUID %llX not found during GID UNAVAIL event", 5893 event_details->ie_gid.gid_guid); 5894 return; 5895 } 5896 5897 /* 5898 * If this GID is DM capable, we'll have to check whether this DGID 5899 * is reachable via another port. 5900 */ 5901 if (gid_info->gl_is_dm_capable == B_TRUE) { 5902 event_arg = (ibdm_saa_event_arg_t *)kmem_alloc( 5903 sizeof (ibdm_saa_event_arg_t), KM_SLEEP); 5904 event_arg->ibmf_saa_handle = ibmf_saa_handle; 5905 event_arg->ibmf_saa_event = ibmf_saa_event; 5906 bcopy(event_details, &event_arg->event_details, 5907 sizeof (ibmf_saa_event_details_t)); 5908 event_arg->callback_arg = callback_arg; 5909 5910 if (taskq_dispatch(system_taskq, ibdm_saa_event_taskq, 5911 (void *)event_arg, TQ_NOSLEEP) == NULL) { 5912 IBTF_DPRINTF_L2("ibdm", "\tsaa_event_cb: " 5913 "taskq_dispatch failed"); 5914 ibdm_free_saa_event_arg(event_arg); 5915 return; 5916 } 5917 } 5918 } 5919 5920 /* 5921 * Handle a new GID discovered by GID_AVAILABLE saa event. 5922 */ 5923 void 5924 ibdm_saa_handle_new_gid(void *arg) 5925 { 5926 ibdm_dp_gidinfo_t *gid_info; 5927 ibdm_hca_list_t *hca_list = NULL; 5928 ibdm_port_attr_t *port = NULL; 5929 ibdm_ioc_info_t *ioc_list = NULL; 5930 5931 IBTF_DPRINTF_L4(ibdm_string, "\tsaa_handle_new_gid(%p)", arg); 5932 5933 gid_info = (ibdm_dp_gidinfo_t *)arg; 5934 5935 /* 5936 * Ensure that no other sweep / probe has completed 5937 * probing this gid. 5938 */ 5939 mutex_enter(&gid_info->gl_mutex); 5940 if (gid_info->gl_state != IBDM_GID_PROBE_NOT_DONE) { 5941 mutex_exit(&gid_info->gl_mutex); 5942 return; 5943 } 5944 mutex_exit(&gid_info->gl_mutex); 5945 5946 /* 5947 * Parse HCAs to fill gl_hca_list 5948 */ 5949 mutex_enter(&ibdm.ibdm_hl_mutex); 5950 for (ibdm_get_next_port(&hca_list, &port, 1); port; 5951 ibdm_get_next_port(&hca_list, &port, 1)) { 5952 if (ibdm_port_reachable(port->pa_sa_hdl, 5953 gid_info->gl_portguid) == B_TRUE) { 5954 ibdm_addto_glhcalist(gid_info, hca_list); 5955 } 5956 } 5957 mutex_exit(&ibdm.ibdm_hl_mutex); 5958 5959 /* 5960 * Ensure no other probe / sweep fabric is in 5961 * progress. 5962 */ 5963 mutex_enter(&ibdm.ibdm_mutex); 5964 while (ibdm.ibdm_busy & IBDM_BUSY) 5965 cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex); 5966 ibdm.ibdm_busy |= IBDM_BUSY; 5967 mutex_exit(&ibdm.ibdm_mutex); 5968 5969 /* 5970 * New IOU probe it, to check if new IOCs 5971 */ 5972 IBTF_DPRINTF_L4(ibdm_string, "\tsaa_handle_new_gid: " 5973 "new GID : probing"); 5974 mutex_enter(&ibdm.ibdm_mutex); 5975 ibdm.ibdm_ngid_probes_in_progress++; 5976 mutex_exit(&ibdm.ibdm_mutex); 5977 mutex_enter(&gid_info->gl_mutex); 5978 gid_info->gl_reprobe_flag = 0; 5979 gid_info->gl_state = IBDM_GID_PROBE_NOT_DONE; 5980 mutex_exit(&gid_info->gl_mutex); 5981 ibdm_probe_gid_thread((void *)gid_info); 5982 5983 mutex_enter(&ibdm.ibdm_mutex); 5984 ibdm_wait_probe_completion(); 5985 mutex_exit(&ibdm.ibdm_mutex); 5986 5987 if (gid_info->gl_iou == NULL) { 5988 mutex_enter(&ibdm.ibdm_mutex); 5989 ibdm.ibdm_busy &= ~IBDM_BUSY; 5990 cv_broadcast(&ibdm.ibdm_busy_cv); 5991 mutex_exit(&ibdm.ibdm_mutex); 5992 return; 5993 } 5994 5995 /* 5996 * Update GID list in all IOCs affected by this 5997 */ 5998 ioc_list = ibdm_update_ioc_gidlist(gid_info, 1); 5999 6000 /* 6001 * Pass on the IOCs with updated GIDs to IBnexus 6002 */ 6003 if (ioc_list) { 6004 mutex_enter(&ibdm.ibdm_ibnex_mutex); 6005 if (ibdm.ibdm_ibnex_callback != NULL) { 6006 (*ibdm.ibdm_ibnex_callback)((void *)ioc_list, 6007 IBDM_EVENT_IOC_PROP_UPDATE); 6008 } 6009 mutex_exit(&ibdm.ibdm_ibnex_mutex); 6010 } 6011 6012 mutex_enter(&ibdm.ibdm_mutex); 6013 ibdm.ibdm_busy &= ~IBDM_BUSY; 6014 cv_broadcast(&ibdm.ibdm_busy_cv); 6015 mutex_exit(&ibdm.ibdm_mutex); 6016 } 6017 6018 /* 6019 * ibdm_saa_event_taskq : 6020 * GID_UNAVAILABLE Event handling requires ibdm_hl_mutex to be 6021 * held. The GID_UNAVAILABLE handling is done in a taskq to 6022 * prevent deadlocks with HCA port down notifications which hold 6023 * ibdm_hl_mutex. 6024 */ 6025 void 6026 ibdm_saa_event_taskq(void *arg) 6027 { 6028 ibdm_saa_event_arg_t *event_arg; 6029 ibmf_saa_handle_t ibmf_saa_handle; 6030 ibmf_saa_subnet_event_t ibmf_saa_event; 6031 ibmf_saa_event_details_t *event_details; 6032 void *callback_arg; 6033 6034 ibdm_dp_gidinfo_t *gid_info; 6035 ibdm_port_attr_t *hca_port, *port = NULL; 6036 ibdm_hca_list_t *hca_list = NULL; 6037 int sa_handle_valid = 0; 6038 ibdm_ioc_info_t *ioc_list = NULL; 6039 6040 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*event_arg)); 6041 6042 event_arg = (ibdm_saa_event_arg_t *)arg; 6043 ibmf_saa_handle = event_arg->ibmf_saa_handle; 6044 ibmf_saa_event = event_arg->ibmf_saa_event; 6045 event_details = &event_arg->event_details; 6046 callback_arg = event_arg->callback_arg; 6047 6048 ASSERT(callback_arg != NULL); 6049 ASSERT(ibmf_saa_event == IBMF_SAA_EVENT_GID_UNAVAILABLE); 6050 IBTF_DPRINTF_L4("ibdm", "\tsaa_event_taskq(%x, %x, %x, %x)", 6051 ibmf_saa_handle, ibmf_saa_event, event_details, 6052 callback_arg); 6053 6054 hca_port = (ibdm_port_attr_t *)callback_arg; 6055 6056 /* Check if the port_attr is still valid */ 6057 mutex_enter(&ibdm.ibdm_hl_mutex); 6058 for (ibdm_get_next_port(&hca_list, &port, 0); port; 6059 ibdm_get_next_port(&hca_list, &port, 0)) { 6060 if (port == hca_port && port->pa_port_guid == 6061 hca_port->pa_port_guid) { 6062 if (ibmf_saa_handle == hca_port->pa_sa_hdl) 6063 sa_handle_valid = 1; 6064 break; 6065 } 6066 } 6067 mutex_exit(&ibdm.ibdm_hl_mutex); 6068 if (sa_handle_valid == 0) { 6069 ibdm_free_saa_event_arg(event_arg); 6070 return; 6071 } 6072 6073 if (hca_port && (hca_port->pa_sa_hdl == NULL || 6074 ibmf_saa_handle != hca_port->pa_sa_hdl)) { 6075 ibdm_free_saa_event_arg(event_arg); 6076 return; 6077 } 6078 hca_list = NULL; 6079 port = NULL; 6080 6081 /* 6082 * Check if the GID is visible to other HCA ports. 6083 * Return if so. 6084 */ 6085 mutex_enter(&ibdm.ibdm_hl_mutex); 6086 for (ibdm_get_next_port(&hca_list, &port, 1); port; 6087 ibdm_get_next_port(&hca_list, &port, 1)) { 6088 if (ibdm_port_reachable(port->pa_sa_hdl, 6089 event_details->ie_gid.gid_guid) == B_TRUE) { 6090 mutex_exit(&ibdm.ibdm_hl_mutex); 6091 ibdm_free_saa_event_arg(event_arg); 6092 return; 6093 } 6094 } 6095 mutex_exit(&ibdm.ibdm_hl_mutex); 6096 6097 /* 6098 * Ensure no other probe / sweep fabric is in 6099 * progress. 6100 */ 6101 mutex_enter(&ibdm.ibdm_mutex); 6102 while (ibdm.ibdm_busy & IBDM_BUSY) 6103 cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex); 6104 ibdm.ibdm_busy |= IBDM_BUSY; 6105 mutex_exit(&ibdm.ibdm_mutex); 6106 6107 /* 6108 * If this GID is no longer in GID list, return 6109 * GID_UNAVAILABLE may be reported for multiple HCA 6110 * ports. 6111 */ 6112 mutex_enter(&ibdm.ibdm_mutex); 6113 gid_info = ibdm.ibdm_dp_gidlist_head; 6114 while (gid_info) { 6115 if (gid_info->gl_portguid == 6116 event_details->ie_gid.gid_guid) { 6117 break; 6118 } 6119 gid_info = gid_info->gl_next; 6120 } 6121 mutex_exit(&ibdm.ibdm_mutex); 6122 if (gid_info == NULL) { 6123 mutex_enter(&ibdm.ibdm_mutex); 6124 ibdm.ibdm_busy &= ~IBDM_BUSY; 6125 cv_broadcast(&ibdm.ibdm_busy_cv); 6126 mutex_exit(&ibdm.ibdm_mutex); 6127 ibdm_free_saa_event_arg(event_arg); 6128 return; 6129 } 6130 6131 IBTF_DPRINTF_L4("ibdm", "\tGID (prefix %x, guid %llx) " 6132 "Unavailable notification", 6133 event_details->ie_gid.gid_prefix, 6134 event_details->ie_gid.gid_guid); 6135 6136 /* 6137 * Update GID list in all IOCs affected by this 6138 */ 6139 if (gid_info->gl_state == IBDM_GID_PROBING_SKIPPED || 6140 gid_info->gl_state == IBDM_GID_PROBING_COMPLETE) 6141 ioc_list = ibdm_update_ioc_gidlist(gid_info, 0); 6142 6143 /* 6144 * Remove GID from the global GID list 6145 * Handle the case where all port GIDs for an 6146 * IOU have been hot-removed. Check both gid_info 6147 * & ioc_info for checking ngids. 6148 */ 6149 mutex_enter(&ibdm.ibdm_mutex); 6150 if (gid_info->gl_iou != NULL && gid_info->gl_ngids == 0) { 6151 mutex_enter(&gid_info->gl_mutex); 6152 (void) ibdm_free_iou_info(gid_info, &gid_info->gl_iou); 6153 mutex_exit(&gid_info->gl_mutex); 6154 } 6155 if (gid_info->gl_prev != NULL) 6156 gid_info->gl_prev->gl_next = gid_info->gl_next; 6157 if (gid_info->gl_next != NULL) 6158 gid_info->gl_next->gl_prev = gid_info->gl_prev; 6159 6160 if (gid_info == ibdm.ibdm_dp_gidlist_head) 6161 ibdm.ibdm_dp_gidlist_head = gid_info->gl_next; 6162 if (gid_info == ibdm.ibdm_dp_gidlist_tail) 6163 ibdm.ibdm_dp_gidlist_tail = gid_info->gl_prev; 6164 ibdm.ibdm_ngids--; 6165 6166 ibdm.ibdm_busy &= ~IBDM_BUSY; 6167 cv_broadcast(&ibdm.ibdm_busy_cv); 6168 mutex_exit(&ibdm.ibdm_mutex); 6169 6170 /* free the hca_list on this gid_info */ 6171 ibdm_delete_glhca_list(gid_info); 6172 6173 mutex_destroy(&gid_info->gl_mutex); 6174 kmem_free(gid_info, sizeof (ibdm_dp_gidinfo_t)); 6175 6176 /* 6177 * Pass on the IOCs with updated GIDs to IBnexus 6178 */ 6179 if (ioc_list) { 6180 IBTF_DPRINTF_L4("ibdm", "\tGID_UNAVAILABLE " 6181 "IOC_PROP_UPDATE for %p\n", ioc_list); 6182 mutex_enter(&ibdm.ibdm_ibnex_mutex); 6183 if (ibdm.ibdm_ibnex_callback != NULL) { 6184 (*ibdm.ibdm_ibnex_callback)((void *) 6185 ioc_list, IBDM_EVENT_IOC_PROP_UPDATE); 6186 } 6187 mutex_exit(&ibdm.ibdm_ibnex_mutex); 6188 } 6189 6190 ibdm_free_saa_event_arg(event_arg); 6191 } 6192 6193 6194 static int 6195 ibdm_cmp_gid_list(ibdm_gid_t *new, ibdm_gid_t *prev) 6196 { 6197 ibdm_gid_t *scan_new, *scan_prev; 6198 int cmp_failed = 0; 6199 6200 ASSERT(new != NULL); 6201 ASSERT(prev != NULL); 6202 6203 /* 6204 * Search for each new gid anywhere in the prev GID list. 6205 * Note that the gid list could have been re-ordered. 6206 */ 6207 for (scan_new = new; scan_new; scan_new = scan_new->gid_next) { 6208 for (scan_prev = prev, cmp_failed = 1; scan_prev; 6209 scan_prev = scan_prev->gid_next) { 6210 if (scan_prev->gid_dgid_hi == scan_new->gid_dgid_hi && 6211 scan_prev->gid_dgid_lo == scan_new->gid_dgid_lo) { 6212 cmp_failed = 0; 6213 break; 6214 } 6215 } 6216 6217 if (cmp_failed) 6218 return (1); 6219 } 6220 return (0); 6221 } 6222 6223 /* 6224 * This is always called in a single thread 6225 * This function updates the gid_list and serv_list of IOC 6226 * The current gid_list is in ioc_info_t(contains only port 6227 * guids for which probe is done) & gidinfo_t(other port gids) 6228 * The gids in both locations are used for comparision. 6229 */ 6230 static void 6231 ibdm_reprobe_update_port_srv(ibdm_ioc_info_t *ioc, ibdm_dp_gidinfo_t *gidinfo) 6232 { 6233 ibdm_gid_t *cur_gid_list; 6234 uint_t cur_nportgids; 6235 6236 ASSERT(MUTEX_HELD(&ibdm.ibdm_mutex)); 6237 6238 ioc->ioc_info_updated.ib_prop_updated = 0; 6239 6240 6241 /* Current GID list in gid_info only */ 6242 cur_gid_list = gidinfo->gl_gid; 6243 cur_nportgids = gidinfo->gl_ngids; 6244 6245 if (ioc->ioc_prev_serv_cnt != 6246 ioc->ioc_profile.ioc_service_entries || 6247 ibdm_serv_cmp(&ioc->ioc_serv[0], &ioc->ioc_prev_serv[0], 6248 ioc->ioc_prev_serv_cnt)) 6249 ioc->ioc_info_updated.ib_srv_prop_updated = 1; 6250 6251 if (ioc->ioc_prev_nportgids != cur_nportgids || 6252 ioc->ioc_prev_gid_list == NULL || cur_gid_list == NULL) { 6253 ioc->ioc_info_updated.ib_gid_prop_updated = 1; 6254 } else if (ibdm_cmp_gid_list(ioc->ioc_prev_gid_list, cur_gid_list)) { 6255 ioc->ioc_info_updated.ib_gid_prop_updated = 1; 6256 } 6257 6258 /* Zero out previous entries */ 6259 ibdm_free_gid_list(ioc->ioc_prev_gid_list); 6260 if (ioc->ioc_prev_serv) 6261 kmem_free(ioc->ioc_prev_serv, ioc->ioc_prev_serv_cnt * 6262 sizeof (ibdm_srvents_info_t)); 6263 ioc->ioc_prev_serv_cnt = 0; 6264 ioc->ioc_prev_nportgids = 0; 6265 ioc->ioc_prev_serv = NULL; 6266 ioc->ioc_prev_gid_list = NULL; 6267 } 6268 6269 /* 6270 * Handle GID removal. This returns gid_info of an GID for the same 6271 * node GUID, if found. For an GID with IOU information, the same 6272 * gid_info is returned if no gid_info with same node_guid is found. 6273 */ 6274 static ibdm_dp_gidinfo_t * 6275 ibdm_handle_gid_rm(ibdm_dp_gidinfo_t *rm_gid) 6276 { 6277 ibdm_dp_gidinfo_t *gid_list; 6278 6279 IBTF_DPRINTF_L4("ibdm", "\thandle_gid_rm(0x%p)", rm_gid); 6280 6281 if (rm_gid->gl_iou == NULL) { 6282 IBTF_DPRINTF_L4("ibdm", "\thandle_gid_rm NO iou"); 6283 /* 6284 * Search for a GID with same node_guid and 6285 * gl_iou != NULL 6286 */ 6287 for (gid_list = ibdm.ibdm_dp_gidlist_head; gid_list; 6288 gid_list = gid_list->gl_next) { 6289 if (gid_list->gl_iou != NULL && (gid_list->gl_nodeguid 6290 == rm_gid->gl_nodeguid)) 6291 break; 6292 } 6293 6294 if (gid_list) 6295 ibdm_rmfrom_glgid_list(gid_list, rm_gid); 6296 6297 IBTF_DPRINTF_L4("ibdm", "\thandle_gid_rm ret %p", gid_list); 6298 return (gid_list); 6299 } else { 6300 /* 6301 * Search for a GID with same node_guid and 6302 * gl_iou == NULL 6303 */ 6304 IBTF_DPRINTF_L4("ibdm", "\thandle_gid_rm with iou"); 6305 for (gid_list = ibdm.ibdm_dp_gidlist_head; gid_list; 6306 gid_list = gid_list->gl_next) { 6307 if (gid_list->gl_iou == NULL && (gid_list->gl_nodeguid 6308 == rm_gid->gl_nodeguid)) 6309 break; 6310 } 6311 6312 if (gid_list) { 6313 /* 6314 * Copy the following fields from rm_gid : 6315 * 1. gl_state 6316 * 2. gl_iou 6317 * 3. gl_gid & gl_ngids 6318 * 6319 * Note : Function is synchronized by 6320 * ibdm_busy flag. 6321 * 6322 * Note : Redirect info is initialized if 6323 * any MADs for the GID fail 6324 */ 6325 IBTF_DPRINTF_L4("ibdm", "\thandle_gid_rm " 6326 "copying info to GID with gl_iou != NULl"); 6327 gid_list->gl_state = rm_gid->gl_state; 6328 gid_list->gl_iou = rm_gid->gl_iou; 6329 gid_list->gl_gid = rm_gid->gl_gid; 6330 gid_list->gl_ngids = rm_gid->gl_ngids; 6331 6332 /* Remove the GID from gl_gid list */ 6333 ibdm_rmfrom_glgid_list(gid_list, rm_gid); 6334 } else { 6335 /* 6336 * Handle a case where all GIDs to the IOU have 6337 * been removed. 6338 */ 6339 IBTF_DPRINTF_L4("ibdm", "\thandle_gid_rm 0 GID " 6340 "to IOU"); 6341 6342 ibdm_rmfrom_glgid_list(rm_gid, rm_gid); 6343 return (rm_gid); 6344 } 6345 IBTF_DPRINTF_L4("ibdm", "\thandle_gid_rm ret %p", gid_list); 6346 return (gid_list); 6347 } 6348 } 6349 6350 static void 6351 ibdm_rmfrom_glgid_list(ibdm_dp_gidinfo_t *gid_info, 6352 ibdm_dp_gidinfo_t *rm_gid) 6353 { 6354 ibdm_gid_t *tmp, *prev; 6355 6356 IBTF_DPRINTF_L4("ibdm", "\trmfrom_glgid (%p, %p)", 6357 gid_info, rm_gid); 6358 6359 for (tmp = gid_info->gl_gid, prev = NULL; tmp; ) { 6360 if (tmp->gid_dgid_hi == rm_gid->gl_dgid_hi && 6361 tmp->gid_dgid_lo == rm_gid->gl_dgid_lo) { 6362 if (prev == NULL) 6363 gid_info->gl_gid = tmp->gid_next; 6364 else 6365 prev->gid_next = tmp->gid_next; 6366 6367 kmem_free(tmp, sizeof (ibdm_gid_t)); 6368 gid_info->gl_ngids--; 6369 break; 6370 } else { 6371 prev = tmp; 6372 tmp = tmp->gid_next; 6373 } 6374 } 6375 } 6376 6377 static void 6378 ibdm_addto_gidlist(ibdm_gid_t **src_ptr, ibdm_gid_t *dest) 6379 { 6380 ibdm_gid_t *head = NULL, *new, *tail; 6381 6382 /* First copy the destination */ 6383 for (; dest; dest = dest->gid_next) { 6384 new = kmem_zalloc(sizeof (ibdm_gid_t), KM_SLEEP); 6385 new->gid_dgid_hi = dest->gid_dgid_hi; 6386 new->gid_dgid_lo = dest->gid_dgid_lo; 6387 new->gid_next = head; 6388 head = new; 6389 } 6390 6391 /* Insert this to the source */ 6392 if (*src_ptr == NULL) 6393 *src_ptr = head; 6394 else { 6395 for (tail = *src_ptr; tail->gid_next != NULL; 6396 tail = tail->gid_next) 6397 ; 6398 6399 tail->gid_next = head; 6400 } 6401 } 6402 6403 static void 6404 ibdm_free_gid_list(ibdm_gid_t *head) 6405 { 6406 ibdm_gid_t *delete; 6407 6408 for (delete = head; delete; ) { 6409 head = delete->gid_next; 6410 kmem_free(delete, sizeof (ibdm_gid_t)); 6411 delete = head; 6412 } 6413 } 6414 6415 /* 6416 * This function rescans the DM capable GIDs (gl_state is 6417 * GID_PROBE_COMPLETE or IBDM_GID_PROBING_SKIPPED.This 6418 * basically checks if the DM capable GID is reachable. If 6419 * not this is handled the same way as GID_UNAVAILABLE, 6420 * except that notifications are not send to IBnexus. 6421 * 6422 * This function also initializes the ioc_prev_list for 6423 * a particular IOC (when called from probe_ioc, with 6424 * ioc_guidp != NULL) or all IOCs for the gid (called from 6425 * sweep_fabric, ioc_guidp == NULL). 6426 */ 6427 static void 6428 ibdm_rescan_gidlist(ib_guid_t *ioc_guidp) 6429 { 6430 ibdm_dp_gidinfo_t *gid_info, *tmp; 6431 int ii, niocs, found; 6432 ibdm_hca_list_t *hca_list = NULL; 6433 ibdm_port_attr_t *port = NULL; 6434 ibdm_ioc_info_t *ioc_list; 6435 6436 for (gid_info = ibdm.ibdm_dp_gidlist_head; gid_info; ) { 6437 found = 0; 6438 if (gid_info->gl_state != IBDM_GID_PROBING_SKIPPED && 6439 gid_info->gl_state != IBDM_GID_PROBING_COMPLETE) { 6440 gid_info = gid_info->gl_next; 6441 continue; 6442 } 6443 6444 /* 6445 * Check if the GID is visible to any HCA ports. 6446 * Return if so. 6447 */ 6448 mutex_enter(&ibdm.ibdm_hl_mutex); 6449 for (ibdm_get_next_port(&hca_list, &port, 1); port; 6450 ibdm_get_next_port(&hca_list, &port, 1)) { 6451 if (ibdm_port_reachable(port->pa_sa_hdl, 6452 gid_info->gl_dgid_lo) == B_TRUE) { 6453 found = 1; 6454 break; 6455 } 6456 } 6457 mutex_exit(&ibdm.ibdm_hl_mutex); 6458 6459 if (found) { 6460 if (gid_info->gl_iou == NULL) { 6461 gid_info = gid_info->gl_next; 6462 continue; 6463 } 6464 6465 /* Intialize the ioc_prev_gid_list */ 6466 niocs = 6467 gid_info->gl_iou->iou_info.iou_num_ctrl_slots; 6468 for (ii = 0; ii < niocs; ii++) { 6469 ioc_list = IBDM_GIDINFO2IOCINFO(gid_info, ii); 6470 6471 if (ioc_guidp == NULL || (*ioc_guidp == 6472 ioc_list->ioc_profile.ioc_guid)) { 6473 /* Add info of GIDs in gid_info also */ 6474 ibdm_addto_gidlist( 6475 &ioc_list->ioc_prev_gid_list, 6476 gid_info->gl_gid); 6477 ioc_list->ioc_prev_nportgids = 6478 gid_info->gl_ngids; 6479 } 6480 } 6481 gid_info = gid_info->gl_next; 6482 continue; 6483 } 6484 6485 IBTF_DPRINTF_L4("ibdm", "\trescan_gidlist " 6486 "deleted port GUID %llx", 6487 gid_info->gl_dgid_lo); 6488 6489 /* 6490 * Update GID list in all IOCs affected by this 6491 */ 6492 ioc_list = ibdm_update_ioc_gidlist(gid_info, 0); 6493 6494 /* 6495 * Remove GID from the global GID list 6496 * Handle the case where all port GIDs for an 6497 * IOU have been hot-removed. 6498 */ 6499 mutex_enter(&ibdm.ibdm_mutex); 6500 if (gid_info->gl_iou != NULL && gid_info->gl_ngids == 0) { 6501 mutex_enter(&gid_info->gl_mutex); 6502 (void) ibdm_free_iou_info(gid_info, &gid_info->gl_iou); 6503 mutex_exit(&gid_info->gl_mutex); 6504 } 6505 6506 tmp = gid_info->gl_next; 6507 if (gid_info->gl_prev != NULL) 6508 gid_info->gl_prev->gl_next = gid_info->gl_next; 6509 if (gid_info->gl_next != NULL) 6510 gid_info->gl_next->gl_prev = gid_info->gl_prev; 6511 6512 if (gid_info == ibdm.ibdm_dp_gidlist_head) 6513 ibdm.ibdm_dp_gidlist_head = gid_info->gl_next; 6514 if (gid_info == ibdm.ibdm_dp_gidlist_tail) 6515 ibdm.ibdm_dp_gidlist_tail = gid_info->gl_prev; 6516 ibdm.ibdm_ngids--; 6517 mutex_exit(&ibdm.ibdm_mutex); 6518 6519 /* free the hca_list on this gid_info */ 6520 ibdm_delete_glhca_list(gid_info); 6521 6522 mutex_destroy(&gid_info->gl_mutex); 6523 kmem_free(gid_info, sizeof (ibdm_dp_gidinfo_t)); 6524 6525 gid_info = tmp; 6526 6527 /* 6528 * Pass on the IOCs with updated GIDs to IBnexus 6529 */ 6530 if (ioc_list) { 6531 IBTF_DPRINTF_L4("ibdm", "\trescan_gidlist " 6532 "IOC_PROP_UPDATE for %p\n", ioc_list); 6533 mutex_enter(&ibdm.ibdm_ibnex_mutex); 6534 if (ibdm.ibdm_ibnex_callback != NULL) { 6535 (*ibdm.ibdm_ibnex_callback)((void *) 6536 ioc_list, IBDM_EVENT_IOC_PROP_UPDATE); 6537 } 6538 mutex_exit(&ibdm.ibdm_ibnex_mutex); 6539 } 6540 } 6541 } 6542 6543 /* 6544 * This function notifies IBnex of IOCs on this GID. 6545 * Notification is for GIDs with gl_reprobe_flag set. 6546 * The flag is set when IOC probe / fabric sweep 6547 * probes a GID starting from CLASS port info. 6548 * 6549 * IBnexus will have information of a reconnected IOC 6550 * if it had probed it before. If this is a new IOC, 6551 * IBnexus ignores the notification. 6552 * 6553 * This function should be called with no locks held. 6554 */ 6555 static void 6556 ibdm_notify_newgid_iocs(ibdm_dp_gidinfo_t *gid_info) 6557 { 6558 ibdm_ioc_info_t *ioc_list; 6559 6560 if (gid_info->gl_reprobe_flag == 0 || 6561 gid_info->gl_iou == NULL) 6562 return; 6563 6564 ioc_list = ibdm_update_ioc_gidlist(gid_info, -1); 6565 6566 /* 6567 * Pass on the IOCs with updated GIDs to IBnexus 6568 */ 6569 if (ioc_list) { 6570 mutex_enter(&ibdm.ibdm_ibnex_mutex); 6571 if (ibdm.ibdm_ibnex_callback != NULL) { 6572 (*ibdm.ibdm_ibnex_callback)((void *)ioc_list, 6573 IBDM_EVENT_IOC_PROP_UPDATE); 6574 } 6575 mutex_exit(&ibdm.ibdm_ibnex_mutex); 6576 } 6577 } 6578 6579 6580 static void 6581 ibdm_free_saa_event_arg(ibdm_saa_event_arg_t *arg) 6582 { 6583 if (arg != NULL) 6584 kmem_free(arg, sizeof (ibdm_saa_event_arg_t)); 6585 } 6586 6587 /* 6588 * This function parses the list of HCAs and HCA ports 6589 * to return the port_attr of the next HCA port. A port 6590 * connected to IB fabric (port_state active) is returned, 6591 * if connected_flag is set. 6592 */ 6593 static void 6594 ibdm_get_next_port(ibdm_hca_list_t **inp_hcap, 6595 ibdm_port_attr_t **inp_portp, int connect_flag) 6596 { 6597 int ii; 6598 ibdm_port_attr_t *port, *next_port = NULL; 6599 ibdm_port_attr_t *inp_port; 6600 ibdm_hca_list_t *hca_list; 6601 int found = 0; 6602 6603 ASSERT(MUTEX_HELD(&ibdm.ibdm_hl_mutex)); 6604 IBTF_DPRINTF_L4(ibdm_string, "\tget_next_port(%p, %p, %x)", 6605 inp_hcap, inp_portp, connect_flag); 6606 6607 hca_list = *inp_hcap; 6608 inp_port = *inp_portp; 6609 6610 if (hca_list == NULL) 6611 hca_list = ibdm.ibdm_hca_list_head; 6612 6613 for (; hca_list; hca_list = hca_list->hl_next) { 6614 for (ii = 0; ii < hca_list->hl_nports; ii++) { 6615 port = &hca_list->hl_port_attr[ii]; 6616 6617 /* 6618 * inp_port != NULL; 6619 * Skip till we find the matching port 6620 */ 6621 if (inp_port && !found) { 6622 if (inp_port == port) 6623 found = 1; 6624 continue; 6625 } 6626 6627 if (!connect_flag) { 6628 next_port = port; 6629 break; 6630 } 6631 6632 if (port->pa_sa_hdl == NULL) 6633 ibdm_initialize_port(port); 6634 if (port->pa_sa_hdl == NULL) 6635 (void) ibdm_fini_port(port); 6636 else if (next_port == NULL && 6637 port->pa_sa_hdl != NULL && 6638 port->pa_state == IBT_PORT_ACTIVE) { 6639 next_port = port; 6640 break; 6641 } 6642 } 6643 6644 if (next_port) 6645 break; 6646 } 6647 6648 IBTF_DPRINTF_L4(ibdm_string, "\tget_next_port : " 6649 "returns hca_list %p port %p", hca_list, next_port); 6650 *inp_hcap = hca_list; 6651 *inp_portp = next_port; 6652 } 6653 6654 static void 6655 ibdm_add_to_gl_gid(ibdm_dp_gidinfo_t *nodegid, ibdm_dp_gidinfo_t *addgid) 6656 { 6657 ibdm_gid_t *tmp; 6658 6659 tmp = kmem_zalloc(sizeof (ibdm_gid_t), KM_SLEEP); 6660 tmp->gid_dgid_hi = addgid->gl_dgid_hi; 6661 tmp->gid_dgid_lo = addgid->gl_dgid_lo; 6662 6663 mutex_enter(&nodegid->gl_mutex); 6664 tmp->gid_next = nodegid->gl_gid; 6665 nodegid->gl_gid = tmp; 6666 nodegid->gl_ngids++; 6667 mutex_exit(&nodegid->gl_mutex); 6668 } 6669 6670 static void 6671 ibdm_addto_glhcalist(ibdm_dp_gidinfo_t *gid_info, 6672 ibdm_hca_list_t *hca) 6673 { 6674 ibdm_hca_list_t *head, *prev = NULL, *temp; 6675 6676 IBTF_DPRINTF_L4(ibdm_string, "\taddto_glhcalist(%p, %p) " 6677 ": gl_hca_list %p", gid_info, hca, gid_info->gl_hca_list); 6678 ASSERT(!MUTEX_HELD(&gid_info->gl_mutex)); 6679 6680 mutex_enter(&gid_info->gl_mutex); 6681 head = gid_info->gl_hca_list; 6682 if (head == NULL) { 6683 head = ibdm_dup_hca_attr(hca); 6684 head->hl_next = NULL; 6685 gid_info->gl_hca_list = head; 6686 mutex_exit(&gid_info->gl_mutex); 6687 IBTF_DPRINTF_L4(ibdm_string, "\tadd_to_glhcalist: " 6688 "gid %p, gl_hca_list %p", gid_info, 6689 gid_info->gl_hca_list); 6690 return; 6691 } 6692 6693 /* Check if already in the list */ 6694 while (head) { 6695 if (head->hl_hca_guid == hca->hl_hca_guid) { 6696 mutex_exit(&gid_info->gl_mutex); 6697 IBTF_DPRINTF_L4(ibdm_string, 6698 "\taddto_glhcalist : gid %p hca %p dup", 6699 gid_info, hca); 6700 return; 6701 } 6702 prev = head; 6703 head = head->hl_next; 6704 } 6705 6706 /* Add this HCA to gl_hca_list */ 6707 temp = ibdm_dup_hca_attr(hca); 6708 temp->hl_next = NULL; 6709 prev->hl_next = temp; 6710 mutex_exit(&gid_info->gl_mutex); 6711 6712 IBTF_DPRINTF_L4(ibdm_string, "\tadd_to_glhcalist: " 6713 "gid %p, gl_hca_list %p", gid_info, gid_info->gl_hca_list); 6714 } 6715 6716 static void 6717 ibdm_delete_glhca_list(ibdm_dp_gidinfo_t *gid_info) 6718 { 6719 ASSERT(!MUTEX_HELD(&gid_info->gl_mutex)); 6720 ASSERT(!MUTEX_HELD(&ibdm.ibdm_mutex)); 6721 6722 mutex_enter(&gid_info->gl_mutex); 6723 if (gid_info->gl_hca_list) 6724 ibdm_ibnex_free_hca_list(gid_info->gl_hca_list); 6725 gid_info->gl_hca_list = NULL; 6726 mutex_exit(&gid_info->gl_mutex); 6727 } 6728 6729 6730 static void 6731 ibdm_reset_all_dgids(ibmf_saa_handle_t port_sa_hdl) 6732 { 6733 IBTF_DPRINTF_L4(ibdm_string, "\treset_all_dgids(%X)", 6734 port_sa_hdl); 6735 6736 if (ibdm_enumerate_iocs == 0) 6737 return; 6738 6739 ASSERT(!MUTEX_HELD(&ibdm.ibdm_mutex)); 6740 ASSERT(!MUTEX_HELD(&ibdm.ibdm_hl_mutex)); 6741 6742 /* Check : Not busy in another probe / sweep */ 6743 mutex_enter(&ibdm.ibdm_mutex); 6744 if ((ibdm.ibdm_busy & IBDM_BUSY) == 0) { 6745 ibdm_dp_gidinfo_t *gid_info; 6746 6747 ibdm.ibdm_busy |= IBDM_BUSY; 6748 mutex_exit(&ibdm.ibdm_mutex); 6749 6750 /* 6751 * Check if any GID is using the SA & IBMF handle 6752 * of HCA port going down. Reset ibdm_dp_gidinfo_t 6753 * using another HCA port which can reach the GID. 6754 * This is for DM capable GIDs only, no need to do 6755 * this for others 6756 * 6757 * Delete the GID if no alternate HCA port to reach 6758 * it is found. 6759 */ 6760 for (gid_info = ibdm.ibdm_dp_gidlist_head; gid_info; ) { 6761 ibdm_dp_gidinfo_t *tmp; 6762 6763 IBTF_DPRINTF_L4(ibdm_string, "\tevent_hdlr " 6764 "checking gidinfo %p", gid_info); 6765 6766 if (gid_info->gl_sa_hdl == port_sa_hdl) { 6767 IBTF_DPRINTF_L3(ibdm_string, 6768 "\tevent_hdlr: down HCA port hdl " 6769 "matches gid %p", gid_info); 6770 6771 /* 6772 * The non-DM GIDs can come back 6773 * with a new subnet prefix, when 6774 * the HCA port commes up again. To 6775 * avoid issues, delete non-DM 6776 * capable GIDs, if the gid was 6777 * discovered using the HCA port 6778 * going down. This is ensured by 6779 * setting gl_disconnected to 1. 6780 */ 6781 if (gid_info->gl_is_dm_capable == B_FALSE) 6782 gid_info->gl_disconnected = 1; 6783 else 6784 ibdm_reset_gidinfo(gid_info); 6785 6786 if (gid_info->gl_disconnected) { 6787 IBTF_DPRINTF_L3(ibdm_string, 6788 "\tevent_hdlr: deleting" 6789 " gid %p", gid_info); 6790 tmp = gid_info; 6791 gid_info = gid_info->gl_next; 6792 ibdm_delete_gidinfo(tmp); 6793 } else 6794 gid_info = gid_info->gl_next; 6795 } else 6796 gid_info = gid_info->gl_next; 6797 } 6798 6799 mutex_enter(&ibdm.ibdm_mutex); 6800 ibdm.ibdm_busy &= ~IBDM_BUSY; 6801 cv_signal(&ibdm.ibdm_busy_cv); 6802 } 6803 mutex_exit(&ibdm.ibdm_mutex); 6804 } 6805 6806 static void 6807 ibdm_reset_gidinfo(ibdm_dp_gidinfo_t *gidinfo) 6808 { 6809 ibdm_hca_list_t *hca_list = NULL; 6810 ibdm_port_attr_t *port = NULL; 6811 int gid_reinited = 0; 6812 sa_node_record_t *nr, *tmp; 6813 sa_portinfo_record_t *pi; 6814 size_t nr_len = 0, pi_len = 0; 6815 size_t path_len; 6816 ib_gid_t sgid, dgid; 6817 int ret, ii, nrecords; 6818 sa_path_record_t *path; 6819 uint8_t npaths = 1; 6820 ibdm_pkey_tbl_t *pkey_tbl; 6821 6822 IBTF_DPRINTF_L4(ibdm_string, "\treset_gidinfo(%p)", gidinfo); 6823 6824 /* 6825 * Get list of all the ports reachable from the local known HCA 6826 * ports which are active 6827 */ 6828 mutex_enter(&ibdm.ibdm_hl_mutex); 6829 for (ibdm_get_next_port(&hca_list, &port, 1); port; 6830 ibdm_get_next_port(&hca_list, &port, 1)) { 6831 6832 6833 /* 6834 * Get the path and re-populate the gidinfo. 6835 * Getting the path is the same probe_ioc 6836 * Init the gid info as in ibdm_create_gidinfo() 6837 */ 6838 nr = ibdm_get_node_records(port->pa_sa_hdl, &nr_len, 6839 gidinfo->gl_nodeguid); 6840 if (nr == NULL) { 6841 IBTF_DPRINTF_L4(ibdm_string, 6842 "\treset_gidinfo : no records"); 6843 continue; 6844 } 6845 6846 nrecords = (nr_len / sizeof (sa_node_record_t)); 6847 for (tmp = nr, ii = 0; (ii < nrecords); ii++, tmp++) { 6848 if (tmp->NodeInfo.PortGUID == gidinfo->gl_portguid) 6849 break; 6850 } 6851 6852 if (ii == nrecords) { 6853 IBTF_DPRINTF_L4(ibdm_string, 6854 "\treset_gidinfo : no record for portguid"); 6855 kmem_free(nr, nr_len); 6856 continue; 6857 } 6858 6859 pi = ibdm_get_portinfo(port->pa_sa_hdl, &pi_len, tmp->LID); 6860 if (pi == NULL) { 6861 IBTF_DPRINTF_L4(ibdm_string, 6862 "\treset_gidinfo : no portinfo"); 6863 kmem_free(nr, nr_len); 6864 continue; 6865 } 6866 6867 sgid.gid_prefix = port->pa_sn_prefix; 6868 sgid.gid_guid = port->pa_port_guid; 6869 dgid.gid_prefix = pi->PortInfo.GidPrefix; 6870 dgid.gid_guid = tmp->NodeInfo.PortGUID; 6871 6872 ret = ibmf_saa_gid_to_pathrecords(port->pa_sa_hdl, sgid, dgid, 6873 IBMF_SAA_PKEY_WC, 0, B_TRUE, &npaths, 0, &path_len, &path); 6874 6875 if ((ret != IBMF_SUCCESS) || path == NULL) { 6876 IBTF_DPRINTF_L4(ibdm_string, 6877 "\treset_gidinfo : no paths"); 6878 kmem_free(pi, pi_len); 6879 kmem_free(nr, nr_len); 6880 continue; 6881 } 6882 6883 gidinfo->gl_dgid_hi = path->DGID.gid_prefix; 6884 gidinfo->gl_dgid_lo = path->DGID.gid_guid; 6885 gidinfo->gl_sgid_hi = path->SGID.gid_prefix; 6886 gidinfo->gl_sgid_lo = path->SGID.gid_guid; 6887 gidinfo->gl_p_key = path->P_Key; 6888 gidinfo->gl_sa_hdl = port->pa_sa_hdl; 6889 gidinfo->gl_ibmf_hdl = port->pa_ibmf_hdl; 6890 gidinfo->gl_slid = path->SLID; 6891 gidinfo->gl_dlid = path->DLID; 6892 /* Reset redirect info, next MAD will set if redirected */ 6893 gidinfo->gl_redirected = 0; 6894 gidinfo->gl_devid = (*tmp).NodeInfo.DeviceID; 6895 gidinfo->gl_SL = path->SL; 6896 6897 gidinfo->gl_qp_hdl = IBMF_QP_HANDLE_DEFAULT; 6898 for (ii = 0; ii < port->pa_npkeys; ii++) { 6899 if (port->pa_pkey_tbl == NULL) 6900 break; 6901 6902 pkey_tbl = &port->pa_pkey_tbl[ii]; 6903 if ((gidinfo->gl_p_key == pkey_tbl->pt_pkey) && 6904 (pkey_tbl->pt_qp_hdl != NULL)) { 6905 gidinfo->gl_qp_hdl = pkey_tbl->pt_qp_hdl; 6906 break; 6907 } 6908 } 6909 6910 if (gidinfo->gl_qp_hdl == NULL) 6911 IBTF_DPRINTF_L2(ibdm_string, 6912 "\treset_gid_info: No matching Pkey"); 6913 else 6914 gid_reinited = 1; 6915 6916 kmem_free(path, path_len); 6917 kmem_free(pi, pi_len); 6918 kmem_free(nr, nr_len); 6919 break; 6920 } 6921 mutex_exit(&ibdm.ibdm_hl_mutex); 6922 6923 if (!gid_reinited) 6924 gidinfo->gl_disconnected = 1; 6925 } 6926 6927 static void 6928 ibdm_delete_gidinfo(ibdm_dp_gidinfo_t *gidinfo) 6929 { 6930 ibdm_ioc_info_t *ioc_list; 6931 int in_gidlist = 0; 6932 6933 /* 6934 * Check if gidinfo has been inserted into the 6935 * ibdm_dp_gidlist_head list. gl_next or gl_prev 6936 * != NULL, if gidinfo is the list. 6937 */ 6938 if (gidinfo->gl_prev != NULL || 6939 gidinfo->gl_next != NULL || 6940 ibdm.ibdm_dp_gidlist_head == gidinfo) 6941 in_gidlist = 1; 6942 6943 ioc_list = ibdm_update_ioc_gidlist(gidinfo, 0); 6944 6945 /* 6946 * Remove GID from the global GID list 6947 * Handle the case where all port GIDs for an 6948 * IOU have been hot-removed. 6949 */ 6950 mutex_enter(&ibdm.ibdm_mutex); 6951 if (gidinfo->gl_iou != NULL && gidinfo->gl_ngids == 0) { 6952 mutex_enter(&gidinfo->gl_mutex); 6953 (void) ibdm_free_iou_info(gidinfo, &gidinfo->gl_iou); 6954 mutex_exit(&gidinfo->gl_mutex); 6955 } 6956 6957 /* Delete gl_hca_list */ 6958 mutex_exit(&ibdm.ibdm_mutex); 6959 ibdm_delete_glhca_list(gidinfo); 6960 mutex_enter(&ibdm.ibdm_mutex); 6961 6962 if (in_gidlist) { 6963 if (gidinfo->gl_prev != NULL) 6964 gidinfo->gl_prev->gl_next = gidinfo->gl_next; 6965 if (gidinfo->gl_next != NULL) 6966 gidinfo->gl_next->gl_prev = gidinfo->gl_prev; 6967 6968 if (gidinfo == ibdm.ibdm_dp_gidlist_head) 6969 ibdm.ibdm_dp_gidlist_head = gidinfo->gl_next; 6970 if (gidinfo == ibdm.ibdm_dp_gidlist_tail) 6971 ibdm.ibdm_dp_gidlist_tail = gidinfo->gl_prev; 6972 ibdm.ibdm_ngids--; 6973 } 6974 mutex_exit(&ibdm.ibdm_mutex); 6975 6976 mutex_destroy(&gidinfo->gl_mutex); 6977 cv_destroy(&gidinfo->gl_probe_cv); 6978 kmem_free(gidinfo, sizeof (ibdm_dp_gidinfo_t)); 6979 6980 /* 6981 * Pass on the IOCs with updated GIDs to IBnexus 6982 */ 6983 if (ioc_list) { 6984 IBTF_DPRINTF_L4("ibdm", "\tdelete_gidinfo " 6985 "IOC_PROP_UPDATE for %p\n", ioc_list); 6986 mutex_enter(&ibdm.ibdm_ibnex_mutex); 6987 if (ibdm.ibdm_ibnex_callback != NULL) { 6988 (*ibdm.ibdm_ibnex_callback)((void *) 6989 ioc_list, IBDM_EVENT_IOC_PROP_UPDATE); 6990 } 6991 mutex_exit(&ibdm.ibdm_ibnex_mutex); 6992 } 6993 } 6994 6995 6996 static void 6997 ibdm_fill_srv_attr_mod(ib_mad_hdr_t *hdr, ibdm_timeout_cb_args_t *cb_args) 6998 { 6999 uint32_t attr_mod; 7000 7001 attr_mod = (cb_args->cb_ioc_num + 1) << 16; 7002 attr_mod |= cb_args->cb_srvents_start; 7003 attr_mod |= (cb_args->cb_srvents_end) << 8; 7004 hdr->AttributeModifier = h2b32(attr_mod); 7005 } 7006 7007 static void 7008 ibdm_bump_transactionID(ibdm_dp_gidinfo_t *gid_info) 7009 { 7010 ASSERT(MUTEX_HELD(&gid_info->gl_mutex)); 7011 gid_info->gl_transactionID++; 7012 if (gid_info->gl_transactionID == gid_info->gl_max_transactionID) { 7013 IBTF_DPRINTF_L4(ibdm_string, 7014 "\tbump_transactionID(%p), wrapup", gid_info); 7015 gid_info->gl_transactionID = gid_info->gl_min_transactionID; 7016 } 7017 } 7018 7019 /* 7020 * gl_prev_iou is set for *non-reprobe* sweeep requests, which 7021 * detected that ChangeID in IOU info has changed. The service 7022 * entry also may have changed. Check if service entry in IOC 7023 * has changed wrt the prev iou, if so notify to IB Nexus. 7024 */ 7025 static ibdm_ioc_info_t * 7026 ibdm_handle_prev_iou() 7027 { 7028 ibdm_dp_gidinfo_t *gid_info; 7029 ibdm_ioc_info_t *ioc_list_head = NULL, *ioc_list; 7030 ibdm_ioc_info_t *prev_ioc, *ioc; 7031 int ii, jj, niocs, prev_niocs; 7032 7033 ASSERT(MUTEX_HELD(&ibdm.ibdm_mutex)); 7034 7035 IBTF_DPRINTF_L4(ibdm_string, "\thandle_prev_iou enter"); 7036 for (gid_info = ibdm.ibdm_dp_gidlist_head; gid_info; 7037 gid_info = gid_info->gl_next) { 7038 if (gid_info->gl_prev_iou == NULL) 7039 continue; 7040 7041 IBTF_DPRINTF_L4(ibdm_string, "\thandle_prev_iou gid %p", 7042 gid_info); 7043 niocs = gid_info->gl_iou->iou_info.iou_num_ctrl_slots; 7044 prev_niocs = 7045 gid_info->gl_prev_iou->iou_info.iou_num_ctrl_slots; 7046 for (ii = 0; ii < niocs; ii++) { 7047 ioc = IBDM_GIDINFO2IOCINFO(gid_info, ii); 7048 7049 /* Find matching IOC */ 7050 for (jj = 0; jj < prev_niocs; jj++) { 7051 prev_ioc = (ibdm_ioc_info_t *) 7052 &gid_info->gl_prev_iou->iou_ioc_info[jj]; 7053 if (prev_ioc->ioc_profile.ioc_guid == 7054 ioc->ioc_profile.ioc_guid) 7055 break; 7056 } 7057 if (jj == prev_niocs) 7058 prev_ioc = NULL; 7059 if (ioc == NULL || prev_ioc == NULL) 7060 continue; 7061 if ((ioc->ioc_profile.ioc_service_entries != 7062 prev_ioc->ioc_profile.ioc_service_entries) || 7063 ibdm_serv_cmp(&ioc->ioc_serv[0], 7064 &prev_ioc->ioc_serv[0], 7065 ioc->ioc_profile.ioc_service_entries) != 0) { 7066 IBTF_DPRINTF_L4(ibdm_string, 7067 "/thandle_prev_iou modified IOC: " 7068 "current ioc %p, old ioc %p", 7069 ioc, prev_ioc); 7070 mutex_enter(&gid_info->gl_mutex); 7071 ioc_list = ibdm_dup_ioc_info(ioc, gid_info); 7072 mutex_exit(&gid_info->gl_mutex); 7073 ioc_list->ioc_info_updated.ib_prop_updated 7074 = 0; 7075 ioc_list->ioc_info_updated.ib_srv_prop_updated 7076 = 1; 7077 7078 if (ioc_list_head == NULL) 7079 ioc_list_head = ioc_list; 7080 else { 7081 ioc_list_head->ioc_next = ioc_list; 7082 ioc_list_head = ioc_list; 7083 } 7084 } 7085 } 7086 7087 mutex_enter(&gid_info->gl_mutex); 7088 (void) ibdm_free_iou_info(gid_info, &gid_info->gl_prev_iou); 7089 mutex_exit(&gid_info->gl_mutex); 7090 } 7091 IBTF_DPRINTF_L4(ibdm_string, "\thandle_prev_iouret %p", 7092 ioc_list_head); 7093 return (ioc_list_head); 7094 } 7095 7096 /* 7097 * Compares two service entries lists, returns 0 if same, returns 1 7098 * if no match. 7099 */ 7100 static int 7101 ibdm_serv_cmp(ibdm_srvents_info_t *serv1, ibdm_srvents_info_t *serv2, 7102 int nserv) 7103 { 7104 int ii; 7105 7106 IBTF_DPRINTF_L4(ibdm_string, "\tserv_cmp: enter"); 7107 for (ii = 0; ii < nserv; ii++, serv1++, serv2++) { 7108 if (serv1->se_attr.srv_id != serv2->se_attr.srv_id || 7109 bcmp(serv1->se_attr.srv_name, 7110 serv2->se_attr.srv_name, 7111 IB_DM_MAX_SVC_NAME_LEN) != 0) { 7112 IBTF_DPRINTF_L4(ibdm_string, "\tserv_cmp: ret 1"); 7113 return (1); 7114 } 7115 } 7116 IBTF_DPRINTF_L4(ibdm_string, "\tserv_cmp: ret 0"); 7117 return (0); 7118 } 7119 7120 /* For debugging purpose only */ 7121 #ifdef DEBUG 7122 void 7123 ibdm_dump_mad_hdr(ib_mad_hdr_t *mad_hdr) 7124 { 7125 IBTF_DPRINTF_L4("ibdm", "\t\t MAD Header info"); 7126 IBTF_DPRINTF_L4("ibdm", "\t\t ---------------"); 7127 7128 IBTF_DPRINTF_L4("ibdm", "\tBase version : 0x%x" 7129 "\tMgmt Class : 0x%x", mad_hdr->BaseVersion, mad_hdr->MgmtClass); 7130 IBTF_DPRINTF_L4("ibdm", "\tClass version : 0x%x" 7131 "\tR Method : 0x%x", 7132 mad_hdr->ClassVersion, mad_hdr->R_Method); 7133 IBTF_DPRINTF_L4("ibdm", "\tMAD Status : 0x%x" 7134 "\tTransaction ID : 0x%llx", 7135 b2h16(mad_hdr->Status), b2h64(mad_hdr->TransactionID)); 7136 IBTF_DPRINTF_L4("ibdm", "\t Attribute ID : 0x%x" 7137 "\tAttribute Modified : 0x%lx", 7138 b2h16(mad_hdr->AttributeID), b2h32(mad_hdr->AttributeModifier)); 7139 } 7140 7141 7142 void 7143 ibdm_dump_ibmf_msg(ibmf_msg_t *ibmf_msg, int flag) 7144 { 7145 ib_mad_hdr_t *mad_hdr; 7146 7147 IBTF_DPRINTF_L4("ibdm", "\t\t(IBMF_PKT): Local address info"); 7148 IBTF_DPRINTF_L4("ibdm", "\t\t ------------------"); 7149 7150 IBTF_DPRINTF_L4("ibdm", "\tLocal Lid : 0x%x\tRemote Lid : 0x%x" 7151 " Remote Qp : 0x%x", ibmf_msg->im_local_addr.ia_local_lid, 7152 ibmf_msg->im_local_addr.ia_remote_lid, 7153 ibmf_msg->im_local_addr.ia_remote_qno); 7154 IBTF_DPRINTF_L4("ibdm", "\tP_key : 0x%x\tQ_key : 0x%x" 7155 " SL : 0x%x", ibmf_msg->im_local_addr.ia_p_key, 7156 ibmf_msg->im_local_addr.ia_q_key, 7157 ibmf_msg->im_local_addr.ia_service_level); 7158 7159 if (flag) 7160 mad_hdr = (ib_mad_hdr_t *)IBDM_OUT_IBMFMSG_MADHDR(ibmf_msg); 7161 else 7162 mad_hdr = IBDM_IN_IBMFMSG_MADHDR(ibmf_msg); 7163 7164 ibdm_dump_mad_hdr(mad_hdr); 7165 } 7166 7167 7168 void 7169 ibdm_dump_path_info(sa_path_record_t *path) 7170 { 7171 IBTF_DPRINTF_L4("ibdm", "\t\t Path information"); 7172 IBTF_DPRINTF_L4("ibdm", "\t\t ----------------"); 7173 7174 IBTF_DPRINTF_L4("ibdm", "\t DGID hi : %llx\tDGID lo : %llx", 7175 path->DGID.gid_prefix, path->DGID.gid_guid); 7176 IBTF_DPRINTF_L4("ibdm", "\t SGID hi : %llx\tSGID lo : %llx", 7177 path->SGID.gid_prefix, path->SGID.gid_guid); 7178 IBTF_DPRINTF_L4("ibdm", "\t SLID : %x\t\tDlID : %x", 7179 path->SLID, path->DLID); 7180 IBTF_DPRINTF_L4("ibdm", "\t P Key : %x\t\tSL : %x", 7181 path->P_Key, path->SL); 7182 } 7183 7184 7185 void 7186 ibdm_dump_classportinfo(ib_mad_classportinfo_t *classportinfo) 7187 { 7188 IBTF_DPRINTF_L4("ibdm", "\t\t CLASSPORT INFO"); 7189 IBTF_DPRINTF_L4("ibdm", "\t\t --------------"); 7190 7191 IBTF_DPRINTF_L4("ibdm", "\t Response Time Value : 0x%x", 7192 ((b2h32(classportinfo->RespTimeValue)) & 0x1F)); 7193 7194 IBTF_DPRINTF_L4("ibdm", "\t Redirected GID hi : 0x%llx", 7195 b2h64(classportinfo->RedirectGID_hi)); 7196 IBTF_DPRINTF_L4("ibdm", "\t Redirected GID lo : 0x%llx", 7197 b2h64(classportinfo->RedirectGID_lo)); 7198 IBTF_DPRINTF_L4("ibdm", "\t Redirected TC : 0x%x", 7199 classportinfo->RedirectTC); 7200 IBTF_DPRINTF_L4("ibdm", "\t Redirected SL : 0x%x", 7201 classportinfo->RedirectSL); 7202 IBTF_DPRINTF_L4("ibdm", "\t Redirected FL : 0x%x", 7203 classportinfo->RedirectFL); 7204 IBTF_DPRINTF_L4("ibdm", "\t Redirected LID : 0x%x", 7205 b2h16(classportinfo->RedirectLID)); 7206 IBTF_DPRINTF_L4("ibdm", "\t Redirected P KEY : 0x%x", 7207 b2h16(classportinfo->RedirectP_Key)); 7208 IBTF_DPRINTF_L4("ibdm", "\t Redirected QP : 0x%x", 7209 classportinfo->RedirectQP); 7210 IBTF_DPRINTF_L4("ibdm", "\t Redirected Q KEY : 0x%x", 7211 b2h32(classportinfo->RedirectQ_Key)); 7212 IBTF_DPRINTF_L4("ibdm", "\t Trap GID hi : 0x%llx", 7213 b2h64(classportinfo->TrapGID_hi)); 7214 IBTF_DPRINTF_L4("ibdm", "\t Trap GID lo : 0x%llx", 7215 b2h64(classportinfo->TrapGID_lo)); 7216 IBTF_DPRINTF_L4("ibdm", "\t Trap TC : 0x%x", 7217 classportinfo->TrapTC); 7218 IBTF_DPRINTF_L4("ibdm", "\t Trap SL : 0x%x", 7219 classportinfo->TrapSL); 7220 IBTF_DPRINTF_L4("ibdm", "\t Trap FL : 0x%x", 7221 classportinfo->TrapFL); 7222 IBTF_DPRINTF_L4("ibdm", "\t Trap LID : 0x%x", 7223 b2h16(classportinfo->TrapLID)); 7224 IBTF_DPRINTF_L4("ibdm", "\t Trap P_Key : 0x%x", 7225 b2h16(classportinfo->TrapP_Key)); 7226 IBTF_DPRINTF_L4("ibdm", "\t Trap HL : 0x%x", 7227 classportinfo->TrapHL); 7228 IBTF_DPRINTF_L4("ibdm", "\t Trap QP : 0x%x", 7229 classportinfo->TrapQP); 7230 IBTF_DPRINTF_L4("ibdm", "\t Trap Q_Key : 0x%x", 7231 b2h32(classportinfo->TrapQ_Key)); 7232 } 7233 7234 7235 void 7236 ibdm_dump_iounitinfo(ib_dm_io_unitinfo_t *iou_info) 7237 { 7238 IBTF_DPRINTF_L4("ibdm", "\t\t I/O UnitInfo"); 7239 IBTF_DPRINTF_L4("ibdm", "\t\t ------------"); 7240 7241 IBTF_DPRINTF_L4("ibdm", "\tChange ID : 0x%x", 7242 b2h16(iou_info->iou_changeid)); 7243 IBTF_DPRINTF_L4("ibdm", "\t#of ctrl slots : %d", 7244 iou_info->iou_num_ctrl_slots); 7245 IBTF_DPRINTF_L4("ibdm", "\tIOU flag : 0x%x", 7246 iou_info->iou_flag); 7247 IBTF_DPRINTF_L4("ibdm", "\tContrl list byte 0 : 0x%x", 7248 iou_info->iou_ctrl_list[0]); 7249 IBTF_DPRINTF_L4("ibdm", "\tContrl list byte 1 : 0x%x", 7250 iou_info->iou_ctrl_list[1]); 7251 IBTF_DPRINTF_L4("ibdm", "\tContrl list byte 2 : 0x%x", 7252 iou_info->iou_ctrl_list[2]); 7253 } 7254 7255 7256 void 7257 ibdm_dump_ioc_profile(ib_dm_ioc_ctrl_profile_t *ioc) 7258 { 7259 IBTF_DPRINTF_L4("ibdm", "\t\t IOC Controller Profile"); 7260 IBTF_DPRINTF_L4("ibdm", "\t\t ----------------------"); 7261 7262 IBTF_DPRINTF_L4("ibdm", "\tIOC Guid : %llx", ioc->ioc_guid); 7263 IBTF_DPRINTF_L4("ibdm", "\tVendorID : 0x%x", ioc->ioc_vendorid); 7264 IBTF_DPRINTF_L4("ibdm", "\tDevice Id : 0x%x", ioc->ioc_deviceid); 7265 IBTF_DPRINTF_L4("ibdm", "\tDevice Ver : 0x%x", ioc->ioc_device_ver); 7266 IBTF_DPRINTF_L4("ibdm", "\tSubsys ID : 0x%x", ioc->ioc_subsys_id); 7267 IBTF_DPRINTF_L4("ibdm", "\tIO class : 0x%x", ioc->ioc_io_class); 7268 IBTF_DPRINTF_L4("ibdm", "\tIO subclass : 0x%x", ioc->ioc_io_subclass); 7269 IBTF_DPRINTF_L4("ibdm", "\tProtocol : 0x%x", ioc->ioc_protocol); 7270 IBTF_DPRINTF_L4("ibdm", "\tProtocolV : 0x%x", ioc->ioc_protocol_ver); 7271 IBTF_DPRINTF_L4("ibdm", "\tmsg qdepth : %d", ioc->ioc_send_msg_qdepth); 7272 IBTF_DPRINTF_L4("ibdm", "\trdma qdepth : %d", 7273 ioc->ioc_rdma_read_qdepth); 7274 IBTF_DPRINTF_L4("ibdm", "\tsndmsg sz : %d", ioc->ioc_send_msg_sz); 7275 IBTF_DPRINTF_L4("ibdm", "\trdma xfersz : %d", ioc->ioc_rdma_xfer_sz); 7276 IBTF_DPRINTF_L4("ibdm", "\topcal mask : 0x%x", 7277 ioc->ioc_ctrl_opcap_mask); 7278 IBTF_DPRINTF_L4("ibdm", "\tsrventries : %x", ioc->ioc_service_entries); 7279 } 7280 7281 7282 void 7283 ibdm_dump_service_entries(ib_dm_srv_t *srv_ents) 7284 { 7285 IBTF_DPRINTF_L4("ibdm", 7286 "\thandle_srventry_mad: service id : %llx", srv_ents->srv_id); 7287 7288 IBTF_DPRINTF_L4("ibdm", "\thandle_srventry_mad: " 7289 "Service Name : %s", srv_ents->srv_name); 7290 } 7291 7292 int ibdm_allow_sweep_fabric_timestamp = 1; 7293 7294 void 7295 ibdm_dump_sweep_fabric_timestamp(int flag) 7296 { 7297 static hrtime_t x; 7298 if (flag) { 7299 if (ibdm_allow_sweep_fabric_timestamp) { 7300 IBTF_DPRINTF_L4("ibdm", "\tTime taken to complete " 7301 "sweep %lld ms", ((gethrtime() - x)/ 1000000)); 7302 } 7303 x = 0; 7304 } else 7305 x = gethrtime(); 7306 } 7307 #endif 7308