1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2007 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 #pragma ident "%Z%%M% %I% %E% SMI" 27 28 /* 29 * ibdm.c 30 * 31 * This file contains the InifiniBand Device Manager (IBDM) support functions. 32 * IB nexus driver will only be the client for the IBDM module. 33 * 34 * IBDM registers with IBTF for HCA arrival/removal notification. 35 * IBDM registers with SA access to send DM MADs to discover the IOC's behind 36 * the IOU's. 37 * 38 * IB nexus driver registers with IBDM to find the information about the 39 * HCA's and IOC's (behind the IOU) present on the IB fabric. 40 */ 41 42 #include <sys/systm.h> 43 #include <sys/taskq.h> 44 #include <sys/ib/mgt/ibdm/ibdm_impl.h> 45 #include <sys/modctl.h> 46 47 /* Function Prototype declarations */ 48 static int ibdm_free_iou_info(ibdm_dp_gidinfo_t *, ibdm_iou_info_t **); 49 static int ibdm_fini(void); 50 static int ibdm_init(void); 51 static int ibdm_get_reachable_ports(ibdm_port_attr_t *, 52 ibdm_hca_list_t *); 53 static ibdm_dp_gidinfo_t *ibdm_check_dgid(ib_guid_t, ib_sn_prefix_t); 54 static ibdm_dp_gidinfo_t *ibdm_check_dest_nodeguid(ibdm_dp_gidinfo_t *); 55 static int ibdm_send_classportinfo(ibdm_dp_gidinfo_t *); 56 static int ibdm_send_iounitinfo(ibdm_dp_gidinfo_t *); 57 static int ibdm_is_dev_mgt_supported(ibdm_dp_gidinfo_t *); 58 static int ibdm_get_node_port_guids(ibmf_saa_handle_t, ib_lid_t, 59 ib_guid_t *, ib_guid_t *); 60 static int ibdm_retry_command(ibdm_timeout_cb_args_t *); 61 static int ibdm_get_diagcode(ibdm_dp_gidinfo_t *, int); 62 static int ibdm_verify_mad_status(ib_mad_hdr_t *); 63 static int ibdm_handle_redirection(ibmf_msg_t *, 64 ibdm_dp_gidinfo_t *, int *); 65 static void ibdm_wait_probe_completion(void); 66 static void ibdm_sweep_fabric(int); 67 static void ibdm_probe_gid_thread(void *); 68 static void ibdm_wakeup_probe_gid_cv(void); 69 static void ibdm_port_attr_ibmf_init(ibdm_port_attr_t *, ib_pkey_t, int); 70 static int ibdm_port_attr_ibmf_fini(ibdm_port_attr_t *, int); 71 static void ibdm_update_port_attr(ibdm_port_attr_t *); 72 static void ibdm_handle_hca_attach(ib_guid_t); 73 static void ibdm_handle_srventry_mad(ibmf_msg_t *, 74 ibdm_dp_gidinfo_t *, int *); 75 static void ibdm_ibmf_recv_cb(ibmf_handle_t, ibmf_msg_t *, void *); 76 static void ibdm_recv_incoming_mad(void *); 77 static void ibdm_process_incoming_mad(ibmf_handle_t, ibmf_msg_t *, void *); 78 static void ibdm_ibmf_send_cb(ibmf_handle_t, ibmf_msg_t *, void *); 79 static void ibdm_pkt_timeout_hdlr(void *arg); 80 static void ibdm_initialize_port(ibdm_port_attr_t *); 81 static void ibdm_handle_diagcode(ibmf_msg_t *, ibdm_dp_gidinfo_t *, int *); 82 static void ibdm_probe_gid(ibdm_dp_gidinfo_t *); 83 static void ibdm_alloc_send_buffers(ibmf_msg_t *); 84 static void ibdm_free_send_buffers(ibmf_msg_t *); 85 static void ibdm_handle_hca_detach(ib_guid_t); 86 static int ibdm_fini_port(ibdm_port_attr_t *); 87 static int ibdm_uninit_hca(ibdm_hca_list_t *); 88 static void ibdm_handle_iounitinfo(ibmf_handle_t, 89 ibmf_msg_t *, ibdm_dp_gidinfo_t *, int *); 90 static void ibdm_handle_ioc_profile(ibmf_handle_t, 91 ibmf_msg_t *, ibdm_dp_gidinfo_t *, int *); 92 static void ibdm_event_hdlr(void *, ibt_hca_hdl_t, 93 ibt_async_code_t, ibt_async_event_t *); 94 static void ibdm_handle_classportinfo(ibmf_handle_t, 95 ibmf_msg_t *, ibdm_dp_gidinfo_t *, int *); 96 static void ibdm_update_ioc_port_gidlist(ibdm_ioc_info_t *, 97 ibdm_dp_gidinfo_t *); 98 99 static ibdm_hca_list_t *ibdm_dup_hca_attr(ibdm_hca_list_t *); 100 static ibdm_ioc_info_t *ibdm_dup_ioc_info(ibdm_ioc_info_t *, 101 ibdm_dp_gidinfo_t *gid_list); 102 static void ibdm_probe_ioc(ib_guid_t, ib_guid_t, int); 103 static ibdm_ioc_info_t *ibdm_is_ioc_present(ib_guid_t, 104 ibdm_dp_gidinfo_t *, int *); 105 static ibdm_port_attr_t *ibdm_get_port_attr(ibt_async_event_t *, 106 ibdm_hca_list_t **); 107 static sa_node_record_t *ibdm_get_node_records(ibmf_saa_handle_t, 108 size_t *, ib_guid_t); 109 static sa_portinfo_record_t *ibdm_get_portinfo(ibmf_saa_handle_t, size_t *, 110 ib_lid_t); 111 static ibdm_dp_gidinfo_t *ibdm_create_gid_info(ibdm_port_attr_t *, 112 ib_gid_t, ib_gid_t); 113 static ibdm_dp_gidinfo_t *ibdm_find_gid(ib_guid_t, ib_guid_t); 114 static int ibdm_send_ioc_profile(ibdm_dp_gidinfo_t *, uint8_t); 115 static ibdm_ioc_info_t *ibdm_update_ioc_gidlist(ibdm_dp_gidinfo_t *, int); 116 static void ibdm_saa_event_cb(ibmf_saa_handle_t, ibmf_saa_subnet_event_t, 117 ibmf_saa_event_details_t *, void *); 118 static void ibdm_reprobe_update_port_srv(ibdm_ioc_info_t *, 119 ibdm_dp_gidinfo_t *); 120 static ibdm_dp_gidinfo_t *ibdm_handle_gid_rm(ibdm_dp_gidinfo_t *); 121 static void ibdm_rmfrom_glgid_list(ibdm_dp_gidinfo_t *, 122 ibdm_dp_gidinfo_t *); 123 static void ibdm_addto_gidlist(ibdm_gid_t **, ibdm_gid_t *); 124 static void ibdm_free_gid_list(ibdm_gid_t *); 125 static void ibdm_rescan_gidlist(ib_guid_t *ioc_guid); 126 static void ibdm_notify_newgid_iocs(ibdm_dp_gidinfo_t *); 127 static void ibdm_saa_event_taskq(void *); 128 static void ibdm_free_saa_event_arg(ibdm_saa_event_arg_t *); 129 static void ibdm_get_next_port(ibdm_hca_list_t **, 130 ibdm_port_attr_t **, int); 131 static void ibdm_add_to_gl_gid(ibdm_dp_gidinfo_t *, 132 ibdm_dp_gidinfo_t *); 133 static void ibdm_addto_glhcalist(ibdm_dp_gidinfo_t *, 134 ibdm_hca_list_t *); 135 static void ibdm_delete_glhca_list(ibdm_dp_gidinfo_t *); 136 static void ibdm_saa_handle_new_gid(void *); 137 static void ibdm_reset_all_dgids(ibmf_saa_handle_t); 138 static void ibdm_reset_gidinfo(ibdm_dp_gidinfo_t *); 139 static void ibdm_delete_gidinfo(ibdm_dp_gidinfo_t *); 140 static void ibdm_fill_srv_attr_mod(ib_mad_hdr_t *, ibdm_timeout_cb_args_t *); 141 static void ibdm_bump_transactionID(ibdm_dp_gidinfo_t *); 142 static ibdm_ioc_info_t *ibdm_handle_prev_iou(); 143 static int ibdm_serv_cmp(ibdm_srvents_info_t *, ibdm_srvents_info_t *, 144 int); 145 146 int ibdm_dft_timeout = IBDM_DFT_TIMEOUT; 147 int ibdm_dft_retry_cnt = IBDM_DFT_NRETRIES; 148 #ifdef DEBUG 149 int ibdm_ignore_saa_event = 0; 150 #endif 151 152 /* Modload support */ 153 static struct modlmisc ibdm_modlmisc = { 154 &mod_miscops, 155 "InfiniBand Device Manager %I%", 156 }; 157 158 struct modlinkage ibdm_modlinkage = { 159 MODREV_1, 160 (void *)&ibdm_modlmisc, 161 NULL 162 }; 163 164 static ibt_clnt_modinfo_t ibdm_ibt_modinfo = { 165 IBTI_V2, 166 IBT_DM, 167 ibdm_event_hdlr, 168 NULL, 169 "ibdm" 170 }; 171 172 /* Global variables */ 173 ibdm_t ibdm; 174 int ibdm_taskq_enable = IBDM_ENABLE_TASKQ_HANDLING; 175 char *ibdm_string = "ibdm"; 176 177 _NOTE(SCHEME_PROTECTS_DATA("Serialized access by cv", 178 ibdm.ibdm_dp_gidlist_head)) 179 180 /* 181 * _init 182 * Loadable module init, called before any other module. 183 * Initialize mutex 184 * Register with IBTF 185 */ 186 int 187 _init(void) 188 { 189 int err; 190 191 IBTF_DPRINTF_L4("ibdm", "\t_init: addr of ibdm %p", &ibdm); 192 193 if ((err = ibdm_init()) != IBDM_SUCCESS) { 194 IBTF_DPRINTF_L2("ibdm", "_init: ibdm_init failed 0x%x", err); 195 (void) ibdm_fini(); 196 return (DDI_FAILURE); 197 } 198 199 if ((err = mod_install(&ibdm_modlinkage)) != 0) { 200 IBTF_DPRINTF_L2("ibdm", "_init: mod_install failed 0x%x", err); 201 (void) ibdm_fini(); 202 } 203 return (err); 204 } 205 206 207 int 208 _fini(void) 209 { 210 int err; 211 212 if ((err = ibdm_fini()) != IBDM_SUCCESS) { 213 IBTF_DPRINTF_L2("ibdm", "_fini: ibdm_fini failed 0x%x", err); 214 (void) ibdm_init(); 215 return (EBUSY); 216 } 217 218 if ((err = mod_remove(&ibdm_modlinkage)) != 0) { 219 IBTF_DPRINTF_L2("ibdm", "_fini: mod_remove failed 0x%x", err); 220 (void) ibdm_init(); 221 } 222 return (err); 223 } 224 225 226 int 227 _info(struct modinfo *modinfop) 228 { 229 return (mod_info(&ibdm_modlinkage, modinfop)); 230 } 231 232 233 /* 234 * ibdm_init(): 235 * Register with IBTF 236 * Allocate memory for the HCAs 237 * Allocate minor-nodes for the HCAs 238 */ 239 static int 240 ibdm_init(void) 241 { 242 int i, hca_count; 243 ib_guid_t *hca_guids; 244 ibt_status_t status; 245 246 IBTF_DPRINTF_L4("ibdm", "\tibdm_init:"); 247 if (!(ibdm.ibdm_state & IBDM_LOCKS_ALLOCED)) { 248 mutex_init(&ibdm.ibdm_mutex, NULL, MUTEX_DEFAULT, NULL); 249 mutex_init(&ibdm.ibdm_hl_mutex, NULL, MUTEX_DEFAULT, NULL); 250 mutex_init(&ibdm.ibdm_ibnex_mutex, NULL, MUTEX_DEFAULT, NULL); 251 mutex_enter(&ibdm.ibdm_mutex); 252 ibdm.ibdm_state |= IBDM_LOCKS_ALLOCED; 253 } 254 255 if (!(ibdm.ibdm_state & IBDM_IBT_ATTACHED)) { 256 if ((status = ibt_attach(&ibdm_ibt_modinfo, NULL, NULL, 257 (void *)&ibdm.ibdm_ibt_clnt_hdl)) != IBT_SUCCESS) { 258 IBTF_DPRINTF_L2("ibdm", "ibdm_init: ibt_attach " 259 "failed %x", status); 260 mutex_exit(&ibdm.ibdm_mutex); 261 return (IBDM_FAILURE); 262 } 263 264 ibdm.ibdm_state |= IBDM_IBT_ATTACHED; 265 mutex_exit(&ibdm.ibdm_mutex); 266 } 267 268 269 if (!(ibdm.ibdm_state & IBDM_HCA_ATTACHED)) { 270 hca_count = ibt_get_hca_list(&hca_guids); 271 IBTF_DPRINTF_L4("ibdm", "ibdm_init: num_hcas = %d", hca_count); 272 for (i = 0; i < hca_count; i++) 273 (void) ibdm_handle_hca_attach(hca_guids[i]); 274 if (hca_count) 275 ibt_free_hca_list(hca_guids, hca_count); 276 277 mutex_enter(&ibdm.ibdm_mutex); 278 ibdm.ibdm_state |= IBDM_HCA_ATTACHED; 279 mutex_exit(&ibdm.ibdm_mutex); 280 } 281 282 if (!(ibdm.ibdm_state & IBDM_CVS_ALLOCED)) { 283 cv_init(&ibdm.ibdm_probe_cv, NULL, CV_DRIVER, NULL); 284 cv_init(&ibdm.ibdm_busy_cv, NULL, CV_DRIVER, NULL); 285 mutex_enter(&ibdm.ibdm_mutex); 286 ibdm.ibdm_state |= IBDM_CVS_ALLOCED; 287 mutex_exit(&ibdm.ibdm_mutex); 288 } 289 return (IBDM_SUCCESS); 290 } 291 292 293 static int 294 ibdm_free_iou_info(ibdm_dp_gidinfo_t *gid_info, ibdm_iou_info_t **ioup) 295 { 296 int ii, k, niocs; 297 size_t size; 298 ibdm_gid_t *delete, *head; 299 timeout_id_t timeout_id; 300 ibdm_ioc_info_t *ioc; 301 ibdm_iou_info_t *gl_iou = *ioup; 302 303 ASSERT(mutex_owned(&gid_info->gl_mutex)); 304 if (gl_iou == NULL) { 305 IBTF_DPRINTF_L4("ibdm", "\tibdm_free_iou_info: No IOU"); 306 return (0); 307 } 308 309 niocs = gl_iou->iou_info.iou_num_ctrl_slots; 310 IBTF_DPRINTF_L4("ibdm", "\tfree_iou_info: gid_info = %p, niocs %d", 311 gid_info, niocs); 312 313 for (ii = 0; ii < niocs; ii++) { 314 ioc = (ibdm_ioc_info_t *)&gl_iou->iou_ioc_info[ii]; 315 316 /* handle the case where an ioc_timeout_id is scheduled */ 317 if (ioc->ioc_timeout_id) { 318 timeout_id = ioc->ioc_timeout_id; 319 mutex_exit(&gid_info->gl_mutex); 320 IBTF_DPRINTF_L5("ibdm", "free_iou_info: " 321 "ioc_timeout_id = 0x%x", timeout_id); 322 if (untimeout(timeout_id) == -1) { 323 IBTF_DPRINTF_L2("ibdm", "free_iou_info: " 324 "untimeout ioc_timeout_id failed"); 325 mutex_enter(&gid_info->gl_mutex); 326 return (-1); 327 } 328 mutex_enter(&gid_info->gl_mutex); 329 ioc->ioc_timeout_id = 0; 330 } 331 332 /* handle the case where an ioc_dc_timeout_id is scheduled */ 333 if (ioc->ioc_dc_timeout_id) { 334 timeout_id = ioc->ioc_dc_timeout_id; 335 mutex_exit(&gid_info->gl_mutex); 336 IBTF_DPRINTF_L5("ibdm", "free_iou_info: " 337 "ioc_dc_timeout_id = 0x%x", timeout_id); 338 if (untimeout(timeout_id) == -1) { 339 IBTF_DPRINTF_L2("ibdm", "free_iou_info: " 340 "untimeout ioc_dc_timeout_id failed"); 341 mutex_enter(&gid_info->gl_mutex); 342 return (-1); 343 } 344 mutex_enter(&gid_info->gl_mutex); 345 ioc->ioc_dc_timeout_id = 0; 346 } 347 348 /* handle the case where serv[k].se_timeout_id is scheduled */ 349 for (k = 0; k < ioc->ioc_profile.ioc_service_entries; k++) { 350 if (ioc->ioc_serv[k].se_timeout_id) { 351 timeout_id = ioc->ioc_serv[k].se_timeout_id; 352 mutex_exit(&gid_info->gl_mutex); 353 IBTF_DPRINTF_L5("ibdm", "free_iou_info: " 354 "ioc->ioc_serv[%d].se_timeout_id = 0x%x", 355 k, timeout_id); 356 if (untimeout(timeout_id) == -1) { 357 IBTF_DPRINTF_L2("ibdm", "free_iou_info:" 358 " untimeout se_timeout_id failed"); 359 mutex_enter(&gid_info->gl_mutex); 360 return (-1); 361 } 362 mutex_enter(&gid_info->gl_mutex); 363 ioc->ioc_serv[k].se_timeout_id = 0; 364 } 365 } 366 367 /* delete GID list in IOC */ 368 head = ioc->ioc_gid_list; 369 while (head) { 370 IBTF_DPRINTF_L4("ibdm", "\tibdm_free_iou_info: " 371 "Deleting gid_list struct %p", head); 372 delete = head; 373 head = head->gid_next; 374 kmem_free(delete, sizeof (ibdm_gid_t)); 375 } 376 ioc->ioc_gid_list = NULL; 377 378 /* delete ioc_serv */ 379 size = ioc->ioc_profile.ioc_service_entries * 380 sizeof (ibdm_srvents_info_t); 381 if (ioc->ioc_serv && size) { 382 kmem_free(ioc->ioc_serv, size); 383 ioc->ioc_serv = NULL; 384 } 385 } 386 387 IBTF_DPRINTF_L4("ibdm", "\tibdm_free_iou_info: deleting IOU & IOC"); 388 size = sizeof (ibdm_iou_info_t) + niocs * sizeof (ibdm_ioc_info_t); 389 kmem_free(gl_iou, size); 390 *ioup = NULL; 391 return (0); 392 } 393 394 395 /* 396 * ibdm_fini(): 397 * Un-register with IBTF 398 * De allocate memory for the GID info 399 */ 400 static int 401 ibdm_fini() 402 { 403 int ii; 404 ibdm_hca_list_t *hca_list, *temp; 405 ibdm_dp_gidinfo_t *gid_info, *tmp; 406 ibdm_gid_t *head, *delete; 407 408 IBTF_DPRINTF_L4("ibdm", "\tibdm_fini"); 409 410 mutex_enter(&ibdm.ibdm_hl_mutex); 411 if (ibdm.ibdm_state & IBDM_IBT_ATTACHED) { 412 if (ibt_detach(ibdm.ibdm_ibt_clnt_hdl) != IBT_SUCCESS) { 413 IBTF_DPRINTF_L2("ibdm", "\t_fini: ibt_detach failed"); 414 mutex_exit(&ibdm.ibdm_hl_mutex); 415 return (IBDM_FAILURE); 416 } 417 ibdm.ibdm_state &= ~IBDM_IBT_ATTACHED; 418 ibdm.ibdm_ibt_clnt_hdl = NULL; 419 } 420 421 hca_list = ibdm.ibdm_hca_list_head; 422 IBTF_DPRINTF_L4("ibdm", "\tibdm_fini: nhcas %d", ibdm.ibdm_hca_count); 423 for (ii = 0; ii < ibdm.ibdm_hca_count; ii++) { 424 temp = hca_list; 425 hca_list = hca_list->hl_next; 426 IBTF_DPRINTF_L4("ibdm", "\tibdm_fini: hca %p", temp); 427 if (ibdm_uninit_hca(temp) != IBDM_SUCCESS) { 428 IBTF_DPRINTF_L2("ibdm", "\tibdm_fini: " 429 "uninit_hca %p failed", temp); 430 mutex_exit(&ibdm.ibdm_hl_mutex); 431 return (IBDM_FAILURE); 432 } 433 } 434 mutex_exit(&ibdm.ibdm_hl_mutex); 435 436 mutex_enter(&ibdm.ibdm_mutex); 437 if (ibdm.ibdm_state & IBDM_HCA_ATTACHED) 438 ibdm.ibdm_state &= ~IBDM_HCA_ATTACHED; 439 440 gid_info = ibdm.ibdm_dp_gidlist_head; 441 while (gid_info) { 442 mutex_enter(&gid_info->gl_mutex); 443 (void) ibdm_free_iou_info(gid_info, &gid_info->gl_iou); 444 mutex_exit(&gid_info->gl_mutex); 445 ibdm_delete_glhca_list(gid_info); 446 447 tmp = gid_info; 448 gid_info = gid_info->gl_next; 449 mutex_destroy(&tmp->gl_mutex); 450 head = tmp->gl_gid; 451 while (head) { 452 IBTF_DPRINTF_L4("ibdm", 453 "\tibdm_fini: Deleting gid structs"); 454 delete = head; 455 head = head->gid_next; 456 kmem_free(delete, sizeof (ibdm_gid_t)); 457 } 458 kmem_free(tmp, sizeof (ibdm_dp_gidinfo_t)); 459 } 460 mutex_exit(&ibdm.ibdm_mutex); 461 462 if (ibdm.ibdm_state & IBDM_LOCKS_ALLOCED) { 463 ibdm.ibdm_state &= ~IBDM_LOCKS_ALLOCED; 464 mutex_destroy(&ibdm.ibdm_mutex); 465 mutex_destroy(&ibdm.ibdm_hl_mutex); 466 mutex_destroy(&ibdm.ibdm_ibnex_mutex); 467 } 468 if (ibdm.ibdm_state & IBDM_CVS_ALLOCED) { 469 ibdm.ibdm_state &= ~IBDM_CVS_ALLOCED; 470 cv_destroy(&ibdm.ibdm_probe_cv); 471 cv_destroy(&ibdm.ibdm_busy_cv); 472 } 473 return (IBDM_SUCCESS); 474 } 475 476 477 /* 478 * ibdm_event_hdlr() 479 * 480 * IBDM registers this asynchronous event handler at the time of 481 * ibt_attach. IBDM support the following async events. For other 482 * event, simply returns success. 483 * IBT_HCA_ATTACH_EVENT: 484 * Retrieves the information about all the port that are 485 * present on this HCA, allocates the port attributes 486 * structure and calls IB nexus callback routine with 487 * the port attributes structure as an input argument. 488 * IBT_HCA_DETACH_EVENT: 489 * Retrieves the information about all the ports that are 490 * present on this HCA and calls IB nexus callback with 491 * port guid as an argument 492 * IBT_EVENT_PORT_UP: 493 * Register with IBMF and SA access 494 * Setup IBMF receive callback routine 495 * IBT_EVENT_PORT_DOWN: 496 * Un-Register with IBMF and SA access 497 * Teardown IBMF receive callback routine 498 */ 499 /*ARGSUSED*/ 500 static void 501 ibdm_event_hdlr(void *clnt_hdl, 502 ibt_hca_hdl_t hca_hdl, ibt_async_code_t code, ibt_async_event_t *event) 503 { 504 ibdm_hca_list_t *hca_list; 505 ibdm_port_attr_t *port; 506 ibmf_saa_handle_t port_sa_hdl; 507 508 IBTF_DPRINTF_L4("ibdm", "\tevent_hdlr: async code 0x%x", code); 509 510 switch (code) { 511 case IBT_HCA_ATTACH_EVENT: /* New HCA registered with IBTF */ 512 ibdm_handle_hca_attach(event->ev_hca_guid); 513 break; 514 515 case IBT_HCA_DETACH_EVENT: /* HCA unregistered with IBTF */ 516 ibdm_handle_hca_detach(event->ev_hca_guid); 517 mutex_enter(&ibdm.ibdm_ibnex_mutex); 518 if (ibdm.ibdm_ibnex_callback != NULL) { 519 (*ibdm.ibdm_ibnex_callback)((void *) 520 &event->ev_hca_guid, IBDM_EVENT_HCA_REMOVED); 521 } 522 mutex_exit(&ibdm.ibdm_ibnex_mutex); 523 break; 524 525 case IBT_EVENT_PORT_UP: 526 IBTF_DPRINTF_L4("ibdm", "\tevent_hdlr: PORT_UP"); 527 mutex_enter(&ibdm.ibdm_hl_mutex); 528 port = ibdm_get_port_attr(event, &hca_list); 529 if (port == NULL) { 530 IBTF_DPRINTF_L2("ibdm", 531 "\tevent_hdlr: HCA not present"); 532 mutex_exit(&ibdm.ibdm_hl_mutex); 533 break; 534 } 535 ibdm_initialize_port(port); 536 hca_list->hl_nports_active++; 537 mutex_exit(&ibdm.ibdm_hl_mutex); 538 break; 539 540 case IBT_ERROR_PORT_DOWN: 541 IBTF_DPRINTF_L4("ibdm", "\tevent_hdlr: PORT_DOWN"); 542 mutex_enter(&ibdm.ibdm_hl_mutex); 543 port = ibdm_get_port_attr(event, &hca_list); 544 if (port == NULL) { 545 IBTF_DPRINTF_L2("ibdm", 546 "\tevent_hdlr: HCA not present"); 547 mutex_exit(&ibdm.ibdm_hl_mutex); 548 break; 549 } 550 hca_list->hl_nports_active--; 551 port_sa_hdl = port->pa_sa_hdl; 552 (void) ibdm_fini_port(port); 553 port->pa_state = IBT_PORT_DOWN; 554 mutex_exit(&ibdm.ibdm_hl_mutex); 555 ibdm_reset_all_dgids(port_sa_hdl); 556 break; 557 558 default: /* Ignore all other events/errors */ 559 break; 560 } 561 } 562 563 564 /* 565 * ibdm_initialize_port() 566 * Register with IBMF 567 * Register with SA access 568 * Register a receive callback routine with IBMF. IBMF invokes 569 * this routine whenever a MAD arrives at this port. 570 * Update the port attributes 571 */ 572 static void 573 ibdm_initialize_port(ibdm_port_attr_t *port) 574 { 575 int ii; 576 uint_t nports, size; 577 uint_t pkey_idx; 578 ib_pkey_t pkey; 579 ibt_hca_portinfo_t *pinfop; 580 ibmf_register_info_t ibmf_reg; 581 ibmf_saa_subnet_event_args_t event_args; 582 583 IBTF_DPRINTF_L4("ibdm", "\tinitialize_port:"); 584 ASSERT(MUTEX_HELD(&ibdm.ibdm_hl_mutex)); 585 586 /* Check whether the port is active */ 587 if (ibt_get_port_state(port->pa_hca_hdl, port->pa_port_num, NULL, 588 NULL) != IBT_SUCCESS) 589 return; 590 591 if (port->pa_sa_hdl != NULL) 592 return; 593 594 if (ibt_query_hca_ports(port->pa_hca_hdl, port->pa_port_num, 595 &pinfop, &nports, &size) != IBT_SUCCESS) { 596 /* This should not occur */ 597 port->pa_npkeys = 0; 598 port->pa_pkey_tbl = NULL; 599 return; 600 } 601 port->pa_sn_prefix = pinfop->p_sgid_tbl[0].gid_prefix; 602 603 port->pa_state = pinfop->p_linkstate; 604 port->pa_npkeys = pinfop->p_pkey_tbl_sz; 605 port->pa_pkey_tbl = (ibdm_pkey_tbl_t *)kmem_zalloc( 606 port->pa_npkeys * sizeof (ibdm_pkey_tbl_t), KM_SLEEP); 607 608 for (pkey_idx = 0; pkey_idx < port->pa_npkeys; pkey_idx++) 609 port->pa_pkey_tbl[pkey_idx].pt_pkey = 610 pinfop->p_pkey_tbl[pkey_idx]; 611 612 ibt_free_portinfo(pinfop, size); 613 614 event_args.is_event_callback = ibdm_saa_event_cb; 615 event_args.is_event_callback_arg = port; 616 if (ibmf_sa_session_open(port->pa_port_guid, 0, &event_args, 617 IBMF_VERSION, 0, &port->pa_sa_hdl) != IBMF_SUCCESS) { 618 IBTF_DPRINTF_L2("ibdm", "\tinitialize_port: " 619 "sa access registration failed"); 620 return; 621 } 622 ibmf_reg.ir_ci_guid = port->pa_hca_guid; 623 ibmf_reg.ir_port_num = port->pa_port_num; 624 ibmf_reg.ir_client_class = DEV_MGT_MANAGER; 625 626 if (ibmf_register(&ibmf_reg, IBMF_VERSION, 0, NULL, NULL, 627 &port->pa_ibmf_hdl, &port->pa_ibmf_caps) != IBMF_SUCCESS) { 628 IBTF_DPRINTF_L2("ibdm", "\tinitialize_port: " 629 "IBMF registration failed"); 630 (void) ibdm_fini_port(port); 631 return; 632 } 633 if (ibmf_setup_async_cb(port->pa_ibmf_hdl, IBMF_QP_HANDLE_DEFAULT, 634 ibdm_ibmf_recv_cb, 0, 0) != IBMF_SUCCESS) { 635 IBTF_DPRINTF_L2("ibdm", "\tinitialize_port: " 636 "IBMF setup recv cb failed"); 637 (void) ibdm_fini_port(port); 638 return; 639 } 640 641 for (ii = 0; ii < port->pa_npkeys; ii++) { 642 pkey = port->pa_pkey_tbl[ii].pt_pkey; 643 if (IBDM_INVALID_PKEY(pkey)) { 644 port->pa_pkey_tbl[ii].pt_qp_hdl = NULL; 645 continue; 646 } 647 ibdm_port_attr_ibmf_init(port, pkey, ii); 648 } 649 } 650 651 652 /* 653 * ibdm_port_attr_ibmf_init: 654 * With IBMF - Alloc QP Handle and Setup Async callback 655 */ 656 static void 657 ibdm_port_attr_ibmf_init(ibdm_port_attr_t *port, ib_pkey_t pkey, int ii) 658 { 659 int ret; 660 661 if ((ret = ibmf_alloc_qp(port->pa_ibmf_hdl, pkey, IB_GSI_QKEY, 662 IBMF_ALT_QP_MAD_NO_RMPP, &port->pa_pkey_tbl[ii].pt_qp_hdl)) != 663 IBMF_SUCCESS) { 664 IBTF_DPRINTF_L2("ibdm", "\tport_attr_ibmf_init: " 665 "IBMF failed to alloc qp %d", ret); 666 port->pa_pkey_tbl[ii].pt_qp_hdl = NULL; 667 return; 668 } 669 670 IBTF_DPRINTF_L4("ibdm", "\tport_attr_ibmf_init: QP handle is %p", 671 port->pa_ibmf_hdl); 672 673 if ((ret = ibmf_setup_async_cb(port->pa_ibmf_hdl, 674 port->pa_pkey_tbl[ii].pt_qp_hdl, ibdm_ibmf_recv_cb, 0, 0)) != 675 IBMF_SUCCESS) { 676 IBTF_DPRINTF_L2("ibdm", "\tport_attr_ibmf_init: " 677 "IBMF setup recv cb failed %d", ret); 678 (void) ibmf_free_qp(port->pa_ibmf_hdl, 679 &port->pa_pkey_tbl[ii].pt_qp_hdl, 0); 680 port->pa_pkey_tbl[ii].pt_qp_hdl = NULL; 681 } 682 } 683 684 685 /* 686 * ibdm_get_port_attr() 687 * Get port attributes from HCA guid and port number 688 * Return pointer to ibdm_port_attr_t on Success 689 * and NULL on failure 690 */ 691 static ibdm_port_attr_t * 692 ibdm_get_port_attr(ibt_async_event_t *event, ibdm_hca_list_t **retval) 693 { 694 ibdm_hca_list_t *hca_list; 695 ibdm_port_attr_t *port_attr; 696 int ii; 697 698 IBTF_DPRINTF_L4("ibdm", "\tget_port_attr: port# %d", event->ev_port); 699 ASSERT(MUTEX_HELD(&ibdm.ibdm_hl_mutex)); 700 hca_list = ibdm.ibdm_hca_list_head; 701 while (hca_list) { 702 if (hca_list->hl_hca_guid == event->ev_hca_guid) { 703 for (ii = 0; ii < hca_list->hl_nports; ii++) { 704 port_attr = &hca_list->hl_port_attr[ii]; 705 if (port_attr->pa_port_num == event->ev_port) { 706 *retval = hca_list; 707 return (port_attr); 708 } 709 } 710 } 711 hca_list = hca_list->hl_next; 712 } 713 return (NULL); 714 } 715 716 717 /* 718 * ibdm_update_port_attr() 719 * Update the port attributes 720 */ 721 static void 722 ibdm_update_port_attr(ibdm_port_attr_t *port) 723 { 724 uint_t nports, size; 725 uint_t pkey_idx; 726 ibt_hca_portinfo_t *portinfop; 727 728 IBTF_DPRINTF_L4("ibdm", "\tupdate_port_attr: Begin"); 729 if (ibt_query_hca_ports(port->pa_hca_hdl, 730 port->pa_port_num, &portinfop, &nports, &size) != IBT_SUCCESS) { 731 /* This should not occur */ 732 port->pa_npkeys = 0; 733 port->pa_pkey_tbl = NULL; 734 return; 735 } 736 port->pa_sn_prefix = portinfop->p_sgid_tbl[0].gid_prefix; 737 738 port->pa_state = portinfop->p_linkstate; 739 740 /* 741 * PKey information in portinfo valid only if port is 742 * ACTIVE. Bail out if not. 743 */ 744 if (port->pa_state != IBT_PORT_ACTIVE) { 745 port->pa_npkeys = 0; 746 port->pa_pkey_tbl = NULL; 747 ibt_free_portinfo(portinfop, size); 748 return; 749 } 750 751 port->pa_npkeys = portinfop->p_pkey_tbl_sz; 752 port->pa_pkey_tbl = (ibdm_pkey_tbl_t *)kmem_zalloc( 753 port->pa_npkeys * sizeof (ibdm_pkey_tbl_t), KM_SLEEP); 754 755 for (pkey_idx = 0; pkey_idx < port->pa_npkeys; pkey_idx++) { 756 port->pa_pkey_tbl[pkey_idx].pt_pkey = 757 portinfop->p_pkey_tbl[pkey_idx]; 758 } 759 ibt_free_portinfo(portinfop, size); 760 } 761 762 763 /* 764 * ibdm_handle_hca_attach() 765 */ 766 static void 767 ibdm_handle_hca_attach(ib_guid_t hca_guid) 768 { 769 uint_t size; 770 uint_t ii, nports; 771 ibt_status_t status; 772 ibt_hca_hdl_t hca_hdl; 773 ibt_hca_attr_t *hca_attr; 774 ibdm_hca_list_t *hca_list, *temp; 775 ibdm_port_attr_t *port_attr; 776 ibt_hca_portinfo_t *portinfop; 777 778 IBTF_DPRINTF_L4("ibdm", 779 "\thandle_hca_attach: hca_guid = 0x%llX", hca_guid); 780 781 /* open the HCA first */ 782 if ((status = ibt_open_hca(ibdm.ibdm_ibt_clnt_hdl, hca_guid, 783 &hca_hdl)) != IBT_SUCCESS) { 784 IBTF_DPRINTF_L2("ibdm", "\thandle_hca_attach: " 785 "open_hca failed, status 0x%x", status); 786 return; 787 } 788 789 hca_attr = (ibt_hca_attr_t *) 790 kmem_alloc(sizeof (ibt_hca_attr_t), KM_SLEEP); 791 /* ibt_query_hca always returns IBT_SUCCESS */ 792 (void) ibt_query_hca(hca_hdl, hca_attr); 793 794 IBTF_DPRINTF_L4("ibdm", "\tvid: 0x%x, pid: 0x%x, ver: 0x%x," 795 " #ports: %d", hca_attr->hca_vendor_id, hca_attr->hca_device_id, 796 hca_attr->hca_version_id, hca_attr->hca_nports); 797 798 if ((status = ibt_query_hca_ports(hca_hdl, 0, &portinfop, &nports, 799 &size)) != IBT_SUCCESS) { 800 IBTF_DPRINTF_L2("ibdm", "\thandle_hca_attach: " 801 "ibt_query_hca_ports failed, status 0x%x", status); 802 kmem_free(hca_attr, sizeof (ibt_hca_attr_t)); 803 (void) ibt_close_hca(hca_hdl); 804 return; 805 } 806 hca_list = (ibdm_hca_list_t *) 807 kmem_zalloc((sizeof (ibdm_hca_list_t)), KM_SLEEP); 808 hca_list->hl_port_attr = (ibdm_port_attr_t *)kmem_zalloc( 809 (sizeof (ibdm_port_attr_t) * hca_attr->hca_nports), KM_SLEEP); 810 hca_list->hl_hca_guid = hca_attr->hca_node_guid; 811 hca_list->hl_nports = hca_attr->hca_nports; 812 hca_list->hl_attach_time = ddi_get_time(); 813 hca_list->hl_hca_hdl = hca_hdl; 814 815 /* 816 * Init a dummy port attribute for the HCA node 817 * This is for Per-HCA Node. Initialize port_attr : 818 * hca_guid & port_guid -> hca_guid 819 * npkeys, pkey_tbl is NULL 820 * port_num, sn_prefix is 0 821 * vendorid, product_id, dev_version from HCA 822 * pa_state is IBT_PORT_ACTIVE 823 */ 824 hca_list->hl_hca_port_attr = (ibdm_port_attr_t *)kmem_zalloc( 825 sizeof (ibdm_port_attr_t), KM_SLEEP); 826 port_attr = hca_list->hl_hca_port_attr; 827 port_attr->pa_vendorid = hca_attr->hca_vendor_id; 828 port_attr->pa_productid = hca_attr->hca_device_id; 829 port_attr->pa_dev_version = hca_attr->hca_version_id; 830 port_attr->pa_hca_guid = hca_attr->hca_node_guid; 831 port_attr->pa_hca_hdl = hca_list->hl_hca_hdl; 832 port_attr->pa_port_guid = hca_attr->hca_node_guid; 833 port_attr->pa_state = IBT_PORT_ACTIVE; 834 835 836 for (ii = 0; ii < nports; ii++) { 837 port_attr = &hca_list->hl_port_attr[ii]; 838 port_attr->pa_vendorid = hca_attr->hca_vendor_id; 839 port_attr->pa_productid = hca_attr->hca_device_id; 840 port_attr->pa_dev_version = hca_attr->hca_version_id; 841 port_attr->pa_hca_guid = hca_attr->hca_node_guid; 842 port_attr->pa_hca_hdl = hca_list->hl_hca_hdl; 843 port_attr->pa_port_guid = portinfop[ii].p_sgid_tbl->gid_guid; 844 port_attr->pa_sn_prefix = portinfop[ii].p_sgid_tbl->gid_prefix; 845 port_attr->pa_port_num = portinfop[ii].p_port_num; 846 port_attr->pa_state = portinfop[ii].p_linkstate; 847 848 /* 849 * Register with IBMF, SA access when the port is in 850 * ACTIVE state. Also register a callback routine 851 * with IBMF to receive incoming DM MAD's. 852 * The IBDM event handler takes care of registration of 853 * port which are not active. 854 */ 855 IBTF_DPRINTF_L4("ibdm", 856 "\thandle_hca_attach: port guid %llx Port state 0x%x", 857 port_attr->pa_port_guid, portinfop[ii].p_linkstate); 858 859 if (portinfop[ii].p_linkstate == IBT_PORT_ACTIVE) { 860 mutex_enter(&ibdm.ibdm_hl_mutex); 861 hca_list->hl_nports_active++; 862 ibdm_initialize_port(port_attr); 863 mutex_exit(&ibdm.ibdm_hl_mutex); 864 } 865 } 866 mutex_enter(&ibdm.ibdm_hl_mutex); 867 for (temp = ibdm.ibdm_hca_list_head; temp; temp = temp->hl_next) { 868 if (temp->hl_hca_guid == hca_guid) { 869 IBTF_DPRINTF_L2("ibdm", "hca_attach: HCA %llX " 870 "already seen by IBDM", hca_guid); 871 mutex_exit(&ibdm.ibdm_hl_mutex); 872 (void) ibdm_uninit_hca(hca_list); 873 return; 874 } 875 } 876 ibdm.ibdm_hca_count++; 877 if (ibdm.ibdm_hca_list_head == NULL) { 878 ibdm.ibdm_hca_list_head = hca_list; 879 ibdm.ibdm_hca_list_tail = hca_list; 880 } else { 881 ibdm.ibdm_hca_list_tail->hl_next = hca_list; 882 ibdm.ibdm_hca_list_tail = hca_list; 883 } 884 mutex_exit(&ibdm.ibdm_hl_mutex); 885 mutex_enter(&ibdm.ibdm_ibnex_mutex); 886 if (ibdm.ibdm_ibnex_callback != NULL) { 887 (*ibdm.ibdm_ibnex_callback)((void *) 888 &hca_guid, IBDM_EVENT_HCA_ADDED); 889 } 890 mutex_exit(&ibdm.ibdm_ibnex_mutex); 891 892 kmem_free(hca_attr, sizeof (ibt_hca_attr_t)); 893 ibt_free_portinfo(portinfop, size); 894 } 895 896 897 /* 898 * ibdm_handle_hca_detach() 899 */ 900 static void 901 ibdm_handle_hca_detach(ib_guid_t hca_guid) 902 { 903 ibdm_hca_list_t *head, *prev = NULL; 904 size_t len; 905 ibdm_dp_gidinfo_t *gidinfo; 906 907 IBTF_DPRINTF_L4("ibdm", 908 "\thandle_hca_detach: hca_guid = 0x%llx", hca_guid); 909 910 /* Make sure no probes are running */ 911 mutex_enter(&ibdm.ibdm_mutex); 912 while (ibdm.ibdm_busy & IBDM_BUSY) 913 cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex); 914 ibdm.ibdm_busy |= IBDM_BUSY; 915 mutex_exit(&ibdm.ibdm_mutex); 916 917 mutex_enter(&ibdm.ibdm_hl_mutex); 918 head = ibdm.ibdm_hca_list_head; 919 while (head) { 920 if (head->hl_hca_guid == hca_guid) { 921 if (prev == NULL) 922 ibdm.ibdm_hca_list_head = head->hl_next; 923 else 924 prev->hl_next = head->hl_next; 925 ibdm.ibdm_hca_count--; 926 break; 927 } 928 prev = head; 929 head = head->hl_next; 930 } 931 mutex_exit(&ibdm.ibdm_hl_mutex); 932 if (ibdm_uninit_hca(head) != IBDM_SUCCESS) 933 (void) ibdm_handle_hca_attach(hca_guid); 934 935 /* 936 * Now clean up the HCA lists in the gidlist. 937 */ 938 for (gidinfo = ibdm.ibdm_dp_gidlist_head; gidinfo; gidinfo = 939 gidinfo->gl_next) { 940 prev = NULL; 941 head = gidinfo->gl_hca_list; 942 while (head) { 943 if (head->hl_hca_guid == hca_guid) { 944 if (prev == NULL) 945 gidinfo->gl_hca_list = 946 head->hl_next; 947 else 948 prev->hl_next = head->hl_next; 949 950 len = sizeof (ibdm_hca_list_t) + 951 (head->hl_nports * 952 sizeof (ibdm_port_attr_t)); 953 kmem_free(head, len); 954 955 break; 956 } 957 prev = head; 958 head = head->hl_next; 959 } 960 } 961 962 mutex_enter(&ibdm.ibdm_mutex); 963 ibdm.ibdm_busy &= ~IBDM_BUSY; 964 cv_broadcast(&ibdm.ibdm_busy_cv); 965 mutex_exit(&ibdm.ibdm_mutex); 966 } 967 968 969 static int 970 ibdm_uninit_hca(ibdm_hca_list_t *head) 971 { 972 int ii; 973 ibdm_port_attr_t *port_attr; 974 975 for (ii = 0; ii < head->hl_nports; ii++) { 976 port_attr = &head->hl_port_attr[ii]; 977 if (ibdm_fini_port(port_attr) != IBDM_SUCCESS) { 978 IBTF_DPRINTF_L2("ibdm", "uninit_hca: HCA %p port 0x%x " 979 "ibdm_fini_port() failed", head, ii); 980 return (IBDM_FAILURE); 981 } 982 } 983 if (head->hl_hca_hdl) 984 if (ibt_close_hca(head->hl_hca_hdl) != IBT_SUCCESS) 985 return (IBDM_FAILURE); 986 kmem_free(head->hl_port_attr, 987 head->hl_nports * sizeof (ibdm_port_attr_t)); 988 kmem_free(head->hl_hca_port_attr, sizeof (ibdm_port_attr_t)); 989 kmem_free(head, sizeof (ibdm_hca_list_t)); 990 return (IBDM_SUCCESS); 991 } 992 993 994 /* 995 * For each port on the HCA, 996 * 1) Teardown IBMF receive callback function 997 * 2) Unregister with IBMF 998 * 3) Unregister with SA access 999 */ 1000 static int 1001 ibdm_fini_port(ibdm_port_attr_t *port_attr) 1002 { 1003 int ii, ibmf_status; 1004 1005 for (ii = 0; ii < port_attr->pa_npkeys; ii++) { 1006 if (port_attr->pa_pkey_tbl == NULL) 1007 break; 1008 if (!port_attr->pa_pkey_tbl[ii].pt_qp_hdl) 1009 continue; 1010 if (ibdm_port_attr_ibmf_fini(port_attr, ii) != IBDM_SUCCESS) { 1011 IBTF_DPRINTF_L4("ibdm", "\tfini_port: " 1012 "ibdm_port_attr_ibmf_fini failed for " 1013 "port pkey 0x%x", ii); 1014 return (IBDM_FAILURE); 1015 } 1016 } 1017 1018 if (port_attr->pa_ibmf_hdl) { 1019 ibmf_status = ibmf_tear_down_async_cb(port_attr->pa_ibmf_hdl, 1020 IBMF_QP_HANDLE_DEFAULT, 0); 1021 if (ibmf_status != IBMF_SUCCESS) { 1022 IBTF_DPRINTF_L4("ibdm", "\tfini_port: " 1023 "ibmf_tear_down_async_cb failed %d", ibmf_status); 1024 return (IBDM_FAILURE); 1025 } 1026 1027 ibmf_status = ibmf_unregister(&port_attr->pa_ibmf_hdl, 0); 1028 if (ibmf_status != IBMF_SUCCESS) { 1029 IBTF_DPRINTF_L4("ibdm", "\tfini_port: " 1030 "ibmf_unregister failed %d", ibmf_status); 1031 return (IBDM_FAILURE); 1032 } 1033 1034 port_attr->pa_ibmf_hdl = NULL; 1035 } 1036 1037 if (port_attr->pa_sa_hdl) { 1038 ibmf_status = ibmf_sa_session_close(&port_attr->pa_sa_hdl, 0); 1039 if (ibmf_status != IBMF_SUCCESS) { 1040 IBTF_DPRINTF_L4("ibdm", "\tfini_port: " 1041 "ibmf_sa_session_close failed %d", ibmf_status); 1042 return (IBDM_FAILURE); 1043 } 1044 port_attr->pa_sa_hdl = NULL; 1045 } 1046 1047 if (port_attr->pa_pkey_tbl != NULL) { 1048 kmem_free(port_attr->pa_pkey_tbl, 1049 port_attr->pa_npkeys * sizeof (ibdm_pkey_tbl_t)); 1050 port_attr->pa_pkey_tbl = NULL; 1051 port_attr->pa_npkeys = 0; 1052 } 1053 1054 return (IBDM_SUCCESS); 1055 } 1056 1057 1058 /* 1059 * ibdm_port_attr_ibmf_fini: 1060 * With IBMF - Tear down Async callback and free QP Handle 1061 */ 1062 static int 1063 ibdm_port_attr_ibmf_fini(ibdm_port_attr_t *port_attr, int ii) 1064 { 1065 int ibmf_status; 1066 1067 IBTF_DPRINTF_L5("ibdm", "\tport_attr_ibmf_fini:"); 1068 1069 if (port_attr->pa_pkey_tbl[ii].pt_qp_hdl) { 1070 ibmf_status = ibmf_tear_down_async_cb(port_attr->pa_ibmf_hdl, 1071 port_attr->pa_pkey_tbl[ii].pt_qp_hdl, 0); 1072 if (ibmf_status != IBMF_SUCCESS) { 1073 IBTF_DPRINTF_L4("ibdm", "\tport_attr_ibmf_fini: " 1074 "ibmf_tear_down_async_cb failed %d", ibmf_status); 1075 return (IBDM_FAILURE); 1076 } 1077 ibmf_status = ibmf_free_qp(port_attr->pa_ibmf_hdl, 1078 &port_attr->pa_pkey_tbl[ii].pt_qp_hdl, 0); 1079 if (ibmf_status != IBMF_SUCCESS) { 1080 IBTF_DPRINTF_L4("ibdm", "\tport_attr_ibmf_fini: " 1081 "ibmf_free_qp failed %d", ibmf_status); 1082 return (IBDM_FAILURE); 1083 } 1084 port_attr->pa_pkey_tbl[ii].pt_qp_hdl = NULL; 1085 } 1086 return (IBDM_SUCCESS); 1087 } 1088 1089 1090 /* 1091 * ibdm_gid_decr_pending: 1092 * decrement gl_pending_cmds. If zero wakeup sleeping threads 1093 */ 1094 static void 1095 ibdm_gid_decr_pending(ibdm_dp_gidinfo_t *gidinfo) 1096 { 1097 mutex_enter(&ibdm.ibdm_mutex); 1098 mutex_enter(&gidinfo->gl_mutex); 1099 if (--gidinfo->gl_pending_cmds == 0) { 1100 /* 1101 * Handle DGID getting removed. 1102 */ 1103 if (gidinfo->gl_disconnected) { 1104 mutex_exit(&gidinfo->gl_mutex); 1105 mutex_exit(&ibdm.ibdm_mutex); 1106 1107 IBTF_DPRINTF_L3(ibdm_string, "\tgid_decr_pending: " 1108 "gidinfo %p hot removal", gidinfo); 1109 ibdm_delete_gidinfo(gidinfo); 1110 1111 mutex_enter(&ibdm.ibdm_mutex); 1112 ibdm.ibdm_ngid_probes_in_progress--; 1113 ibdm_wait_probe_completion(); 1114 mutex_exit(&ibdm.ibdm_mutex); 1115 return; 1116 } 1117 mutex_exit(&gidinfo->gl_mutex); 1118 mutex_exit(&ibdm.ibdm_mutex); 1119 ibdm_notify_newgid_iocs(gidinfo); 1120 mutex_enter(&ibdm.ibdm_mutex); 1121 mutex_enter(&gidinfo->gl_mutex); 1122 1123 ibdm.ibdm_ngid_probes_in_progress--; 1124 ibdm_wait_probe_completion(); 1125 } 1126 mutex_exit(&gidinfo->gl_mutex); 1127 mutex_exit(&ibdm.ibdm_mutex); 1128 } 1129 1130 1131 /* 1132 * ibdm_wait_probe_completion: 1133 * wait for probing to complete 1134 */ 1135 static void 1136 ibdm_wait_probe_completion(void) 1137 { 1138 ASSERT(MUTEX_HELD(&ibdm.ibdm_mutex)); 1139 if (ibdm.ibdm_ngid_probes_in_progress) { 1140 IBTF_DPRINTF_L4("ibdm", "\twait for probe complete"); 1141 ibdm.ibdm_busy |= IBDM_PROBE_IN_PROGRESS; 1142 while (ibdm.ibdm_busy & IBDM_PROBE_IN_PROGRESS) 1143 cv_wait(&ibdm.ibdm_probe_cv, &ibdm.ibdm_mutex); 1144 } 1145 } 1146 1147 1148 /* 1149 * ibdm_wakeup_probe_gid_cv: 1150 * wakeup waiting threads (based on ibdm_ngid_probes_in_progress) 1151 */ 1152 static void 1153 ibdm_wakeup_probe_gid_cv(void) 1154 { 1155 ASSERT(MUTEX_HELD(&ibdm.ibdm_mutex)); 1156 if (!ibdm.ibdm_ngid_probes_in_progress) { 1157 IBTF_DPRINTF_L4("ibdm", "wakeup_probe_gid_thread: Wakeup"); 1158 ibdm.ibdm_busy &= ~IBDM_PROBE_IN_PROGRESS; 1159 cv_broadcast(&ibdm.ibdm_probe_cv); 1160 } 1161 1162 } 1163 1164 1165 /* 1166 * ibdm_sweep_fabric(reprobe_flag) 1167 * Find all possible Managed IOU's and their IOC's that are visible 1168 * to the host. The algorithm used is as follows 1169 * 1170 * Send a "bus walk" request for each port on the host HCA to SA access 1171 * SA returns complete set of GID's that are reachable from 1172 * source port. This is done in parallel. 1173 * 1174 * Initialize GID state to IBDM_GID_PROBE_NOT_DONE 1175 * 1176 * Sort the GID list and eliminate duplicate GID's 1177 * 1) Use DGID for sorting 1178 * 2) use PortGuid for sorting 1179 * Send SA query to retrieve NodeRecord and 1180 * extract PortGuid from that. 1181 * 1182 * Set GID state to IBDM_GID_PROBE_FAILED to all the ports that dont 1183 * support DM MAD's 1184 * Send a "Portinfo" query to get the port capabilities and 1185 * then check for DM MAD's support 1186 * 1187 * Send "ClassPortInfo" request for all the GID's in parallel, 1188 * set the GID state to IBDM_GET_CLASSPORTINFO and wait on the 1189 * cv_signal to complete. 1190 * 1191 * When DM agent on the remote GID sends back the response, IBMF 1192 * invokes DM callback routine. 1193 * 1194 * If the response is proper, send "IOUnitInfo" request and set 1195 * GID state to IBDM_GET_IOUNITINFO. 1196 * 1197 * If the response is proper, send "IocProfileInfo" request to 1198 * all the IOC simultaneously and set GID state to IBDM_GET_IOC_DETAILS. 1199 * 1200 * Send request to get Service entries simultaneously 1201 * 1202 * Signal the waiting thread when received response for all the commands. 1203 * 1204 * Set the GID state to IBDM_GID_PROBE_FAILED when received a error 1205 * response during the probing period. 1206 * 1207 * Note: 1208 * ibdm.ibdm_ngid_probes_in_progress and ibdm_gid_list_t:gl_pending_cmds 1209 * keep track of number commands in progress at any point of time. 1210 * MAD transaction ID is used to identify a particular GID 1211 * TBD: Consider registering the IBMF receive callback on demand 1212 * 1213 * Note: This routine must be called with ibdm.ibdm_mutex held 1214 * TBD: Re probe the failure GID (for certain failures) when requested 1215 * for fabric sweep next time 1216 * 1217 * Parameters : If reprobe_flag is set, All IOCs will be reprobed. 1218 */ 1219 static void 1220 ibdm_sweep_fabric(int reprobe_flag) 1221 { 1222 int ii; 1223 int new_paths = 0; 1224 uint8_t niocs; 1225 taskqid_t tid; 1226 ibdm_ioc_info_t *ioc; 1227 ibdm_hca_list_t *hca_list = NULL; 1228 ibdm_port_attr_t *port = NULL; 1229 ibdm_dp_gidinfo_t *gid_info; 1230 1231 IBTF_DPRINTF_L4("ibdm", "\tsweep_fabric: Enter"); 1232 ASSERT(MUTEX_HELD(&ibdm.ibdm_mutex)); 1233 1234 /* 1235 * Check whether a sweep already in progress. If so, just 1236 * wait for the fabric sweep to complete 1237 */ 1238 while (ibdm.ibdm_busy & IBDM_BUSY) 1239 cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex); 1240 ibdm.ibdm_busy |= IBDM_BUSY; 1241 mutex_exit(&ibdm.ibdm_mutex); 1242 1243 ibdm_dump_sweep_fabric_timestamp(0); 1244 1245 /* Rescan the GID list for any removed GIDs for reprobe */ 1246 if (reprobe_flag) 1247 ibdm_rescan_gidlist(NULL); 1248 1249 /* 1250 * Get list of all the ports reachable from the local known HCA 1251 * ports which are active 1252 */ 1253 mutex_enter(&ibdm.ibdm_hl_mutex); 1254 for (ibdm_get_next_port(&hca_list, &port, 1); port; 1255 ibdm_get_next_port(&hca_list, &port, 1)) { 1256 /* 1257 * Get PATHS to all the reachable ports from 1258 * SGID and update the global ibdm structure. 1259 */ 1260 new_paths = ibdm_get_reachable_ports(port, hca_list); 1261 ibdm.ibdm_ngids += new_paths; 1262 } 1263 mutex_exit(&ibdm.ibdm_hl_mutex); 1264 1265 mutex_enter(&ibdm.ibdm_mutex); 1266 ibdm.ibdm_ngid_probes_in_progress += ibdm.ibdm_ngids; 1267 mutex_exit(&ibdm.ibdm_mutex); 1268 1269 /* Send a request to probe GIDs asynchronously. */ 1270 for (gid_info = ibdm.ibdm_dp_gidlist_head; gid_info; 1271 gid_info = gid_info->gl_next) { 1272 mutex_enter(&gid_info->gl_mutex); 1273 gid_info->gl_reprobe_flag = reprobe_flag; 1274 mutex_exit(&gid_info->gl_mutex); 1275 1276 /* process newly encountered GIDs */ 1277 tid = taskq_dispatch(system_taskq, ibdm_probe_gid_thread, 1278 (void *)gid_info, TQ_NOSLEEP); 1279 IBTF_DPRINTF_L4("ibdm", "\tsweep_fabric: gid_info = %p" 1280 " taskq_id = %x", gid_info, tid); 1281 /* taskq failed to dispatch call it directly */ 1282 if (tid == NULL) 1283 ibdm_probe_gid_thread((void *)gid_info); 1284 } 1285 1286 mutex_enter(&ibdm.ibdm_mutex); 1287 ibdm_wait_probe_completion(); 1288 1289 /* 1290 * Update the properties, if reprobe_flag is set 1291 * Skip if gl_reprobe_flag is set, this will be 1292 * a re-inserted / new GID, for which notifications 1293 * have already been send. 1294 */ 1295 if (reprobe_flag) { 1296 for (gid_info = ibdm.ibdm_dp_gidlist_head; gid_info; 1297 gid_info = gid_info->gl_next) { 1298 if (gid_info->gl_iou == NULL) 1299 continue; 1300 if (gid_info->gl_reprobe_flag) { 1301 gid_info->gl_reprobe_flag = 0; 1302 continue; 1303 } 1304 1305 niocs = gid_info->gl_iou->iou_info.iou_num_ctrl_slots; 1306 for (ii = 0; ii < niocs; ii++) { 1307 ioc = IBDM_GIDINFO2IOCINFO(gid_info, ii); 1308 if (ioc) 1309 ibdm_reprobe_update_port_srv(ioc, 1310 gid_info); 1311 } 1312 } 1313 } else if (ibdm.ibdm_prev_iou) { 1314 ibdm_ioc_info_t *ioc_list; 1315 1316 /* 1317 * Get the list of IOCs which have changed. 1318 * If any IOCs have changed, Notify IBNexus 1319 */ 1320 ibdm.ibdm_prev_iou = 0; 1321 ioc_list = ibdm_handle_prev_iou(); 1322 if (ioc_list) { 1323 if (ibdm.ibdm_ibnex_callback != NULL) { 1324 (*ibdm.ibdm_ibnex_callback)( 1325 (void *)ioc_list, 1326 IBDM_EVENT_IOC_PROP_UPDATE); 1327 } 1328 } 1329 } 1330 1331 ibdm_dump_sweep_fabric_timestamp(1); 1332 1333 ibdm.ibdm_busy &= ~IBDM_BUSY; 1334 cv_broadcast(&ibdm.ibdm_busy_cv); 1335 IBTF_DPRINTF_L5("ibdm", "\tsweep_fabric: EXIT"); 1336 } 1337 1338 1339 /* 1340 * ibdm_probe_gid_thread: 1341 * thread that does the actual work for sweeping the fabric 1342 * for a given GID 1343 */ 1344 static void 1345 ibdm_probe_gid_thread(void *args) 1346 { 1347 int reprobe_flag; 1348 ib_guid_t node_guid; 1349 ib_guid_t port_guid; 1350 ibdm_dp_gidinfo_t *gid_info; 1351 1352 gid_info = (ibdm_dp_gidinfo_t *)args; 1353 reprobe_flag = gid_info->gl_reprobe_flag; 1354 IBTF_DPRINTF_L4("ibdm", "\tprobe_gid_thread: gid_info = %p, flag = %d", 1355 gid_info, reprobe_flag); 1356 ASSERT(gid_info != NULL); 1357 ASSERT(gid_info->gl_pending_cmds == 0); 1358 1359 if (gid_info->gl_state != IBDM_GID_PROBE_NOT_DONE && 1360 reprobe_flag == 0) { 1361 /* 1362 * This GID may have been already probed. Send 1363 * in a CLP to check if IOUnitInfo changed? 1364 * Explicitly set gl_reprobe_flag to 0 so that 1365 * IBnex is not notified on completion 1366 */ 1367 if (gid_info->gl_state == IBDM_GID_PROBING_COMPLETE) { 1368 IBTF_DPRINTF_L4("ibdm", "\tprobe_gid_thread: " 1369 "get new IOCs information"); 1370 mutex_enter(&gid_info->gl_mutex); 1371 gid_info->gl_pending_cmds++; 1372 gid_info->gl_state = IBDM_GET_IOUNITINFO; 1373 gid_info->gl_reprobe_flag = 0; 1374 mutex_exit(&gid_info->gl_mutex); 1375 if (ibdm_send_iounitinfo(gid_info) != IBDM_SUCCESS) { 1376 mutex_enter(&gid_info->gl_mutex); 1377 gid_info->gl_pending_cmds = 0; 1378 mutex_exit(&gid_info->gl_mutex); 1379 mutex_enter(&ibdm.ibdm_mutex); 1380 --ibdm.ibdm_ngid_probes_in_progress; 1381 ibdm_wakeup_probe_gid_cv(); 1382 mutex_exit(&ibdm.ibdm_mutex); 1383 } 1384 } else { 1385 mutex_enter(&ibdm.ibdm_mutex); 1386 --ibdm.ibdm_ngid_probes_in_progress; 1387 ibdm_wakeup_probe_gid_cv(); 1388 mutex_exit(&ibdm.ibdm_mutex); 1389 } 1390 return; 1391 } else if (reprobe_flag && gid_info->gl_state == 1392 IBDM_GID_PROBING_COMPLETE) { 1393 /* 1394 * Reprobe all IOCs for the GID which has completed 1395 * probe. Skip other port GIDs to same IOU. 1396 * Explicitly set gl_reprobe_flag to 0 so that 1397 * IBnex is not notified on completion 1398 */ 1399 ibdm_ioc_info_t *ioc_info; 1400 uint8_t niocs, ii; 1401 1402 ASSERT(gid_info->gl_iou); 1403 mutex_enter(&gid_info->gl_mutex); 1404 niocs = gid_info->gl_iou->iou_info.iou_num_ctrl_slots; 1405 gid_info->gl_state = IBDM_GET_IOC_DETAILS; 1406 gid_info->gl_pending_cmds += niocs; 1407 gid_info->gl_reprobe_flag = 0; 1408 mutex_exit(&gid_info->gl_mutex); 1409 for (ii = 0; ii < niocs; ii++) { 1410 uchar_t slot_info; 1411 ib_dm_io_unitinfo_t *giou_info; 1412 1413 /* 1414 * Check whether IOC is present in the slot 1415 * Series of nibbles (in the field 1416 * iou_ctrl_list) represents a slot in the 1417 * IOU. 1418 * Byte format: 76543210 1419 * Bits 0-3 of first byte represent Slot 2 1420 * bits 4-7 of first byte represent slot 1, 1421 * bits 0-3 of second byte represent slot 4 1422 * and so on 1423 * Each 4-bit nibble has the following meaning 1424 * 0x0 : IOC not installed 1425 * 0x1 : IOC is present 1426 * 0xf : Slot does not exist 1427 * and all other values are reserved. 1428 */ 1429 ioc_info = IBDM_GIDINFO2IOCINFO(gid_info, ii); 1430 giou_info = &gid_info->gl_iou->iou_info; 1431 slot_info = giou_info->iou_ctrl_list[(ii/2)]; 1432 if ((ii % 2) == 0) 1433 slot_info = (slot_info >> 4); 1434 1435 if ((slot_info & 0xf) != 1) { 1436 ioc_info->ioc_state = 1437 IBDM_IOC_STATE_PROBE_FAILED; 1438 ibdm_gid_decr_pending(gid_info); 1439 continue; 1440 } 1441 1442 if (ibdm_send_ioc_profile(gid_info, ii) != 1443 IBDM_SUCCESS) { 1444 ibdm_gid_decr_pending(gid_info); 1445 } 1446 } 1447 1448 return; 1449 } else if (gid_info->gl_state != IBDM_GID_PROBE_NOT_DONE) { 1450 mutex_enter(&ibdm.ibdm_mutex); 1451 --ibdm.ibdm_ngid_probes_in_progress; 1452 ibdm_wakeup_probe_gid_cv(); 1453 mutex_exit(&ibdm.ibdm_mutex); 1454 return; 1455 } 1456 1457 mutex_enter(&gid_info->gl_mutex); 1458 gid_info->gl_pending_cmds++; 1459 gid_info->gl_state = IBDM_GET_CLASSPORTINFO; 1460 mutex_exit(&gid_info->gl_mutex); 1461 1462 /* 1463 * Check whether the destination GID supports DM agents. If 1464 * not, stop probing the GID and continue with the next GID 1465 * in the list. 1466 */ 1467 if (ibdm_is_dev_mgt_supported(gid_info) != IBDM_SUCCESS) { 1468 mutex_enter(&gid_info->gl_mutex); 1469 gid_info->gl_pending_cmds = 0; 1470 gid_info->gl_state = IBDM_GID_PROBING_FAILED; 1471 mutex_exit(&gid_info->gl_mutex); 1472 ibdm_delete_glhca_list(gid_info); 1473 mutex_enter(&ibdm.ibdm_mutex); 1474 --ibdm.ibdm_ngid_probes_in_progress; 1475 ibdm_wakeup_probe_gid_cv(); 1476 mutex_exit(&ibdm.ibdm_mutex); 1477 return; 1478 } 1479 1480 /* Get the nodeguid and portguid of the port */ 1481 if (ibdm_get_node_port_guids(gid_info->gl_sa_hdl, gid_info->gl_dlid, 1482 &node_guid, &port_guid) != IBDM_SUCCESS) { 1483 mutex_enter(&gid_info->gl_mutex); 1484 gid_info->gl_pending_cmds = 0; 1485 gid_info->gl_state = IBDM_GID_PROBING_FAILED; 1486 mutex_exit(&gid_info->gl_mutex); 1487 ibdm_delete_glhca_list(gid_info); 1488 mutex_enter(&ibdm.ibdm_mutex); 1489 --ibdm.ibdm_ngid_probes_in_progress; 1490 ibdm_wakeup_probe_gid_cv(); 1491 mutex_exit(&ibdm.ibdm_mutex); 1492 return; 1493 } 1494 1495 /* 1496 * Check whether we already knew about this NodeGuid 1497 * If so, do not probe the GID and continue with the 1498 * next GID in the gid list. Set the GID state to 1499 * probing done. 1500 */ 1501 mutex_enter(&ibdm.ibdm_mutex); 1502 gid_info->gl_nodeguid = node_guid; 1503 gid_info->gl_portguid = port_guid; 1504 if (ibdm_check_dest_nodeguid(gid_info) != NULL) { 1505 mutex_exit(&ibdm.ibdm_mutex); 1506 mutex_enter(&gid_info->gl_mutex); 1507 gid_info->gl_pending_cmds = 0; 1508 gid_info->gl_state = IBDM_GID_PROBING_SKIPPED; 1509 mutex_exit(&gid_info->gl_mutex); 1510 ibdm_delete_glhca_list(gid_info); 1511 mutex_enter(&ibdm.ibdm_mutex); 1512 --ibdm.ibdm_ngid_probes_in_progress; 1513 ibdm_wakeup_probe_gid_cv(); 1514 mutex_exit(&ibdm.ibdm_mutex); 1515 return; 1516 } 1517 ibdm_add_to_gl_gid(gid_info, gid_info); 1518 mutex_exit(&ibdm.ibdm_mutex); 1519 1520 /* 1521 * New or reinserted GID : Enable notification to IBnex 1522 */ 1523 mutex_enter(&gid_info->gl_mutex); 1524 gid_info->gl_reprobe_flag = 1; 1525 mutex_exit(&gid_info->gl_mutex); 1526 1527 /* 1528 * Send ClassPortInfo request to the GID asynchronously. 1529 */ 1530 if (ibdm_send_classportinfo(gid_info) != IBDM_SUCCESS) { 1531 mutex_enter(&gid_info->gl_mutex); 1532 gid_info->gl_state = IBDM_GID_PROBING_FAILED; 1533 gid_info->gl_pending_cmds = 0; 1534 mutex_exit(&gid_info->gl_mutex); 1535 ibdm_delete_glhca_list(gid_info); 1536 mutex_enter(&ibdm.ibdm_mutex); 1537 --ibdm.ibdm_ngid_probes_in_progress; 1538 ibdm_wakeup_probe_gid_cv(); 1539 mutex_exit(&ibdm.ibdm_mutex); 1540 return; 1541 } 1542 } 1543 1544 1545 /* 1546 * ibdm_check_dest_nodeguid 1547 * Searches for the NodeGuid in the GID list 1548 * Returns matching gid_info if found and otherwise NULL 1549 * 1550 * This function is called to handle new GIDs discovered 1551 * during device sweep / probe or for GID_AVAILABLE event. 1552 * 1553 * Parameter : 1554 * gid_info GID to check 1555 */ 1556 static ibdm_dp_gidinfo_t * 1557 ibdm_check_dest_nodeguid(ibdm_dp_gidinfo_t *gid_info) 1558 { 1559 ibdm_dp_gidinfo_t *gid_list; 1560 ibdm_gid_t *tmp; 1561 1562 IBTF_DPRINTF_L4("ibdm", "\tcheck_dest_nodeguid"); 1563 1564 gid_list = ibdm.ibdm_dp_gidlist_head; 1565 while (gid_list) { 1566 if ((gid_list != gid_info) && 1567 (gid_info->gl_nodeguid == gid_list->gl_nodeguid)) { 1568 IBTF_DPRINTF_L4("ibdm", 1569 "\tcheck_dest_nodeguid: NodeGuid is present"); 1570 1571 /* Add to gid_list */ 1572 tmp = kmem_zalloc(sizeof (ibdm_gid_t), 1573 KM_SLEEP); 1574 tmp->gid_dgid_hi = gid_info->gl_dgid_hi; 1575 tmp->gid_dgid_lo = gid_info->gl_dgid_lo; 1576 tmp->gid_next = gid_list->gl_gid; 1577 gid_list->gl_gid = tmp; 1578 gid_list->gl_ngids++; 1579 return (gid_list); 1580 } 1581 1582 gid_list = gid_list->gl_next; 1583 } 1584 1585 return (NULL); 1586 } 1587 1588 1589 /* 1590 * ibdm_is_dev_mgt_supported 1591 * Get the PortInfo attribute (SA Query) 1592 * Check "CompatabilityMask" field in the Portinfo. 1593 * Return IBDM_SUCCESS if DM MAD's supported (if bit 19 set) 1594 * by the port, otherwise IBDM_FAILURE 1595 */ 1596 static int 1597 ibdm_is_dev_mgt_supported(ibdm_dp_gidinfo_t *gid_info) 1598 { 1599 int ret; 1600 size_t length = 0; 1601 sa_portinfo_record_t req, *resp = NULL; 1602 ibmf_saa_access_args_t qargs; 1603 1604 bzero(&req, sizeof (sa_portinfo_record_t)); 1605 req.EndportLID = gid_info->gl_dlid; 1606 1607 qargs.sq_attr_id = SA_PORTINFORECORD_ATTRID; 1608 qargs.sq_access_type = IBMF_SAA_RETRIEVE; 1609 qargs.sq_component_mask = SA_PORTINFO_COMPMASK_PORTLID; 1610 qargs.sq_template = &req; 1611 qargs.sq_callback = NULL; 1612 qargs.sq_callback_arg = NULL; 1613 1614 ret = ibmf_sa_access(gid_info->gl_sa_hdl, 1615 &qargs, 0, &length, (void **)&resp); 1616 1617 if ((ret != IBMF_SUCCESS) || (length == 0) || (resp == NULL)) { 1618 IBTF_DPRINTF_L2("ibdm", "\tis_dev_mgt_supported:" 1619 "failed to get PORTINFO attribute %d", ret); 1620 return (IBDM_FAILURE); 1621 } 1622 1623 if (resp->PortInfo.CapabilityMask & SM_CAP_MASK_IS_DM_SUPPD) { 1624 IBTF_DPRINTF_L4("ibdm", "\tis_dev_mgt_supported: SUPPD !!"); 1625 ret = IBDM_SUCCESS; 1626 } else { 1627 IBTF_DPRINTF_L4("ibdm", "\tis_dev_mgt_supported: " 1628 "Not SUPPD !!, cap 0x%x", resp->PortInfo.CapabilityMask); 1629 ret = IBDM_FAILURE; 1630 } 1631 kmem_free(resp, length); 1632 return (ret); 1633 } 1634 1635 1636 /* 1637 * ibdm_get_node_port_guids() 1638 * Get the NodeInfoRecord of the port 1639 * Save NodeGuid and PortGUID values in the GID list structure. 1640 * Return IBDM_SUCCESS/IBDM_FAILURE 1641 */ 1642 static int 1643 ibdm_get_node_port_guids(ibmf_saa_handle_t sa_hdl, ib_lid_t dlid, 1644 ib_guid_t *node_guid, ib_guid_t *port_guid) 1645 { 1646 int ret; 1647 size_t length = 0; 1648 sa_node_record_t req, *resp = NULL; 1649 ibmf_saa_access_args_t qargs; 1650 1651 IBTF_DPRINTF_L4("ibdm", "\tget_node_port_guids"); 1652 1653 bzero(&req, sizeof (sa_node_record_t)); 1654 req.LID = dlid; 1655 1656 qargs.sq_attr_id = SA_NODERECORD_ATTRID; 1657 qargs.sq_access_type = IBMF_SAA_RETRIEVE; 1658 qargs.sq_component_mask = SA_NODEINFO_COMPMASK_NODELID; 1659 qargs.sq_template = &req; 1660 qargs.sq_callback = NULL; 1661 qargs.sq_callback_arg = NULL; 1662 1663 ret = ibmf_sa_access(sa_hdl, &qargs, 0, &length, (void **)&resp); 1664 if ((ret != IBMF_SUCCESS) || (length == 0) || (resp == NULL)) { 1665 IBTF_DPRINTF_L2("ibdm", "\tget_node_port_guids:" 1666 " SA Retrieve Failed: %d", ret); 1667 return (IBDM_FAILURE); 1668 } 1669 IBTF_DPRINTF_L4("ibdm", "\tget_node_port_guids: NodeGuid %llx Port" 1670 "GUID %llx", resp->NodeInfo.NodeGUID, resp->NodeInfo.NodeGUID); 1671 1672 *node_guid = resp->NodeInfo.NodeGUID; 1673 *port_guid = resp->NodeInfo.PortGUID; 1674 kmem_free(resp, length); 1675 return (IBDM_SUCCESS); 1676 } 1677 1678 1679 /* 1680 * ibdm_get_reachable_ports() 1681 * Get list of the destination GID (and its path records) by 1682 * querying the SA access. 1683 * 1684 * Returns Number paths 1685 */ 1686 static int 1687 ibdm_get_reachable_ports(ibdm_port_attr_t *portinfo, ibdm_hca_list_t *hca) 1688 { 1689 uint_t ii, jj, nrecs; 1690 uint_t npaths = 0; 1691 size_t length; 1692 ib_gid_t sgid; 1693 ibdm_pkey_tbl_t *pkey_tbl; 1694 sa_path_record_t *result; 1695 sa_path_record_t *precp; 1696 ibdm_dp_gidinfo_t *gid_info; 1697 1698 ASSERT(MUTEX_HELD(&ibdm.ibdm_hl_mutex)); 1699 IBTF_DPRINTF_L4("ibdm", "\tget_reachable_ports: portinfo %p", portinfo); 1700 1701 sgid.gid_prefix = portinfo->pa_sn_prefix; 1702 sgid.gid_guid = portinfo->pa_port_guid; 1703 1704 /* get reversible paths */ 1705 if (portinfo->pa_sa_hdl && ibmf_saa_paths_from_gid(portinfo->pa_sa_hdl, 1706 sgid, IBMF_SAA_PKEY_WC, B_TRUE, 0, &nrecs, &length, &result) 1707 != IBMF_SUCCESS) { 1708 IBTF_DPRINTF_L2("ibdm", 1709 "\tget_reachable_ports: Getting path records failed"); 1710 return (0); 1711 } 1712 1713 for (ii = 0; ii < nrecs; ii++) { 1714 precp = &result[ii]; 1715 if ((gid_info = ibdm_check_dgid(precp->DGID.gid_guid, 1716 precp->DGID.gid_prefix)) != NULL) { 1717 IBTF_DPRINTF_L5("ibdm", "\tget_reachable_ports: " 1718 "Already exists nrecs %d, ii %d", nrecs, ii); 1719 ibdm_addto_glhcalist(gid_info, hca); 1720 continue; 1721 } 1722 /* 1723 * This is a new GID. Allocate a GID structure and 1724 * initialize the structure 1725 * gl_state is initialized to IBDM_GID_PROBE_NOT_DONE (0) 1726 * by kmem_zalloc call 1727 */ 1728 gid_info = kmem_zalloc(sizeof (ibdm_dp_gidinfo_t), KM_SLEEP); 1729 mutex_init(&gid_info->gl_mutex, NULL, MUTEX_DEFAULT, NULL); 1730 gid_info->gl_dgid_hi = precp->DGID.gid_prefix; 1731 gid_info->gl_dgid_lo = precp->DGID.gid_guid; 1732 gid_info->gl_sgid_hi = precp->SGID.gid_prefix; 1733 gid_info->gl_sgid_lo = precp->SGID.gid_guid; 1734 gid_info->gl_p_key = precp->P_Key; 1735 gid_info->gl_sa_hdl = portinfo->pa_sa_hdl; 1736 gid_info->gl_ibmf_hdl = portinfo->pa_ibmf_hdl; 1737 gid_info->gl_slid = precp->SLID; 1738 gid_info->gl_dlid = precp->DLID; 1739 gid_info->gl_transactionID = (++ibdm.ibdm_transactionID) 1740 << IBDM_GID_TRANSACTIONID_SHIFT; 1741 gid_info->gl_min_transactionID = gid_info->gl_transactionID; 1742 gid_info->gl_max_transactionID = (ibdm.ibdm_transactionID +1) 1743 << IBDM_GID_TRANSACTIONID_SHIFT; 1744 ibdm_addto_glhcalist(gid_info, hca); 1745 1746 ibdm_dump_path_info(precp); 1747 1748 gid_info->gl_qp_hdl = NULL; 1749 ASSERT(portinfo->pa_pkey_tbl != NULL && 1750 portinfo->pa_npkeys != 0); 1751 1752 for (jj = 0; jj < portinfo->pa_npkeys; jj++) { 1753 pkey_tbl = &portinfo->pa_pkey_tbl[jj]; 1754 if ((gid_info->gl_p_key == pkey_tbl->pt_pkey) && 1755 (pkey_tbl->pt_qp_hdl != NULL)) { 1756 gid_info->gl_qp_hdl = pkey_tbl->pt_qp_hdl; 1757 break; 1758 } 1759 } 1760 1761 /* 1762 * QP handle for GID not initialized. No matching Pkey 1763 * was found!! ibdm should *not* hit this case. Flag an 1764 * error and drop the GID if ibdm does encounter this. 1765 */ 1766 if (gid_info->gl_qp_hdl == NULL) { 1767 IBTF_DPRINTF_L2(ibdm_string, 1768 "\tget_reachable_ports: No matching Pkey"); 1769 ibdm_delete_gidinfo(gid_info); 1770 continue; 1771 } 1772 if (ibdm.ibdm_dp_gidlist_head == NULL) { 1773 ibdm.ibdm_dp_gidlist_head = gid_info; 1774 ibdm.ibdm_dp_gidlist_tail = gid_info; 1775 } else { 1776 ibdm.ibdm_dp_gidlist_tail->gl_next = gid_info; 1777 gid_info->gl_prev = ibdm.ibdm_dp_gidlist_tail; 1778 ibdm.ibdm_dp_gidlist_tail = gid_info; 1779 } 1780 npaths++; 1781 } 1782 kmem_free(result, length); 1783 IBTF_DPRINTF_L4("ibdm", "\tget_reachable_ports: npaths = %d", npaths); 1784 return (npaths); 1785 } 1786 1787 1788 /* 1789 * ibdm_check_dgid() 1790 * Look in the global list to check whether we know this DGID already 1791 * Return IBDM_GID_PRESENT/IBDM_GID_NOT_PRESENT 1792 */ 1793 static ibdm_dp_gidinfo_t * 1794 ibdm_check_dgid(ib_guid_t guid, ib_sn_prefix_t prefix) 1795 { 1796 ibdm_dp_gidinfo_t *gid_list; 1797 1798 for (gid_list = ibdm.ibdm_dp_gidlist_head; gid_list; 1799 gid_list = gid_list->gl_next) { 1800 if ((guid == gid_list->gl_dgid_lo) && 1801 (prefix == gid_list->gl_dgid_hi)) { 1802 break; 1803 } 1804 } 1805 return (gid_list); 1806 } 1807 1808 1809 /* 1810 * ibdm_find_gid() 1811 * Look in the global list to find a GID entry with matching 1812 * port & node GUID. 1813 * Return pointer to gidinfo if found, else return NULL 1814 */ 1815 static ibdm_dp_gidinfo_t * 1816 ibdm_find_gid(ib_guid_t nodeguid, ib_guid_t portguid) 1817 { 1818 ibdm_dp_gidinfo_t *gid_list; 1819 1820 IBTF_DPRINTF_L4("ibdm", "ibdm_find_gid(%llx, %llx)\n", 1821 nodeguid, portguid); 1822 1823 for (gid_list = ibdm.ibdm_dp_gidlist_head; gid_list; 1824 gid_list = gid_list->gl_next) { 1825 if ((portguid == gid_list->gl_portguid) && 1826 (nodeguid == gid_list->gl_nodeguid)) { 1827 break; 1828 } 1829 } 1830 1831 IBTF_DPRINTF_L4("ibdm", "ibdm_find_gid : returned %p\n", 1832 gid_list); 1833 return (gid_list); 1834 } 1835 1836 1837 /* 1838 * ibdm_send_classportinfo() 1839 * Send classportinfo request. When the request is completed 1840 * IBMF calls ibdm_classportinfo_cb routine to inform about 1841 * the completion. 1842 * Returns IBDM_SUCCESS/IBDM_FAILURE 1843 */ 1844 static int 1845 ibdm_send_classportinfo(ibdm_dp_gidinfo_t *gid_info) 1846 { 1847 ibmf_msg_t *msg; 1848 ib_mad_hdr_t *hdr; 1849 ibdm_timeout_cb_args_t *cb_args; 1850 1851 IBTF_DPRINTF_L4("ibdm", 1852 "\tsend_classportinfo: gid info 0x%p", gid_info); 1853 1854 /* 1855 * Send command to get classportinfo attribute. Allocate a IBMF 1856 * packet and initialize the packet. 1857 */ 1858 if (ibmf_alloc_msg(gid_info->gl_ibmf_hdl, IBMF_ALLOC_SLEEP, 1859 &msg) != IBMF_SUCCESS) { 1860 IBTF_DPRINTF_L4("ibdm", "\tsend_classportinfo: pkt alloc fail"); 1861 return (IBDM_FAILURE); 1862 } 1863 1864 ibdm_alloc_send_buffers(msg); 1865 1866 msg->im_local_addr.ia_local_lid = gid_info->gl_slid; 1867 msg->im_local_addr.ia_remote_lid = gid_info->gl_dlid; 1868 msg->im_local_addr.ia_remote_qno = 1; 1869 msg->im_local_addr.ia_p_key = gid_info->gl_p_key; 1870 msg->im_local_addr.ia_q_key = IB_GSI_QKEY; 1871 1872 hdr = IBDM_OUT_IBMFMSG_MADHDR(msg); 1873 hdr->BaseVersion = MAD_CLASS_BASE_VERS_1; 1874 hdr->MgmtClass = MAD_MGMT_CLASS_DEV_MGT; 1875 hdr->ClassVersion = IB_DM_CLASS_VERSION_1; 1876 hdr->R_Method = IB_DM_DEVMGT_METHOD_GET; 1877 hdr->Status = 0; 1878 hdr->TransactionID = h2b64(gid_info->gl_transactionID); 1879 hdr->AttributeID = h2b16(IB_DM_ATTR_CLASSPORTINFO); 1880 hdr->AttributeModifier = 0; 1881 1882 cb_args = &gid_info->gl_cpi_cb_args; 1883 cb_args->cb_gid_info = gid_info; 1884 cb_args->cb_retry_count = ibdm_dft_retry_cnt; 1885 cb_args->cb_req_type = IBDM_REQ_TYPE_CLASSPORTINFO; 1886 1887 gid_info->gl_timeout_id = timeout(ibdm_pkt_timeout_hdlr, 1888 cb_args, IBDM_TIMEOUT_VALUE(ibdm_dft_timeout)); 1889 1890 IBTF_DPRINTF_L5("ibdm", "\tsend_classportinfo: " 1891 "timeout id %x", gid_info->gl_timeout_id); 1892 1893 if (ibmf_msg_transport(gid_info->gl_ibmf_hdl, gid_info->gl_qp_hdl, 1894 msg, NULL, ibdm_ibmf_send_cb, cb_args, 0) != IBMF_SUCCESS) { 1895 IBTF_DPRINTF_L2("ibdm", 1896 "\tsend_classportinfo: ibmf send failed"); 1897 ibdm_ibmf_send_cb(gid_info->gl_ibmf_hdl, msg, cb_args); 1898 } 1899 1900 return (IBDM_SUCCESS); 1901 } 1902 1903 1904 /* 1905 * ibdm_handle_classportinfo() 1906 * Invoked by the IBMF when the classportinfo request is completed. 1907 */ 1908 static void 1909 ibdm_handle_classportinfo(ibmf_handle_t ibmf_hdl, 1910 ibmf_msg_t *msg, ibdm_dp_gidinfo_t *gid_info, int *flag) 1911 { 1912 void *data; 1913 timeout_id_t timeout_id; 1914 ib_mad_hdr_t *hdr; 1915 ibdm_mad_classportinfo_t *cpi; 1916 1917 IBTF_DPRINTF_L4("ibdm", "\thandle_classportinfo:ibmf hdl " 1918 "%p msg %p gid info %p", ibmf_hdl, msg, gid_info); 1919 1920 if (IBDM_IN_IBMFMSG_ATTR(msg) != IB_DM_ATTR_CLASSPORTINFO) { 1921 IBTF_DPRINTF_L4("ibdm", "\thandle_classportinfo: " 1922 "Not a ClassPortInfo resp"); 1923 *flag |= IBDM_IBMF_PKT_UNEXP_RESP; 1924 return; 1925 } 1926 1927 /* 1928 * Verify whether timeout handler is created/active. 1929 * If created/ active, cancel the timeout handler 1930 */ 1931 mutex_enter(&gid_info->gl_mutex); 1932 ibdm_bump_transactionID(gid_info); 1933 if (gid_info->gl_state != IBDM_GET_CLASSPORTINFO) { 1934 IBTF_DPRINTF_L2("ibdm", "\thandle_classportinfo:DUP resp"); 1935 *flag |= IBDM_IBMF_PKT_DUP_RESP; 1936 mutex_exit(&gid_info->gl_mutex); 1937 return; 1938 } 1939 gid_info->gl_iou_cb_args.cb_req_type = 0; 1940 if (gid_info->gl_timeout_id) { 1941 timeout_id = gid_info->gl_timeout_id; 1942 mutex_exit(&gid_info->gl_mutex); 1943 IBTF_DPRINTF_L5("ibdm", "handle_ioclassportinfo: " 1944 "gl_timeout_id = 0x%x", timeout_id); 1945 if (untimeout(timeout_id) == -1) { 1946 IBTF_DPRINTF_L2("ibdm", "handle_classportinfo: " 1947 "untimeout gl_timeout_id failed"); 1948 } 1949 mutex_enter(&gid_info->gl_mutex); 1950 gid_info->gl_timeout_id = 0; 1951 } 1952 gid_info->gl_state = IBDM_GET_IOUNITINFO; 1953 gid_info->gl_pending_cmds++; 1954 mutex_exit(&gid_info->gl_mutex); 1955 1956 data = msg->im_msgbufs_recv.im_bufs_cl_data; 1957 cpi = (ibdm_mad_classportinfo_t *)data; 1958 1959 /* 1960 * Cache the "RespTimeValue" and redirection information in the 1961 * global gid list data structure. This cached information will 1962 * be used to send any further requests to the GID. 1963 */ 1964 gid_info->gl_resp_timeout = 1965 (b2h32(cpi->RespTimeValue) & 0x1F); 1966 1967 gid_info->gl_redirected = ((IBDM_IN_IBMFMSG_STATUS(msg) & 1968 MAD_STATUS_REDIRECT_REQUIRED) ? B_TRUE: B_FALSE); 1969 gid_info->gl_redirect_dlid = b2h16(cpi->RedirectLID); 1970 gid_info->gl_redirect_QP = (b2h32(cpi->RedirectQP) & 0xffffff); 1971 gid_info->gl_redirect_pkey = b2h16(cpi->RedirectP_Key); 1972 gid_info->gl_redirect_qkey = b2h32(cpi->RedirectQ_Key); 1973 gid_info->gl_redirectGID_hi = b2h64(cpi->RedirectGID_hi); 1974 gid_info->gl_redirectGID_lo = b2h64(cpi->RedirectGID_lo); 1975 1976 ibdm_dump_classportinfo(cpi); 1977 1978 /* 1979 * Send IOUnitInfo request 1980 * Reuse previously allocated IBMF packet for sending ClassPortInfo 1981 * Check whether DM agent on the remote node requested redirection 1982 * If so, send the request to the redirect DGID/DLID/PKEY/QP. 1983 */ 1984 ibdm_alloc_send_buffers(msg); 1985 msg->im_local_addr.ia_local_lid = gid_info->gl_slid; 1986 msg->im_local_addr.ia_remote_lid = gid_info->gl_dlid; 1987 1988 if (gid_info->gl_redirected == B_TRUE) { 1989 if (gid_info->gl_redirect_dlid != 0) { 1990 msg->im_local_addr.ia_remote_lid = 1991 gid_info->gl_redirect_dlid; 1992 } 1993 msg->im_local_addr.ia_remote_qno = gid_info->gl_redirect_QP; 1994 msg->im_local_addr.ia_p_key = gid_info->gl_redirect_pkey; 1995 msg->im_local_addr.ia_q_key = gid_info->gl_redirect_qkey; 1996 } else { 1997 msg->im_local_addr.ia_remote_qno = 1; 1998 msg->im_local_addr.ia_p_key = gid_info->gl_p_key; 1999 msg->im_local_addr.ia_q_key = IB_GSI_QKEY; 2000 } 2001 2002 hdr = IBDM_OUT_IBMFMSG_MADHDR(msg); 2003 hdr->BaseVersion = MAD_CLASS_BASE_VERS_1; 2004 hdr->MgmtClass = MAD_MGMT_CLASS_DEV_MGT; 2005 hdr->ClassVersion = IB_DM_CLASS_VERSION_1; 2006 hdr->R_Method = IB_DM_DEVMGT_METHOD_GET; 2007 hdr->Status = 0; 2008 hdr->TransactionID = h2b64(gid_info->gl_transactionID); 2009 hdr->AttributeID = h2b16(IB_DM_ATTR_IO_UNITINFO); 2010 hdr->AttributeModifier = 0; 2011 2012 gid_info->gl_iou_cb_args.cb_req_type = IBDM_REQ_TYPE_IOUINFO; 2013 gid_info->gl_iou_cb_args.cb_gid_info = gid_info; 2014 gid_info->gl_iou_cb_args.cb_retry_count = ibdm_dft_retry_cnt; 2015 2016 gid_info->gl_timeout_id = timeout(ibdm_pkt_timeout_hdlr, 2017 &gid_info->gl_iou_cb_args, IBDM_TIMEOUT_VALUE(ibdm_dft_timeout)); 2018 2019 IBTF_DPRINTF_L5("ibdm", "handle_classportinfo:" 2020 "timeout %x", gid_info->gl_timeout_id); 2021 2022 if (ibmf_msg_transport(ibmf_hdl, gid_info->gl_qp_hdl, msg, NULL, 2023 ibdm_ibmf_send_cb, &gid_info->gl_iou_cb_args, 0) != IBMF_SUCCESS) { 2024 IBTF_DPRINTF_L2("ibdm", 2025 "\thandle_classportinfo: msg transport failed"); 2026 ibdm_ibmf_send_cb(ibmf_hdl, msg, &gid_info->gl_iou_cb_args); 2027 } 2028 (*flag) |= IBDM_IBMF_PKT_REUSED; 2029 } 2030 2031 2032 /* 2033 * ibdm_send_iounitinfo: 2034 * Sends a DM request to get IOU unitinfo. 2035 */ 2036 static int 2037 ibdm_send_iounitinfo(ibdm_dp_gidinfo_t *gid_info) 2038 { 2039 ibmf_msg_t *msg; 2040 ib_mad_hdr_t *hdr; 2041 2042 IBTF_DPRINTF_L4("ibdm", "\tsend_iounitinfo: gid info 0x%p", gid_info); 2043 2044 /* 2045 * Send command to get iounitinfo attribute. Allocate a IBMF 2046 * packet and initialize the packet. 2047 */ 2048 if (ibmf_alloc_msg(gid_info->gl_ibmf_hdl, IBMF_ALLOC_SLEEP, &msg) != 2049 IBMF_SUCCESS) { 2050 IBTF_DPRINTF_L4("ibdm", "\tsend_iounitinfo: pkt alloc fail"); 2051 return (IBDM_FAILURE); 2052 } 2053 2054 mutex_enter(&gid_info->gl_mutex); 2055 ibdm_bump_transactionID(gid_info); 2056 mutex_exit(&gid_info->gl_mutex); 2057 2058 2059 ibdm_alloc_send_buffers(msg); 2060 msg->im_local_addr.ia_local_lid = gid_info->gl_slid; 2061 msg->im_local_addr.ia_remote_lid = gid_info->gl_dlid; 2062 msg->im_local_addr.ia_remote_qno = 1; 2063 msg->im_local_addr.ia_p_key = gid_info->gl_p_key; 2064 msg->im_local_addr.ia_q_key = IB_GSI_QKEY; 2065 2066 hdr = IBDM_OUT_IBMFMSG_MADHDR(msg); 2067 hdr->BaseVersion = MAD_CLASS_BASE_VERS_1; 2068 hdr->MgmtClass = MAD_MGMT_CLASS_DEV_MGT; 2069 hdr->ClassVersion = IB_DM_CLASS_VERSION_1; 2070 hdr->R_Method = IB_DM_DEVMGT_METHOD_GET; 2071 hdr->Status = 0; 2072 hdr->TransactionID = h2b64(gid_info->gl_transactionID); 2073 hdr->AttributeID = h2b16(IB_DM_ATTR_IO_UNITINFO); 2074 hdr->AttributeModifier = 0; 2075 2076 gid_info->gl_iou_cb_args.cb_gid_info = gid_info; 2077 gid_info->gl_iou_cb_args.cb_retry_count = ibdm_dft_retry_cnt; 2078 gid_info->gl_iou_cb_args.cb_req_type = IBDM_REQ_TYPE_IOUINFO; 2079 2080 gid_info->gl_timeout_id = timeout(ibdm_pkt_timeout_hdlr, 2081 &gid_info->gl_iou_cb_args, IBDM_TIMEOUT_VALUE(ibdm_dft_timeout)); 2082 2083 IBTF_DPRINTF_L5("ibdm", "send_iouunitinfo:" 2084 "timeout %x", gid_info->gl_timeout_id); 2085 2086 if (ibmf_msg_transport(gid_info->gl_ibmf_hdl, gid_info->gl_qp_hdl, msg, 2087 NULL, ibdm_ibmf_send_cb, &gid_info->gl_iou_cb_args, 0) != 2088 IBMF_SUCCESS) { 2089 IBTF_DPRINTF_L2("ibdm", "\tsend_iounitinfo: ibmf send failed"); 2090 ibdm_ibmf_send_cb(gid_info->gl_ibmf_hdl, 2091 msg, &gid_info->gl_iou_cb_args); 2092 } 2093 return (IBDM_SUCCESS); 2094 } 2095 2096 /* 2097 * ibdm_handle_iounitinfo() 2098 * Invoked by the IBMF when IO Unitinfo request is completed. 2099 */ 2100 static void 2101 ibdm_handle_iounitinfo(ibmf_handle_t ibmf_hdl, 2102 ibmf_msg_t *msg, ibdm_dp_gidinfo_t *gid_info, int *flag) 2103 { 2104 int ii, first = B_TRUE; 2105 int num_iocs; 2106 size_t size; 2107 uchar_t slot_info; 2108 timeout_id_t timeout_id; 2109 ib_mad_hdr_t *hdr; 2110 ibdm_ioc_info_t *ioc_info; 2111 ib_dm_io_unitinfo_t *iou_info; 2112 ib_dm_io_unitinfo_t *giou_info; 2113 ibdm_timeout_cb_args_t *cb_args; 2114 2115 IBTF_DPRINTF_L4("ibdm", "\thandle_iouintinfo:" 2116 " ibmf hdl %p pkt %p gid info %p", ibmf_hdl, msg, gid_info); 2117 2118 if (IBDM_IN_IBMFMSG_ATTR(msg) != IB_DM_ATTR_IO_UNITINFO) { 2119 IBTF_DPRINTF_L4("ibdm", "\thandle_iounitinfo: " 2120 "Unexpected response"); 2121 (*flag) |= IBDM_IBMF_PKT_UNEXP_RESP; 2122 return; 2123 } 2124 2125 mutex_enter(&gid_info->gl_mutex); 2126 if (gid_info->gl_state != IBDM_GET_IOUNITINFO) { 2127 IBTF_DPRINTF_L4("ibdm", 2128 "\thandle_iounitinfo: DUP resp"); 2129 mutex_exit(&gid_info->gl_mutex); 2130 (*flag) = IBDM_IBMF_PKT_DUP_RESP; 2131 return; 2132 } 2133 gid_info->gl_iou_cb_args.cb_req_type = 0; 2134 if (gid_info->gl_timeout_id) { 2135 timeout_id = gid_info->gl_timeout_id; 2136 mutex_exit(&gid_info->gl_mutex); 2137 IBTF_DPRINTF_L5("ibdm", "handle_iounitinfo: " 2138 "gl_timeout_id = 0x%x", timeout_id); 2139 if (untimeout(timeout_id) == -1) { 2140 IBTF_DPRINTF_L2("ibdm", "handle_iounitinfo: " 2141 "untimeout gl_timeout_id failed"); 2142 } 2143 mutex_enter(&gid_info->gl_mutex); 2144 gid_info->gl_timeout_id = 0; 2145 } 2146 gid_info->gl_state = IBDM_GET_IOC_DETAILS; 2147 2148 iou_info = IBDM_IN_IBMFMSG2IOU(msg); 2149 ibdm_dump_iounitinfo(iou_info); 2150 num_iocs = iou_info->iou_num_ctrl_slots; 2151 /* 2152 * check if number of IOCs reported is zero? if yes, return. 2153 * when num_iocs are reported zero internal IOC database needs 2154 * to be updated. To ensure that save the number of IOCs in 2155 * the new field "gl_num_iocs". Use a new field instead of 2156 * "giou_info->iou_num_ctrl_slots" as that would prevent 2157 * an unnecessary kmem_alloc/kmem_free when num_iocs is 0. 2158 */ 2159 if (num_iocs == 0 && gid_info->gl_num_iocs == 0) { 2160 IBTF_DPRINTF_L4("ibdm", "\thandle_iounitinfo: no IOC's"); 2161 mutex_exit(&gid_info->gl_mutex); 2162 return; 2163 } 2164 IBTF_DPRINTF_L4("ibdm", "\thandle_iounitinfo: num_iocs = %d", num_iocs); 2165 2166 /* 2167 * if there is an existing gl_iou (IOU has been probed before) 2168 * check if the "iou_changeid" is same as saved entry in 2169 * "giou_info->iou_changeid". 2170 * (note: this logic can prevent IOC enumeration if a given 2171 * vendor doesn't support setting iou_changeid field for its IOU) 2172 * 2173 * if there is an existing gl_iou and iou_changeid has changed : 2174 * free up existing gl_iou info and its related structures. 2175 * reallocate gl_iou info all over again. 2176 * if we donot free this up; then this leads to memory leaks 2177 */ 2178 if (gid_info->gl_iou) { 2179 giou_info = &gid_info->gl_iou->iou_info; 2180 if (b2h16(iou_info->iou_changeid) == 2181 giou_info->iou_changeid) { 2182 IBTF_DPRINTF_L3("ibdm", 2183 "\thandle_iounitinfo: no IOCs changed"); 2184 gid_info->gl_state = IBDM_GID_PROBING_COMPLETE; 2185 mutex_exit(&gid_info->gl_mutex); 2186 return; 2187 } 2188 2189 /* 2190 * Store the iou info as prev_iou to be used after 2191 * sweep is done. 2192 */ 2193 ASSERT(gid_info->gl_prev_iou == NULL); 2194 IBTF_DPRINTF_L4(ibdm_string, 2195 "\thandle_iounitinfo: setting gl_prev_iou %p", 2196 gid_info->gl_prev_iou); 2197 gid_info->gl_prev_iou = gid_info->gl_iou; 2198 ibdm.ibdm_prev_iou = 1; 2199 gid_info->gl_iou = NULL; 2200 } 2201 2202 size = sizeof (ibdm_iou_info_t) + num_iocs * sizeof (ibdm_ioc_info_t); 2203 gid_info->gl_iou = (ibdm_iou_info_t *)kmem_zalloc(size, KM_SLEEP); 2204 giou_info = &gid_info->gl_iou->iou_info; 2205 gid_info->gl_iou->iou_ioc_info = (ibdm_ioc_info_t *) 2206 ((char *)gid_info->gl_iou + sizeof (ibdm_iou_info_t)); 2207 2208 giou_info->iou_num_ctrl_slots = gid_info->gl_num_iocs = num_iocs; 2209 giou_info->iou_flag = iou_info->iou_flag; 2210 bcopy(iou_info->iou_ctrl_list, giou_info->iou_ctrl_list, 128); 2211 giou_info->iou_changeid = b2h16(iou_info->iou_changeid); 2212 gid_info->gl_pending_cmds += num_iocs; 2213 gid_info->gl_pending_cmds += 1; /* for diag code */ 2214 mutex_exit(&gid_info->gl_mutex); 2215 2216 if (ibdm_get_diagcode(gid_info, 0) != IBDM_SUCCESS) { 2217 mutex_enter(&gid_info->gl_mutex); 2218 gid_info->gl_pending_cmds--; 2219 mutex_exit(&gid_info->gl_mutex); 2220 } 2221 /* 2222 * Parallelize getting IOC controller profiles from here. 2223 * Allocate IBMF packets and send commands to get IOC profile for 2224 * each IOC present on the IOU. 2225 */ 2226 for (ii = 0; ii < num_iocs; ii++) { 2227 /* 2228 * Check whether IOC is present in the slot 2229 * Series of nibbles (in the field iou_ctrl_list) represents 2230 * a slot in the IOU. 2231 * Byte format: 76543210 2232 * Bits 0-3 of first byte represent Slot 2 2233 * bits 4-7 of first byte represent slot 1, 2234 * bits 0-3 of second byte represent slot 4 and so on 2235 * Each 4-bit nibble has the following meaning 2236 * 0x0 : IOC not installed 2237 * 0x1 : IOC is present 2238 * 0xf : Slot does not exist 2239 * and all other values are reserved. 2240 */ 2241 ioc_info = IBDM_GIDINFO2IOCINFO(gid_info, ii); 2242 slot_info = giou_info->iou_ctrl_list[(ii/2)]; 2243 if ((ii % 2) == 0) 2244 slot_info = (slot_info >> 4); 2245 2246 if ((slot_info & 0xf) != 1) { 2247 IBTF_DPRINTF_L4("ibdm", "\thandle_iouintinfo: " 2248 "No IOC is present in the slot = %d", ii); 2249 ioc_info->ioc_state = IBDM_IOC_STATE_PROBE_FAILED; 2250 mutex_enter(&gid_info->gl_mutex); 2251 gid_info->gl_pending_cmds--; 2252 mutex_exit(&gid_info->gl_mutex); 2253 continue; 2254 } 2255 2256 mutex_enter(&gid_info->gl_mutex); 2257 ibdm_bump_transactionID(gid_info); 2258 mutex_exit(&gid_info->gl_mutex); 2259 2260 /* 2261 * Re use the already allocated packet (for IOUnitinfo) to 2262 * send the first IOC controller attribute. Allocate new 2263 * IBMF packets for the rest of the IOC's 2264 */ 2265 if (first != B_TRUE) { 2266 msg = NULL; 2267 if (ibmf_alloc_msg(ibmf_hdl, IBMF_ALLOC_SLEEP, 2268 &msg) != IBMF_SUCCESS) { 2269 IBTF_DPRINTF_L4("ibdm", "\thandle_iouintinfo: " 2270 "IBMF packet allocation failed"); 2271 mutex_enter(&gid_info->gl_mutex); 2272 gid_info->gl_pending_cmds--; 2273 mutex_exit(&gid_info->gl_mutex); 2274 continue; 2275 } 2276 2277 } 2278 2279 /* allocate send buffers for all messages */ 2280 ibdm_alloc_send_buffers(msg); 2281 2282 msg->im_local_addr.ia_local_lid = gid_info->gl_slid; 2283 msg->im_local_addr.ia_remote_lid = gid_info->gl_dlid; 2284 if (gid_info->gl_redirected == B_TRUE) { 2285 if (gid_info->gl_redirect_dlid != 0) { 2286 msg->im_local_addr.ia_remote_lid = 2287 gid_info->gl_redirect_dlid; 2288 } 2289 msg->im_local_addr.ia_remote_qno = 2290 gid_info->gl_redirect_QP; 2291 msg->im_local_addr.ia_p_key = 2292 gid_info->gl_redirect_pkey; 2293 msg->im_local_addr.ia_q_key = 2294 gid_info->gl_redirect_qkey; 2295 } else { 2296 msg->im_local_addr.ia_remote_qno = 1; 2297 msg->im_local_addr.ia_p_key = gid_info->gl_p_key; 2298 msg->im_local_addr.ia_q_key = IB_GSI_QKEY; 2299 } 2300 2301 hdr = IBDM_OUT_IBMFMSG_MADHDR(msg); 2302 hdr->BaseVersion = MAD_CLASS_BASE_VERS_1; 2303 hdr->MgmtClass = MAD_MGMT_CLASS_DEV_MGT; 2304 hdr->ClassVersion = IB_DM_CLASS_VERSION_1; 2305 hdr->R_Method = IB_DM_DEVMGT_METHOD_GET; 2306 hdr->Status = 0; 2307 hdr->TransactionID = h2b64(gid_info->gl_transactionID); 2308 hdr->AttributeID = h2b16(IB_DM_ATTR_IOC_CTRL_PROFILE); 2309 hdr->AttributeModifier = h2b32(ii + 1); 2310 2311 ioc_info->ioc_state = IBDM_IOC_STATE_PROBE_INVALID; 2312 cb_args = &ioc_info->ioc_cb_args; 2313 cb_args->cb_gid_info = gid_info; 2314 cb_args->cb_retry_count = ibdm_dft_retry_cnt; 2315 cb_args->cb_req_type = IBDM_REQ_TYPE_IOCINFO; 2316 cb_args->cb_ioc_num = ii; 2317 2318 ioc_info->ioc_timeout_id = timeout(ibdm_pkt_timeout_hdlr, 2319 cb_args, IBDM_TIMEOUT_VALUE(ibdm_dft_timeout)); 2320 2321 IBTF_DPRINTF_L5("ibdm", "\thandle_iounitinfo:" 2322 "timeout 0x%x, ioc_num %d", ioc_info->ioc_timeout_id, ii); 2323 2324 if (ibmf_msg_transport(ibmf_hdl, gid_info->gl_qp_hdl, msg, 2325 NULL, ibdm_ibmf_send_cb, cb_args, 0) != IBMF_SUCCESS) { 2326 IBTF_DPRINTF_L2("ibdm", 2327 "\thandle_iounitinfo: msg transport failed"); 2328 ibdm_ibmf_send_cb(ibmf_hdl, msg, cb_args); 2329 } 2330 (*flag) |= IBDM_IBMF_PKT_REUSED; 2331 first = B_FALSE; 2332 gid_info->gl_iou->iou_niocs_probe_in_progress++; 2333 } 2334 } 2335 2336 2337 /* 2338 * ibdm_handle_ioc_profile() 2339 * Invoked by the IBMF when the IOCControllerProfile request 2340 * gets completed 2341 */ 2342 static void 2343 ibdm_handle_ioc_profile(ibmf_handle_t ibmf_hdl, 2344 ibmf_msg_t *msg, ibdm_dp_gidinfo_t *gid_info, int *flag) 2345 { 2346 int first = B_TRUE, reprobe = 0; 2347 uint_t ii, ioc_no, srv_start; 2348 uint_t nserv_entries; 2349 timeout_id_t timeout_id; 2350 ib_mad_hdr_t *hdr; 2351 ibdm_ioc_info_t *ioc_info; 2352 ibdm_timeout_cb_args_t *cb_args; 2353 ib_dm_ioc_ctrl_profile_t *ioc, *gioc; 2354 2355 IBTF_DPRINTF_L4("ibdm", "\thandle_ioc_profile:" 2356 " ibmf hdl %p msg %p gid info %p", ibmf_hdl, msg, gid_info); 2357 2358 ioc = IBDM_IN_IBMFMSG2IOC(msg); 2359 /* 2360 * Check whether we know this IOC already 2361 * This will return NULL if reprobe is in progress 2362 * IBDM_IOC_STATE_REPROBE_PROGRESS will be set. 2363 * Do not hold mutexes here. 2364 */ 2365 if (ibdm_is_ioc_present(ioc->ioc_guid, gid_info, flag) != NULL) { 2366 IBTF_DPRINTF_L4("ibdm", "\thandle_ioc_profile:" 2367 "IOC guid %llx is present", ioc->ioc_guid); 2368 return; 2369 } 2370 ioc_no = IBDM_IN_IBMFMSG_ATTRMOD(msg); 2371 IBTF_DPRINTF_L4("ibdm", "\thandle_ioc_profile: ioc_no = %d", ioc_no-1); 2372 2373 /* Make sure that IOC index is with the valid range */ 2374 if (IBDM_IS_IOC_NUM_INVALID(ioc_no, gid_info)) { 2375 IBTF_DPRINTF_L2("ibdm", "\thandle_ioc_profile: " 2376 "IOC index Out of range, index %d", ioc); 2377 (*flag) |= IBDM_IBMF_PKT_UNEXP_RESP; 2378 return; 2379 } 2380 ioc_info = &gid_info->gl_iou->iou_ioc_info[ioc_no - 1]; 2381 ioc_info->ioc_iou_info = gid_info->gl_iou; 2382 2383 mutex_enter(&gid_info->gl_mutex); 2384 if (ioc_info->ioc_state == IBDM_IOC_STATE_REPROBE_PROGRESS) { 2385 reprobe = 1; 2386 ioc_info->ioc_prev_serv = ioc_info->ioc_serv; 2387 ioc_info->ioc_serv = NULL; 2388 ioc_info->ioc_prev_serv_cnt = 2389 ioc_info->ioc_profile.ioc_service_entries; 2390 } else if (ioc_info->ioc_state != IBDM_IOC_STATE_PROBE_INVALID) { 2391 IBTF_DPRINTF_L2("ibdm", "\thandle_ioc_profile: DUP response" 2392 "ioc %d, ioc_state %x", ioc_no - 1, ioc_info->ioc_state); 2393 mutex_exit(&gid_info->gl_mutex); 2394 (*flag) |= IBDM_IBMF_PKT_DUP_RESP; 2395 return; 2396 } 2397 ioc_info->ioc_cb_args.cb_req_type = 0; 2398 if (ioc_info->ioc_timeout_id) { 2399 timeout_id = ioc_info->ioc_timeout_id; 2400 mutex_exit(&gid_info->gl_mutex); 2401 IBTF_DPRINTF_L5("ibdm", "handle_ioc_profile: " 2402 "ioc_timeout_id = 0x%x", timeout_id); 2403 if (untimeout(timeout_id) == -1) { 2404 IBTF_DPRINTF_L2("ibdm", "handle_ioc_profile: " 2405 "untimeout ioc_timeout_id failed"); 2406 } 2407 mutex_enter(&gid_info->gl_mutex); 2408 ioc_info->ioc_timeout_id = 0; 2409 } 2410 2411 ioc_info->ioc_state = IBDM_IOC_STATE_PROBE_SUCCESS; 2412 if (reprobe == 0) { 2413 ioc_info->ioc_iou_guid = gid_info->gl_nodeguid; 2414 ioc_info->ioc_nodeguid = gid_info->gl_nodeguid; 2415 } 2416 2417 /* 2418 * Save all the IOC information in the global structures. 2419 * Note the wire format is Big Endian and the Sparc process also 2420 * big endian. So, there is no need to convert the data fields 2421 * The conversion routines used below are ineffective on Sparc 2422 * machines where as they will be effective on little endian 2423 * machines such as Intel processors. 2424 */ 2425 gioc = (ib_dm_ioc_ctrl_profile_t *)&ioc_info->ioc_profile; 2426 2427 /* 2428 * Restrict updates to onlyport GIDs and service entries during reprobe 2429 */ 2430 if (reprobe == 0) { 2431 gioc->ioc_guid = b2h64(ioc->ioc_guid); 2432 gioc->ioc_vendorid = 2433 ((b2h32(ioc->ioc_vendorid) & IB_DM_VENDORID_MASK) 2434 >> IB_DM_VENDORID_SHIFT); 2435 gioc->ioc_deviceid = b2h32(ioc->ioc_deviceid); 2436 gioc->ioc_device_ver = b2h16(ioc->ioc_device_ver); 2437 gioc->ioc_subsys_vendorid = 2438 ((b2h32(ioc->ioc_subsys_vendorid) & IB_DM_VENDORID_MASK) 2439 >> IB_DM_VENDORID_SHIFT); 2440 gioc->ioc_subsys_id = b2h32(ioc->ioc_subsys_id); 2441 gioc->ioc_io_class = b2h16(ioc->ioc_io_class); 2442 gioc->ioc_io_subclass = b2h16(ioc->ioc_io_subclass); 2443 gioc->ioc_protocol = b2h16(ioc->ioc_protocol); 2444 gioc->ioc_protocol_ver = b2h16(ioc->ioc_protocol_ver); 2445 gioc->ioc_send_msg_qdepth = 2446 b2h16(ioc->ioc_send_msg_qdepth); 2447 gioc->ioc_rdma_read_qdepth = 2448 b2h16(ioc->ioc_rdma_read_qdepth); 2449 gioc->ioc_send_msg_sz = b2h32(ioc->ioc_send_msg_sz); 2450 gioc->ioc_rdma_xfer_sz = b2h32(ioc->ioc_rdma_xfer_sz); 2451 gioc->ioc_ctrl_opcap_mask = ioc->ioc_ctrl_opcap_mask; 2452 bcopy(ioc->ioc_id_string, gioc->ioc_id_string, 2453 IB_DM_IOC_ID_STRING_LEN); 2454 2455 ioc_info->ioc_iou_diagcode = gid_info->gl_iou->iou_diagcode; 2456 ioc_info->ioc_iou_dc_valid = gid_info->gl_iou->iou_dc_valid; 2457 ioc_info->ioc_diagdeviceid = (IB_DM_IOU_DEVICEID_MASK & 2458 gid_info->gl_iou->iou_info.iou_flag) ? B_TRUE : B_FALSE; 2459 2460 if (ioc_info->ioc_diagdeviceid == B_TRUE) 2461 gid_info->gl_pending_cmds++; 2462 } 2463 gioc->ioc_service_entries = ioc->ioc_service_entries; 2464 gid_info->gl_pending_cmds += (gioc->ioc_service_entries/4); 2465 if (gioc->ioc_service_entries % 4) 2466 gid_info->gl_pending_cmds++; 2467 2468 mutex_exit(&gid_info->gl_mutex); 2469 2470 ibdm_dump_ioc_profile(gioc); 2471 2472 if ((ioc_info->ioc_diagdeviceid == B_TRUE) && (reprobe == 0)) { 2473 if (ibdm_get_diagcode(gid_info, ioc_no) != IBDM_SUCCESS) { 2474 mutex_enter(&gid_info->gl_mutex); 2475 gid_info->gl_pending_cmds--; 2476 mutex_exit(&gid_info->gl_mutex); 2477 } 2478 } 2479 ioc_info->ioc_serv = (ibdm_srvents_info_t *)kmem_zalloc( 2480 (gioc->ioc_service_entries * sizeof (ibdm_srvents_info_t)), 2481 KM_SLEEP); 2482 2483 /* 2484 * In one single request, maximum number of requests that can be 2485 * obtained is 4. If number of service entries are more than four, 2486 * calculate number requests needed and send them parallelly. 2487 */ 2488 nserv_entries = ioc->ioc_service_entries; 2489 ii = 0; 2490 while (nserv_entries) { 2491 mutex_enter(&gid_info->gl_mutex); 2492 ibdm_bump_transactionID(gid_info); 2493 mutex_exit(&gid_info->gl_mutex); 2494 2495 if (first != B_TRUE) { 2496 if (ibmf_alloc_msg(ibmf_hdl, IBMF_ALLOC_SLEEP, 2497 &msg) != IBMF_SUCCESS) { 2498 continue; 2499 } 2500 2501 } 2502 ibdm_alloc_send_buffers(msg); 2503 msg->im_local_addr.ia_local_lid = gid_info->gl_slid; 2504 msg->im_local_addr.ia_remote_lid = gid_info->gl_dlid; 2505 if (gid_info->gl_redirected == B_TRUE) { 2506 if (gid_info->gl_redirect_dlid != 0) { 2507 msg->im_local_addr.ia_remote_lid = 2508 gid_info->gl_redirect_dlid; 2509 } 2510 msg->im_local_addr.ia_remote_qno = 2511 gid_info->gl_redirect_QP; 2512 msg->im_local_addr.ia_p_key = 2513 gid_info->gl_redirect_pkey; 2514 msg->im_local_addr.ia_q_key = 2515 gid_info->gl_redirect_qkey; 2516 } else { 2517 msg->im_local_addr.ia_remote_qno = 1; 2518 msg->im_local_addr.ia_p_key = gid_info->gl_p_key; 2519 msg->im_local_addr.ia_q_key = IB_GSI_QKEY; 2520 } 2521 2522 hdr = IBDM_OUT_IBMFMSG_MADHDR(msg); 2523 hdr->BaseVersion = MAD_CLASS_BASE_VERS_1; 2524 hdr->MgmtClass = MAD_MGMT_CLASS_DEV_MGT; 2525 hdr->ClassVersion = IB_DM_CLASS_VERSION_1; 2526 hdr->R_Method = IB_DM_DEVMGT_METHOD_GET; 2527 hdr->Status = 0; 2528 hdr->TransactionID = h2b64(gid_info->gl_transactionID); 2529 hdr->AttributeID = h2b16(IB_DM_ATTR_SERVICE_ENTRIES); 2530 2531 srv_start = ii * 4; 2532 cb_args = &ioc_info->ioc_serv[srv_start].se_cb_args; 2533 cb_args->cb_gid_info = gid_info; 2534 cb_args->cb_retry_count = ibdm_dft_retry_cnt; 2535 cb_args->cb_req_type = IBDM_REQ_TYPE_SRVENTS; 2536 cb_args->cb_srvents_start = srv_start; 2537 cb_args->cb_ioc_num = ioc_no - 1; 2538 2539 if (nserv_entries >= IBDM_MAX_SERV_ENTRIES_PER_REQ) { 2540 nserv_entries -= IBDM_MAX_SERV_ENTRIES_PER_REQ; 2541 cb_args->cb_srvents_end = (cb_args->cb_srvents_start + 2542 IBDM_MAX_SERV_ENTRIES_PER_REQ - 1); 2543 } else { 2544 cb_args->cb_srvents_end = 2545 (cb_args->cb_srvents_start + nserv_entries - 1); 2546 nserv_entries = 0; 2547 } 2548 ibdm_fill_srv_attr_mod(hdr, cb_args); 2549 2550 ioc_info->ioc_serv[srv_start].se_timeout_id = timeout( 2551 ibdm_pkt_timeout_hdlr, cb_args, 2552 IBDM_TIMEOUT_VALUE(ibdm_dft_timeout)); 2553 2554 IBTF_DPRINTF_L5("ibdm", "\thandle_ioc_profile:" 2555 "timeout %x, ioc %d srv %d", 2556 ioc_info->ioc_serv[srv_start].se_timeout_id, 2557 ioc_no - 1, srv_start); 2558 2559 if (ibmf_msg_transport(ibmf_hdl, gid_info->gl_qp_hdl, msg, 2560 NULL, ibdm_ibmf_send_cb, cb_args, 0) != IBMF_SUCCESS) { 2561 IBTF_DPRINTF_L2("ibdm", 2562 "\thandle_ioc_profile: msg send failed"); 2563 ibdm_ibmf_send_cb(ibmf_hdl, msg, cb_args); 2564 } 2565 (*flag) |= IBDM_IBMF_PKT_REUSED; 2566 first = B_FALSE; 2567 ii++; 2568 } 2569 } 2570 2571 2572 /* 2573 * ibdm_handle_srventry_mad() 2574 */ 2575 static void 2576 ibdm_handle_srventry_mad(ibmf_msg_t *msg, 2577 ibdm_dp_gidinfo_t *gid_info, int *flag) 2578 { 2579 uint_t ii, ioc_no, attrmod; 2580 uint_t nentries, start, end; 2581 timeout_id_t timeout_id; 2582 ib_dm_srv_t *srv_ents; 2583 ibdm_ioc_info_t *ioc_info; 2584 ibdm_srvents_info_t *gsrv_ents; 2585 2586 IBTF_DPRINTF_L4("ibdm", "\thandle_srventry_mad:" 2587 " IBMF msg %p gid info %p", msg, gid_info); 2588 2589 srv_ents = IBDM_IN_IBMFMSG2SRVENT(msg); 2590 /* 2591 * Get the start and end index of the service entries 2592 * Upper 16 bits identify the IOC 2593 * Lower 16 bits specify the range of service entries 2594 * LSB specifies (Big endian) end of the range 2595 * MSB specifies (Big endian) start of the range 2596 */ 2597 attrmod = IBDM_IN_IBMFMSG_ATTRMOD(msg); 2598 ioc_no = ((attrmod >> 16) & IBDM_16_BIT_MASK); 2599 end = ((attrmod >> 8) & IBDM_8_BIT_MASK); 2600 start = (attrmod & IBDM_8_BIT_MASK); 2601 2602 /* Make sure that IOC index is with the valid range */ 2603 if ((ioc_no < 1) | 2604 (ioc_no > gid_info->gl_iou->iou_info.iou_num_ctrl_slots)) { 2605 IBTF_DPRINTF_L2("ibdm", "\thandle_srventry_mad: " 2606 "IOC index Out of range, index %d", ioc_no); 2607 (*flag) |= IBDM_IBMF_PKT_UNEXP_RESP; 2608 return; 2609 } 2610 ioc_info = IBDM_GIDINFO2IOCINFO(gid_info, (ioc_no -1)); 2611 2612 /* 2613 * Make sure that the "start" and "end" service indexes are 2614 * with in the valid range 2615 */ 2616 nentries = ioc_info->ioc_profile.ioc_service_entries; 2617 if ((start > end) | (start >= nentries) | (end >= nentries)) { 2618 IBTF_DPRINTF_L2("ibdm", "\thandle_srventry_mad: " 2619 "Attr modifier 0x%x, #Serv entries %d", attrmod, nentries); 2620 (*flag) |= IBDM_IBMF_PKT_UNEXP_RESP; 2621 return; 2622 } 2623 gsrv_ents = &ioc_info->ioc_serv[start]; 2624 mutex_enter(&gid_info->gl_mutex); 2625 if (gsrv_ents->se_state != IBDM_SE_INVALID) { 2626 IBTF_DPRINTF_L2("ibdm", "\thandle_srventry_mad: " 2627 "already known, ioc %d, srv %d, se_state %x", 2628 ioc_no - 1, start, gsrv_ents->se_state); 2629 mutex_exit(&gid_info->gl_mutex); 2630 (*flag) |= IBDM_IBMF_PKT_DUP_RESP; 2631 return; 2632 } 2633 ioc_info->ioc_serv[start].se_cb_args.cb_req_type = 0; 2634 if (ioc_info->ioc_serv[start].se_timeout_id) { 2635 IBTF_DPRINTF_L2("ibdm", 2636 "\thandle_srventry_mad: ioc %d start %d", ioc_no, start); 2637 timeout_id = ioc_info->ioc_serv[start].se_timeout_id; 2638 mutex_exit(&gid_info->gl_mutex); 2639 IBTF_DPRINTF_L5("ibdm", "handle_srverntry_mad: " 2640 "se_timeout_id = 0x%x", timeout_id); 2641 if (untimeout(timeout_id) == -1) { 2642 IBTF_DPRINTF_L2("ibdm", "handle_srventry_mad: " 2643 "untimeout se_timeout_id failed"); 2644 } 2645 mutex_enter(&gid_info->gl_mutex); 2646 ioc_info->ioc_serv[start].se_timeout_id = 0; 2647 } 2648 2649 gsrv_ents->se_state = IBDM_SE_VALID; 2650 mutex_exit(&gid_info->gl_mutex); 2651 for (ii = start; ii <= end; ii++, srv_ents++, gsrv_ents++) { 2652 gsrv_ents->se_attr.srv_id = b2h64(srv_ents->srv_id); 2653 bcopy(srv_ents->srv_name, 2654 gsrv_ents->se_attr.srv_name, IB_DM_MAX_SVC_NAME_LEN); 2655 ibdm_dump_service_entries(&gsrv_ents->se_attr); 2656 } 2657 } 2658 2659 2660 /* 2661 * ibdm_get_diagcode: 2662 * Send request to get IOU/IOC diag code 2663 * Returns IBDM_SUCCESS/IBDM_FAILURE 2664 */ 2665 static int 2666 ibdm_get_diagcode(ibdm_dp_gidinfo_t *gid_info, int attr) 2667 { 2668 ibmf_msg_t *msg; 2669 ib_mad_hdr_t *hdr; 2670 ibdm_ioc_info_t *ioc; 2671 ibdm_timeout_cb_args_t *cb_args; 2672 timeout_id_t *timeout_id; 2673 2674 IBTF_DPRINTF_L4("ibdm", "\tget_diagcode: gid info %p, attr = %d", 2675 gid_info, attr); 2676 2677 if (ibmf_alloc_msg(gid_info->gl_ibmf_hdl, IBMF_ALLOC_SLEEP, 2678 &msg) != IBMF_SUCCESS) { 2679 IBTF_DPRINTF_L4("ibdm", "\tget_diagcode: pkt alloc fail"); 2680 return (IBDM_FAILURE); 2681 } 2682 2683 ibdm_alloc_send_buffers(msg); 2684 2685 mutex_enter(&gid_info->gl_mutex); 2686 ibdm_bump_transactionID(gid_info); 2687 mutex_exit(&gid_info->gl_mutex); 2688 2689 msg->im_local_addr.ia_local_lid = gid_info->gl_slid; 2690 msg->im_local_addr.ia_remote_lid = gid_info->gl_dlid; 2691 if (gid_info->gl_redirected == B_TRUE) { 2692 if (gid_info->gl_redirect_dlid != 0) { 2693 msg->im_local_addr.ia_remote_lid = 2694 gid_info->gl_redirect_dlid; 2695 } 2696 2697 msg->im_local_addr.ia_remote_qno = gid_info->gl_redirect_QP; 2698 msg->im_local_addr.ia_p_key = gid_info->gl_redirect_pkey; 2699 msg->im_local_addr.ia_q_key = gid_info->gl_redirect_qkey; 2700 } else { 2701 msg->im_local_addr.ia_remote_qno = 1; 2702 msg->im_local_addr.ia_p_key = gid_info->gl_p_key; 2703 msg->im_local_addr.ia_q_key = IB_GSI_QKEY; 2704 } 2705 2706 hdr = IBDM_OUT_IBMFMSG_MADHDR(msg); 2707 hdr->BaseVersion = MAD_CLASS_BASE_VERS_1; 2708 hdr->MgmtClass = MAD_MGMT_CLASS_DEV_MGT; 2709 hdr->ClassVersion = IB_DM_CLASS_VERSION_1; 2710 hdr->R_Method = IB_DM_DEVMGT_METHOD_GET; 2711 hdr->Status = 0; 2712 hdr->TransactionID = h2b64(gid_info->gl_transactionID); 2713 2714 hdr->AttributeID = h2b16(IB_DM_ATTR_DIAG_CODE); 2715 hdr->AttributeModifier = h2b32(attr); 2716 2717 if (attr == 0) { 2718 cb_args = &gid_info->gl_iou_cb_args; 2719 gid_info->gl_iou->iou_dc_valid = B_FALSE; 2720 cb_args->cb_ioc_num = 0; 2721 cb_args->cb_req_type = IBDM_REQ_TYPE_IOU_DIAGCODE; 2722 timeout_id = &gid_info->gl_timeout_id; 2723 } else { 2724 ioc = IBDM_GIDINFO2IOCINFO(gid_info, (attr - 1)); 2725 ioc->ioc_dc_valid = B_FALSE; 2726 cb_args = &ioc->ioc_dc_cb_args; 2727 cb_args->cb_ioc_num = attr - 1; 2728 cb_args->cb_req_type = IBDM_REQ_TYPE_IOC_DIAGCODE; 2729 timeout_id = &ioc->ioc_dc_timeout_id; 2730 } 2731 cb_args->cb_gid_info = gid_info; 2732 cb_args->cb_retry_count = ibdm_dft_retry_cnt; 2733 cb_args->cb_srvents_start = 0; 2734 2735 2736 *timeout_id = timeout(ibdm_pkt_timeout_hdlr, 2737 cb_args, IBDM_TIMEOUT_VALUE(ibdm_dft_timeout)); 2738 2739 IBTF_DPRINTF_L5("ibdm", "\tget_diagcode:" 2740 "timeout %x, ioc %d", *timeout_id, cb_args->cb_ioc_num); 2741 2742 if (ibmf_msg_transport(gid_info->gl_ibmf_hdl, gid_info->gl_qp_hdl, 2743 msg, NULL, ibdm_ibmf_send_cb, cb_args, 0) != IBMF_SUCCESS) { 2744 IBTF_DPRINTF_L2("ibdm", "\tget_diagcode: ibmf send failed"); 2745 ibdm_ibmf_send_cb(gid_info->gl_ibmf_hdl, msg, cb_args); 2746 } 2747 return (IBDM_SUCCESS); 2748 } 2749 2750 /* 2751 * ibdm_handle_diagcode: 2752 * Process the DiagCode MAD response and update local DM 2753 * data structure. 2754 */ 2755 static void 2756 ibdm_handle_diagcode(ibmf_msg_t *ibmf_msg, 2757 ibdm_dp_gidinfo_t *gid_info, int *flag) 2758 { 2759 uint16_t attrmod, *diagcode; 2760 ibdm_iou_info_t *iou; 2761 ibdm_ioc_info_t *ioc; 2762 timeout_id_t timeout_id; 2763 ibdm_timeout_cb_args_t *cb_args; 2764 2765 diagcode = (uint16_t *)ibmf_msg->im_msgbufs_recv.im_bufs_cl_data; 2766 2767 mutex_enter(&gid_info->gl_mutex); 2768 attrmod = IBDM_IN_IBMFMSG_ATTRMOD(ibmf_msg); 2769 iou = gid_info->gl_iou; 2770 if (attrmod == 0) { 2771 if (iou->iou_dc_valid != B_FALSE) { 2772 (*flag) |= IBDM_IBMF_PKT_DUP_RESP; 2773 IBTF_DPRINTF_L4("ibdm", 2774 "\thandle_diagcode: Duplicate IOU DiagCode"); 2775 mutex_exit(&gid_info->gl_mutex); 2776 return; 2777 } 2778 cb_args = &gid_info->gl_iou_cb_args; 2779 cb_args->cb_req_type = 0; 2780 iou->iou_diagcode = b2h16(*diagcode); 2781 iou->iou_dc_valid = B_TRUE; 2782 if (gid_info->gl_timeout_id) { 2783 timeout_id = gid_info->gl_timeout_id; 2784 mutex_exit(&gid_info->gl_mutex); 2785 IBTF_DPRINTF_L5("ibdm", "\thandle_diagcode: " 2786 "gl_timeout_id = 0x%x", timeout_id); 2787 if (untimeout(timeout_id) == -1) { 2788 IBTF_DPRINTF_L2("ibdm", "handle_diagcode: " 2789 "untimeout gl_timeout_id failed"); 2790 } 2791 mutex_enter(&gid_info->gl_mutex); 2792 gid_info->gl_timeout_id = 0; 2793 } 2794 } else { 2795 ioc = IBDM_GIDINFO2IOCINFO(gid_info, (attrmod - 1)); 2796 if (ioc->ioc_dc_valid != B_FALSE) { 2797 (*flag) |= IBDM_IBMF_PKT_DUP_RESP; 2798 IBTF_DPRINTF_L4("ibdm", 2799 "\thandle_diagcode: Duplicate IOC DiagCode"); 2800 mutex_exit(&gid_info->gl_mutex); 2801 return; 2802 } 2803 cb_args = &ioc->ioc_dc_cb_args; 2804 cb_args->cb_req_type = 0; 2805 ioc->ioc_diagcode = b2h16(*diagcode); 2806 ioc->ioc_dc_valid = B_TRUE; 2807 timeout_id = iou->iou_ioc_info[attrmod - 1].ioc_dc_timeout_id; 2808 if (timeout_id) { 2809 mutex_exit(&gid_info->gl_mutex); 2810 IBTF_DPRINTF_L5("ibdm", "handle_diagcode: " 2811 "timeout_id = 0x%x", timeout_id); 2812 if (untimeout(timeout_id) == -1) { 2813 IBTF_DPRINTF_L2("ibdm", "\thandle_diagcode: " 2814 "untimeout ioc_dc_timeout_id failed"); 2815 } 2816 mutex_enter(&gid_info->gl_mutex); 2817 iou->iou_ioc_info[attrmod - 1].ioc_dc_timeout_id = 0; 2818 } 2819 } 2820 mutex_exit(&gid_info->gl_mutex); 2821 2822 IBTF_DPRINTF_L4("ibdm", "\thandle_diagcode: DiagCode : 0x%x" 2823 "attrmod : 0x%x", b2h16(*diagcode), attrmod); 2824 } 2825 2826 2827 /* 2828 * ibdm_is_ioc_present() 2829 * Return ibdm_ioc_info_t if IOC guid is found in the global gid list 2830 */ 2831 static ibdm_ioc_info_t * 2832 ibdm_is_ioc_present(ib_guid_t ioc_guid, 2833 ibdm_dp_gidinfo_t *gid_info, int *flag) 2834 { 2835 int ii; 2836 ibdm_ioc_info_t *ioc; 2837 ibdm_dp_gidinfo_t *head; 2838 ib_dm_io_unitinfo_t *iou; 2839 2840 mutex_enter(&ibdm.ibdm_mutex); 2841 head = ibdm.ibdm_dp_gidlist_head; 2842 while (head) { 2843 mutex_enter(&head->gl_mutex); 2844 if (head->gl_iou == NULL) { 2845 mutex_exit(&head->gl_mutex); 2846 head = head->gl_next; 2847 continue; 2848 } 2849 iou = &head->gl_iou->iou_info; 2850 for (ii = 0; ii < iou->iou_num_ctrl_slots; ii++) { 2851 ioc = IBDM_GIDINFO2IOCINFO(head, ii); 2852 if ((ioc->ioc_state == IBDM_IOC_STATE_PROBE_SUCCESS) && 2853 (ioc->ioc_profile.ioc_guid == ioc_guid)) { 2854 if (gid_info == head) { 2855 *flag |= IBDM_IBMF_PKT_DUP_RESP; 2856 } else if (ibdm_check_dgid(head->gl_dgid_lo, 2857 head->gl_dgid_hi) != NULL) { 2858 IBTF_DPRINTF_L4("ibdm", "\tis_ioc_" 2859 "present: gid not present"); 2860 ibdm_add_to_gl_gid(gid_info, head); 2861 } 2862 mutex_exit(&head->gl_mutex); 2863 mutex_exit(&ibdm.ibdm_mutex); 2864 return (ioc); 2865 } 2866 } 2867 mutex_exit(&head->gl_mutex); 2868 head = head->gl_next; 2869 } 2870 mutex_exit(&ibdm.ibdm_mutex); 2871 return (NULL); 2872 } 2873 2874 2875 /* 2876 * ibdm_ibmf_send_cb() 2877 * IBMF invokes this callback routine after posting the DM MAD to 2878 * the HCA. 2879 */ 2880 /*ARGSUSED*/ 2881 static void 2882 ibdm_ibmf_send_cb(ibmf_handle_t ibmf_hdl, ibmf_msg_t *ibmf_msg, void *arg) 2883 { 2884 ibdm_dump_ibmf_msg(ibmf_msg, 1); 2885 ibdm_free_send_buffers(ibmf_msg); 2886 if (ibmf_free_msg(ibmf_hdl, &ibmf_msg) != IBMF_SUCCESS) { 2887 IBTF_DPRINTF_L4("ibdm", 2888 "\tibmf_send_cb: IBMF free msg failed"); 2889 } 2890 } 2891 2892 2893 /* 2894 * ibdm_ibmf_recv_cb() 2895 * Invoked by the IBMF when a response to the one of the DM requests 2896 * is received. 2897 */ 2898 /*ARGSUSED*/ 2899 static void 2900 ibdm_ibmf_recv_cb(ibmf_handle_t ibmf_hdl, ibmf_msg_t *msg, void *arg) 2901 { 2902 ibdm_taskq_args_t *taskq_args; 2903 2904 /* 2905 * If the taskq enable is set then dispatch a taskq to process 2906 * the MAD, otherwise just process it on this thread 2907 */ 2908 if (ibdm_taskq_enable != IBDM_ENABLE_TASKQ_HANDLING) { 2909 ibdm_process_incoming_mad(ibmf_hdl, msg, arg); 2910 return; 2911 } 2912 2913 /* 2914 * create a taskq and dispatch it to process the incoming MAD 2915 */ 2916 taskq_args = kmem_alloc(sizeof (ibdm_taskq_args_t), KM_NOSLEEP); 2917 if (taskq_args == NULL) { 2918 IBTF_DPRINTF_L2("ibdm", "ibmf_recv_cb: kmem_alloc failed for" 2919 "taskq_args"); 2920 if (ibmf_free_msg(ibmf_hdl, &msg) != IBMF_SUCCESS) { 2921 IBTF_DPRINTF_L4("ibmf_recv_cb", 2922 "\tibmf_recv_cb: IBMF free msg failed"); 2923 } 2924 return; 2925 } 2926 taskq_args->tq_ibmf_handle = ibmf_hdl; 2927 taskq_args->tq_ibmf_msg = msg; 2928 taskq_args->tq_args = arg; 2929 2930 if (taskq_dispatch(system_taskq, ibdm_recv_incoming_mad, taskq_args, 2931 TQ_NOSLEEP) == 0) { 2932 IBTF_DPRINTF_L2("ibdm", "ibmf_recv_cb: taskq_dispatch failed"); 2933 if (ibmf_free_msg(ibmf_hdl, &msg) != IBMF_SUCCESS) { 2934 IBTF_DPRINTF_L4("ibmf_recv_cb", 2935 "\tibmf_recv_cb: IBMF free msg failed"); 2936 } 2937 kmem_free(taskq_args, sizeof (ibdm_taskq_args_t)); 2938 return; 2939 } 2940 2941 /* taskq_args are deleted in ibdm_recv_incoming_mad() */ 2942 } 2943 2944 2945 void 2946 ibdm_recv_incoming_mad(void *args) 2947 { 2948 ibdm_taskq_args_t *taskq_args; 2949 2950 taskq_args = (ibdm_taskq_args_t *)args; 2951 2952 IBTF_DPRINTF_L4("ibdm", "\tibdm_recv_incoming_mad: " 2953 "Processing incoming MAD via taskq"); 2954 2955 ibdm_process_incoming_mad(taskq_args->tq_ibmf_handle, 2956 taskq_args->tq_ibmf_msg, taskq_args->tq_args); 2957 2958 kmem_free(taskq_args, sizeof (ibdm_taskq_args_t)); 2959 } 2960 2961 2962 /* 2963 * Calls ibdm_process_incoming_mad with all function arguments extracted 2964 * from args 2965 */ 2966 /*ARGSUSED*/ 2967 static void 2968 ibdm_process_incoming_mad(ibmf_handle_t ibmf_hdl, ibmf_msg_t *msg, void *arg) 2969 { 2970 int flag = 0; 2971 int ret; 2972 uint64_t transaction_id; 2973 ib_mad_hdr_t *hdr; 2974 ibdm_dp_gidinfo_t *gid_info = NULL; 2975 2976 IBTF_DPRINTF_L4("ibdm", 2977 "\tprocess_incoming_mad: ibmf hdl %p pkt %p", ibmf_hdl, msg); 2978 ibdm_dump_ibmf_msg(msg, 0); 2979 2980 /* 2981 * IBMF calls this routine for every DM MAD that arrives at this port. 2982 * But we handle only the responses for requests we sent. We drop all 2983 * the DM packets that does not have response bit set in the MAD 2984 * header(this eliminates all the requests sent to this port). 2985 * We handle only DM class version 1 MAD's 2986 */ 2987 hdr = IBDM_IN_IBMFMSG_MADHDR(msg); 2988 if (ibdm_verify_mad_status(hdr) != IBDM_SUCCESS) { 2989 if (ibmf_free_msg(ibmf_hdl, &msg) != IBMF_SUCCESS) { 2990 IBTF_DPRINTF_L2("ibdm", "\tprocess_incoming_mad: " 2991 "IBMF free msg failed DM request drop it"); 2992 } 2993 return; 2994 } 2995 2996 transaction_id = b2h64(hdr->TransactionID); 2997 2998 mutex_enter(&ibdm.ibdm_mutex); 2999 gid_info = ibdm.ibdm_dp_gidlist_head; 3000 while (gid_info) { 3001 if ((gid_info->gl_transactionID & 3002 IBDM_GID_TRANSACTIONID_MASK) == 3003 (transaction_id & IBDM_GID_TRANSACTIONID_MASK)) 3004 break; 3005 gid_info = gid_info->gl_next; 3006 } 3007 mutex_exit(&ibdm.ibdm_mutex); 3008 3009 if (gid_info == NULL) { 3010 /* Drop the packet */ 3011 IBTF_DPRINTF_L2("ibdm", "process_incoming_mad: transaction ID" 3012 " does not match: 0x%llx", transaction_id); 3013 if (ibmf_free_msg(ibmf_hdl, &msg) != IBMF_SUCCESS) { 3014 IBTF_DPRINTF_L2("ibdm", "process_incoming_mad: " 3015 "IBMF free msg failed DM request drop it"); 3016 } 3017 return; 3018 } 3019 3020 /* Handle redirection for all the MAD's, except ClassPortInfo */ 3021 if (((IBDM_IN_IBMFMSG_STATUS(msg) & MAD_STATUS_REDIRECT_REQUIRED)) && 3022 (IBDM_IN_IBMFMSG_ATTR(msg) != IB_DM_ATTR_CLASSPORTINFO)) { 3023 ret = ibdm_handle_redirection(msg, gid_info, &flag); 3024 if (ret == IBDM_SUCCESS) { 3025 return; 3026 } 3027 } else { 3028 uint_t gl_state; 3029 3030 mutex_enter(&gid_info->gl_mutex); 3031 gl_state = gid_info->gl_state; 3032 mutex_exit(&gid_info->gl_mutex); 3033 3034 switch (gl_state) { 3035 case IBDM_GET_CLASSPORTINFO: 3036 ibdm_handle_classportinfo( 3037 ibmf_hdl, msg, gid_info, &flag); 3038 break; 3039 3040 case IBDM_GET_IOUNITINFO: 3041 ibdm_handle_iounitinfo(ibmf_hdl, msg, gid_info, &flag); 3042 break; 3043 3044 case IBDM_GET_IOC_DETAILS: 3045 switch (IBDM_IN_IBMFMSG_ATTR(msg)) { 3046 3047 case IB_DM_ATTR_SERVICE_ENTRIES: 3048 ibdm_handle_srventry_mad(msg, gid_info, &flag); 3049 break; 3050 3051 case IB_DM_ATTR_IOC_CTRL_PROFILE: 3052 ibdm_handle_ioc_profile( 3053 ibmf_hdl, msg, gid_info, &flag); 3054 break; 3055 3056 case IB_DM_ATTR_DIAG_CODE: 3057 ibdm_handle_diagcode(msg, gid_info, &flag); 3058 break; 3059 3060 default: 3061 IBTF_DPRINTF_L2("ibdm", "process_incoming_mad: " 3062 "Error state, wrong attribute :-("); 3063 (void) ibmf_free_msg(ibmf_hdl, &msg); 3064 return; 3065 } 3066 break; 3067 default: 3068 IBTF_DPRINTF_L2("ibdm", 3069 "process_incoming_mad: Dropping the packet" 3070 " gl_state %x", gl_state); 3071 if (ibmf_free_msg(ibmf_hdl, &msg) != IBMF_SUCCESS) { 3072 IBTF_DPRINTF_L2("ibdm", "process_incoming_mad: " 3073 "IBMF free msg failed DM request drop it"); 3074 } 3075 return; 3076 } 3077 } 3078 3079 if ((flag & IBDM_IBMF_PKT_DUP_RESP) || 3080 (flag & IBDM_IBMF_PKT_UNEXP_RESP)) { 3081 IBTF_DPRINTF_L2("ibdm", 3082 "\tprocess_incoming_mad:Dup/unexp resp : 0x%x", flag); 3083 if (ibmf_free_msg(ibmf_hdl, &msg) != IBMF_SUCCESS) { 3084 IBTF_DPRINTF_L2("ibdm", "process_incoming_mad: " 3085 "IBMF free msg failed DM request drop it"); 3086 } 3087 return; 3088 } 3089 3090 mutex_enter(&gid_info->gl_mutex); 3091 if (gid_info->gl_pending_cmds < 1) { 3092 IBTF_DPRINTF_L2("ibdm", 3093 "\tprocess_incoming_mad: pending commands negative"); 3094 } 3095 if (--gid_info->gl_pending_cmds) { 3096 IBTF_DPRINTF_L4("ibdm", "\tprocess_incoming_mad: " 3097 "gid_info %p pending cmds %d", 3098 gid_info, gid_info->gl_pending_cmds); 3099 mutex_exit(&gid_info->gl_mutex); 3100 } else { 3101 IBTF_DPRINTF_L4("ibdm", "\tprocess_incoming_mad: Probing DONE"); 3102 gid_info->gl_state = IBDM_GID_PROBING_COMPLETE; 3103 mutex_exit(&gid_info->gl_mutex); 3104 ibdm_notify_newgid_iocs(gid_info); 3105 mutex_enter(&ibdm.ibdm_mutex); 3106 if (--ibdm.ibdm_ngid_probes_in_progress == 0) { 3107 IBTF_DPRINTF_L4("ibdm", 3108 "\tprocess_incoming_mad: Wakeup"); 3109 ibdm.ibdm_busy &= ~IBDM_PROBE_IN_PROGRESS; 3110 cv_broadcast(&ibdm.ibdm_probe_cv); 3111 } 3112 mutex_exit(&ibdm.ibdm_mutex); 3113 } 3114 3115 /* 3116 * Do not deallocate the IBMF packet if atleast one request 3117 * is posted. IBMF packet is reused. 3118 */ 3119 if (!(flag & IBDM_IBMF_PKT_REUSED)) { 3120 if (ibmf_free_msg(ibmf_hdl, &msg) != IBMF_SUCCESS) { 3121 IBTF_DPRINTF_L2("ibdm", "\tprocess_incoming_mad: " 3122 "IBMF free msg failed DM request drop it"); 3123 } 3124 } 3125 } 3126 3127 3128 /* 3129 * ibdm_verify_mad_status() 3130 * Verifies the MAD status 3131 * Returns IBDM_SUCCESS if status is correct 3132 * Returns IBDM_FAILURE for bogus MAD status 3133 */ 3134 static int 3135 ibdm_verify_mad_status(ib_mad_hdr_t *hdr) 3136 { 3137 int ret = 0; 3138 3139 if ((hdr->R_Method != IB_DM_DEVMGT_METHOD_GET_RESP) || 3140 (hdr->ClassVersion != IB_DM_CLASS_VERSION_1)) { 3141 return (IBDM_FAILURE); 3142 } 3143 3144 if (b2h16(hdr->Status) == 0) 3145 ret = IBDM_SUCCESS; 3146 else if ((b2h16(hdr->Status) & 0x1f) == MAD_STATUS_REDIRECT_REQUIRED) 3147 ret = IBDM_SUCCESS; 3148 else { 3149 IBTF_DPRINTF_L2("ibdm", 3150 "\tverify_mad_status: Status : 0x%x", b2h16(hdr->Status)); 3151 ret = IBDM_FAILURE; 3152 } 3153 return (ret); 3154 } 3155 3156 3157 3158 /* 3159 * ibdm_handle_redirection() 3160 * Returns IBDM_SUCCESS/IBDM_FAILURE 3161 */ 3162 static int 3163 ibdm_handle_redirection(ibmf_msg_t *msg, 3164 ibdm_dp_gidinfo_t *gid_info, int *flag) 3165 { 3166 int attrmod, ioc_no, start; 3167 void *data; 3168 timeout_id_t *timeout_id; 3169 ib_mad_hdr_t *hdr; 3170 ibdm_ioc_info_t *ioc = NULL; 3171 ibdm_timeout_cb_args_t *cb_args; 3172 ibdm_mad_classportinfo_t *cpi; 3173 3174 IBTF_DPRINTF_L4("ibdm", "\thandle_redirection: Enter"); 3175 mutex_enter(&gid_info->gl_mutex); 3176 switch (gid_info->gl_state) { 3177 case IBDM_GET_IOUNITINFO: 3178 cb_args = &gid_info->gl_iou_cb_args; 3179 timeout_id = &gid_info->gl_timeout_id; 3180 break; 3181 3182 case IBDM_GET_IOC_DETAILS: 3183 attrmod = IBDM_IN_IBMFMSG_ATTRMOD(msg); 3184 switch (IBDM_IN_IBMFMSG_ATTR(msg)) { 3185 3186 case IB_DM_ATTR_DIAG_CODE: 3187 if (attrmod == 0) { 3188 cb_args = &gid_info->gl_iou_cb_args; 3189 timeout_id = &gid_info->gl_timeout_id; 3190 break; 3191 } 3192 if (IBDM_IS_IOC_NUM_INVALID(attrmod, gid_info)) { 3193 IBTF_DPRINTF_L2("ibdm", "\thandle_redirction:" 3194 "IOC# Out of range %d", attrmod); 3195 (*flag) |= IBDM_IBMF_PKT_UNEXP_RESP; 3196 mutex_exit(&gid_info->gl_mutex); 3197 return (IBDM_FAILURE); 3198 } 3199 ioc = IBDM_GIDINFO2IOCINFO(gid_info, (attrmod -1)); 3200 cb_args = &ioc->ioc_dc_cb_args; 3201 timeout_id = &ioc->ioc_dc_timeout_id; 3202 break; 3203 3204 case IB_DM_ATTR_IOC_CTRL_PROFILE: 3205 if (IBDM_IS_IOC_NUM_INVALID(attrmod, gid_info)) { 3206 IBTF_DPRINTF_L2("ibdm", "\thandle_redirction:" 3207 "IOC# Out of range %d", attrmod); 3208 (*flag) |= IBDM_IBMF_PKT_UNEXP_RESP; 3209 mutex_exit(&gid_info->gl_mutex); 3210 return (IBDM_FAILURE); 3211 } 3212 ioc = IBDM_GIDINFO2IOCINFO(gid_info, (attrmod -1)); 3213 cb_args = &ioc->ioc_cb_args; 3214 timeout_id = &ioc->ioc_timeout_id; 3215 break; 3216 3217 case IB_DM_ATTR_SERVICE_ENTRIES: 3218 ioc_no = ((attrmod >> 16) & IBDM_16_BIT_MASK); 3219 if (IBDM_IS_IOC_NUM_INVALID(ioc_no, gid_info)) { 3220 IBTF_DPRINTF_L2("ibdm", "\thandle_redirction:" 3221 "IOC# Out of range %d", ioc_no); 3222 (*flag) |= IBDM_IBMF_PKT_UNEXP_RESP; 3223 mutex_exit(&gid_info->gl_mutex); 3224 return (IBDM_FAILURE); 3225 } 3226 start = (attrmod & IBDM_8_BIT_MASK); 3227 ioc = IBDM_GIDINFO2IOCINFO(gid_info, (ioc_no -1)); 3228 if (start > ioc->ioc_profile.ioc_service_entries) { 3229 IBTF_DPRINTF_L2("ibdm", "\thandle_redirction:" 3230 " SE index Out of range %d", start); 3231 (*flag) |= IBDM_IBMF_PKT_UNEXP_RESP; 3232 mutex_exit(&gid_info->gl_mutex); 3233 return (IBDM_FAILURE); 3234 } 3235 cb_args = &ioc->ioc_serv[start].se_cb_args; 3236 timeout_id = &ioc->ioc_serv[start].se_timeout_id; 3237 break; 3238 3239 default: 3240 /* ERROR State */ 3241 IBTF_DPRINTF_L2("ibdm", 3242 "\thandle_redirection: wrong attribute :-("); 3243 (*flag) |= IBDM_IBMF_PKT_UNEXP_RESP; 3244 mutex_exit(&gid_info->gl_mutex); 3245 return (IBDM_FAILURE); 3246 } 3247 break; 3248 default: 3249 /* ERROR State */ 3250 IBTF_DPRINTF_L2("ibdm", 3251 "\thandle_redirection: Error state :-("); 3252 (*flag) |= IBDM_IBMF_PKT_UNEXP_RESP; 3253 mutex_exit(&gid_info->gl_mutex); 3254 return (IBDM_FAILURE); 3255 } 3256 if ((*timeout_id) != 0) { 3257 mutex_exit(&gid_info->gl_mutex); 3258 if (untimeout(*timeout_id) == -1) { 3259 IBTF_DPRINTF_L2("ibdm", "\thandle_redirection: " 3260 "untimeout failed %x", *timeout_id); 3261 } else { 3262 IBTF_DPRINTF_L5("ibdm", 3263 "\thandle_redirection: timeout %x", *timeout_id); 3264 } 3265 mutex_enter(&gid_info->gl_mutex); 3266 *timeout_id = 0; 3267 } 3268 3269 data = msg->im_msgbufs_recv.im_bufs_cl_data; 3270 cpi = (ibdm_mad_classportinfo_t *)data; 3271 3272 gid_info->gl_resp_timeout = 3273 (b2h32(cpi->RespTimeValue) & 0x1F); 3274 3275 gid_info->gl_redirected = B_TRUE; 3276 gid_info->gl_redirect_dlid = b2h16(cpi->RedirectLID); 3277 gid_info->gl_redirect_QP = (b2h32(cpi->RedirectQP) & 0xffffff); 3278 gid_info->gl_redirect_pkey = b2h16(cpi->RedirectP_Key); 3279 gid_info->gl_redirect_qkey = b2h32(cpi->RedirectQ_Key); 3280 gid_info->gl_redirectGID_hi = b2h64(cpi->RedirectGID_hi); 3281 gid_info->gl_redirectGID_lo = b2h64(cpi->RedirectGID_lo); 3282 3283 if (gid_info->gl_redirect_dlid != 0) { 3284 msg->im_local_addr.ia_remote_lid = 3285 gid_info->gl_redirect_dlid; 3286 } 3287 ibdm_bump_transactionID(gid_info); 3288 mutex_exit(&gid_info->gl_mutex); 3289 3290 ibdm_alloc_send_buffers(msg); 3291 3292 hdr = IBDM_OUT_IBMFMSG_MADHDR(msg); 3293 hdr->BaseVersion = MAD_CLASS_BASE_VERS_1; 3294 hdr->MgmtClass = MAD_MGMT_CLASS_DEV_MGT; 3295 hdr->ClassVersion = IB_DM_CLASS_VERSION_1; 3296 hdr->R_Method = IB_DM_DEVMGT_METHOD_GET; 3297 hdr->Status = 0; 3298 hdr->TransactionID = h2b64(gid_info->gl_transactionID); 3299 hdr->AttributeID = 3300 msg->im_msgbufs_recv.im_bufs_mad_hdr->AttributeID; 3301 hdr->AttributeModifier = 3302 msg->im_msgbufs_recv.im_bufs_mad_hdr->AttributeModifier; 3303 3304 msg->im_local_addr.ia_remote_qno = gid_info->gl_redirect_QP; 3305 msg->im_local_addr.ia_p_key = gid_info->gl_redirect_pkey; 3306 msg->im_local_addr.ia_q_key = gid_info->gl_redirect_qkey; 3307 3308 *timeout_id = timeout(ibdm_pkt_timeout_hdlr, 3309 cb_args, IBDM_TIMEOUT_VALUE(ibdm_dft_timeout)); 3310 3311 IBTF_DPRINTF_L5("ibdm", "\thandle_redirect:" 3312 "timeout %x", *timeout_id); 3313 3314 if (ibmf_msg_transport(gid_info->gl_ibmf_hdl, gid_info->gl_qp_hdl, 3315 msg, NULL, ibdm_ibmf_send_cb, cb_args, 0) != IBMF_SUCCESS) { 3316 IBTF_DPRINTF_L4("ibdm", "\thandle_redirection:" 3317 "message transport failed"); 3318 ibdm_ibmf_send_cb(gid_info->gl_ibmf_hdl, msg, cb_args); 3319 } 3320 (*flag) |= IBDM_IBMF_PKT_REUSED; 3321 IBTF_DPRINTF_L4("ibdm", "\thandle_redirection: Exit"); 3322 return (IBDM_SUCCESS); 3323 } 3324 3325 3326 /* 3327 * ibdm_pkt_timeout_hdlr 3328 * This timeout handler is registed for every IBMF packet that is 3329 * sent through the IBMF. It gets called when no response is received 3330 * within the specified time for the packet. No retries for the failed 3331 * commands currently. Drops the failed IBMF packet and update the 3332 * pending list commands. 3333 */ 3334 static void 3335 ibdm_pkt_timeout_hdlr(void *arg) 3336 { 3337 int probe_done = B_FALSE; 3338 ibdm_iou_info_t *iou; 3339 ibdm_ioc_info_t *ioc; 3340 ibdm_timeout_cb_args_t *cb_args = arg; 3341 ibdm_dp_gidinfo_t *gid_info; 3342 int srv_ent; 3343 uint_t new_gl_state; 3344 3345 IBTF_DPRINTF_L2("ibdm", "\tpkt_timeout_hdlr: gid_info: %p " 3346 "rtype 0x%x iocidx 0x%x srvidx %d", cb_args->cb_gid_info, 3347 cb_args->cb_req_type, cb_args->cb_ioc_num, 3348 cb_args->cb_srvents_start); 3349 3350 gid_info = cb_args->cb_gid_info; 3351 mutex_enter(&gid_info->gl_mutex); 3352 3353 if ((gid_info->gl_state == IBDM_GID_PROBING_COMPLETE) || 3354 (cb_args->cb_req_type == 0)) { 3355 3356 IBTF_DPRINTF_L2("ibdm", "\tpkt_timeout_hdlr: req completed" 3357 "rtype 0x%x iocidx 0x%x srvidx %d", cb_args->cb_req_type, 3358 cb_args->cb_ioc_num, cb_args->cb_srvents_start); 3359 3360 if (gid_info->gl_timeout_id) 3361 gid_info->gl_timeout_id = 0; 3362 mutex_exit(&gid_info->gl_mutex); 3363 return; 3364 } 3365 if (cb_args->cb_retry_count) { 3366 cb_args->cb_retry_count--; 3367 if (ibdm_retry_command(cb_args) == IBDM_SUCCESS) { 3368 if (gid_info->gl_timeout_id) 3369 gid_info->gl_timeout_id = 0; 3370 mutex_exit(&gid_info->gl_mutex); 3371 return; 3372 } 3373 cb_args->cb_retry_count = 0; 3374 } 3375 3376 IBTF_DPRINTF_L2("ibdm", "\tpkt_timeout_hdlr: command failed: gid %p" 3377 " rtype 0x%x iocidx 0x%x srvidx %d", cb_args->cb_gid_info, 3378 cb_args->cb_req_type, cb_args->cb_ioc_num, 3379 cb_args->cb_srvents_start); 3380 3381 new_gl_state = IBDM_GID_PROBING_COMPLETE; 3382 switch (cb_args->cb_req_type) { 3383 3384 case IBDM_REQ_TYPE_CLASSPORTINFO: 3385 case IBDM_REQ_TYPE_IOUINFO: 3386 new_gl_state = IBDM_GID_PROBING_FAILED; 3387 if (--gid_info->gl_pending_cmds == 0) 3388 probe_done = B_TRUE; 3389 if (gid_info->gl_timeout_id) 3390 gid_info->gl_timeout_id = 0; 3391 mutex_exit(&gid_info->gl_mutex); 3392 ibdm_delete_glhca_list(gid_info); 3393 mutex_enter(&gid_info->gl_mutex); 3394 break; 3395 case IBDM_REQ_TYPE_IOCINFO: 3396 iou = gid_info->gl_iou; 3397 ioc = &iou->iou_ioc_info[cb_args->cb_ioc_num]; 3398 ioc->ioc_state = IBDM_IOC_STATE_PROBE_FAILED; 3399 if (--gid_info->gl_pending_cmds == 0) 3400 probe_done = B_TRUE; 3401 #ifndef __lock_lint 3402 if (ioc->ioc_timeout_id) 3403 ioc->ioc_timeout_id = 0; 3404 #endif 3405 break; 3406 case IBDM_REQ_TYPE_SRVENTS: 3407 iou = gid_info->gl_iou; 3408 ioc = &iou->iou_ioc_info[cb_args->cb_ioc_num]; 3409 ioc->ioc_state = IBDM_IOC_STATE_PROBE_FAILED; 3410 if (--gid_info->gl_pending_cmds == 0) 3411 probe_done = B_TRUE; 3412 srv_ent = cb_args->cb_srvents_start; 3413 #ifndef __lock_lint 3414 if (ioc->ioc_serv[srv_ent].se_timeout_id) 3415 ioc->ioc_serv[srv_ent].se_timeout_id = 0; 3416 #endif 3417 break; 3418 case IBDM_REQ_TYPE_IOU_DIAGCODE: 3419 iou = gid_info->gl_iou; 3420 iou->iou_dc_valid = B_FALSE; 3421 if (--gid_info->gl_pending_cmds == 0) 3422 probe_done = B_TRUE; 3423 if (gid_info->gl_timeout_id) 3424 gid_info->gl_timeout_id = 0; 3425 break; 3426 case IBDM_REQ_TYPE_IOC_DIAGCODE: 3427 iou = gid_info->gl_iou; 3428 ioc = &iou->iou_ioc_info[cb_args->cb_ioc_num]; 3429 ioc->ioc_dc_valid = B_FALSE; 3430 if (--gid_info->gl_pending_cmds == 0) 3431 probe_done = B_TRUE; 3432 #ifndef __lock_lint 3433 if (ioc->ioc_dc_timeout_id) 3434 ioc->ioc_dc_timeout_id = 0; 3435 #endif 3436 break; 3437 } 3438 if (probe_done == B_TRUE) { 3439 gid_info->gl_state = new_gl_state; 3440 mutex_exit(&gid_info->gl_mutex); 3441 ibdm_notify_newgid_iocs(gid_info); 3442 mutex_enter(&ibdm.ibdm_mutex); 3443 if (--ibdm.ibdm_ngid_probes_in_progress == 0) { 3444 IBTF_DPRINTF_L4("ibdm", "\tpkt_timeout_hdlr: Wakeup"); 3445 ibdm.ibdm_busy &= ~IBDM_PROBE_IN_PROGRESS; 3446 cv_broadcast(&ibdm.ibdm_probe_cv); 3447 } 3448 mutex_exit(&ibdm.ibdm_mutex); 3449 } else 3450 mutex_exit(&gid_info->gl_mutex); 3451 } 3452 3453 3454 /* 3455 * ibdm_retry_command() 3456 * Retries the failed command. 3457 * Returns IBDM_FAILURE/IBDM_SUCCESS 3458 */ 3459 static int 3460 ibdm_retry_command(ibdm_timeout_cb_args_t *cb_args) 3461 { 3462 int ret; 3463 ibmf_msg_t *msg; 3464 ib_mad_hdr_t *hdr; 3465 ibdm_dp_gidinfo_t *gid_info = cb_args->cb_gid_info; 3466 timeout_id_t *timeout_id; 3467 ibdm_ioc_info_t *ioc; 3468 int ioc_no; 3469 3470 IBTF_DPRINTF_L2("ibdm", "\tretry_command: gid_info: %p " 3471 "rtype 0x%x iocidx 0x%x srvidx %d", cb_args->cb_gid_info, 3472 cb_args->cb_req_type, cb_args->cb_ioc_num, 3473 cb_args->cb_srvents_start); 3474 3475 ret = ibmf_alloc_msg(gid_info->gl_ibmf_hdl, IBMF_ALLOC_NOSLEEP, &msg); 3476 3477 3478 /* 3479 * Reset the gid if alloc_msg failed with BAD_HANDLE 3480 * ibdm_reset_gidinfo reinits the gid_info 3481 */ 3482 if (ret == IBMF_BAD_HANDLE) { 3483 IBTF_DPRINTF_L3(ibdm_string, "\tretry_command: gid %p hdl bad", 3484 gid_info); 3485 3486 mutex_exit(&gid_info->gl_mutex); 3487 ibdm_reset_gidinfo(gid_info); 3488 mutex_enter(&gid_info->gl_mutex); 3489 3490 /* Retry alloc */ 3491 ret = ibmf_alloc_msg(gid_info->gl_ibmf_hdl, IBMF_ALLOC_NOSLEEP, 3492 &msg); 3493 } 3494 3495 if (ret != IBDM_SUCCESS) { 3496 IBTF_DPRINTF_L2("ibdm", "\tretry_command: alloc failed: %p " 3497 "rtype 0x%x iocidx 0x%x srvidx %d", cb_args->cb_gid_info, 3498 cb_args->cb_req_type, cb_args->cb_ioc_num, 3499 cb_args->cb_srvents_start); 3500 return (IBDM_FAILURE); 3501 } 3502 3503 ibdm_alloc_send_buffers(msg); 3504 3505 ibdm_bump_transactionID(gid_info); 3506 3507 msg->im_local_addr.ia_local_lid = gid_info->gl_slid; 3508 msg->im_local_addr.ia_remote_lid = gid_info->gl_dlid; 3509 if (gid_info->gl_redirected == B_TRUE) { 3510 if (gid_info->gl_redirect_dlid != 0) { 3511 msg->im_local_addr.ia_remote_lid = 3512 gid_info->gl_redirect_dlid; 3513 } 3514 msg->im_local_addr.ia_remote_qno = gid_info->gl_redirect_QP; 3515 msg->im_local_addr.ia_p_key = gid_info->gl_redirect_pkey; 3516 msg->im_local_addr.ia_q_key = gid_info->gl_redirect_qkey; 3517 } else { 3518 msg->im_local_addr.ia_remote_qno = 1; 3519 msg->im_local_addr.ia_p_key = gid_info->gl_p_key; 3520 msg->im_local_addr.ia_q_key = IB_GSI_QKEY; 3521 } 3522 hdr = IBDM_OUT_IBMFMSG_MADHDR(msg); 3523 hdr->BaseVersion = MAD_CLASS_BASE_VERS_1; 3524 hdr->MgmtClass = MAD_MGMT_CLASS_DEV_MGT; 3525 hdr->ClassVersion = IB_DM_CLASS_VERSION_1; 3526 hdr->R_Method = IB_DM_DEVMGT_METHOD_GET; 3527 hdr->Status = 0; 3528 hdr->TransactionID = h2b64(gid_info->gl_transactionID); 3529 3530 switch (cb_args->cb_req_type) { 3531 case IBDM_REQ_TYPE_CLASSPORTINFO: 3532 hdr->AttributeID = h2b16(IB_DM_ATTR_CLASSPORTINFO); 3533 hdr->AttributeModifier = 0; 3534 timeout_id = &gid_info->gl_timeout_id; 3535 break; 3536 case IBDM_REQ_TYPE_IOUINFO: 3537 hdr->AttributeID = h2b16(IB_DM_ATTR_IO_UNITINFO); 3538 hdr->AttributeModifier = 0; 3539 timeout_id = &gid_info->gl_timeout_id; 3540 break; 3541 case IBDM_REQ_TYPE_IOCINFO: 3542 hdr->AttributeID = h2b16(IB_DM_ATTR_IOC_CTRL_PROFILE); 3543 hdr->AttributeModifier = h2b32(cb_args->cb_ioc_num + 1); 3544 ioc = IBDM_GIDINFO2IOCINFO(gid_info, cb_args->cb_ioc_num); 3545 timeout_id = &ioc->ioc_timeout_id; 3546 break; 3547 case IBDM_REQ_TYPE_SRVENTS: 3548 hdr->AttributeID = h2b16(IB_DM_ATTR_SERVICE_ENTRIES); 3549 ibdm_fill_srv_attr_mod(hdr, cb_args); 3550 ioc = IBDM_GIDINFO2IOCINFO(gid_info, cb_args->cb_ioc_num); 3551 timeout_id = 3552 &ioc->ioc_serv[cb_args->cb_srvents_start].se_timeout_id; 3553 break; 3554 case IBDM_REQ_TYPE_IOU_DIAGCODE: 3555 hdr->AttributeID = h2b16(IB_DM_ATTR_DIAG_CODE); 3556 hdr->AttributeModifier = 0; 3557 timeout_id = &gid_info->gl_timeout_id; 3558 break; 3559 case IBDM_REQ_TYPE_IOC_DIAGCODE: 3560 hdr->AttributeID = h2b16(IB_DM_ATTR_DIAG_CODE); 3561 hdr->AttributeModifier = h2b32(cb_args->cb_ioc_num + 1); 3562 ioc_no = cb_args->cb_ioc_num; 3563 ioc = &gid_info->gl_iou->iou_ioc_info[ioc_no]; 3564 timeout_id = &ioc->ioc_dc_timeout_id; 3565 break; 3566 } 3567 3568 mutex_exit(&gid_info->gl_mutex); 3569 3570 *timeout_id = timeout(ibdm_pkt_timeout_hdlr, 3571 cb_args, IBDM_TIMEOUT_VALUE(ibdm_dft_timeout)); 3572 3573 IBTF_DPRINTF_L5("ibdm", "\tretry_command: %p,%x,%d,%d:" 3574 "timeout %x", cb_args->cb_req_type, cb_args->cb_ioc_num, 3575 cb_args->cb_srvents_start, *timeout_id); 3576 3577 if (ibmf_msg_transport(gid_info->gl_ibmf_hdl, 3578 gid_info->gl_qp_hdl, msg, NULL, ibdm_ibmf_send_cb, 3579 cb_args, 0) != IBMF_SUCCESS) { 3580 IBTF_DPRINTF_L2("ibdm", "\tretry_command: send failed: %p " 3581 "rtype 0x%x iocidx 0x%x srvidx %d", cb_args->cb_gid_info, 3582 cb_args->cb_req_type, cb_args->cb_ioc_num, 3583 cb_args->cb_srvents_start); 3584 ibdm_ibmf_send_cb(gid_info->gl_ibmf_hdl, msg, cb_args); 3585 } 3586 mutex_enter(&gid_info->gl_mutex); 3587 return (IBDM_SUCCESS); 3588 } 3589 3590 3591 /* 3592 * ibdm_update_ioc_port_gidlist() 3593 */ 3594 static void 3595 ibdm_update_ioc_port_gidlist(ibdm_ioc_info_t *dest, 3596 ibdm_dp_gidinfo_t *gid_info) 3597 { 3598 int ii, ngid_ents; 3599 ibdm_gid_t *tmp; 3600 ibdm_hca_list_t *gid_hca_head, *temp; 3601 ibdm_hca_list_t *ioc_head = NULL; 3602 3603 IBTF_DPRINTF_L5("ibdm", "\tupdate_ioc_port_gidlist: Enter"); 3604 3605 ngid_ents = gid_info->gl_ngids; 3606 dest->ioc_nportgids = ngid_ents; 3607 dest->ioc_gid_list = kmem_zalloc(sizeof (ibdm_gid_t) * 3608 ngid_ents, KM_SLEEP); 3609 tmp = gid_info->gl_gid; 3610 for (ii = 0; (ii < ngid_ents) && (tmp); ii++) { 3611 dest->ioc_gid_list[ii].gid_dgid_hi = tmp->gid_dgid_hi; 3612 dest->ioc_gid_list[ii].gid_dgid_lo = tmp->gid_dgid_lo; 3613 tmp = tmp->gid_next; 3614 } 3615 3616 gid_hca_head = gid_info->gl_hca_list; 3617 while (gid_hca_head) { 3618 temp = ibdm_dup_hca_attr(gid_hca_head); 3619 temp->hl_next = ioc_head; 3620 ioc_head = temp; 3621 gid_hca_head = gid_hca_head->hl_next; 3622 } 3623 dest->ioc_hca_list = ioc_head; 3624 } 3625 3626 3627 /* 3628 * ibdm_alloc_send_buffers() 3629 * Allocates memory for the IBMF send buffer 3630 */ 3631 static void 3632 ibdm_alloc_send_buffers(ibmf_msg_t *msgp) 3633 { 3634 msgp->im_msgbufs_send.im_bufs_mad_hdr = 3635 kmem_zalloc(IBDM_MAD_SIZE, KM_SLEEP); 3636 msgp->im_msgbufs_send.im_bufs_cl_data = (uchar_t *) 3637 msgp->im_msgbufs_send.im_bufs_mad_hdr + sizeof (ib_mad_hdr_t); 3638 msgp->im_msgbufs_send.im_bufs_cl_data_len = 3639 IBDM_MAD_SIZE - sizeof (ib_mad_hdr_t); 3640 } 3641 3642 3643 /* 3644 * ibdm_alloc_send_buffers() 3645 * De-allocates memory for the IBMF send buffer 3646 */ 3647 static void 3648 ibdm_free_send_buffers(ibmf_msg_t *msgp) 3649 { 3650 if (msgp->im_msgbufs_send.im_bufs_mad_hdr != NULL) 3651 kmem_free(msgp->im_msgbufs_send.im_bufs_mad_hdr, IBDM_MAD_SIZE); 3652 } 3653 3654 /* 3655 * ibdm_probe_ioc() 3656 * 1. Gets the node records for the port GUID. This detects all the port 3657 * to the IOU. 3658 * 2. Selectively probes all the IOC, given it's node GUID 3659 * 3. In case of reprobe, only the IOC to be reprobed is send the IOC 3660 * Controller Profile asynchronously 3661 */ 3662 /*ARGSUSED*/ 3663 static void 3664 ibdm_probe_ioc(ib_guid_t nodeguid, ib_guid_t ioc_guid, int reprobe_flag) 3665 { 3666 int ii, nrecords; 3667 size_t nr_len = 0, pi_len = 0; 3668 ib_gid_t sgid, dgid; 3669 ibdm_hca_list_t *hca_list = NULL; 3670 sa_node_record_t *nr, *tmp; 3671 ibdm_port_attr_t *port = NULL; 3672 ibdm_dp_gidinfo_t *reprobe_gid, *new_gid, *node_gid; 3673 ibdm_dp_gidinfo_t *temp_gidinfo; 3674 ibdm_gid_t *temp_gid; 3675 sa_portinfo_record_t *pi; 3676 3677 IBTF_DPRINTF_L4("ibdm", "\tprobe_ioc(%llx, %llx, %x): Begin", 3678 nodeguid, ioc_guid, reprobe_flag); 3679 3680 /* Rescan the GID list for any removed GIDs for reprobe */ 3681 if (reprobe_flag) 3682 ibdm_rescan_gidlist(&ioc_guid); 3683 3684 mutex_enter(&ibdm.ibdm_hl_mutex); 3685 for (ibdm_get_next_port(&hca_list, &port, 1); port; 3686 ibdm_get_next_port(&hca_list, &port, 1)) { 3687 reprobe_gid = new_gid = node_gid = NULL; 3688 3689 nr = ibdm_get_node_records(port->pa_sa_hdl, &nr_len, nodeguid); 3690 if (nr == NULL) { 3691 IBTF_DPRINTF_L4("ibdm", "\tprobe_ioc: no records"); 3692 continue; 3693 } 3694 nrecords = (nr_len / sizeof (sa_node_record_t)); 3695 for (tmp = nr, ii = 0; (ii < nrecords); ii++, tmp++) { 3696 pi = ibdm_get_portinfo( 3697 port->pa_sa_hdl, &pi_len, tmp->LID); 3698 3699 if ((pi) && (pi->PortInfo.CapabilityMask & 3700 SM_CAP_MASK_IS_DM_SUPPD)) { 3701 /* 3702 * For reprobes: Check if GID, already in 3703 * the list. If so, set the state to SKIPPED 3704 */ 3705 if (((temp_gidinfo = ibdm_find_gid(nodeguid, 3706 tmp->NodeInfo.PortGUID)) != NULL) && 3707 temp_gidinfo->gl_state == 3708 IBDM_GID_PROBING_COMPLETE) { 3709 ASSERT(reprobe_gid == NULL); 3710 ibdm_addto_glhcalist(temp_gidinfo, 3711 hca_list); 3712 reprobe_gid = temp_gidinfo; 3713 kmem_free(pi, pi_len); 3714 continue; 3715 } else if (temp_gidinfo != NULL) { 3716 kmem_free(pi, pi_len); 3717 ibdm_addto_glhcalist(temp_gidinfo, 3718 hca_list); 3719 continue; 3720 } 3721 3722 IBTF_DPRINTF_L4("ibdm", "\tprobe_ioc : " 3723 "create_gid : prefix %llx, guid %llx\n", 3724 pi->PortInfo.GidPrefix, 3725 tmp->NodeInfo.PortGUID); 3726 3727 sgid.gid_prefix = port->pa_sn_prefix; 3728 sgid.gid_guid = port->pa_port_guid; 3729 dgid.gid_prefix = pi->PortInfo.GidPrefix; 3730 dgid.gid_guid = tmp->NodeInfo.PortGUID; 3731 new_gid = ibdm_create_gid_info(port, sgid, 3732 dgid); 3733 if (new_gid == NULL) { 3734 IBTF_DPRINTF_L2("ibdm", "\tprobe_ioc: " 3735 "create_gid_info failed\n"); 3736 kmem_free(pi, pi_len); 3737 continue; 3738 } 3739 if (node_gid == NULL) { 3740 node_gid = new_gid; 3741 ibdm_add_to_gl_gid(node_gid, node_gid); 3742 } else { 3743 IBTF_DPRINTF_L4("ibdm", 3744 "\tprobe_ioc: new gid"); 3745 temp_gid = kmem_zalloc( 3746 sizeof (ibdm_gid_t), KM_SLEEP); 3747 temp_gid->gid_dgid_hi = 3748 new_gid->gl_dgid_hi; 3749 temp_gid->gid_dgid_lo = 3750 new_gid->gl_dgid_lo; 3751 temp_gid->gid_next = node_gid->gl_gid; 3752 node_gid->gl_gid = temp_gid; 3753 node_gid->gl_ngids++; 3754 } 3755 new_gid->gl_nodeguid = nodeguid; 3756 new_gid->gl_portguid = dgid.gid_guid; 3757 ibdm_addto_glhcalist(new_gid, hca_list); 3758 3759 /* 3760 * Set the state to skipped as all these 3761 * gids point to the same node. 3762 * We (re)probe only one GID below and reset 3763 * state appropriately 3764 */ 3765 new_gid->gl_state = IBDM_GID_PROBING_SKIPPED; 3766 kmem_free(pi, pi_len); 3767 } 3768 } 3769 kmem_free(nr, nr_len); 3770 3771 IBTF_DPRINTF_L4("ibdm", "\tprobe_ioc : reprobe_flag %d " 3772 "reprobe_gid %p new_gid %p node_gid %p", 3773 reprobe_flag, reprobe_gid, new_gid, node_gid); 3774 3775 if (reprobe_flag != 0 && reprobe_gid != NULL) { 3776 int niocs, jj; 3777 ibdm_ioc_info_t *tmp_ioc; 3778 int ioc_matched = 0; 3779 3780 mutex_exit(&ibdm.ibdm_hl_mutex); 3781 mutex_enter(&reprobe_gid->gl_mutex); 3782 reprobe_gid->gl_state = IBDM_GET_IOC_DETAILS; 3783 niocs = 3784 reprobe_gid->gl_iou->iou_info.iou_num_ctrl_slots; 3785 reprobe_gid->gl_pending_cmds++; 3786 mutex_exit(&reprobe_gid->gl_mutex); 3787 3788 for (jj = 0; jj < niocs; jj++) { 3789 tmp_ioc = 3790 IBDM_GIDINFO2IOCINFO(reprobe_gid, jj); 3791 if (tmp_ioc->ioc_profile.ioc_guid != ioc_guid) 3792 continue; 3793 3794 ioc_matched = 1; 3795 3796 /* 3797 * Explicitly set gl_reprobe_flag to 0 so that 3798 * IBnex is not notified on completion 3799 */ 3800 mutex_enter(&reprobe_gid->gl_mutex); 3801 reprobe_gid->gl_reprobe_flag = 0; 3802 mutex_exit(&reprobe_gid->gl_mutex); 3803 3804 mutex_enter(&ibdm.ibdm_mutex); 3805 ibdm.ibdm_ngid_probes_in_progress++; 3806 mutex_exit(&ibdm.ibdm_mutex); 3807 if (ibdm_send_ioc_profile(reprobe_gid, jj) != 3808 IBDM_SUCCESS) { 3809 IBTF_DPRINTF_L4("ibdm", 3810 "\tprobe_ioc: " 3811 "send_ioc_profile failed " 3812 "for ioc %d", jj); 3813 ibdm_gid_decr_pending(reprobe_gid); 3814 break; 3815 } 3816 mutex_enter(&ibdm.ibdm_mutex); 3817 ibdm_wait_probe_completion(); 3818 mutex_exit(&ibdm.ibdm_mutex); 3819 break; 3820 } 3821 if (ioc_matched == 0) 3822 ibdm_gid_decr_pending(reprobe_gid); 3823 else { 3824 mutex_enter(&ibdm.ibdm_hl_mutex); 3825 break; 3826 } 3827 } else if (new_gid != NULL) { 3828 mutex_exit(&ibdm.ibdm_hl_mutex); 3829 node_gid = node_gid ? node_gid : new_gid; 3830 3831 /* 3832 * New or reinserted GID : Enable notification 3833 * to IBnex 3834 */ 3835 mutex_enter(&node_gid->gl_mutex); 3836 node_gid->gl_reprobe_flag = 1; 3837 mutex_exit(&node_gid->gl_mutex); 3838 3839 ibdm_probe_gid(node_gid); 3840 3841 mutex_enter(&ibdm.ibdm_hl_mutex); 3842 } 3843 } 3844 mutex_exit(&ibdm.ibdm_hl_mutex); 3845 IBTF_DPRINTF_L4("ibdm", "\tprobe_ioc : End\n"); 3846 } 3847 3848 3849 /* 3850 * ibdm_probe_gid() 3851 * Selectively probes the GID 3852 */ 3853 static void 3854 ibdm_probe_gid(ibdm_dp_gidinfo_t *gid_info) 3855 { 3856 IBTF_DPRINTF_L4("ibdm", "\tprobe_gid:"); 3857 mutex_enter(&gid_info->gl_mutex); 3858 gid_info->gl_pending_cmds++; 3859 gid_info->gl_state = IBDM_GET_CLASSPORTINFO; 3860 mutex_exit(&gid_info->gl_mutex); 3861 if (ibdm_send_classportinfo(gid_info) != IBDM_SUCCESS) { 3862 mutex_enter(&gid_info->gl_mutex); 3863 gid_info->gl_state = IBDM_GID_PROBING_FAILED; 3864 --gid_info->gl_pending_cmds; 3865 mutex_exit(&gid_info->gl_mutex); 3866 ibdm_delete_glhca_list(gid_info); 3867 gid_info = gid_info->gl_next; 3868 return; 3869 } 3870 mutex_enter(&ibdm.ibdm_mutex); 3871 ibdm.ibdm_ngid_probes_in_progress++; 3872 gid_info = gid_info->gl_next; 3873 3874 ibdm_wait_probe_completion(); 3875 mutex_exit(&ibdm.ibdm_mutex); 3876 IBTF_DPRINTF_L4("ibdm", "\tprobe_gid: Wakeup signal received"); 3877 } 3878 3879 3880 /* 3881 * ibdm_create_gid_info() 3882 * Allocates a gid_info structure and initializes 3883 * Returns pointer to the structure on success 3884 * and NULL on failure 3885 */ 3886 static ibdm_dp_gidinfo_t * 3887 ibdm_create_gid_info(ibdm_port_attr_t *port, ib_gid_t sgid, ib_gid_t dgid) 3888 { 3889 uint8_t ii, npaths; 3890 sa_path_record_t *path; 3891 size_t len; 3892 ibdm_pkey_tbl_t *pkey_tbl; 3893 ibdm_dp_gidinfo_t *gid_info = NULL; 3894 int ret; 3895 3896 IBTF_DPRINTF_L4("ibdm", "\tcreate_gid_info: Begin"); 3897 npaths = 1; 3898 3899 /* query for reversible paths */ 3900 if (port->pa_sa_hdl) 3901 ret = ibmf_saa_gid_to_pathrecords(port->pa_sa_hdl, 3902 sgid, dgid, IBMF_SAA_PKEY_WC, 0, B_TRUE, &npaths, 0, 3903 &len, &path); 3904 else 3905 return (NULL); 3906 3907 if (ret == IBMF_SUCCESS && path) { 3908 ibdm_dump_path_info(path); 3909 3910 gid_info = kmem_zalloc( 3911 sizeof (ibdm_dp_gidinfo_t), KM_SLEEP); 3912 mutex_init(&gid_info->gl_mutex, NULL, MUTEX_DEFAULT, NULL); 3913 gid_info->gl_dgid_hi = path->DGID.gid_prefix; 3914 gid_info->gl_dgid_lo = path->DGID.gid_guid; 3915 gid_info->gl_sgid_hi = path->SGID.gid_prefix; 3916 gid_info->gl_sgid_lo = path->SGID.gid_guid; 3917 gid_info->gl_p_key = path->P_Key; 3918 gid_info->gl_sa_hdl = port->pa_sa_hdl; 3919 gid_info->gl_ibmf_hdl = port->pa_ibmf_hdl; 3920 gid_info->gl_slid = path->SLID; 3921 gid_info->gl_dlid = path->DLID; 3922 gid_info->gl_transactionID = (++ibdm.ibdm_transactionID) 3923 << IBDM_GID_TRANSACTIONID_SHIFT; 3924 gid_info->gl_min_transactionID = gid_info->gl_transactionID; 3925 gid_info->gl_max_transactionID = (ibdm.ibdm_transactionID +1) 3926 << IBDM_GID_TRANSACTIONID_SHIFT; 3927 3928 gid_info->gl_qp_hdl = IBMF_QP_HANDLE_DEFAULT; 3929 for (ii = 0; ii < port->pa_npkeys; ii++) { 3930 if (port->pa_pkey_tbl == NULL) 3931 break; 3932 3933 pkey_tbl = &port->pa_pkey_tbl[ii]; 3934 if ((gid_info->gl_p_key == pkey_tbl->pt_pkey) && 3935 (pkey_tbl->pt_qp_hdl != NULL)) { 3936 gid_info->gl_qp_hdl = pkey_tbl->pt_qp_hdl; 3937 break; 3938 } 3939 } 3940 kmem_free(path, len); 3941 3942 /* 3943 * QP handle for GID not initialized. No matching Pkey 3944 * was found!! ibdm should *not* hit this case. Flag an 3945 * error and drop the GID if ibdm does encounter this. 3946 */ 3947 if (gid_info->gl_qp_hdl == NULL) { 3948 IBTF_DPRINTF_L2(ibdm_string, 3949 "\tcreate_gid_info: No matching Pkey"); 3950 ibdm_delete_gidinfo(gid_info); 3951 return (NULL); 3952 } 3953 3954 ibdm.ibdm_ngids++; 3955 if (ibdm.ibdm_dp_gidlist_head == NULL) { 3956 ibdm.ibdm_dp_gidlist_head = gid_info; 3957 ibdm.ibdm_dp_gidlist_tail = gid_info; 3958 } else { 3959 ibdm.ibdm_dp_gidlist_tail->gl_next = gid_info; 3960 gid_info->gl_prev = ibdm.ibdm_dp_gidlist_tail; 3961 ibdm.ibdm_dp_gidlist_tail = gid_info; 3962 } 3963 } 3964 3965 return (gid_info); 3966 } 3967 3968 3969 /* 3970 * ibdm_get_node_records 3971 * Sends a SA query to get the NODE record 3972 * Returns pointer to the sa_node_record_t on success 3973 * and NULL on failure 3974 */ 3975 static sa_node_record_t * 3976 ibdm_get_node_records(ibmf_saa_handle_t sa_hdl, size_t *length, ib_guid_t guid) 3977 { 3978 sa_node_record_t req, *resp = NULL; 3979 ibmf_saa_access_args_t args; 3980 int ret; 3981 3982 IBTF_DPRINTF_L4("ibdm", "\tget_node_records: Begin"); 3983 3984 bzero(&req, sizeof (sa_node_record_t)); 3985 req.NodeInfo.NodeGUID = guid; 3986 3987 args.sq_attr_id = SA_NODERECORD_ATTRID; 3988 args.sq_access_type = IBMF_SAA_RETRIEVE; 3989 args.sq_component_mask = SA_NODEINFO_COMPMASK_NODEGUID; 3990 args.sq_template = &req; 3991 args.sq_callback = NULL; 3992 args.sq_callback_arg = NULL; 3993 3994 ret = ibmf_sa_access(sa_hdl, &args, 0, length, (void **) &resp); 3995 if (ret != IBMF_SUCCESS) { 3996 IBTF_DPRINTF_L2("ibdm", "\tget_node_records:" 3997 " SA Retrieve Failed: %d", ret); 3998 return (NULL); 3999 } 4000 if ((resp == NULL) || (*length == 0)) { 4001 IBTF_DPRINTF_L2("ibdm", "\tget_node_records: No records"); 4002 return (NULL); 4003 } 4004 4005 IBTF_DPRINTF_L4("ibdm", "\tget_node_records: NodeGuid %llx " 4006 "PortGUID %llx", resp->NodeInfo.NodeGUID, resp->NodeInfo.PortGUID); 4007 4008 return (resp); 4009 } 4010 4011 4012 /* 4013 * ibdm_get_portinfo() 4014 * Sends a SA query to get the PortInfo record 4015 * Returns pointer to the sa_portinfo_record_t on success 4016 * and NULL on failure 4017 */ 4018 static sa_portinfo_record_t * 4019 ibdm_get_portinfo(ibmf_saa_handle_t sa_hdl, size_t *length, ib_lid_t lid) 4020 { 4021 sa_portinfo_record_t req, *resp = NULL; 4022 ibmf_saa_access_args_t args; 4023 int ret; 4024 4025 IBTF_DPRINTF_L4("ibdm", "\tget_portinfo: Begin"); 4026 4027 bzero(&req, sizeof (sa_portinfo_record_t)); 4028 req.EndportLID = lid; 4029 4030 args.sq_attr_id = SA_PORTINFORECORD_ATTRID; 4031 args.sq_access_type = IBMF_SAA_RETRIEVE; 4032 args.sq_component_mask = SA_PORTINFO_COMPMASK_PORTLID; 4033 args.sq_template = &req; 4034 args.sq_callback = NULL; 4035 args.sq_callback_arg = NULL; 4036 4037 ret = ibmf_sa_access(sa_hdl, &args, 0, length, (void **) &resp); 4038 if (ret != IBMF_SUCCESS) { 4039 IBTF_DPRINTF_L2("ibdm", "\tget_portinfo:" 4040 " SA Retrieve Failed: 0x%X", ret); 4041 return (NULL); 4042 } 4043 if ((*length == 0) || (resp == NULL)) 4044 return (NULL); 4045 4046 IBTF_DPRINTF_L4("ibdm", "\tget_portinfo: GidPrefix %llx Cap 0x%x", 4047 resp->PortInfo.GidPrefix, resp->PortInfo.CapabilityMask); 4048 return (resp); 4049 } 4050 4051 4052 /* 4053 * ibdm_ibnex_register_callback 4054 * IB nexus callback routine for HCA attach and detach notification 4055 */ 4056 void 4057 ibdm_ibnex_register_callback(ibdm_callback_t ibnex_dm_callback) 4058 { 4059 IBTF_DPRINTF_L4("ibdm", "\tibnex_register_callbacks"); 4060 mutex_enter(&ibdm.ibdm_ibnex_mutex); 4061 ibdm.ibdm_ibnex_callback = ibnex_dm_callback; 4062 mutex_exit(&ibdm.ibdm_ibnex_mutex); 4063 } 4064 4065 4066 /* 4067 * ibdm_ibnex_unregister_callbacks 4068 */ 4069 void 4070 ibdm_ibnex_unregister_callback() 4071 { 4072 IBTF_DPRINTF_L4("ibdm", "\tibnex_unregister_callbacks"); 4073 mutex_enter(&ibdm.ibdm_ibnex_mutex); 4074 ibdm.ibdm_ibnex_callback = NULL; 4075 mutex_exit(&ibdm.ibdm_ibnex_mutex); 4076 } 4077 4078 4079 /* 4080 * ibdm_ibnex_get_waittime() 4081 * Calculates the wait time based on the last HCA attach time 4082 */ 4083 time_t 4084 ibdm_ibnex_get_waittime(ib_guid_t hca_guid, int *dft_wait) 4085 { 4086 int ii; 4087 time_t temp, wait_time = 0; 4088 ibdm_hca_list_t *hca; 4089 4090 IBTF_DPRINTF_L4("ibdm", "\tibnex_get_waittime hcaguid:%llx" 4091 "\tport settling time %d", hca_guid, *dft_wait); 4092 4093 mutex_enter(&ibdm.ibdm_hl_mutex); 4094 hca = ibdm.ibdm_hca_list_head; 4095 4096 if (hca_guid) { 4097 for (ii = 0; ii < ibdm.ibdm_hca_count; ii++) { 4098 if ((hca_guid == hca->hl_hca_guid) && 4099 (hca->hl_nports != hca->hl_nports_active)) { 4100 wait_time = 4101 ddi_get_time() - hca->hl_attach_time; 4102 wait_time = ((wait_time >= *dft_wait) ? 4103 0 : (*dft_wait - wait_time)); 4104 break; 4105 } 4106 hca = hca->hl_next; 4107 } 4108 mutex_exit(&ibdm.ibdm_hl_mutex); 4109 IBTF_DPRINTF_L4("ibdm", "\tibnex_get_waittime %llx", wait_time); 4110 return (wait_time); 4111 } 4112 4113 for (ii = 0; ii < ibdm.ibdm_hca_count; ii++) { 4114 if (hca->hl_nports != hca->hl_nports_active) { 4115 temp = ddi_get_time() - hca->hl_attach_time; 4116 temp = ((temp >= *dft_wait) ? 0 : (*dft_wait - temp)); 4117 wait_time = (temp > wait_time) ? temp : wait_time; 4118 } 4119 } 4120 mutex_exit(&ibdm.ibdm_hl_mutex); 4121 IBTF_DPRINTF_L4("ibdm", "\tibnex_get_waittime %llx", wait_time); 4122 return (wait_time); 4123 } 4124 4125 4126 /* 4127 * ibdm_ibnex_probe_hcaport 4128 * Probes the presence of HCA port (with HCA dip and port number) 4129 * Returns port attributes structure on SUCCESS 4130 */ 4131 ibdm_port_attr_t * 4132 ibdm_ibnex_probe_hcaport(ib_guid_t hca_guid, uint8_t port_num) 4133 { 4134 int ii, jj; 4135 ibdm_hca_list_t *hca_list; 4136 ibdm_port_attr_t *port_attr; 4137 4138 IBTF_DPRINTF_L4("ibdm", "\tibnex_probe_hcaport:"); 4139 4140 mutex_enter(&ibdm.ibdm_hl_mutex); 4141 hca_list = ibdm.ibdm_hca_list_head; 4142 for (ii = 0; ii < ibdm.ibdm_hca_count; ii++) { 4143 if (hca_list->hl_hca_guid == hca_guid) { 4144 for (jj = 0; jj < hca_list->hl_nports; jj++) { 4145 if (hca_list->hl_port_attr[jj].pa_port_num == 4146 port_num) { 4147 break; 4148 } 4149 } 4150 if (jj != hca_list->hl_nports) 4151 break; 4152 } 4153 hca_list = hca_list->hl_next; 4154 } 4155 if (ii == ibdm.ibdm_hca_count) { 4156 IBTF_DPRINTF_L2("ibdm", "\tibnex_probe_hcaport: not found"); 4157 mutex_exit(&ibdm.ibdm_hl_mutex); 4158 return (NULL); 4159 } 4160 port_attr = (ibdm_port_attr_t *)kmem_zalloc( 4161 sizeof (ibdm_port_attr_t), KM_SLEEP); 4162 bcopy((char *)&hca_list->hl_port_attr[jj], 4163 port_attr, sizeof (ibdm_port_attr_t)); 4164 ibdm_update_port_attr(port_attr); 4165 4166 mutex_exit(&ibdm.ibdm_hl_mutex); 4167 return (port_attr); 4168 } 4169 4170 4171 /* 4172 * ibdm_ibnex_get_port_attrs 4173 * Scan all HCAs for a matching port_guid. 4174 * Returns "port attributes" structure on success. 4175 */ 4176 ibdm_port_attr_t * 4177 ibdm_ibnex_get_port_attrs(ib_guid_t port_guid) 4178 { 4179 int ii, jj; 4180 ibdm_hca_list_t *hca_list; 4181 ibdm_port_attr_t *port_attr; 4182 4183 IBTF_DPRINTF_L4("ibdm", "\tibnex_get_port_attrs:"); 4184 4185 mutex_enter(&ibdm.ibdm_hl_mutex); 4186 hca_list = ibdm.ibdm_hca_list_head; 4187 4188 for (ii = 0; ii < ibdm.ibdm_hca_count; ii++) { 4189 for (jj = 0; jj < hca_list->hl_nports; jj++) { 4190 if (hca_list->hl_port_attr[jj].pa_port_guid == 4191 port_guid) { 4192 break; 4193 } 4194 } 4195 if (jj != hca_list->hl_nports) 4196 break; 4197 hca_list = hca_list->hl_next; 4198 } 4199 4200 if (ii == ibdm.ibdm_hca_count) { 4201 IBTF_DPRINTF_L2("ibdm", "\tibnex_get_port_attrs: not found"); 4202 mutex_exit(&ibdm.ibdm_hl_mutex); 4203 return (NULL); 4204 } 4205 4206 port_attr = (ibdm_port_attr_t *)kmem_alloc(sizeof (ibdm_port_attr_t), 4207 KM_SLEEP); 4208 bcopy((char *)&hca_list->hl_port_attr[jj], port_attr, 4209 sizeof (ibdm_port_attr_t)); 4210 ibdm_update_port_attr(port_attr); 4211 4212 mutex_exit(&ibdm.ibdm_hl_mutex); 4213 return (port_attr); 4214 } 4215 4216 4217 /* 4218 * ibdm_ibnex_free_port_attr() 4219 */ 4220 void 4221 ibdm_ibnex_free_port_attr(ibdm_port_attr_t *port_attr) 4222 { 4223 IBTF_DPRINTF_L4("ibdm", "\tibnex_free_port_attr:"); 4224 if (port_attr) { 4225 if (port_attr->pa_pkey_tbl != NULL) { 4226 kmem_free(port_attr->pa_pkey_tbl, 4227 (port_attr->pa_npkeys * sizeof (ibdm_pkey_tbl_t))); 4228 } 4229 kmem_free(port_attr, sizeof (ibdm_port_attr_t)); 4230 } 4231 } 4232 4233 4234 /* 4235 * ibdm_ibnex_get_hca_list() 4236 * Returns portinfo for all the port for all the HCA's 4237 */ 4238 void 4239 ibdm_ibnex_get_hca_list(ibdm_hca_list_t **hca, int *count) 4240 { 4241 ibdm_hca_list_t *head = NULL, *temp, *temp1; 4242 int ii; 4243 4244 IBTF_DPRINTF_L4("ibdm", "\tibnex_get_hca_list:"); 4245 4246 mutex_enter(&ibdm.ibdm_hl_mutex); 4247 temp = ibdm.ibdm_hca_list_head; 4248 for (ii = 0; ii < ibdm.ibdm_hca_count; ii++) { 4249 temp1 = ibdm_dup_hca_attr(temp); 4250 temp1->hl_next = head; 4251 head = temp1; 4252 temp = temp->hl_next; 4253 } 4254 *count = ibdm.ibdm_hca_count; 4255 *hca = head; 4256 mutex_exit(&ibdm.ibdm_hl_mutex); 4257 } 4258 4259 4260 /* 4261 * ibdm_ibnex_get_hca_info_by_guid() 4262 */ 4263 ibdm_hca_list_t * 4264 ibdm_ibnex_get_hca_info_by_guid(ib_guid_t hca_guid) 4265 { 4266 ibdm_hca_list_t *head = NULL, *hca = NULL; 4267 4268 IBTF_DPRINTF_L4("ibdm", "\tibnex_get_hca_info_by_dip"); 4269 4270 mutex_enter(&ibdm.ibdm_hl_mutex); 4271 head = ibdm.ibdm_hca_list_head; 4272 while (head) { 4273 if (head->hl_hca_guid == hca_guid) { 4274 hca = ibdm_dup_hca_attr(head); 4275 hca->hl_next = NULL; 4276 break; 4277 } 4278 head = head->hl_next; 4279 } 4280 mutex_exit(&ibdm.ibdm_hl_mutex); 4281 IBTF_DPRINTF_L4("ibdm", "\tibnex_get_hca_info_by_dip %p", hca); 4282 return (hca); 4283 } 4284 4285 4286 /* 4287 * ibdm_dup_hca_attr() 4288 * Allocate a new HCA attribute strucuture and initialize 4289 * hca attribute structure with the incoming HCA attributes 4290 * returned the allocated hca attributes. 4291 */ 4292 static ibdm_hca_list_t * 4293 ibdm_dup_hca_attr(ibdm_hca_list_t *in_hca) 4294 { 4295 int len; 4296 ibdm_hca_list_t *out_hca; 4297 4298 len = sizeof (ibdm_hca_list_t) + 4299 (in_hca->hl_nports * sizeof (ibdm_port_attr_t)); 4300 IBTF_DPRINTF_L4("ibdm", "\tdup_hca_attr len %d", len); 4301 out_hca = (ibdm_hca_list_t *)kmem_alloc(len, KM_SLEEP); 4302 bcopy((char *)in_hca, 4303 (char *)out_hca, sizeof (ibdm_hca_list_t)); 4304 if (in_hca->hl_nports) { 4305 out_hca->hl_port_attr = (ibdm_port_attr_t *) 4306 ((char *)out_hca + sizeof (ibdm_hca_list_t)); 4307 bcopy((char *)in_hca->hl_port_attr, 4308 (char *)out_hca->hl_port_attr, 4309 (in_hca->hl_nports * sizeof (ibdm_port_attr_t))); 4310 for (len = 0; len < out_hca->hl_nports; len++) 4311 ibdm_update_port_attr(&out_hca->hl_port_attr[len]); 4312 } 4313 return (out_hca); 4314 } 4315 4316 4317 /* 4318 * ibdm_ibnex_free_hca_list() 4319 * Free one/more HCA lists 4320 */ 4321 void 4322 ibdm_ibnex_free_hca_list(ibdm_hca_list_t *hca_list) 4323 { 4324 int ii; 4325 size_t len; 4326 ibdm_hca_list_t *temp; 4327 ibdm_port_attr_t *port; 4328 4329 IBTF_DPRINTF_L4("ibdm", "\tibnex_free_hca_list:"); 4330 ASSERT(hca_list); 4331 while (hca_list) { 4332 temp = hca_list; 4333 hca_list = hca_list->hl_next; 4334 for (ii = 0; ii < temp->hl_nports; ii++) { 4335 port = &temp->hl_port_attr[ii]; 4336 len = (port->pa_npkeys * sizeof (ibdm_pkey_tbl_t)); 4337 if (len != 0) 4338 kmem_free(port->pa_pkey_tbl, len); 4339 } 4340 len = sizeof (ibdm_hca_list_t) + (temp->hl_nports * 4341 sizeof (ibdm_port_attr_t)); 4342 kmem_free(temp, len); 4343 } 4344 } 4345 4346 4347 /* 4348 * ibdm_ibnex_probe_iocguid() 4349 * Probes the IOC on the fabric and returns the IOC information 4350 * if present. Otherwise, NULL is returned 4351 */ 4352 /* ARGSUSED */ 4353 ibdm_ioc_info_t * 4354 ibdm_ibnex_probe_ioc(ib_guid_t iou, ib_guid_t ioc_guid, int reprobe_flag) 4355 { 4356 int k; 4357 ibdm_ioc_info_t *ioc_info; 4358 ibdm_dp_gidinfo_t *gid_info; 4359 4360 IBTF_DPRINTF_L4("ibdm", "\tibnex_probe_ioc: (%llX, %llX, %d) Begin", 4361 iou, ioc_guid, reprobe_flag); 4362 /* Check whether we know this already */ 4363 ioc_info = ibdm_ibnex_get_ioc_info(ioc_guid); 4364 if (ioc_info == NULL) { 4365 mutex_enter(&ibdm.ibdm_mutex); 4366 while (ibdm.ibdm_busy & IBDM_BUSY) 4367 cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex); 4368 ibdm.ibdm_busy |= IBDM_BUSY; 4369 mutex_exit(&ibdm.ibdm_mutex); 4370 ibdm_probe_ioc(iou, ioc_guid, 0); 4371 mutex_enter(&ibdm.ibdm_mutex); 4372 ibdm.ibdm_busy &= ~IBDM_BUSY; 4373 cv_broadcast(&ibdm.ibdm_busy_cv); 4374 mutex_exit(&ibdm.ibdm_mutex); 4375 ioc_info = ibdm_ibnex_get_ioc_info(ioc_guid); 4376 } else if (reprobe_flag) { /* Handle Reprobe for the IOC */ 4377 /* Free the ioc_list before reprobe; and cancel any timers */ 4378 mutex_enter(&ibdm.ibdm_mutex); 4379 if (ioc_info->ioc_timeout_id) { 4380 IBTF_DPRINTF_L5("ibdm", "\tprobe_ioc: " 4381 "ioc_timeout_id = 0x%x", 4382 ioc_info->ioc_timeout_id); 4383 if (untimeout(ioc_info->ioc_timeout_id) == -1) { 4384 IBTF_DPRINTF_L2("ibdm", "\tprobe_ioc: " 4385 "untimeout ioc_timeout_id failed"); 4386 } 4387 ioc_info->ioc_timeout_id = 0; 4388 } 4389 if (ioc_info->ioc_dc_timeout_id) { 4390 IBTF_DPRINTF_L5("ibdm", "\tprobe_ioc: " 4391 "ioc_dc_timeout_id = 0x%x", 4392 ioc_info->ioc_dc_timeout_id); 4393 if (untimeout(ioc_info->ioc_dc_timeout_id) == -1) { 4394 IBTF_DPRINTF_L2("ibdm", "\tprobe_ioc: " 4395 "untimeout ioc_dc_timeout_id failed"); 4396 } 4397 ioc_info->ioc_dc_timeout_id = 0; 4398 } 4399 for (k = 0; k < ioc_info->ioc_profile.ioc_service_entries; k++) 4400 if (ioc_info->ioc_serv[k].se_timeout_id) { 4401 IBTF_DPRINTF_L5("ibdm", "\tprobe_ioc: " 4402 "ioc_info->ioc_serv[k].se_timeout_id = %x", 4403 k, ioc_info->ioc_serv[k].se_timeout_id); 4404 if (untimeout(ioc_info->ioc_serv[k]. 4405 se_timeout_id) == -1) { 4406 IBTF_DPRINTF_L2("ibdm", "\tprobe_ioc: " 4407 "untimeout se_timeout_id %d " 4408 "failed", k); 4409 } 4410 ioc_info->ioc_serv[k].se_timeout_id = 0; 4411 } 4412 mutex_exit(&ibdm.ibdm_mutex); 4413 ibdm_ibnex_free_ioc_list(ioc_info); 4414 4415 mutex_enter(&ibdm.ibdm_mutex); 4416 while (ibdm.ibdm_busy & IBDM_BUSY) 4417 cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex); 4418 ibdm.ibdm_busy |= IBDM_BUSY; 4419 mutex_exit(&ibdm.ibdm_mutex); 4420 4421 ibdm_probe_ioc(iou, ioc_guid, 1); 4422 4423 /* 4424 * Skip if gl_reprobe_flag is set, this will be 4425 * a re-inserted / new GID, for which notifications 4426 * have already been send. 4427 */ 4428 for (gid_info = ibdm.ibdm_dp_gidlist_head; gid_info; 4429 gid_info = gid_info->gl_next) { 4430 uint8_t ii, niocs; 4431 ibdm_ioc_info_t *ioc; 4432 4433 if (gid_info->gl_iou == NULL) 4434 continue; 4435 4436 if (gid_info->gl_reprobe_flag) { 4437 gid_info->gl_reprobe_flag = 0; 4438 continue; 4439 } 4440 4441 niocs = gid_info->gl_iou->iou_info.iou_num_ctrl_slots; 4442 for (ii = 0; ii < niocs; ii++) { 4443 ioc = IBDM_GIDINFO2IOCINFO(gid_info, ii); 4444 if (ioc->ioc_profile.ioc_guid == ioc_guid) { 4445 mutex_enter(&ibdm.ibdm_mutex); 4446 ibdm_reprobe_update_port_srv(ioc, 4447 gid_info); 4448 mutex_exit(&ibdm.ibdm_mutex); 4449 } 4450 } 4451 } 4452 mutex_enter(&ibdm.ibdm_mutex); 4453 ibdm.ibdm_busy &= ~IBDM_BUSY; 4454 cv_broadcast(&ibdm.ibdm_busy_cv); 4455 mutex_exit(&ibdm.ibdm_mutex); 4456 4457 ioc_info = ibdm_ibnex_get_ioc_info(ioc_guid); 4458 } 4459 return (ioc_info); 4460 } 4461 4462 4463 /* 4464 * ibdm_ibnex_get_ioc_info() 4465 * Returns pointer to ibdm_ioc_info_t if it finds 4466 * matching record for the ioc_guid, otherwise NULL 4467 * is returned 4468 */ 4469 ibdm_ioc_info_t * 4470 ibdm_ibnex_get_ioc_info(ib_guid_t ioc_guid) 4471 { 4472 int ii; 4473 ibdm_ioc_info_t *ioc = NULL, *tmp = NULL; 4474 ibdm_dp_gidinfo_t *gid_list; 4475 ib_dm_io_unitinfo_t *iou; 4476 4477 IBTF_DPRINTF_L4("ibdm", "\tibnex_get_ioc_info: GUID %llx", ioc_guid); 4478 4479 mutex_enter(&ibdm.ibdm_mutex); 4480 while (ibdm.ibdm_busy & IBDM_BUSY) 4481 cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex); 4482 ibdm.ibdm_busy |= IBDM_BUSY; 4483 4484 gid_list = ibdm.ibdm_dp_gidlist_head; 4485 while (gid_list) { 4486 mutex_enter(&gid_list->gl_mutex); 4487 if (gid_list->gl_state != IBDM_GID_PROBING_COMPLETE) { 4488 mutex_exit(&gid_list->gl_mutex); 4489 gid_list = gid_list->gl_next; 4490 continue; 4491 } 4492 if (gid_list->gl_iou == NULL) { 4493 IBTF_DPRINTF_L2("ibdm", 4494 "\tget_ioc_info: No IOU info"); 4495 mutex_exit(&gid_list->gl_mutex); 4496 gid_list = gid_list->gl_next; 4497 continue; 4498 } 4499 iou = &gid_list->gl_iou->iou_info; 4500 for (ii = 0; ii < iou->iou_num_ctrl_slots; ii++) { 4501 tmp = IBDM_GIDINFO2IOCINFO(gid_list, ii); 4502 if ((tmp->ioc_profile.ioc_guid == ioc_guid) && 4503 (tmp->ioc_state == IBDM_IOC_STATE_PROBE_SUCCESS)) { 4504 ioc = ibdm_dup_ioc_info(tmp, gid_list); 4505 mutex_exit(&gid_list->gl_mutex); 4506 ibdm.ibdm_busy &= ~IBDM_BUSY; 4507 cv_broadcast(&ibdm.ibdm_busy_cv); 4508 mutex_exit(&ibdm.ibdm_mutex); 4509 IBTF_DPRINTF_L4("ibdm", "\tget_ioc_info: End"); 4510 return (ioc); 4511 } 4512 } 4513 if (ii == iou->iou_num_ctrl_slots) 4514 ioc = NULL; 4515 4516 mutex_exit(&gid_list->gl_mutex); 4517 gid_list = gid_list->gl_next; 4518 } 4519 4520 ibdm.ibdm_busy &= ~IBDM_BUSY; 4521 cv_broadcast(&ibdm.ibdm_busy_cv); 4522 mutex_exit(&ibdm.ibdm_mutex); 4523 IBTF_DPRINTF_L4("ibdm", "\tget_ioc_info: failure End"); 4524 return (ioc); 4525 } 4526 4527 4528 /* 4529 * ibdm_ibnex_get_ioc_count() 4530 * Returns number of ibdm_ioc_info_t it finds 4531 */ 4532 int 4533 ibdm_ibnex_get_ioc_count(void) 4534 { 4535 int count = 0, k; 4536 ibdm_ioc_info_t *ioc; 4537 ibdm_dp_gidinfo_t *gid_list; 4538 4539 mutex_enter(&ibdm.ibdm_mutex); 4540 ibdm_sweep_fabric(0); 4541 4542 while (ibdm.ibdm_busy & IBDM_BUSY) 4543 cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex); 4544 ibdm.ibdm_busy |= IBDM_BUSY; 4545 4546 for (gid_list = ibdm.ibdm_dp_gidlist_head; gid_list; 4547 gid_list = gid_list->gl_next) { 4548 mutex_enter(&gid_list->gl_mutex); 4549 if ((gid_list->gl_state != IBDM_GID_PROBING_COMPLETE) || 4550 (gid_list->gl_iou == NULL)) { 4551 mutex_exit(&gid_list->gl_mutex); 4552 continue; 4553 } 4554 for (k = 0; k < gid_list->gl_iou->iou_info.iou_num_ctrl_slots; 4555 k++) { 4556 ioc = IBDM_GIDINFO2IOCINFO(gid_list, k); 4557 if (ioc->ioc_state == IBDM_IOC_STATE_PROBE_SUCCESS) 4558 ++count; 4559 } 4560 mutex_exit(&gid_list->gl_mutex); 4561 } 4562 ibdm.ibdm_busy &= ~IBDM_BUSY; 4563 cv_broadcast(&ibdm.ibdm_busy_cv); 4564 mutex_exit(&ibdm.ibdm_mutex); 4565 4566 IBTF_DPRINTF_L4("ibdm", "\tget_ioc_count: count = %d", count); 4567 return (count); 4568 } 4569 4570 4571 /* 4572 * ibdm_ibnex_get_ioc_list() 4573 * Returns information about all the IOCs present on the fabric. 4574 * Reprobes the IOCs and the GID list if list_flag is set to REPROBE_ALL. 4575 * Does not sweep fabric if DONOT_PROBE is set 4576 */ 4577 ibdm_ioc_info_t * 4578 ibdm_ibnex_get_ioc_list(ibdm_ibnex_get_ioclist_mtd_t list_flag) 4579 { 4580 int ii; 4581 ibdm_ioc_info_t *ioc_list = NULL, *tmp, *ioc; 4582 ibdm_dp_gidinfo_t *gid_list; 4583 ib_dm_io_unitinfo_t *iou; 4584 4585 IBTF_DPRINTF_L4("ibdm", "\tget_ioc_list: Enter"); 4586 4587 mutex_enter(&ibdm.ibdm_mutex); 4588 if (list_flag != IBDM_IBNEX_DONOT_PROBE) 4589 ibdm_sweep_fabric(list_flag == IBDM_IBNEX_REPROBE_ALL); 4590 4591 while (ibdm.ibdm_busy & IBDM_BUSY) 4592 cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex); 4593 ibdm.ibdm_busy |= IBDM_BUSY; 4594 4595 gid_list = ibdm.ibdm_dp_gidlist_head; 4596 while (gid_list) { 4597 mutex_enter(&gid_list->gl_mutex); 4598 if (gid_list->gl_state != IBDM_GID_PROBING_COMPLETE) { 4599 mutex_exit(&gid_list->gl_mutex); 4600 gid_list = gid_list->gl_next; 4601 continue; 4602 } 4603 if (gid_list->gl_iou == NULL) { 4604 IBTF_DPRINTF_L2("ibdm", 4605 "\tget_ioc_list: No IOU info"); 4606 mutex_exit(&gid_list->gl_mutex); 4607 gid_list = gid_list->gl_next; 4608 continue; 4609 } 4610 iou = &gid_list->gl_iou->iou_info; 4611 for (ii = 0; ii < iou->iou_num_ctrl_slots; ii++) { 4612 ioc = IBDM_GIDINFO2IOCINFO(gid_list, ii); 4613 if (ioc->ioc_state == IBDM_IOC_STATE_PROBE_SUCCESS) { 4614 tmp = ibdm_dup_ioc_info(ioc, gid_list); 4615 tmp->ioc_next = ioc_list; 4616 ioc_list = tmp; 4617 } 4618 } 4619 mutex_exit(&gid_list->gl_mutex); 4620 gid_list = gid_list->gl_next; 4621 } 4622 ibdm.ibdm_busy &= ~IBDM_BUSY; 4623 cv_broadcast(&ibdm.ibdm_busy_cv); 4624 mutex_exit(&ibdm.ibdm_mutex); 4625 4626 IBTF_DPRINTF_L4("ibdm", "\tget_ioc_list: End"); 4627 return (ioc_list); 4628 } 4629 4630 /* 4631 * ibdm_dup_ioc_info() 4632 * Duplicate the IOC information and return the IOC 4633 * information. 4634 */ 4635 static ibdm_ioc_info_t * 4636 ibdm_dup_ioc_info(ibdm_ioc_info_t *in_ioc, ibdm_dp_gidinfo_t *gid_list) 4637 { 4638 ibdm_ioc_info_t *out_ioc; 4639 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*out_ioc)); 4640 4641 out_ioc = kmem_alloc(sizeof (ibdm_ioc_info_t), KM_SLEEP); 4642 bcopy(in_ioc, out_ioc, sizeof (ibdm_ioc_info_t)); 4643 ibdm_update_ioc_port_gidlist(out_ioc, gid_list); 4644 out_ioc->ioc_iou_dc_valid = gid_list->gl_iou->iou_dc_valid; 4645 out_ioc->ioc_iou_diagcode = gid_list->gl_iou->iou_diagcode; 4646 4647 return (out_ioc); 4648 } 4649 4650 4651 /* 4652 * ibdm_free_ioc_list() 4653 * Deallocate memory for IOC list structure 4654 */ 4655 void 4656 ibdm_ibnex_free_ioc_list(ibdm_ioc_info_t *ioc) 4657 { 4658 ibdm_ioc_info_t *temp; 4659 4660 IBTF_DPRINTF_L4("ibdm", "\tibnex_free_ioc_list:"); 4661 while (ioc) { 4662 temp = ioc; 4663 ioc = ioc->ioc_next; 4664 kmem_free(temp->ioc_gid_list, 4665 (sizeof (ibdm_gid_t) * temp->ioc_nportgids)); 4666 if (temp->ioc_hca_list) 4667 ibdm_ibnex_free_hca_list(temp->ioc_hca_list); 4668 kmem_free(temp, sizeof (ibdm_ioc_info_t)); 4669 } 4670 } 4671 4672 4673 /* 4674 * ibdm_ibnex_update_pkey_tbls 4675 * Updates the DM P_Key database. 4676 * NOTE: Two cases are handled here: P_Key being added or removed. 4677 * 4678 * Arguments : NONE 4679 * Return Values : NONE 4680 */ 4681 void 4682 ibdm_ibnex_update_pkey_tbls(void) 4683 { 4684 int h, pp, pidx; 4685 uint_t nports; 4686 uint_t size; 4687 ib_pkey_t new_pkey; 4688 ib_pkey_t *orig_pkey; 4689 ibdm_hca_list_t *hca_list; 4690 ibdm_port_attr_t *port; 4691 ibt_hca_portinfo_t *pinfop; 4692 4693 IBTF_DPRINTF_L4("ibdm", "\tibnex_update_pkey_tbls:"); 4694 4695 mutex_enter(&ibdm.ibdm_hl_mutex); 4696 hca_list = ibdm.ibdm_hca_list_head; 4697 4698 for (h = 0; h < ibdm.ibdm_hca_count; h++) { 4699 4700 /* This updates P_Key Tables for all ports of this HCA */ 4701 (void) ibt_query_hca_ports(hca_list->hl_hca_hdl, 0, &pinfop, 4702 &nports, &size); 4703 4704 /* number of ports shouldn't have changed */ 4705 ASSERT(nports == hca_list->hl_nports); 4706 4707 for (pp = 0; pp < hca_list->hl_nports; pp++) { 4708 port = &hca_list->hl_port_attr[pp]; 4709 4710 /* 4711 * First figure out the P_Keys from IBTL. 4712 * Three things could have happened: 4713 * New P_Keys added 4714 * Existing P_Keys removed 4715 * Both of the above two 4716 * 4717 * Loop through the P_Key Indices and check if a 4718 * give P_Key_Ix matches that of the one seen by 4719 * IBDM. If they match no action is needed. 4720 * 4721 * If they don't match: 4722 * 1. if orig_pkey is invalid and new_pkey is valid 4723 * ---> add new_pkey to DM database 4724 * 2. if orig_pkey is valid and new_pkey is invalid 4725 * ---> remove orig_pkey from DM database 4726 * 3. if orig_pkey and new_pkey are both valid: 4727 * ---> remov orig_pkey from DM database 4728 * ---> add new_pkey to DM database 4729 * 4. if orig_pkey and new_pkey are both invalid: 4730 * ---> do nothing. Updated DM database. 4731 */ 4732 4733 for (pidx = 0; pidx < port->pa_npkeys; pidx++) { 4734 new_pkey = pinfop[pp].p_pkey_tbl[pidx]; 4735 orig_pkey = &port->pa_pkey_tbl[pidx].pt_pkey; 4736 4737 /* keys match - do nothing */ 4738 if (*orig_pkey == new_pkey) 4739 continue; 4740 4741 if (IBDM_INVALID_PKEY(*orig_pkey) && 4742 !IBDM_INVALID_PKEY(new_pkey)) { 4743 /* P_Key was added */ 4744 IBTF_DPRINTF_L5("ibdm", 4745 "\tibnex_update_pkey_tbls: new " 4746 "P_Key added = 0x%x", new_pkey); 4747 *orig_pkey = new_pkey; 4748 ibdm_port_attr_ibmf_init(port, 4749 new_pkey, pp); 4750 } else if (!IBDM_INVALID_PKEY(*orig_pkey) && 4751 IBDM_INVALID_PKEY(new_pkey)) { 4752 /* P_Key was removed */ 4753 IBTF_DPRINTF_L5("ibdm", 4754 "\tibnex_update_pkey_tbls: P_Key " 4755 "removed = 0x%x", *orig_pkey); 4756 *orig_pkey = new_pkey; 4757 (void) ibdm_port_attr_ibmf_fini(port, 4758 pidx); 4759 } else if (!IBDM_INVALID_PKEY(*orig_pkey) && 4760 !IBDM_INVALID_PKEY(new_pkey)) { 4761 /* P_Key were replaced */ 4762 IBTF_DPRINTF_L5("ibdm", 4763 "\tibnex_update_pkey_tbls: P_Key " 4764 "replaced 0x%x with 0x%x", 4765 *orig_pkey, new_pkey); 4766 (void) ibdm_port_attr_ibmf_fini(port, 4767 pidx); 4768 *orig_pkey = new_pkey; 4769 ibdm_port_attr_ibmf_init(port, 4770 new_pkey, pp); 4771 } else { 4772 /* 4773 * P_Keys are invalid 4774 * set anyway to reflect if 4775 * INVALID_FULL was changed to 4776 * INVALID_LIMITED or vice-versa. 4777 */ 4778 *orig_pkey = new_pkey; 4779 } /* end of else */ 4780 4781 } /* loop of p_key index */ 4782 4783 } /* loop of #ports of HCA */ 4784 4785 ibt_free_portinfo(pinfop, size); 4786 hca_list = hca_list->hl_next; 4787 4788 } /* loop for all HCAs in the system */ 4789 4790 mutex_exit(&ibdm.ibdm_hl_mutex); 4791 } 4792 4793 4794 /* 4795 * ibdm_send_ioc_profile() 4796 * Send IOC Controller Profile request. When the request is completed 4797 * IBMF calls ibdm_process_incoming_mad routine to inform about 4798 * the completion. 4799 */ 4800 static int 4801 ibdm_send_ioc_profile(ibdm_dp_gidinfo_t *gid_info, uint8_t ioc_no) 4802 { 4803 ibmf_msg_t *msg; 4804 ib_mad_hdr_t *hdr; 4805 ibdm_ioc_info_t *ioc_info = &(gid_info->gl_iou->iou_ioc_info[ioc_no]); 4806 ibdm_timeout_cb_args_t *cb_args; 4807 4808 IBTF_DPRINTF_L4("ibdm", "\tsend_ioc_profile: " 4809 "gid info 0x%p, ioc_no = %d", gid_info, ioc_no); 4810 4811 /* 4812 * Send command to get IOC profile. 4813 * Allocate a IBMF packet and initialize the packet. 4814 */ 4815 if (ibmf_alloc_msg(gid_info->gl_ibmf_hdl, IBMF_ALLOC_SLEEP, 4816 &msg) != IBMF_SUCCESS) { 4817 IBTF_DPRINTF_L4("ibdm", "\tsend_ioc_profile: pkt alloc fail"); 4818 return (IBDM_FAILURE); 4819 } 4820 4821 ibdm_alloc_send_buffers(msg); 4822 4823 mutex_enter(&gid_info->gl_mutex); 4824 ibdm_bump_transactionID(gid_info); 4825 mutex_exit(&gid_info->gl_mutex); 4826 4827 msg->im_local_addr.ia_local_lid = gid_info->gl_slid; 4828 msg->im_local_addr.ia_remote_lid = gid_info->gl_dlid; 4829 if (gid_info->gl_redirected == B_TRUE) { 4830 if (gid_info->gl_redirect_dlid != 0) { 4831 msg->im_local_addr.ia_remote_lid = 4832 gid_info->gl_redirect_dlid; 4833 } 4834 msg->im_local_addr.ia_remote_qno = gid_info->gl_redirect_QP; 4835 msg->im_local_addr.ia_p_key = gid_info->gl_redirect_pkey; 4836 msg->im_local_addr.ia_q_key = gid_info->gl_redirect_qkey; 4837 } else { 4838 msg->im_local_addr.ia_remote_qno = 1; 4839 msg->im_local_addr.ia_p_key = gid_info->gl_p_key; 4840 msg->im_local_addr.ia_q_key = IB_GSI_QKEY; 4841 } 4842 4843 hdr = IBDM_OUT_IBMFMSG_MADHDR(msg); 4844 hdr->BaseVersion = MAD_CLASS_BASE_VERS_1; 4845 hdr->MgmtClass = MAD_MGMT_CLASS_DEV_MGT; 4846 hdr->ClassVersion = IB_DM_CLASS_VERSION_1; 4847 hdr->R_Method = IB_DM_DEVMGT_METHOD_GET; 4848 hdr->Status = 0; 4849 hdr->TransactionID = h2b64(gid_info->gl_transactionID); 4850 hdr->AttributeID = h2b16(IB_DM_ATTR_IOC_CTRL_PROFILE); 4851 hdr->AttributeModifier = h2b32(ioc_no + 1); 4852 4853 ioc_info->ioc_state = IBDM_IOC_STATE_REPROBE_PROGRESS; 4854 cb_args = &ioc_info->ioc_cb_args; 4855 cb_args->cb_gid_info = gid_info; 4856 cb_args->cb_retry_count = ibdm_dft_retry_cnt; 4857 cb_args->cb_req_type = IBDM_REQ_TYPE_IOCINFO; 4858 cb_args->cb_ioc_num = ioc_no; 4859 4860 ioc_info->ioc_timeout_id = timeout(ibdm_pkt_timeout_hdlr, 4861 cb_args, IBDM_TIMEOUT_VALUE(ibdm_dft_timeout)); 4862 4863 IBTF_DPRINTF_L5("ibdm", "\tsend_ioc_profile:" 4864 "timeout %x", ioc_info->ioc_timeout_id); 4865 4866 if (ibmf_msg_transport(gid_info->gl_ibmf_hdl, gid_info->gl_qp_hdl, msg, 4867 NULL, ibdm_ibmf_send_cb, cb_args, 0) != IBMF_SUCCESS) { 4868 IBTF_DPRINTF_L2("ibdm", 4869 "\tsend_ioc_profile: msg transport failed"); 4870 ibdm_ibmf_send_cb(gid_info->gl_ibmf_hdl, msg, cb_args); 4871 } 4872 ioc_info->ioc_state = IBDM_IOC_STATE_REPROBE_PROGRESS; 4873 return (IBDM_SUCCESS); 4874 } 4875 4876 4877 /* 4878 * ibdm_port_reachable 4879 * Sends a SA query to get the NODE record for port GUID 4880 * Returns IBDM_SUCCESS if the port GID is reachable 4881 */ 4882 static int 4883 ibdm_port_reachable(ibmf_saa_handle_t sa_hdl, ib_guid_t guid, 4884 ib_guid_t *node_guid) 4885 { 4886 sa_node_record_t req, *resp = NULL; 4887 ibmf_saa_access_args_t args; 4888 int ret; 4889 size_t length; 4890 4891 IBTF_DPRINTF_L4("ibdm", "\tport_reachable: port_guid %llx", 4892 guid); 4893 4894 bzero(&req, sizeof (sa_node_record_t)); 4895 req.NodeInfo.PortGUID = guid; 4896 4897 args.sq_attr_id = SA_NODERECORD_ATTRID; 4898 args.sq_access_type = IBMF_SAA_RETRIEVE; 4899 args.sq_component_mask = SA_NODEINFO_COMPMASK_PORTGUID; 4900 args.sq_template = &req; 4901 args.sq_callback = NULL; 4902 args.sq_callback_arg = NULL; 4903 4904 ret = ibmf_sa_access(sa_hdl, &args, 0, &length, (void **) &resp); 4905 if (ret != IBMF_SUCCESS) { 4906 IBTF_DPRINTF_L2("ibdm", "\tport_reachable:" 4907 " SA Retrieve Failed: %d", ret); 4908 return (IBDM_FAILURE); 4909 } 4910 4911 if ((resp == NULL) || (length == 0)) { 4912 IBTF_DPRINTF_L2("ibdm", "\tport_reachable: No records"); 4913 return (IBDM_FAILURE); 4914 } 4915 4916 if (node_guid != NULL) 4917 *node_guid = resp->NodeInfo.NodeGUID; 4918 4919 kmem_free(resp, length); 4920 4921 return (IBDM_SUCCESS); 4922 } 4923 4924 /* 4925 * Update the gidlist for all affected IOCs when GID becomes 4926 * available/unavailable. 4927 * 4928 * Parameters : 4929 * gidinfo - Incoming / Outgoing GID. 4930 * add_flag - 1 for GID added, 0 for GID removed. 4931 * - (-1) : IOC gid list updated, ioc_list required. 4932 * 4933 * This function gets the GID for the node GUID corresponding to the 4934 * port GID. Gets the IOU info 4935 */ 4936 static ibdm_ioc_info_t * 4937 ibdm_update_ioc_gidlist(ibdm_dp_gidinfo_t *gid_info, int avail_flag) 4938 { 4939 ibdm_dp_gidinfo_t *node_gid = NULL; 4940 uint8_t niocs, ii; 4941 ibdm_ioc_info_t *ioc, *ioc_list = NULL, *tmp; 4942 4943 IBTF_DPRINTF_L4("ibdm", "\tupdate_ioc_gidlist"); 4944 4945 switch (avail_flag) { 4946 case 1 : 4947 node_gid = ibdm_check_dest_nodeguid(gid_info); 4948 break; 4949 case 0 : 4950 node_gid = ibdm_handle_gid_rm(gid_info); 4951 break; 4952 case -1 : 4953 node_gid = gid_info; 4954 break; 4955 default : 4956 break; 4957 } 4958 4959 if (node_gid == NULL) { 4960 IBTF_DPRINTF_L4("ibdm", "\tupdate_ioc_gidlist: " 4961 "No node GID found, port gid 0x%p, avail_flag %d", 4962 gid_info, avail_flag); 4963 return (NULL); 4964 } 4965 4966 mutex_enter(&node_gid->gl_mutex); 4967 if ((node_gid->gl_state != IBDM_GID_PROBING_COMPLETE && 4968 node_gid->gl_state != IBDM_GID_PROBING_SKIPPED) || 4969 node_gid->gl_iou == NULL) { 4970 IBTF_DPRINTF_L4("ibdm", "\tupdate_ioc_gidlist " 4971 "gl_state %x, gl_iou %p", node_gid->gl_state, 4972 node_gid->gl_iou); 4973 mutex_exit(&node_gid->gl_mutex); 4974 return (NULL); 4975 } 4976 4977 niocs = node_gid->gl_iou->iou_info.iou_num_ctrl_slots; 4978 IBTF_DPRINTF_L4("ibdm", "\tupdate_ioc_gidlist : niocs %x", 4979 niocs); 4980 for (ii = 0; ii < niocs; ii++) { 4981 ioc = IBDM_GIDINFO2IOCINFO(node_gid, ii); 4982 /* 4983 * Skip IOCs for which probe is not complete or 4984 * reprobe is progress 4985 */ 4986 if (ioc->ioc_state == IBDM_IOC_STATE_PROBE_SUCCESS) { 4987 tmp = ibdm_dup_ioc_info(ioc, node_gid); 4988 tmp->ioc_info_updated.ib_gid_prop_updated = 1; 4989 tmp->ioc_next = ioc_list; 4990 ioc_list = tmp; 4991 } 4992 } 4993 mutex_exit(&node_gid->gl_mutex); 4994 4995 IBTF_DPRINTF_L4("ibdm", "\tupdate_ioc_gidlist : return %p", 4996 ioc_list); 4997 return (ioc_list); 4998 } 4999 5000 /* 5001 * ibdm_saa_event_cb : 5002 * Event handling which does *not* require ibdm_hl_mutex to be 5003 * held are executed in the same thread. This is to prevent 5004 * deadlocks with HCA port down notifications which hold the 5005 * ibdm_hl_mutex. 5006 * 5007 * GID_AVAILABLE event is handled here. A taskq is spawned to 5008 * handle GID_UNAVAILABLE. 5009 * 5010 * A new mutex ibdm_ibnex_mutex has been introduced to protect 5011 * ibnex_callback. This has been done to prevent any possible 5012 * deadlock (described above) while handling GID_AVAILABLE. 5013 * 5014 * IBMF calls the event callback for a HCA port. The SA handle 5015 * for this port would be valid, till the callback returns. 5016 * IBDM calling IBDM using the above SA handle should be valid. 5017 * 5018 * IBDM will additionally check (SA handle != NULL), before 5019 * calling IBMF. 5020 */ 5021 /*ARGSUSED*/ 5022 static void 5023 ibdm_saa_event_cb(ibmf_saa_handle_t ibmf_saa_handle, 5024 ibmf_saa_subnet_event_t ibmf_saa_event, 5025 ibmf_saa_event_details_t *event_details, void *callback_arg) 5026 { 5027 ibdm_saa_event_arg_t *event_arg; 5028 ib_gid_t sgid, dgid; 5029 ibdm_port_attr_t *hca_port; 5030 ibdm_dp_gidinfo_t *gid_info, *node_gid_info = NULL; 5031 ib_guid_t nodeguid; 5032 5033 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*event_arg)); 5034 5035 hca_port = (ibdm_port_attr_t *)callback_arg; 5036 5037 IBTF_DPRINTF_L4("ibdm", "\tsaa_event_cb(%x, %x, %x, %x)\n", 5038 ibmf_saa_handle, ibmf_saa_event, event_details, 5039 callback_arg); 5040 #ifdef DEBUG 5041 if (ibdm_ignore_saa_event) 5042 return; 5043 #endif 5044 5045 if (ibmf_saa_event == IBMF_SAA_EVENT_GID_AVAILABLE) { 5046 /* 5047 * Ensure no other probe / sweep fabric is in 5048 * progress. 5049 */ 5050 mutex_enter(&ibdm.ibdm_mutex); 5051 while (ibdm.ibdm_busy & IBDM_BUSY) 5052 cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex); 5053 ibdm.ibdm_busy |= IBDM_BUSY; 5054 mutex_exit(&ibdm.ibdm_mutex); 5055 5056 /* 5057 * If we already know about this GID, return. 5058 * GID_AVAILABLE may be reported for multiple HCA 5059 * ports. 5060 */ 5061 if ((ibdm_check_dgid(event_details->ie_gid.gid_guid, 5062 event_details->ie_gid.gid_prefix)) != NULL) { 5063 mutex_enter(&ibdm.ibdm_mutex); 5064 ibdm.ibdm_busy &= ~IBDM_BUSY; 5065 cv_broadcast(&ibdm.ibdm_busy_cv); 5066 mutex_exit(&ibdm.ibdm_mutex); 5067 return; 5068 } 5069 5070 IBTF_DPRINTF_L4("ibdm", "\tGID (prefix %x, guid %llx) " 5071 "Insertion notified", 5072 event_details->ie_gid.gid_prefix, 5073 event_details->ie_gid.gid_guid); 5074 5075 /* This is a new gid, insert it to GID list */ 5076 sgid.gid_prefix = hca_port->pa_sn_prefix; 5077 sgid.gid_guid = hca_port->pa_port_guid; 5078 dgid.gid_prefix = event_details->ie_gid.gid_prefix; 5079 dgid.gid_guid = event_details->ie_gid.gid_guid; 5080 gid_info = ibdm_create_gid_info(hca_port, sgid, dgid); 5081 if (gid_info == NULL) { 5082 IBTF_DPRINTF_L4("ibdm", "\tGID_AVAILABLE: " 5083 "create_gid_info returned NULL"); 5084 mutex_enter(&ibdm.ibdm_mutex); 5085 ibdm.ibdm_busy &= ~IBDM_BUSY; 5086 cv_broadcast(&ibdm.ibdm_busy_cv); 5087 mutex_exit(&ibdm.ibdm_mutex); 5088 return; 5089 } 5090 mutex_enter(&gid_info->gl_mutex); 5091 gid_info->gl_state = IBDM_GID_PROBING_SKIPPED; 5092 mutex_exit(&gid_info->gl_mutex); 5093 5094 /* Get the node GUID */ 5095 if (ibdm_port_reachable(ibmf_saa_handle, dgid.gid_guid, 5096 &nodeguid) != IBDM_SUCCESS) { 5097 /* 5098 * Set the state to PROBE_NOT_DONE for the 5099 * next sweep to probe it 5100 */ 5101 IBTF_DPRINTF_L2("ibdm", "\tsaa_event_taskq: " 5102 "Skipping GID : port GUID not found"); 5103 mutex_enter(&gid_info->gl_mutex); 5104 gid_info->gl_state = IBDM_GID_PROBE_NOT_DONE; 5105 mutex_exit(&gid_info->gl_mutex); 5106 mutex_enter(&ibdm.ibdm_mutex); 5107 ibdm.ibdm_busy &= ~IBDM_BUSY; 5108 cv_broadcast(&ibdm.ibdm_busy_cv); 5109 mutex_exit(&ibdm.ibdm_mutex); 5110 return; 5111 } 5112 5113 gid_info->gl_nodeguid = nodeguid; 5114 gid_info->gl_portguid = dgid.gid_guid; 5115 5116 /* 5117 * Get the gid info with the same node GUID. 5118 */ 5119 mutex_enter(&ibdm.ibdm_mutex); 5120 node_gid_info = ibdm.ibdm_dp_gidlist_head; 5121 while (node_gid_info) { 5122 if (node_gid_info->gl_nodeguid == 5123 gid_info->gl_nodeguid && 5124 node_gid_info->gl_iou != NULL) { 5125 break; 5126 } 5127 node_gid_info = node_gid_info->gl_next; 5128 } 5129 mutex_exit(&ibdm.ibdm_mutex); 5130 5131 /* 5132 * Handling a new GID requires filling of gl_hca_list. 5133 * This require ibdm hca_list to be parsed and hence 5134 * holding the ibdm_hl_mutex. Spawning a new thread to 5135 * handle this. 5136 */ 5137 if (node_gid_info == NULL) { 5138 if (taskq_dispatch(system_taskq, 5139 ibdm_saa_handle_new_gid, (void *)gid_info, 5140 TQ_NOSLEEP) == NULL) { 5141 IBTF_DPRINTF_L2("ibdm", "\tsaa_event_cb: " 5142 "new_gid taskq_dispatch failed"); 5143 return; 5144 } 5145 } 5146 5147 mutex_enter(&ibdm.ibdm_mutex); 5148 ibdm.ibdm_busy &= ~IBDM_BUSY; 5149 cv_broadcast(&ibdm.ibdm_busy_cv); 5150 mutex_exit(&ibdm.ibdm_mutex); 5151 return; 5152 } 5153 5154 if (ibmf_saa_event != IBMF_SAA_EVENT_GID_UNAVAILABLE) 5155 return; 5156 5157 event_arg = (ibdm_saa_event_arg_t *)kmem_alloc( 5158 sizeof (ibdm_saa_event_arg_t), KM_SLEEP); 5159 event_arg->ibmf_saa_handle = ibmf_saa_handle; 5160 event_arg->ibmf_saa_event = ibmf_saa_event; 5161 bcopy(event_details, &event_arg->event_details, 5162 sizeof (ibmf_saa_event_details_t)); 5163 event_arg->callback_arg = callback_arg; 5164 5165 if (taskq_dispatch(system_taskq, ibdm_saa_event_taskq, 5166 (void *)event_arg, TQ_NOSLEEP) == NULL) { 5167 IBTF_DPRINTF_L2("ibdm", "\tsaa_event_cb: " 5168 "taskq_dispatch failed"); 5169 ibdm_free_saa_event_arg(event_arg); 5170 return; 5171 } 5172 } 5173 5174 /* 5175 * Handle a new GID discovered by GID_AVAILABLE saa event. 5176 */ 5177 void 5178 ibdm_saa_handle_new_gid(void *arg) 5179 { 5180 ibdm_dp_gidinfo_t *gid_info; 5181 ibdm_hca_list_t *hca_list = NULL; 5182 ibdm_port_attr_t *port = NULL; 5183 ibdm_ioc_info_t *ioc_list = NULL; 5184 5185 IBTF_DPRINTF_L4(ibdm_string, "\tsaa_handle_new_gid(%p)", arg); 5186 5187 gid_info = (ibdm_dp_gidinfo_t *)arg; 5188 5189 /* 5190 * Ensure that no other sweep / probe has completed 5191 * probing this gid. 5192 */ 5193 mutex_enter(&gid_info->gl_mutex); 5194 if (gid_info->gl_state != IBDM_GID_PROBE_NOT_DONE) { 5195 mutex_exit(&gid_info->gl_mutex); 5196 return; 5197 } 5198 mutex_exit(&gid_info->gl_mutex); 5199 5200 /* 5201 * Parse HCAs to fill gl_hca_list 5202 */ 5203 mutex_enter(&ibdm.ibdm_hl_mutex); 5204 for (ibdm_get_next_port(&hca_list, &port, 1); port; 5205 ibdm_get_next_port(&hca_list, &port, 1)) { 5206 if (ibdm_port_reachable(port->pa_sa_hdl, 5207 gid_info->gl_portguid, NULL) == 5208 IBDM_SUCCESS) { 5209 ibdm_addto_glhcalist(gid_info, hca_list); 5210 } 5211 } 5212 mutex_exit(&ibdm.ibdm_hl_mutex); 5213 5214 /* 5215 * Ensure no other probe / sweep fabric is in 5216 * progress. 5217 */ 5218 mutex_enter(&ibdm.ibdm_mutex); 5219 while (ibdm.ibdm_busy & IBDM_BUSY) 5220 cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex); 5221 ibdm.ibdm_busy |= IBDM_BUSY; 5222 mutex_exit(&ibdm.ibdm_mutex); 5223 5224 /* 5225 * New IOU probe it, to check if new IOCs 5226 */ 5227 IBTF_DPRINTF_L4(ibdm_string, "\tsaa_handle_new_gid: " 5228 "new GID : probing"); 5229 mutex_enter(&ibdm.ibdm_mutex); 5230 ibdm.ibdm_ngid_probes_in_progress++; 5231 mutex_exit(&ibdm.ibdm_mutex); 5232 mutex_enter(&gid_info->gl_mutex); 5233 gid_info->gl_reprobe_flag = 0; 5234 gid_info->gl_state = IBDM_GID_PROBE_NOT_DONE; 5235 mutex_exit(&gid_info->gl_mutex); 5236 ibdm_probe_gid_thread((void *)gid_info); 5237 5238 mutex_enter(&ibdm.ibdm_mutex); 5239 ibdm_wait_probe_completion(); 5240 mutex_exit(&ibdm.ibdm_mutex); 5241 5242 if (gid_info->gl_iou == NULL) { 5243 mutex_enter(&ibdm.ibdm_mutex); 5244 ibdm.ibdm_busy &= ~IBDM_BUSY; 5245 cv_broadcast(&ibdm.ibdm_busy_cv); 5246 mutex_exit(&ibdm.ibdm_mutex); 5247 return; 5248 } 5249 5250 /* 5251 * Update GID list in all IOCs affected by this 5252 */ 5253 ioc_list = ibdm_update_ioc_gidlist(gid_info, 1); 5254 5255 /* 5256 * Pass on the IOCs with updated GIDs to IBnexus 5257 */ 5258 if (ioc_list) { 5259 mutex_enter(&ibdm.ibdm_ibnex_mutex); 5260 if (ibdm.ibdm_ibnex_callback != NULL) { 5261 (*ibdm.ibdm_ibnex_callback)((void *)ioc_list, 5262 IBDM_EVENT_IOC_PROP_UPDATE); 5263 } 5264 mutex_exit(&ibdm.ibdm_ibnex_mutex); 5265 } 5266 5267 mutex_enter(&ibdm.ibdm_mutex); 5268 ibdm.ibdm_busy &= ~IBDM_BUSY; 5269 cv_broadcast(&ibdm.ibdm_busy_cv); 5270 mutex_exit(&ibdm.ibdm_mutex); 5271 } 5272 5273 /* 5274 * ibdm_saa_event_taskq : 5275 * GID_UNAVAILABLE Event handling requires ibdm_hl_mutex to be 5276 * held. The GID_UNAVAILABLE handling is done in a taskq to 5277 * prevent deadlocks with HCA port down notifications which hold 5278 * ibdm_hl_mutex. 5279 */ 5280 void 5281 ibdm_saa_event_taskq(void *arg) 5282 { 5283 ibdm_saa_event_arg_t *event_arg; 5284 ibmf_saa_handle_t ibmf_saa_handle; 5285 ibmf_saa_subnet_event_t ibmf_saa_event; 5286 ibmf_saa_event_details_t *event_details; 5287 void *callback_arg; 5288 5289 ibdm_dp_gidinfo_t *gid_info; 5290 ibdm_port_attr_t *hca_port, *port = NULL; 5291 ibdm_hca_list_t *hca_list = NULL; 5292 int sa_handle_valid = 0; 5293 ibdm_ioc_info_t *ioc_list = NULL; 5294 5295 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*event_arg)); 5296 5297 event_arg = (ibdm_saa_event_arg_t *)arg; 5298 ibmf_saa_handle = event_arg->ibmf_saa_handle; 5299 ibmf_saa_event = event_arg->ibmf_saa_event; 5300 event_details = &event_arg->event_details; 5301 callback_arg = event_arg->callback_arg; 5302 5303 ASSERT(callback_arg != NULL); 5304 ASSERT(ibmf_saa_event == IBMF_SAA_EVENT_GID_UNAVAILABLE); 5305 IBTF_DPRINTF_L4("ibdm", "\tsaa_event_taskq(%x, %x, %x, %x)", 5306 ibmf_saa_handle, ibmf_saa_event, event_details, 5307 callback_arg); 5308 5309 hca_port = (ibdm_port_attr_t *)callback_arg; 5310 5311 /* Check if the port_attr is still valid */ 5312 mutex_enter(&ibdm.ibdm_hl_mutex); 5313 for (ibdm_get_next_port(&hca_list, &port, 0); port; 5314 ibdm_get_next_port(&hca_list, &port, 0)) { 5315 if (port == hca_port && port->pa_port_guid == 5316 hca_port->pa_port_guid) { 5317 if (ibmf_saa_handle == hca_port->pa_sa_hdl) 5318 sa_handle_valid = 1; 5319 break; 5320 } 5321 } 5322 mutex_exit(&ibdm.ibdm_hl_mutex); 5323 if (sa_handle_valid == 0) { 5324 ibdm_free_saa_event_arg(event_arg); 5325 return; 5326 } 5327 5328 if (hca_port && (hca_port->pa_sa_hdl == NULL || 5329 ibmf_saa_handle != hca_port->pa_sa_hdl)) { 5330 ibdm_free_saa_event_arg(event_arg); 5331 return; 5332 } 5333 hca_list = NULL; 5334 port = NULL; 5335 5336 /* 5337 * Check if the GID is visible to other HCA ports. 5338 * Return if so. 5339 */ 5340 mutex_enter(&ibdm.ibdm_hl_mutex); 5341 for (ibdm_get_next_port(&hca_list, &port, 1); port; 5342 ibdm_get_next_port(&hca_list, &port, 1)) { 5343 if (ibdm_port_reachable(port->pa_sa_hdl, 5344 event_details->ie_gid.gid_guid, NULL) == 5345 IBDM_SUCCESS) { 5346 mutex_exit(&ibdm.ibdm_hl_mutex); 5347 ibdm_free_saa_event_arg(event_arg); 5348 return; 5349 } 5350 } 5351 mutex_exit(&ibdm.ibdm_hl_mutex); 5352 5353 /* 5354 * Ensure no other probe / sweep fabric is in 5355 * progress. 5356 */ 5357 mutex_enter(&ibdm.ibdm_mutex); 5358 while (ibdm.ibdm_busy & IBDM_BUSY) 5359 cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex); 5360 ibdm.ibdm_busy |= IBDM_BUSY; 5361 mutex_exit(&ibdm.ibdm_mutex); 5362 5363 /* 5364 * If this GID is no longer in GID list, return 5365 * GID_UNAVAILABLE may be reported for multiple HCA 5366 * ports. 5367 */ 5368 mutex_enter(&ibdm.ibdm_mutex); 5369 gid_info = ibdm.ibdm_dp_gidlist_head; 5370 while (gid_info) { 5371 if (gid_info->gl_portguid == 5372 event_details->ie_gid.gid_guid) { 5373 break; 5374 } 5375 gid_info = gid_info->gl_next; 5376 } 5377 mutex_exit(&ibdm.ibdm_mutex); 5378 if (gid_info == NULL) { 5379 mutex_enter(&ibdm.ibdm_mutex); 5380 ibdm.ibdm_busy &= ~IBDM_BUSY; 5381 cv_broadcast(&ibdm.ibdm_busy_cv); 5382 mutex_exit(&ibdm.ibdm_mutex); 5383 ibdm_free_saa_event_arg(event_arg); 5384 return; 5385 } 5386 5387 IBTF_DPRINTF_L4("ibdm", "\tGID (prefix %x, guid %llx) " 5388 "Unavailable notification", 5389 event_details->ie_gid.gid_prefix, 5390 event_details->ie_gid.gid_guid); 5391 5392 /* 5393 * Update GID list in all IOCs affected by this 5394 */ 5395 if (gid_info->gl_state == IBDM_GID_PROBING_SKIPPED || 5396 gid_info->gl_state == IBDM_GID_PROBING_COMPLETE) 5397 ioc_list = ibdm_update_ioc_gidlist(gid_info, 0); 5398 5399 /* 5400 * Remove GID from the global GID list 5401 * Handle the case where all port GIDs for an 5402 * IOU have been hot-removed. Check both gid_info 5403 * & ioc_info for checking ngids. 5404 */ 5405 mutex_enter(&ibdm.ibdm_mutex); 5406 if (gid_info->gl_iou != NULL && gid_info->gl_ngids == 0) { 5407 mutex_enter(&gid_info->gl_mutex); 5408 (void) ibdm_free_iou_info(gid_info, &gid_info->gl_iou); 5409 mutex_exit(&gid_info->gl_mutex); 5410 } 5411 if (gid_info->gl_prev != NULL) 5412 gid_info->gl_prev->gl_next = gid_info->gl_next; 5413 if (gid_info->gl_next != NULL) 5414 gid_info->gl_next->gl_prev = gid_info->gl_prev; 5415 5416 if (gid_info == ibdm.ibdm_dp_gidlist_head) 5417 ibdm.ibdm_dp_gidlist_head = gid_info->gl_next; 5418 if (gid_info == ibdm.ibdm_dp_gidlist_tail) 5419 ibdm.ibdm_dp_gidlist_tail = gid_info->gl_prev; 5420 ibdm.ibdm_ngids--; 5421 5422 ibdm.ibdm_busy &= ~IBDM_BUSY; 5423 cv_broadcast(&ibdm.ibdm_busy_cv); 5424 mutex_exit(&ibdm.ibdm_mutex); 5425 5426 mutex_destroy(&gid_info->gl_mutex); 5427 kmem_free(gid_info, sizeof (ibdm_dp_gidinfo_t)); 5428 5429 /* 5430 * Pass on the IOCs with updated GIDs to IBnexus 5431 */ 5432 if (ioc_list) { 5433 IBTF_DPRINTF_L4("ibdm", "\tGID_UNAVAILABLE " 5434 "IOC_PROP_UPDATE for %p\n", ioc_list); 5435 mutex_enter(&ibdm.ibdm_ibnex_mutex); 5436 if (ibdm.ibdm_ibnex_callback != NULL) { 5437 (*ibdm.ibdm_ibnex_callback)((void *) 5438 ioc_list, IBDM_EVENT_IOC_PROP_UPDATE); 5439 } 5440 mutex_exit(&ibdm.ibdm_ibnex_mutex); 5441 } 5442 5443 ibdm_free_saa_event_arg(event_arg); 5444 } 5445 5446 5447 static int 5448 ibdm_cmp_gid_list(ibdm_gid_t *new, ibdm_gid_t *prev) 5449 { 5450 ibdm_gid_t *scan_new, *scan_prev; 5451 int cmp_failed = 0; 5452 5453 ASSERT(new != NULL); 5454 ASSERT(prev != NULL); 5455 5456 /* 5457 * Search for each new gid anywhere in the prev GID list. 5458 * Note that the gid list could have been re-ordered. 5459 */ 5460 for (scan_new = new; scan_new; scan_new = scan_new->gid_next) { 5461 for (scan_prev = prev, cmp_failed = 1; scan_prev; 5462 scan_prev = scan_prev->gid_next) { 5463 if (scan_prev->gid_dgid_hi == scan_new->gid_dgid_hi && 5464 scan_prev->gid_dgid_lo == scan_new->gid_dgid_lo) { 5465 cmp_failed = 0; 5466 break; 5467 } 5468 } 5469 5470 if (cmp_failed) 5471 return (1); 5472 } 5473 return (0); 5474 } 5475 5476 /* 5477 * This is always called in a single thread 5478 * This function updates the gid_list and serv_list of IOC 5479 * The current gid_list is in ioc_info_t(contains only port 5480 * guids for which probe is done) & gidinfo_t(other port gids) 5481 * The gids in both locations are used for comparision. 5482 */ 5483 static void 5484 ibdm_reprobe_update_port_srv(ibdm_ioc_info_t *ioc, ibdm_dp_gidinfo_t *gidinfo) 5485 { 5486 ibdm_gid_t *cur_gid_list; 5487 uint_t cur_nportgids; 5488 5489 ASSERT(MUTEX_HELD(&ibdm.ibdm_mutex)); 5490 5491 ioc->ioc_info_updated.ib_prop_updated = 0; 5492 5493 5494 /* Current GID list in gid_info only */ 5495 cur_gid_list = gidinfo->gl_gid; 5496 cur_nportgids = gidinfo->gl_ngids; 5497 5498 if (ioc->ioc_prev_serv_cnt != 5499 ioc->ioc_profile.ioc_service_entries || 5500 ibdm_serv_cmp(&ioc->ioc_serv[0], &ioc->ioc_prev_serv[0], 5501 ioc->ioc_prev_serv_cnt)) 5502 ioc->ioc_info_updated.ib_srv_prop_updated = 1; 5503 5504 if (ioc->ioc_prev_nportgids != cur_nportgids || 5505 ioc->ioc_prev_gid_list == NULL || cur_gid_list == NULL) { 5506 ioc->ioc_info_updated.ib_gid_prop_updated = 1; 5507 } else if (ibdm_cmp_gid_list(ioc->ioc_prev_gid_list, cur_gid_list)) { 5508 ioc->ioc_info_updated.ib_gid_prop_updated = 1; 5509 } 5510 5511 /* Zero out previous entries */ 5512 ibdm_free_gid_list(ioc->ioc_prev_gid_list); 5513 if (ioc->ioc_prev_serv) 5514 kmem_free(ioc->ioc_prev_serv, ioc->ioc_prev_serv_cnt * 5515 sizeof (ibdm_srvents_info_t)); 5516 ioc->ioc_prev_serv_cnt = 0; 5517 ioc->ioc_prev_nportgids = 0; 5518 ioc->ioc_prev_serv = NULL; 5519 ioc->ioc_prev_gid_list = NULL; 5520 } 5521 5522 /* 5523 * Handle GID removal. This returns gid_info of an GID for the same 5524 * node GUID, if found. For an GID with IOU information, the same 5525 * gid_info is returned if no gid_info with same node_guid is found. 5526 */ 5527 static ibdm_dp_gidinfo_t * 5528 ibdm_handle_gid_rm(ibdm_dp_gidinfo_t *rm_gid) 5529 { 5530 ibdm_dp_gidinfo_t *gid_list; 5531 5532 IBTF_DPRINTF_L4("ibdm", "\thandle_gid_rm(0x%p)", rm_gid); 5533 5534 if (rm_gid->gl_iou == NULL) { 5535 IBTF_DPRINTF_L4("ibdm", "\thandle_gid_rm NO iou"); 5536 /* 5537 * Search for a GID with same node_guid and 5538 * gl_iou != NULL 5539 */ 5540 for (gid_list = ibdm.ibdm_dp_gidlist_head; gid_list; 5541 gid_list = gid_list->gl_next) { 5542 if (gid_list->gl_iou != NULL && (gid_list->gl_nodeguid 5543 == rm_gid->gl_nodeguid)) 5544 break; 5545 } 5546 5547 if (gid_list) 5548 ibdm_rmfrom_glgid_list(gid_list, rm_gid); 5549 5550 IBTF_DPRINTF_L4("ibdm", "\thandle_gid_rm ret %p", gid_list); 5551 return (gid_list); 5552 } else { 5553 /* 5554 * Search for a GID with same node_guid and 5555 * gl_iou == NULL 5556 */ 5557 IBTF_DPRINTF_L4("ibdm", "\thandle_gid_rm with iou"); 5558 for (gid_list = ibdm.ibdm_dp_gidlist_head; gid_list; 5559 gid_list = gid_list->gl_next) { 5560 if (gid_list->gl_iou == NULL && (gid_list->gl_nodeguid 5561 == rm_gid->gl_nodeguid)) 5562 break; 5563 } 5564 5565 if (gid_list) { 5566 /* 5567 * Copy the following fields from rm_gid : 5568 * 1. gl_state 5569 * 2. gl_iou 5570 * 3. gl_gid & gl_ngids 5571 * 5572 * Note : Function is synchronized by 5573 * ibdm_busy flag. 5574 * 5575 * Note : Redirect info is initialized if 5576 * any MADs for the GID fail 5577 */ 5578 IBTF_DPRINTF_L4("ibdm", "\thandle_gid_rm " 5579 "copying info to GID with gl_iou != NULl"); 5580 gid_list->gl_state = rm_gid->gl_state; 5581 gid_list->gl_iou = rm_gid->gl_iou; 5582 gid_list->gl_gid = rm_gid->gl_gid; 5583 gid_list->gl_ngids = rm_gid->gl_ngids; 5584 5585 /* Remove the GID from gl_gid list */ 5586 ibdm_rmfrom_glgid_list(gid_list, rm_gid); 5587 } else { 5588 /* 5589 * Handle a case where all GIDs to the IOU have 5590 * been removed. 5591 */ 5592 IBTF_DPRINTF_L4("ibdm", "\thandle_gid_rm 0 GID " 5593 "to IOU"); 5594 5595 ibdm_rmfrom_glgid_list(rm_gid, rm_gid); 5596 return (rm_gid); 5597 } 5598 IBTF_DPRINTF_L4("ibdm", "\thandle_gid_rm ret %p", gid_list); 5599 return (gid_list); 5600 } 5601 } 5602 5603 static void 5604 ibdm_rmfrom_glgid_list(ibdm_dp_gidinfo_t *gid_info, 5605 ibdm_dp_gidinfo_t *rm_gid) 5606 { 5607 ibdm_gid_t *tmp, *prev; 5608 5609 IBTF_DPRINTF_L4("ibdm", "\trmfrom_glgid (%p, %p)", 5610 gid_info, rm_gid); 5611 5612 for (tmp = gid_info->gl_gid, prev = NULL; tmp; ) { 5613 if (tmp->gid_dgid_hi == rm_gid->gl_dgid_hi && 5614 tmp->gid_dgid_lo == rm_gid->gl_dgid_lo) { 5615 if (prev == NULL) 5616 gid_info->gl_gid = tmp->gid_next; 5617 else 5618 prev->gid_next = tmp->gid_next; 5619 5620 kmem_free(tmp, sizeof (ibdm_gid_t)); 5621 gid_info->gl_ngids--; 5622 break; 5623 } else { 5624 prev = tmp; 5625 tmp = tmp->gid_next; 5626 } 5627 } 5628 } 5629 5630 static void 5631 ibdm_addto_gidlist(ibdm_gid_t **src_ptr, ibdm_gid_t *dest) 5632 { 5633 ibdm_gid_t *head = NULL, *new, *tail; 5634 5635 /* First copy the destination */ 5636 for (; dest; dest = dest->gid_next) { 5637 new = kmem_zalloc(sizeof (ibdm_gid_t), KM_SLEEP); 5638 new->gid_dgid_hi = dest->gid_dgid_hi; 5639 new->gid_dgid_lo = dest->gid_dgid_lo; 5640 new->gid_next = head; 5641 head = new; 5642 } 5643 5644 /* Insert this to the source */ 5645 if (*src_ptr == NULL) 5646 *src_ptr = head; 5647 else { 5648 for (tail = *src_ptr; tail->gid_next != NULL; 5649 tail = tail->gid_next) 5650 ; 5651 5652 tail->gid_next = head; 5653 } 5654 } 5655 5656 static void 5657 ibdm_free_gid_list(ibdm_gid_t *head) 5658 { 5659 ibdm_gid_t *delete; 5660 5661 for (delete = head; delete; ) { 5662 head = delete->gid_next; 5663 kmem_free(delete, sizeof (ibdm_gid_t)); 5664 delete = head; 5665 } 5666 } 5667 5668 /* 5669 * This function rescans the DM capable GIDs (gl_state is 5670 * GID_PROBE_COMPLETE or IBDM_GID_PROBING_SKIPPED.This 5671 * basically checks if the DM capable GID is reachable. If 5672 * not this is handled the same way as GID_UNAVAILABLE, 5673 * except that notifications are not send to IBnexus. 5674 * 5675 * This function also initializes the ioc_prev_list for 5676 * a particular IOC (when called from probe_ioc, with 5677 * ioc_guidp != NULL) or all IOCs for the gid (called from 5678 * sweep_fabric, ioc_guidp == NULL). 5679 */ 5680 static void 5681 ibdm_rescan_gidlist(ib_guid_t *ioc_guidp) 5682 { 5683 ibdm_dp_gidinfo_t *gid_info, *tmp; 5684 int ii, niocs, found; 5685 ibdm_hca_list_t *hca_list = NULL; 5686 ibdm_port_attr_t *port = NULL; 5687 ibdm_ioc_info_t *ioc_list; 5688 5689 for (gid_info = ibdm.ibdm_dp_gidlist_head; gid_info; ) { 5690 found = 0; 5691 if (gid_info->gl_state != IBDM_GID_PROBING_SKIPPED && 5692 gid_info->gl_state != IBDM_GID_PROBING_COMPLETE) { 5693 gid_info = gid_info->gl_next; 5694 continue; 5695 } 5696 5697 /* 5698 * Check if the GID is visible to any HCA ports. 5699 * Return if so. 5700 */ 5701 mutex_enter(&ibdm.ibdm_hl_mutex); 5702 for (ibdm_get_next_port(&hca_list, &port, 1); port; 5703 ibdm_get_next_port(&hca_list, &port, 1)) { 5704 if (ibdm_port_reachable(port->pa_sa_hdl, 5705 gid_info->gl_dgid_lo, NULL) == IBDM_SUCCESS) { 5706 found = 1; 5707 break; 5708 } 5709 } 5710 mutex_exit(&ibdm.ibdm_hl_mutex); 5711 5712 if (found) { 5713 if (gid_info->gl_iou == NULL) { 5714 gid_info = gid_info->gl_next; 5715 continue; 5716 } 5717 5718 /* Intialize the ioc_prev_gid_list */ 5719 niocs = 5720 gid_info->gl_iou->iou_info.iou_num_ctrl_slots; 5721 for (ii = 0; ii < niocs; ii++) { 5722 ioc_list = IBDM_GIDINFO2IOCINFO(gid_info, ii); 5723 5724 if (ioc_guidp == NULL || (*ioc_guidp == 5725 ioc_list->ioc_profile.ioc_guid)) { 5726 /* Add info of GIDs in gid_info also */ 5727 ibdm_addto_gidlist( 5728 &ioc_list->ioc_prev_gid_list, 5729 gid_info->gl_gid); 5730 ioc_list->ioc_prev_nportgids = 5731 gid_info->gl_ngids; 5732 } 5733 } 5734 gid_info = gid_info->gl_next; 5735 continue; 5736 } 5737 5738 IBTF_DPRINTF_L4("ibdm", "\trescan_gidlist " 5739 "deleted port GUID %llx", 5740 gid_info->gl_dgid_lo); 5741 5742 /* 5743 * Update GID list in all IOCs affected by this 5744 */ 5745 ioc_list = ibdm_update_ioc_gidlist(gid_info, 0); 5746 5747 /* 5748 * Remove GID from the global GID list 5749 * Handle the case where all port GIDs for an 5750 * IOU have been hot-removed. 5751 */ 5752 mutex_enter(&ibdm.ibdm_mutex); 5753 if (gid_info->gl_iou != NULL && gid_info->gl_ngids == 0) { 5754 mutex_enter(&gid_info->gl_mutex); 5755 (void) ibdm_free_iou_info(gid_info, &gid_info->gl_iou); 5756 mutex_exit(&gid_info->gl_mutex); 5757 } 5758 tmp = gid_info->gl_next; 5759 if (gid_info->gl_prev != NULL) 5760 gid_info->gl_prev->gl_next = gid_info->gl_next; 5761 if (gid_info->gl_next != NULL) 5762 gid_info->gl_next->gl_prev = gid_info->gl_prev; 5763 5764 if (gid_info == ibdm.ibdm_dp_gidlist_head) 5765 ibdm.ibdm_dp_gidlist_head = gid_info->gl_next; 5766 if (gid_info == ibdm.ibdm_dp_gidlist_tail) 5767 ibdm.ibdm_dp_gidlist_tail = gid_info->gl_prev; 5768 ibdm.ibdm_ngids--; 5769 5770 mutex_destroy(&gid_info->gl_mutex); 5771 kmem_free(gid_info, sizeof (ibdm_dp_gidinfo_t)); 5772 gid_info = tmp; 5773 5774 mutex_exit(&ibdm.ibdm_mutex); 5775 5776 /* 5777 * Pass on the IOCs with updated GIDs to IBnexus 5778 */ 5779 if (ioc_list) { 5780 IBTF_DPRINTF_L4("ibdm", "\trescan_gidlist " 5781 "IOC_PROP_UPDATE for %p\n", ioc_list); 5782 mutex_enter(&ibdm.ibdm_ibnex_mutex); 5783 if (ibdm.ibdm_ibnex_callback != NULL) { 5784 (*ibdm.ibdm_ibnex_callback)((void *) 5785 ioc_list, IBDM_EVENT_IOC_PROP_UPDATE); 5786 } 5787 mutex_exit(&ibdm.ibdm_ibnex_mutex); 5788 } 5789 } 5790 } 5791 5792 /* 5793 * This function notifies IBnex of IOCs on this GID. 5794 * Notification is for GIDs with gl_reprobe_flag set. 5795 * The flag is set when IOC probe / fabric sweep 5796 * probes a GID starting from CLASS port info. 5797 * 5798 * IBnexus will have information of a reconnected IOC 5799 * if it had probed it before. If this is a new IOC, 5800 * IBnexus ignores the notification. 5801 * 5802 * This function should be called with no locks held. 5803 */ 5804 static void 5805 ibdm_notify_newgid_iocs(ibdm_dp_gidinfo_t *gid_info) 5806 { 5807 ibdm_ioc_info_t *ioc_list; 5808 5809 if (gid_info->gl_reprobe_flag == 0 || 5810 gid_info->gl_iou == NULL) 5811 return; 5812 5813 ioc_list = ibdm_update_ioc_gidlist(gid_info, -1); 5814 5815 /* 5816 * Pass on the IOCs with updated GIDs to IBnexus 5817 */ 5818 if (ioc_list) { 5819 mutex_enter(&ibdm.ibdm_ibnex_mutex); 5820 if (ibdm.ibdm_ibnex_callback != NULL) { 5821 (*ibdm.ibdm_ibnex_callback)((void *)ioc_list, 5822 IBDM_EVENT_IOC_PROP_UPDATE); 5823 } 5824 mutex_exit(&ibdm.ibdm_ibnex_mutex); 5825 } 5826 } 5827 5828 5829 static void 5830 ibdm_free_saa_event_arg(ibdm_saa_event_arg_t *arg) 5831 { 5832 if (arg != NULL) 5833 kmem_free(arg, sizeof (ibdm_saa_event_arg_t)); 5834 } 5835 5836 /* 5837 * This function parses the list of HCAs and HCA ports 5838 * to return the port_attr of the next HCA port. A port 5839 * connected to IB fabric (port_state active) is returned, 5840 * if connected_flag is set. 5841 */ 5842 static void 5843 ibdm_get_next_port(ibdm_hca_list_t **inp_hcap, 5844 ibdm_port_attr_t **inp_portp, int connect_flag) 5845 { 5846 int ii; 5847 ibdm_port_attr_t *port, *next_port = NULL; 5848 ibdm_port_attr_t *inp_port; 5849 ibdm_hca_list_t *hca_list; 5850 int found = 0; 5851 5852 ASSERT(MUTEX_HELD(&ibdm.ibdm_hl_mutex)); 5853 IBTF_DPRINTF_L4(ibdm_string, "\tget_next_port(%p, %p, %x)", 5854 inp_hcap, inp_portp, connect_flag); 5855 5856 hca_list = *inp_hcap; 5857 inp_port = *inp_portp; 5858 5859 if (hca_list == NULL) 5860 hca_list = ibdm.ibdm_hca_list_head; 5861 5862 for (; hca_list; hca_list = hca_list->hl_next) { 5863 for (ii = 0; ii < hca_list->hl_nports; ii++) { 5864 port = &hca_list->hl_port_attr[ii]; 5865 5866 /* 5867 * inp_port != NULL; 5868 * Skip till we find the matching port 5869 */ 5870 if (inp_port && !found) { 5871 if (inp_port == port) 5872 found = 1; 5873 continue; 5874 } 5875 5876 if (!connect_flag) { 5877 next_port = port; 5878 break; 5879 } 5880 5881 if (port->pa_sa_hdl == NULL) 5882 ibdm_initialize_port(port); 5883 if (port->pa_sa_hdl == NULL) 5884 (void) ibdm_fini_port(port); 5885 else if (next_port == NULL && 5886 port->pa_sa_hdl != NULL && 5887 port->pa_state == IBT_PORT_ACTIVE) { 5888 next_port = port; 5889 break; 5890 } 5891 } 5892 5893 if (next_port) 5894 break; 5895 } 5896 5897 IBTF_DPRINTF_L4(ibdm_string, "\tget_next_port : " 5898 "returns hca_list %p port %p", hca_list, next_port); 5899 *inp_hcap = hca_list; 5900 *inp_portp = next_port; 5901 } 5902 5903 static void 5904 ibdm_add_to_gl_gid(ibdm_dp_gidinfo_t *nodegid, ibdm_dp_gidinfo_t *addgid) 5905 { 5906 ibdm_gid_t *tmp; 5907 5908 tmp = kmem_zalloc(sizeof (ibdm_gid_t), KM_SLEEP); 5909 tmp->gid_dgid_hi = addgid->gl_dgid_hi; 5910 tmp->gid_dgid_lo = addgid->gl_dgid_lo; 5911 5912 mutex_enter(&nodegid->gl_mutex); 5913 tmp->gid_next = nodegid->gl_gid; 5914 nodegid->gl_gid = tmp; 5915 nodegid->gl_ngids++; 5916 mutex_exit(&nodegid->gl_mutex); 5917 } 5918 5919 static void 5920 ibdm_addto_glhcalist(ibdm_dp_gidinfo_t *gid_info, 5921 ibdm_hca_list_t *hca) 5922 { 5923 ibdm_hca_list_t *head, *prev = NULL, *temp; 5924 5925 IBTF_DPRINTF_L4(ibdm_string, "\taddto_glhcalist(%p, %p) " 5926 ": gl_hca_list %p", gid_info, hca, gid_info->gl_hca_list); 5927 ASSERT(!MUTEX_HELD(&gid_info->gl_mutex)); 5928 mutex_enter(&gid_info->gl_mutex); 5929 head = gid_info->gl_hca_list; 5930 if (head == NULL) { 5931 head = ibdm_dup_hca_attr(hca); 5932 head->hl_next = NULL; 5933 gid_info->gl_hca_list = head; 5934 mutex_exit(&gid_info->gl_mutex); 5935 IBTF_DPRINTF_L4(ibdm_string, "\tadd_to_glhcalist: " 5936 "gid %p, gl_hca_list %p", gid_info, 5937 gid_info->gl_hca_list); 5938 return; 5939 } 5940 5941 /* Check if already in the list */ 5942 while (head) { 5943 if (head->hl_hca_guid == hca->hl_hca_guid) { 5944 mutex_exit(&gid_info->gl_mutex); 5945 IBTF_DPRINTF_L4(ibdm_string, 5946 "\taddto_glhcalist : gid %p hca %p dup", 5947 gid_info, hca); 5948 return; 5949 } 5950 prev = head; 5951 head = head->hl_next; 5952 } 5953 5954 /* Add this HCA to gl_hca_list */ 5955 temp = ibdm_dup_hca_attr(hca); 5956 temp->hl_next = NULL; 5957 prev->hl_next = temp; 5958 5959 mutex_exit(&gid_info->gl_mutex); 5960 5961 IBTF_DPRINTF_L4(ibdm_string, "\tadd_to_glhcalist: " 5962 "gid %p, gl_hca_list %p", gid_info, gid_info->gl_hca_list); 5963 } 5964 5965 static void 5966 ibdm_delete_glhca_list(ibdm_dp_gidinfo_t *gid_info) 5967 { 5968 ASSERT(!MUTEX_HELD(&gid_info->gl_mutex)); 5969 ASSERT(!MUTEX_HELD(&ibdm.ibdm_mutex)); 5970 5971 mutex_enter(&gid_info->gl_mutex); 5972 if (gid_info->gl_hca_list) 5973 ibdm_ibnex_free_hca_list(gid_info->gl_hca_list); 5974 gid_info->gl_hca_list = NULL; 5975 mutex_exit(&gid_info->gl_mutex); 5976 } 5977 5978 5979 static void 5980 ibdm_reset_all_dgids(ibmf_saa_handle_t port_sa_hdl) 5981 { 5982 IBTF_DPRINTF_L4(ibdm_string, "\treset_all_dgids(%X)", 5983 port_sa_hdl); 5984 5985 ASSERT(!MUTEX_HELD(&ibdm.ibdm_mutex)); 5986 ASSERT(!MUTEX_HELD(&ibdm.ibdm_hl_mutex)); 5987 5988 /* Check : Not busy in another probe / sweep */ 5989 mutex_enter(&ibdm.ibdm_mutex); 5990 if ((ibdm.ibdm_busy & IBDM_BUSY) == 0) { 5991 ibdm_dp_gidinfo_t *gid_info; 5992 5993 ibdm.ibdm_busy |= IBDM_BUSY; 5994 mutex_exit(&ibdm.ibdm_mutex); 5995 5996 /* 5997 * Check if any GID is using the SA & IBMF handle 5998 * of HCA port going down. Reset ibdm_dp_gidinfo_t 5999 * using another HCA port which can reach the GID. 6000 * This is for DM capable GIDs only, no need to do 6001 * this for others 6002 * 6003 * Delete the GID if no alternate HCA port to reach 6004 * it is found. 6005 */ 6006 for (gid_info = ibdm.ibdm_dp_gidlist_head; gid_info; ) { 6007 ibdm_dp_gidinfo_t *tmp; 6008 6009 IBTF_DPRINTF_L4(ibdm_string, "\tevent_hdlr " 6010 "checking gidinfo %p", gid_info); 6011 6012 if (gid_info->gl_sa_hdl == port_sa_hdl) { 6013 IBTF_DPRINTF_L3(ibdm_string, 6014 "\tevent_hdlr: down HCA port hdl " 6015 "matches gid %p", gid_info); 6016 6017 /* 6018 * The non-DM GIDs can come back 6019 * with a new subnet prefix, when 6020 * the HCA port commes up again. To 6021 * avoid issues, delete non-DM 6022 * capable GIDs, if the gid was 6023 * discovered using the HCA port 6024 * going down. This is ensured by 6025 * setting gl_disconnected to 1. 6026 */ 6027 if (gid_info->gl_nodeguid == 0) 6028 gid_info->gl_disconnected = 1; 6029 else 6030 ibdm_reset_gidinfo(gid_info); 6031 6032 if (gid_info->gl_disconnected) { 6033 IBTF_DPRINTF_L3(ibdm_string, 6034 "\tevent_hdlr: deleting" 6035 " gid %p", gid_info); 6036 tmp = gid_info; 6037 gid_info = gid_info->gl_next; 6038 ibdm_delete_gidinfo(tmp); 6039 } else 6040 gid_info = gid_info->gl_next; 6041 } else 6042 gid_info = gid_info->gl_next; 6043 } 6044 6045 mutex_enter(&ibdm.ibdm_mutex); 6046 ibdm.ibdm_busy &= ~IBDM_BUSY; 6047 cv_signal(&ibdm.ibdm_busy_cv); 6048 } 6049 mutex_exit(&ibdm.ibdm_mutex); 6050 } 6051 6052 static void 6053 ibdm_reset_gidinfo(ibdm_dp_gidinfo_t *gidinfo) 6054 { 6055 ibdm_hca_list_t *hca_list = NULL; 6056 ibdm_port_attr_t *port = NULL; 6057 int gid_reinited = 0; 6058 sa_node_record_t *nr, *tmp; 6059 sa_portinfo_record_t *pi; 6060 size_t nr_len = 0, pi_len = 0; 6061 size_t path_len; 6062 ib_gid_t sgid, dgid; 6063 int ret, ii, nrecords; 6064 sa_path_record_t *path; 6065 uint8_t npaths = 1; 6066 ibdm_pkey_tbl_t *pkey_tbl; 6067 6068 IBTF_DPRINTF_L4(ibdm_string, "\treset_gidinfo(%p)", gidinfo); 6069 6070 /* 6071 * Get list of all the ports reachable from the local known HCA 6072 * ports which are active 6073 */ 6074 mutex_enter(&ibdm.ibdm_hl_mutex); 6075 for (ibdm_get_next_port(&hca_list, &port, 1); port; 6076 ibdm_get_next_port(&hca_list, &port, 1)) { 6077 6078 6079 /* 6080 * Get the path and re-populate the gidinfo. 6081 * Getting the path is the same probe_ioc 6082 * Init the gid info as in ibdm_create_gidinfo() 6083 */ 6084 nr = ibdm_get_node_records(port->pa_sa_hdl, &nr_len, 6085 gidinfo->gl_nodeguid); 6086 if (nr == NULL) { 6087 IBTF_DPRINTF_L4(ibdm_string, 6088 "\treset_gidinfo : no records"); 6089 continue; 6090 } 6091 6092 nrecords = (nr_len / sizeof (sa_node_record_t)); 6093 for (tmp = nr, ii = 0; (ii < nrecords); ii++, tmp++) { 6094 if (tmp->NodeInfo.PortGUID == gidinfo->gl_portguid) 6095 break; 6096 } 6097 6098 if (ii == nrecords) { 6099 IBTF_DPRINTF_L4(ibdm_string, 6100 "\treset_gidinfo : no record for portguid"); 6101 kmem_free(nr, nr_len); 6102 continue; 6103 } 6104 6105 pi = ibdm_get_portinfo(port->pa_sa_hdl, &pi_len, tmp->LID); 6106 if (pi == NULL) { 6107 IBTF_DPRINTF_L4(ibdm_string, 6108 "\treset_gidinfo : no portinfo"); 6109 kmem_free(nr, nr_len); 6110 continue; 6111 } 6112 6113 sgid.gid_prefix = port->pa_sn_prefix; 6114 sgid.gid_guid = port->pa_port_guid; 6115 dgid.gid_prefix = pi->PortInfo.GidPrefix; 6116 dgid.gid_guid = tmp->NodeInfo.PortGUID; 6117 6118 ret = ibmf_saa_gid_to_pathrecords(port->pa_sa_hdl, sgid, dgid, 6119 IBMF_SAA_PKEY_WC, 0, B_TRUE, &npaths, 0, &path_len, &path); 6120 6121 if ((ret != IBMF_SUCCESS) || path == NULL) { 6122 IBTF_DPRINTF_L4(ibdm_string, 6123 "\treset_gidinfo : no paths"); 6124 kmem_free(pi, pi_len); 6125 kmem_free(nr, nr_len); 6126 continue; 6127 } 6128 6129 gidinfo->gl_dgid_hi = path->DGID.gid_prefix; 6130 gidinfo->gl_dgid_lo = path->DGID.gid_guid; 6131 gidinfo->gl_sgid_hi = path->SGID.gid_prefix; 6132 gidinfo->gl_sgid_lo = path->SGID.gid_guid; 6133 gidinfo->gl_p_key = path->P_Key; 6134 gidinfo->gl_sa_hdl = port->pa_sa_hdl; 6135 gidinfo->gl_ibmf_hdl = port->pa_ibmf_hdl; 6136 gidinfo->gl_slid = path->SLID; 6137 gidinfo->gl_dlid = path->DLID; 6138 /* Reset redirect info, next MAD will set if redirected */ 6139 gidinfo->gl_redirected = 0; 6140 6141 gidinfo->gl_qp_hdl = IBMF_QP_HANDLE_DEFAULT; 6142 for (ii = 0; ii < port->pa_npkeys; ii++) { 6143 if (port->pa_pkey_tbl == NULL) 6144 break; 6145 6146 pkey_tbl = &port->pa_pkey_tbl[ii]; 6147 if ((gidinfo->gl_p_key == pkey_tbl->pt_pkey) && 6148 (pkey_tbl->pt_qp_hdl != NULL)) { 6149 gidinfo->gl_qp_hdl = pkey_tbl->pt_qp_hdl; 6150 break; 6151 } 6152 } 6153 6154 if (gidinfo->gl_qp_hdl == NULL) 6155 IBTF_DPRINTF_L2(ibdm_string, 6156 "\treset_gid_info: No matching Pkey"); 6157 else 6158 gid_reinited = 1; 6159 6160 kmem_free(path, path_len); 6161 kmem_free(pi, pi_len); 6162 kmem_free(nr, nr_len); 6163 break; 6164 } 6165 mutex_exit(&ibdm.ibdm_hl_mutex); 6166 6167 if (!gid_reinited) 6168 gidinfo->gl_disconnected = 1; 6169 } 6170 6171 static void 6172 ibdm_delete_gidinfo(ibdm_dp_gidinfo_t *gidinfo) 6173 { 6174 ibdm_ioc_info_t *ioc_list; 6175 int in_gidlist = 0; 6176 6177 /* 6178 * Check if gidinfo has been inserted into the 6179 * ibdm_dp_gidlist_head list. gl_next or gl_prev 6180 * != NULL, if gidinfo is the list. 6181 */ 6182 if (gidinfo->gl_prev != NULL || 6183 gidinfo->gl_next != NULL || 6184 ibdm.ibdm_dp_gidlist_head == gidinfo) 6185 in_gidlist = 1; 6186 6187 ioc_list = ibdm_update_ioc_gidlist(gidinfo, 0); 6188 6189 /* 6190 * Remove GID from the global GID list 6191 * Handle the case where all port GIDs for an 6192 * IOU have been hot-removed. 6193 */ 6194 mutex_enter(&ibdm.ibdm_mutex); 6195 if (gidinfo->gl_iou != NULL && gidinfo->gl_ngids == 0) { 6196 mutex_enter(&gidinfo->gl_mutex); 6197 (void) ibdm_free_iou_info(gidinfo, &gidinfo->gl_iou); 6198 mutex_exit(&gidinfo->gl_mutex); 6199 } 6200 6201 /* Delete gl_hca_list */ 6202 mutex_exit(&ibdm.ibdm_mutex); 6203 ibdm_delete_glhca_list(gidinfo); 6204 mutex_enter(&ibdm.ibdm_mutex); 6205 6206 if (in_gidlist) { 6207 if (gidinfo->gl_prev != NULL) 6208 gidinfo->gl_prev->gl_next = gidinfo->gl_next; 6209 if (gidinfo->gl_next != NULL) 6210 gidinfo->gl_next->gl_prev = gidinfo->gl_prev; 6211 6212 if (gidinfo == ibdm.ibdm_dp_gidlist_head) 6213 ibdm.ibdm_dp_gidlist_head = gidinfo->gl_next; 6214 if (gidinfo == ibdm.ibdm_dp_gidlist_tail) 6215 ibdm.ibdm_dp_gidlist_tail = gidinfo->gl_prev; 6216 ibdm.ibdm_ngids--; 6217 } 6218 mutex_exit(&ibdm.ibdm_mutex); 6219 6220 mutex_destroy(&gidinfo->gl_mutex); 6221 kmem_free(gidinfo, sizeof (ibdm_dp_gidinfo_t)); 6222 6223 /* 6224 * Pass on the IOCs with updated GIDs to IBnexus 6225 */ 6226 if (ioc_list) { 6227 IBTF_DPRINTF_L4("ibdm", "\tdelete_gidinfo " 6228 "IOC_PROP_UPDATE for %p\n", ioc_list); 6229 mutex_enter(&ibdm.ibdm_ibnex_mutex); 6230 if (ibdm.ibdm_ibnex_callback != NULL) { 6231 (*ibdm.ibdm_ibnex_callback)((void *) 6232 ioc_list, IBDM_EVENT_IOC_PROP_UPDATE); 6233 } 6234 mutex_exit(&ibdm.ibdm_ibnex_mutex); 6235 } 6236 } 6237 6238 6239 static void 6240 ibdm_fill_srv_attr_mod(ib_mad_hdr_t *hdr, ibdm_timeout_cb_args_t *cb_args) 6241 { 6242 uint32_t attr_mod; 6243 6244 attr_mod = (cb_args->cb_ioc_num + 1) << 16; 6245 attr_mod |= cb_args->cb_srvents_start; 6246 attr_mod |= (cb_args->cb_srvents_end) << 8; 6247 hdr->AttributeModifier = h2b32(attr_mod); 6248 } 6249 6250 static void 6251 ibdm_bump_transactionID(ibdm_dp_gidinfo_t *gid_info) 6252 { 6253 ASSERT(MUTEX_HELD(&gid_info->gl_mutex)); 6254 gid_info->gl_transactionID++; 6255 if (gid_info->gl_transactionID == gid_info->gl_max_transactionID) { 6256 IBTF_DPRINTF_L4(ibdm_string, 6257 "\tbump_transactionID(%p), wrapup", gid_info); 6258 gid_info->gl_transactionID = gid_info->gl_min_transactionID; 6259 } 6260 } 6261 6262 /* 6263 * gl_prev_iou is set for *non-reprobe* sweeep requests, which 6264 * detected that ChangeID in IOU info has changed. The service 6265 * entry also may have changed. Check if service entry in IOC 6266 * has changed wrt the prev iou, if so notify to IB Nexus. 6267 */ 6268 static ibdm_ioc_info_t * 6269 ibdm_handle_prev_iou() 6270 { 6271 ibdm_dp_gidinfo_t *gid_info; 6272 ibdm_ioc_info_t *ioc_list_head = NULL, *ioc_list; 6273 ibdm_ioc_info_t *prev_ioc, *ioc; 6274 int ii, jj, niocs, prev_niocs; 6275 6276 ASSERT(MUTEX_HELD(&ibdm.ibdm_mutex)); 6277 6278 IBTF_DPRINTF_L4(ibdm_string, "\thandle_prev_iou enter"); 6279 for (gid_info = ibdm.ibdm_dp_gidlist_head; gid_info; 6280 gid_info = gid_info->gl_next) { 6281 if (gid_info->gl_prev_iou == NULL) 6282 continue; 6283 6284 IBTF_DPRINTF_L4(ibdm_string, "\thandle_prev_iou gid %p", 6285 gid_info); 6286 niocs = gid_info->gl_iou->iou_info.iou_num_ctrl_slots; 6287 prev_niocs = 6288 gid_info->gl_prev_iou->iou_info.iou_num_ctrl_slots; 6289 for (ii = 0; ii < niocs; ii++) { 6290 ioc = IBDM_GIDINFO2IOCINFO(gid_info, ii); 6291 6292 /* Find matching IOC */ 6293 for (jj = 0; jj < prev_niocs; jj++) { 6294 prev_ioc = (ibdm_ioc_info_t *) 6295 &gid_info->gl_prev_iou->iou_ioc_info[jj]; 6296 if (prev_ioc->ioc_profile.ioc_guid == 6297 ioc->ioc_profile.ioc_guid) 6298 break; 6299 } 6300 if (jj == prev_niocs) 6301 prev_ioc = NULL; 6302 if (ioc == NULL || prev_ioc == NULL) 6303 continue; 6304 if ((ioc->ioc_profile.ioc_service_entries != 6305 prev_ioc->ioc_profile.ioc_service_entries) || 6306 ibdm_serv_cmp(&ioc->ioc_serv[0], 6307 &prev_ioc->ioc_serv[0], 6308 ioc->ioc_profile.ioc_service_entries) != 0) { 6309 IBTF_DPRINTF_L4(ibdm_string, 6310 "/thandle_prev_iou modified IOC: " 6311 "current ioc %p, old ioc %p", 6312 ioc, prev_ioc); 6313 ioc_list = ibdm_dup_ioc_info(ioc, gid_info); 6314 ioc_list->ioc_info_updated.ib_prop_updated 6315 = 0; 6316 ioc_list->ioc_info_updated.ib_srv_prop_updated 6317 = 1; 6318 6319 if (ioc_list_head == NULL) 6320 ioc_list_head = ioc_list; 6321 else { 6322 ioc_list_head->ioc_next = ioc_list; 6323 ioc_list_head = ioc_list; 6324 } 6325 } 6326 } 6327 6328 mutex_enter(&gid_info->gl_mutex); 6329 (void) ibdm_free_iou_info(gid_info, &gid_info->gl_prev_iou); 6330 mutex_exit(&gid_info->gl_mutex); 6331 } 6332 IBTF_DPRINTF_L4(ibdm_string, "\thandle_prev_iouret %p", 6333 ioc_list_head); 6334 return (ioc_list_head); 6335 } 6336 6337 /* 6338 * Compares two service entries lists, returns 0 if same, returns 1 6339 * if no match. 6340 */ 6341 static int 6342 ibdm_serv_cmp(ibdm_srvents_info_t *serv1, ibdm_srvents_info_t *serv2, 6343 int nserv) 6344 { 6345 int ii; 6346 6347 IBTF_DPRINTF_L4(ibdm_string, "\tserv_cmp: enter"); 6348 for (ii = 0; ii < nserv; ii++, serv1++, serv2++) { 6349 if (serv1->se_attr.srv_id != serv2->se_attr.srv_id || 6350 bcmp(serv1->se_attr.srv_name, 6351 serv2->se_attr.srv_name, 6352 IB_DM_MAX_SVC_NAME_LEN) != 0) { 6353 IBTF_DPRINTF_L4(ibdm_string, "\tserv_cmp: ret 1"); 6354 return (1); 6355 } 6356 } 6357 IBTF_DPRINTF_L4(ibdm_string, "\tserv_cmp: ret 0"); 6358 return (0); 6359 } 6360 6361 /* For debugging purpose only */ 6362 #ifdef DEBUG 6363 void 6364 ibdm_dump_ibmf_msg(ibmf_msg_t *ibmf_msg, int flag) 6365 { 6366 ib_mad_hdr_t *mad_hdr; 6367 6368 IBTF_DPRINTF_L4("ibdm", "\t\t(IBMF_PKT): Local address info"); 6369 IBTF_DPRINTF_L4("ibdm", "\t\t ------------------"); 6370 6371 IBTF_DPRINTF_L4("ibdm", "\tLocal Lid : 0x%x\tRemote Lid : 0x%x" 6372 " Remote Qp : 0x%x", ibmf_msg->im_local_addr.ia_local_lid, 6373 ibmf_msg->im_local_addr.ia_remote_lid, 6374 ibmf_msg->im_local_addr.ia_remote_qno); 6375 IBTF_DPRINTF_L4("ibdm", "\tP_key : 0x%x\tQ_key : 0x%x", 6376 ibmf_msg->im_local_addr.ia_p_key, ibmf_msg->im_local_addr.ia_q_key); 6377 6378 if (flag) 6379 mad_hdr = (ib_mad_hdr_t *)IBDM_OUT_IBMFMSG_MADHDR(ibmf_msg); 6380 else 6381 mad_hdr = IBDM_IN_IBMFMSG_MADHDR(ibmf_msg); 6382 6383 IBTF_DPRINTF_L4("ibdm", "\t\t MAD Header info"); 6384 IBTF_DPRINTF_L4("ibdm", "\t\t ---------------"); 6385 6386 IBTF_DPRINTF_L4("ibdm", "\tBase version : 0x%x" 6387 "\tMgmt Class : 0x%x", mad_hdr->BaseVersion, mad_hdr->MgmtClass); 6388 IBTF_DPRINTF_L4("ibdm", "\tClass version : 0x%x" 6389 "\tR Method : 0x%x", 6390 mad_hdr->ClassVersion, mad_hdr->R_Method); 6391 IBTF_DPRINTF_L4("ibdm", "\tMAD Status : 0x%x" 6392 "\tTransaction ID : 0x%llx", 6393 mad_hdr->Status, mad_hdr->TransactionID); 6394 IBTF_DPRINTF_L4("ibdm", "\t Attribute ID : 0x%x" 6395 "\tAttribute Modified : 0x%lx", 6396 mad_hdr->AttributeID, mad_hdr->AttributeModifier); 6397 } 6398 6399 void 6400 ibdm_dump_path_info(sa_path_record_t *path) 6401 { 6402 IBTF_DPRINTF_L4("ibdm", "\t\t Path information"); 6403 IBTF_DPRINTF_L4("ibdm", "\t\t ----------------"); 6404 6405 IBTF_DPRINTF_L4("ibdm", "\t DGID hi : %llx\tDGID lo : %llx", 6406 path->DGID.gid_prefix, path->DGID.gid_guid); 6407 IBTF_DPRINTF_L4("ibdm", "\t SGID hi : %llx\tSGID lo : %llx", 6408 path->SGID.gid_prefix, path->SGID.gid_guid); 6409 IBTF_DPRINTF_L4("ibdm", "\t SLID : %x\tDlID : %x", 6410 path->SLID, path->DLID); 6411 IBTF_DPRINTF_L4("ibdm", "\t P Key : %x", path->P_Key); 6412 } 6413 6414 6415 void 6416 ibdm_dump_classportinfo(ibdm_mad_classportinfo_t *classportinfo) 6417 { 6418 IBTF_DPRINTF_L4("ibdm", "\t\t CLASSPORT INFO"); 6419 IBTF_DPRINTF_L4("ibdm", "\t\t --------------"); 6420 6421 IBTF_DPRINTF_L4("ibdm", "\t Response Time Value : 0x%x", 6422 ((b2h32(classportinfo->RespTimeValue)) & 0x1F)); 6423 6424 IBTF_DPRINTF_L4("ibdm", "\t Redirected QP : 0x%x", 6425 (b2h32(classportinfo->RedirectQP))); 6426 IBTF_DPRINTF_L4("ibdm", "\t Redirected P KEY : 0x%x", 6427 b2h16(classportinfo->RedirectP_Key)); 6428 IBTF_DPRINTF_L4("ibdm", "\t Redirected Q KEY : 0x%x", 6429 b2h16(classportinfo->RedirectQ_Key)); 6430 IBTF_DPRINTF_L4("ibdm", "\t Redirected GID hi : 0x%x", 6431 b2h64(classportinfo->RedirectGID_hi)); 6432 IBTF_DPRINTF_L4("ibdm", "\t Redirected GID lo : 0x%x", 6433 b2h64(classportinfo->RedirectGID_lo)); 6434 } 6435 6436 6437 void 6438 ibdm_dump_iounitinfo(ib_dm_io_unitinfo_t *iou_info) 6439 { 6440 IBTF_DPRINTF_L4("ibdm", "\t\t I/O UnitInfo"); 6441 IBTF_DPRINTF_L4("ibdm", "\t\t ------------"); 6442 6443 IBTF_DPRINTF_L4("ibdm", "\tChange ID : 0x%x", 6444 b2h16(iou_info->iou_changeid)); 6445 IBTF_DPRINTF_L4("ibdm", "\t#of ctrl slots : %d", 6446 iou_info->iou_num_ctrl_slots); 6447 IBTF_DPRINTF_L4("ibdm", "\tIOU flag : 0x%x", 6448 iou_info->iou_flag); 6449 IBTF_DPRINTF_L4("ibdm", "\tContrl list byte 0 : 0x%x", 6450 iou_info->iou_ctrl_list[0]); 6451 IBTF_DPRINTF_L4("ibdm", "\tContrl list byte 1 : 0x%x", 6452 iou_info->iou_ctrl_list[1]); 6453 IBTF_DPRINTF_L4("ibdm", "\tContrl list byte 2 : 0x%x", 6454 iou_info->iou_ctrl_list[2]); 6455 } 6456 6457 6458 void 6459 ibdm_dump_ioc_profile(ib_dm_ioc_ctrl_profile_t *ioc) 6460 { 6461 IBTF_DPRINTF_L4("ibdm", "\t\t IOC Controller Profile"); 6462 IBTF_DPRINTF_L4("ibdm", "\t\t ----------------------"); 6463 6464 IBTF_DPRINTF_L4("ibdm", "\tIOC Guid : %llx", ioc->ioc_guid); 6465 IBTF_DPRINTF_L4("ibdm", "\tVendorID : 0x%x", ioc->ioc_vendorid); 6466 IBTF_DPRINTF_L4("ibdm", "\tDevice Id : 0x%x", ioc->ioc_deviceid); 6467 IBTF_DPRINTF_L4("ibdm", "\tDevice Ver : 0x%x", ioc->ioc_device_ver); 6468 IBTF_DPRINTF_L4("ibdm", "\tSubsys ID : 0x%x", ioc->ioc_subsys_id); 6469 IBTF_DPRINTF_L4("ibdm", "\tIO class : 0x%x", ioc->ioc_io_class); 6470 IBTF_DPRINTF_L4("ibdm", "\tIO subclass : 0x%x", ioc->ioc_io_subclass); 6471 IBTF_DPRINTF_L4("ibdm", "\tProtocol : 0x%x", ioc->ioc_protocol); 6472 IBTF_DPRINTF_L4("ibdm", "\tProtocolV : 0x%x", ioc->ioc_protocol_ver); 6473 IBTF_DPRINTF_L4("ibdm", "\tmsg qdepth : %d", ioc->ioc_send_msg_qdepth); 6474 IBTF_DPRINTF_L4("ibdm", "\trdma qdepth : %d", 6475 ioc->ioc_rdma_read_qdepth); 6476 IBTF_DPRINTF_L4("ibdm", "\tsndmsg sz : %d", ioc->ioc_send_msg_sz); 6477 IBTF_DPRINTF_L4("ibdm", "\trdma xfersz : %d", ioc->ioc_rdma_xfer_sz); 6478 IBTF_DPRINTF_L4("ibdm", "\topcal mask : 0x%x", 6479 ioc->ioc_ctrl_opcap_mask); 6480 IBTF_DPRINTF_L4("ibdm", "\tsrventries : %x", ioc->ioc_service_entries); 6481 } 6482 6483 6484 void 6485 ibdm_dump_service_entries(ib_dm_srv_t *srv_ents) 6486 { 6487 IBTF_DPRINTF_L4("ibdm", 6488 "\thandle_srventry_mad: service id : %llx", srv_ents->srv_id); 6489 6490 IBTF_DPRINTF_L4("ibdm", "\thandle_srventry_mad: " 6491 "Service Name : %s", srv_ents->srv_name); 6492 } 6493 6494 int ibdm_allow_sweep_fabric_timestamp = 1; 6495 6496 void 6497 ibdm_dump_sweep_fabric_timestamp(int flag) 6498 { 6499 static hrtime_t x; 6500 if (flag) { 6501 if (ibdm_allow_sweep_fabric_timestamp) { 6502 IBTF_DPRINTF_L4("ibdm", "\tTime taken to complete " 6503 "sweep %lld ms", ((gethrtime() - x)/ 1000000)); 6504 } 6505 x = 0; 6506 } else 6507 x = gethrtime(); 6508 } 6509 #endif 6510