1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License, Version 1.0 only 6 * (the "License"). You may not use this file except in compliance 7 * with the License. 8 * 9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10 * or http://www.opensolaris.org/os/licensing. 11 * See the License for the specific language governing permissions 12 * and limitations under the License. 13 * 14 * When distributing Covered Code, include this CDDL HEADER in each 15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16 * If applicable, add the following below this CDDL HEADER, with the 17 * fields enclosed by brackets "[]" replaced with your own identifying 18 * information: Portions Copyright [yyyy] [name of copyright owner] 19 * 20 * CDDL HEADER END 21 */ 22 /* 23 * Copyright 2005 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 #pragma ident "%Z%%M% %I% %E% SMI" 28 29 /* 30 * ibdm.c 31 * 32 * This file contains the InifiniBand Device Manager (IBDM) support functions. 33 * IB nexus driver will only be the client for the IBDM module. 34 * 35 * IBDM registers with IBTF for HCA arrival/removal notification. 36 * IBDM registers with SA access to send DM MADs to discover the IOC's behind 37 * the IOU's. 38 * 39 * IB nexus driver registers with IBDM to find the information about the 40 * HCA's and IOC's (behind the IOU) present on the IB fabric. 41 */ 42 43 #include <sys/systm.h> 44 #include <sys/taskq.h> 45 #include <sys/ib/mgt/ibdm/ibdm_impl.h> 46 #include <sys/modctl.h> 47 48 /* Function Prototype declarations */ 49 static int ibdm_free_iou_info(ibdm_dp_gidinfo_t *); 50 static int ibdm_fini(void); 51 static int ibdm_init(void); 52 static int ibdm_get_reachable_ports(ibdm_port_attr_t *, 53 ibdm_hca_list_t *); 54 static ibdm_dp_gidinfo_t *ibdm_check_dgid(ib_guid_t, ib_sn_prefix_t); 55 static ibdm_dp_gidinfo_t *ibdm_check_dest_nodeguid(ibdm_dp_gidinfo_t *); 56 static int ibdm_send_classportinfo(ibdm_dp_gidinfo_t *); 57 static int ibdm_send_iounitinfo(ibdm_dp_gidinfo_t *); 58 static int ibdm_is_dev_mgt_supported(ibdm_dp_gidinfo_t *); 59 static int ibdm_get_node_port_guids(ibmf_saa_handle_t, ib_lid_t, 60 ib_guid_t *, ib_guid_t *); 61 static int ibdm_retry_command(ibdm_timeout_cb_args_t *); 62 static int ibdm_get_diagcode(ibdm_dp_gidinfo_t *, int); 63 static int ibdm_verify_mad_status(ib_mad_hdr_t *); 64 static int ibdm_handle_redirection(ibmf_msg_t *, 65 ibdm_dp_gidinfo_t *, int *); 66 static void ibdm_wait_probe_completion(void); 67 static void ibdm_sweep_fabric(int); 68 static void ibdm_probe_gid_thread(void *); 69 static void ibdm_wakeup_probe_gid_cv(void); 70 static void ibdm_port_attr_ibmf_init(ibdm_port_attr_t *, ib_pkey_t, int); 71 static int ibdm_port_attr_ibmf_fini(ibdm_port_attr_t *, int); 72 static void ibdm_update_port_attr(ibdm_port_attr_t *); 73 static void ibdm_handle_hca_attach(ib_guid_t); 74 static void ibdm_handle_srventry_mad(ibmf_msg_t *, 75 ibdm_dp_gidinfo_t *, int *); 76 static void ibdm_ibmf_recv_cb(ibmf_handle_t, ibmf_msg_t *, void *); 77 static void ibdm_recv_incoming_mad(void *); 78 static void ibdm_process_incoming_mad(ibmf_handle_t, ibmf_msg_t *, void *); 79 static void ibdm_ibmf_send_cb(ibmf_handle_t, ibmf_msg_t *, void *); 80 static void ibdm_pkt_timeout_hdlr(void *arg); 81 static void ibdm_initialize_port(ibdm_port_attr_t *); 82 static void ibdm_handle_diagcode(ibmf_msg_t *, ibdm_dp_gidinfo_t *, int *); 83 static void ibdm_probe_gid(ibdm_dp_gidinfo_t *); 84 static void ibdm_alloc_send_buffers(ibmf_msg_t *); 85 static void ibdm_free_send_buffers(ibmf_msg_t *); 86 static void ibdm_handle_hca_detach(ib_guid_t); 87 static int ibdm_fini_port(ibdm_port_attr_t *); 88 static int ibdm_uninit_hca(ibdm_hca_list_t *); 89 static void ibdm_handle_iounitinfo(ibmf_handle_t, 90 ibmf_msg_t *, ibdm_dp_gidinfo_t *, int *); 91 static void ibdm_handle_ioc_profile(ibmf_handle_t, 92 ibmf_msg_t *, ibdm_dp_gidinfo_t *, int *); 93 static void ibdm_event_hdlr(void *, ibt_hca_hdl_t, 94 ibt_async_code_t, ibt_async_event_t *); 95 static void ibdm_handle_classportinfo(ibmf_handle_t, 96 ibmf_msg_t *, ibdm_dp_gidinfo_t *, int *); 97 static void ibdm_update_ioc_port_gidlist(ibdm_ioc_info_t *, 98 ibdm_dp_gidinfo_t *); 99 100 static ibdm_hca_list_t *ibdm_dup_hca_attr(ibdm_hca_list_t *); 101 static ibdm_ioc_info_t *ibdm_dup_ioc_info(ibdm_ioc_info_t *, 102 ibdm_dp_gidinfo_t *gid_list); 103 static void ibdm_probe_ioc(ib_guid_t, ib_guid_t, int); 104 static ibdm_ioc_info_t *ibdm_is_ioc_present(ib_guid_t, 105 ibdm_dp_gidinfo_t *, int *); 106 static ibdm_port_attr_t *ibdm_get_port_attr(ibt_async_event_t *, 107 ibdm_hca_list_t **); 108 static sa_node_record_t *ibdm_get_node_records(ibmf_saa_handle_t, 109 size_t *, ib_guid_t); 110 static sa_portinfo_record_t *ibdm_get_portinfo(ibmf_saa_handle_t, size_t *, 111 ib_lid_t); 112 static ibdm_dp_gidinfo_t *ibdm_create_gid_info(ibdm_port_attr_t *, 113 ib_gid_t, ib_gid_t); 114 static ibdm_dp_gidinfo_t *ibdm_find_gid(ib_guid_t, ib_guid_t); 115 static int ibdm_send_ioc_profile(ibdm_dp_gidinfo_t *, uint8_t); 116 static ibdm_ioc_info_t *ibdm_update_ioc_gidlist(ibdm_dp_gidinfo_t *, int); 117 static void ibdm_saa_event_cb(ibmf_saa_handle_t, ibmf_saa_subnet_event_t, 118 ibmf_saa_event_details_t *, void *); 119 static void ibdm_reprobe_update_port_srv(ibdm_ioc_info_t *, 120 ibdm_dp_gidinfo_t *); 121 static ibdm_dp_gidinfo_t *ibdm_handle_gid_rm(ibdm_dp_gidinfo_t *); 122 static void ibdm_rmfrom_glgid_list(ibdm_dp_gidinfo_t *, 123 ibdm_dp_gidinfo_t *); 124 static void ibdm_addto_gidlist(ibdm_gid_t **, ibdm_gid_t *); 125 static void ibdm_free_gid_list(ibdm_gid_t *); 126 static void ibdm_rescan_gidlist(ib_guid_t *ioc_guid); 127 static void ibdm_notify_newgid_iocs(ibdm_dp_gidinfo_t *); 128 static void ibdm_saa_event_taskq(void *); 129 static void ibdm_free_saa_event_arg(ibdm_saa_event_arg_t *); 130 static void ibdm_get_next_port(ibdm_hca_list_t **, 131 ibdm_port_attr_t **, int); 132 static void ibdm_add_to_gl_gid(ibdm_dp_gidinfo_t *, 133 ibdm_dp_gidinfo_t *); 134 static void ibdm_addto_glhcalist(ibdm_dp_gidinfo_t *, 135 ibdm_hca_list_t *); 136 static void ibdm_delete_glhca_list(ibdm_dp_gidinfo_t *); 137 static void ibdm_saa_handle_new_gid(void *); 138 static void ibdm_reset_all_dgids(ibmf_saa_handle_t); 139 static void ibdm_reset_gidinfo(ibdm_dp_gidinfo_t *); 140 static void ibdm_delete_gidinfo(ibdm_dp_gidinfo_t *); 141 static void ibdm_fill_srv_attr_mod(ib_mad_hdr_t *, ibdm_timeout_cb_args_t *); 142 static void ibdm_bump_transactionID(ibdm_dp_gidinfo_t *); 143 144 int ibdm_dft_timeout = IBDM_DFT_TIMEOUT; 145 int ibdm_dft_retry_cnt = IBDM_DFT_NRETRIES; 146 #ifdef DEBUG 147 int ibdm_ignore_saa_event = 0; 148 #endif 149 150 /* Modload support */ 151 static struct modlmisc ibdm_modlmisc = { 152 &mod_miscops, 153 "InfiniBand Device Manager %I%", 154 }; 155 156 struct modlinkage ibdm_modlinkage = { 157 MODREV_1, 158 (void *)&ibdm_modlmisc, 159 NULL 160 }; 161 162 static ibt_clnt_modinfo_t ibdm_ibt_modinfo = { 163 IBTI_V2, 164 IBT_DM, 165 ibdm_event_hdlr, 166 NULL, 167 "ibdm" 168 }; 169 170 /* Global variables */ 171 ibdm_t ibdm; 172 int ibdm_taskq_enable = IBDM_ENABLE_TASKQ_HANDLING; 173 char *ibdm_string = "ibdm"; 174 175 _NOTE(SCHEME_PROTECTS_DATA("Serialized access by cv", 176 ibdm.ibdm_dp_gidlist_head)) 177 178 /* 179 * _init 180 * Loadable module init, called before any other module. 181 * Initialize mutex 182 * Register with IBTF 183 */ 184 int 185 _init(void) 186 { 187 int err; 188 189 IBTF_DPRINTF_L4("ibdm", "\t_init: addr of ibdm %p", &ibdm); 190 191 if ((err = ibdm_init()) != IBDM_SUCCESS) { 192 IBTF_DPRINTF_L2("ibdm", "_init: ibdm_init failed 0x%x", err); 193 (void) ibdm_fini(); 194 return (DDI_FAILURE); 195 } 196 197 if ((err = mod_install(&ibdm_modlinkage)) != 0) { 198 IBTF_DPRINTF_L2("ibdm", "_init: mod_install failed 0x%x", err); 199 (void) ibdm_fini(); 200 } 201 return (err); 202 } 203 204 205 int 206 _fini(void) 207 { 208 int err; 209 210 if ((err = ibdm_fini()) != IBDM_SUCCESS) { 211 IBTF_DPRINTF_L2("ibdm", "_fini: ibdm_fini failed 0x%x", err); 212 (void) ibdm_init(); 213 return (EBUSY); 214 } 215 216 if ((err = mod_remove(&ibdm_modlinkage)) != 0) { 217 IBTF_DPRINTF_L2("ibdm", "_fini: mod_remove failed 0x%x", err); 218 (void) ibdm_init(); 219 } 220 return (err); 221 } 222 223 224 int 225 _info(struct modinfo *modinfop) 226 { 227 return (mod_info(&ibdm_modlinkage, modinfop)); 228 } 229 230 231 /* 232 * ibdm_init(): 233 * Register with IBTF 234 * Allocate memory for the HCAs 235 * Allocate minor-nodes for the HCAs 236 */ 237 static int 238 ibdm_init(void) 239 { 240 int i, hca_count; 241 ib_guid_t *hca_guids; 242 ibt_status_t status; 243 244 IBTF_DPRINTF_L4("ibdm", "\tibdm_init:"); 245 if (!(ibdm.ibdm_state & IBDM_LOCKS_ALLOCED)) { 246 mutex_init(&ibdm.ibdm_mutex, NULL, MUTEX_DEFAULT, NULL); 247 mutex_init(&ibdm.ibdm_hl_mutex, NULL, MUTEX_DEFAULT, NULL); 248 mutex_init(&ibdm.ibdm_ibnex_mutex, NULL, MUTEX_DEFAULT, NULL); 249 mutex_enter(&ibdm.ibdm_mutex); 250 ibdm.ibdm_state |= IBDM_LOCKS_ALLOCED; 251 } 252 253 if (!(ibdm.ibdm_state & IBDM_IBT_ATTACHED)) { 254 if ((status = ibt_attach(&ibdm_ibt_modinfo, NULL, NULL, 255 (void *)&ibdm.ibdm_ibt_clnt_hdl)) != IBT_SUCCESS) { 256 IBTF_DPRINTF_L2("ibdm", "ibdm_init: ibt_attach " 257 "failed %x", status); 258 mutex_exit(&ibdm.ibdm_mutex); 259 return (IBDM_FAILURE); 260 } 261 262 ibdm.ibdm_state |= IBDM_IBT_ATTACHED; 263 mutex_exit(&ibdm.ibdm_mutex); 264 } 265 266 267 if (!(ibdm.ibdm_state & IBDM_HCA_ATTACHED)) { 268 hca_count = ibt_get_hca_list(&hca_guids); 269 IBTF_DPRINTF_L4("ibdm", "ibdm_init: num_hcas = %d", hca_count); 270 for (i = 0; i < hca_count; i++) 271 (void) ibdm_handle_hca_attach(hca_guids[i]); 272 if (hca_count) 273 ibt_free_hca_list(hca_guids, hca_count); 274 275 mutex_enter(&ibdm.ibdm_mutex); 276 ibdm.ibdm_state |= IBDM_HCA_ATTACHED; 277 mutex_exit(&ibdm.ibdm_mutex); 278 } 279 280 if (!(ibdm.ibdm_state & IBDM_CVS_ALLOCED)) { 281 cv_init(&ibdm.ibdm_probe_cv, NULL, CV_DRIVER, NULL); 282 cv_init(&ibdm.ibdm_busy_cv, NULL, CV_DRIVER, NULL); 283 mutex_enter(&ibdm.ibdm_mutex); 284 ibdm.ibdm_state |= IBDM_CVS_ALLOCED; 285 mutex_exit(&ibdm.ibdm_mutex); 286 } 287 return (IBDM_SUCCESS); 288 } 289 290 291 static int 292 ibdm_free_iou_info(ibdm_dp_gidinfo_t *gid_info) 293 { 294 int ii, k, niocs; 295 size_t size; 296 ibdm_gid_t *delete, *head; 297 timeout_id_t timeout_id; 298 ibdm_ioc_info_t *ioc; 299 300 ASSERT(mutex_owned(&gid_info->gl_mutex)); 301 if (gid_info->gl_iou == NULL) { 302 IBTF_DPRINTF_L4("ibdm", "\tibdm_free_iou_info: No IOU"); 303 return (0); 304 } 305 306 niocs = gid_info->gl_iou->iou_info.iou_num_ctrl_slots; 307 IBTF_DPRINTF_L4("ibdm", "\tfree_iou_info: gid_info = %p, niocs %d", 308 gid_info, niocs); 309 310 for (ii = 0; ii < niocs; ii++) { 311 ioc = IBDM_GIDINFO2IOCINFO(gid_info, ii); 312 313 /* handle the case where an ioc_timeout_id is scheduled */ 314 if (ioc->ioc_timeout_id) { 315 timeout_id = ioc->ioc_timeout_id; 316 mutex_exit(&gid_info->gl_mutex); 317 IBTF_DPRINTF_L5("ibdm", "free_iou_info: " 318 "ioc_timeout_id = 0x%x", timeout_id); 319 if (untimeout(timeout_id) == -1) { 320 IBTF_DPRINTF_L2("ibdm", "free_iou_info: " 321 "untimeout ioc_timeout_id failed"); 322 mutex_enter(&gid_info->gl_mutex); 323 return (-1); 324 } 325 mutex_enter(&gid_info->gl_mutex); 326 ioc->ioc_timeout_id = 0; 327 } 328 329 /* handle the case where an ioc_dc_timeout_id is scheduled */ 330 if (ioc->ioc_dc_timeout_id) { 331 timeout_id = ioc->ioc_dc_timeout_id; 332 mutex_exit(&gid_info->gl_mutex); 333 IBTF_DPRINTF_L5("ibdm", "free_iou_info: " 334 "ioc_dc_timeout_id = 0x%x", timeout_id); 335 if (untimeout(timeout_id) == -1) { 336 IBTF_DPRINTF_L2("ibdm", "free_iou_info: " 337 "untimeout ioc_dc_timeout_id failed"); 338 mutex_enter(&gid_info->gl_mutex); 339 return (-1); 340 } 341 mutex_enter(&gid_info->gl_mutex); 342 ioc->ioc_dc_timeout_id = 0; 343 } 344 345 /* handle the case where serv[k].se_timeout_id is scheduled */ 346 for (k = 0; k < ioc->ioc_profile.ioc_service_entries; k++) { 347 if (ioc->ioc_serv[k].se_timeout_id) { 348 timeout_id = ioc->ioc_serv[k].se_timeout_id; 349 mutex_exit(&gid_info->gl_mutex); 350 IBTF_DPRINTF_L5("ibdm", "free_iou_info: " 351 "ioc->ioc_serv[%d].se_timeout_id = 0x%x", 352 k, timeout_id); 353 if (untimeout(timeout_id) == -1) { 354 IBTF_DPRINTF_L2("ibdm", "free_iou_info:" 355 " untimeout se_timeout_id failed"); 356 mutex_enter(&gid_info->gl_mutex); 357 return (-1); 358 } 359 mutex_enter(&gid_info->gl_mutex); 360 ioc->ioc_serv[k].se_timeout_id = 0; 361 } 362 } 363 364 /* delete GID list */ 365 head = ioc->ioc_gid_list; 366 while (head) { 367 IBTF_DPRINTF_L4("ibdm", "\tibdm_free_iou_info: " 368 "Deleting gid_list struct %p", head); 369 delete = head; 370 head = head->gid_next; 371 kmem_free(delete, sizeof (ibdm_gid_t)); 372 } 373 ioc->ioc_gid_list = NULL; 374 375 /* delete ioc_serv */ 376 size = ioc->ioc_profile.ioc_service_entries * 377 sizeof (ibdm_srvents_info_t); 378 if (ioc->ioc_serv && size) { 379 kmem_free(ioc->ioc_serv, size); 380 ioc->ioc_serv = NULL; 381 } 382 } 383 384 IBTF_DPRINTF_L4("ibdm", "\tibdm_free_iou_info: deleting IOU & IOC"); 385 size = sizeof (ibdm_iou_info_t) + niocs * sizeof (ibdm_ioc_info_t); 386 kmem_free(gid_info->gl_iou, size); 387 gid_info->gl_iou = NULL; 388 return (0); 389 } 390 391 392 /* 393 * ibdm_fini(): 394 * Un-register with IBTF 395 * De allocate memory for the GID info 396 */ 397 static int 398 ibdm_fini() 399 { 400 int ii; 401 ibdm_hca_list_t *hca_list, *temp; 402 ibdm_dp_gidinfo_t *gid_info, *tmp; 403 ibdm_gid_t *head, *delete; 404 405 IBTF_DPRINTF_L4("ibdm", "\tibdm_fini"); 406 407 mutex_enter(&ibdm.ibdm_hl_mutex); 408 if (ibdm.ibdm_state & IBDM_IBT_ATTACHED) { 409 if (ibt_detach(ibdm.ibdm_ibt_clnt_hdl) != IBT_SUCCESS) { 410 IBTF_DPRINTF_L2("ibdm", "\t_fini: ibt_detach failed"); 411 mutex_exit(&ibdm.ibdm_hl_mutex); 412 return (IBDM_FAILURE); 413 } 414 ibdm.ibdm_state &= ~IBDM_IBT_ATTACHED; 415 ibdm.ibdm_ibt_clnt_hdl = NULL; 416 } 417 418 hca_list = ibdm.ibdm_hca_list_head; 419 IBTF_DPRINTF_L4("ibdm", "\tibdm_fini: nhcas %d", ibdm.ibdm_hca_count); 420 for (ii = 0; ii < ibdm.ibdm_hca_count; ii++) { 421 temp = hca_list; 422 hca_list = hca_list->hl_next; 423 IBTF_DPRINTF_L4("ibdm", "\tibdm_fini: hca %p", temp); 424 if (ibdm_uninit_hca(temp) != IBDM_SUCCESS) { 425 IBTF_DPRINTF_L2("ibdm", "\tibdm_fini: " 426 "uninit_hca %p failed", temp); 427 mutex_exit(&ibdm.ibdm_hl_mutex); 428 return (IBDM_FAILURE); 429 } 430 } 431 mutex_exit(&ibdm.ibdm_hl_mutex); 432 433 mutex_enter(&ibdm.ibdm_mutex); 434 if (ibdm.ibdm_state & IBDM_HCA_ATTACHED) 435 ibdm.ibdm_state &= ~IBDM_HCA_ATTACHED; 436 437 gid_info = ibdm.ibdm_dp_gidlist_head; 438 while (gid_info) { 439 mutex_enter(&gid_info->gl_mutex); 440 (void) ibdm_free_iou_info(gid_info); 441 mutex_exit(&gid_info->gl_mutex); 442 ibdm_delete_glhca_list(gid_info); 443 444 tmp = gid_info; 445 gid_info = gid_info->gl_next; 446 mutex_destroy(&tmp->gl_mutex); 447 head = tmp->gl_gid; 448 while (head) { 449 IBTF_DPRINTF_L4("ibdm", 450 "\tibdm_fini: Deleting gid structs"); 451 delete = head; 452 head = head->gid_next; 453 kmem_free(delete, sizeof (ibdm_gid_t)); 454 } 455 kmem_free(tmp, sizeof (ibdm_dp_gidinfo_t)); 456 } 457 mutex_exit(&ibdm.ibdm_mutex); 458 459 if (ibdm.ibdm_state & IBDM_LOCKS_ALLOCED) { 460 ibdm.ibdm_state &= ~IBDM_LOCKS_ALLOCED; 461 mutex_destroy(&ibdm.ibdm_mutex); 462 mutex_destroy(&ibdm.ibdm_hl_mutex); 463 mutex_destroy(&ibdm.ibdm_ibnex_mutex); 464 } 465 if (ibdm.ibdm_state & IBDM_CVS_ALLOCED) { 466 ibdm.ibdm_state &= ~IBDM_CVS_ALLOCED; 467 cv_destroy(&ibdm.ibdm_probe_cv); 468 cv_destroy(&ibdm.ibdm_busy_cv); 469 } 470 return (IBDM_SUCCESS); 471 } 472 473 474 /* 475 * ibdm_event_hdlr() 476 * 477 * IBDM registers this asynchronous event handler at the time of 478 * ibt_attach. IBDM support the following async events. For other 479 * event, simply returns success. 480 * IBT_HCA_ATTACH_EVENT: 481 * Retrieves the information about all the port that are 482 * present on this HCA, allocates the port attributes 483 * structure and calls IB nexus callback routine with 484 * the port attributes structure as an input argument. 485 * IBT_HCA_DETACH_EVENT: 486 * Retrieves the information about all the ports that are 487 * present on this HCA and calls IB nexus callback with 488 * port guid as an argument 489 * IBT_EVENT_PORT_UP: 490 * Register with IBMF and SA access 491 * Setup IBMF receive callback routine 492 * IBT_EVENT_PORT_DOWN: 493 * Un-Register with IBMF and SA access 494 * Teardown IBMF receive callback routine 495 */ 496 /*ARGSUSED*/ 497 static void 498 ibdm_event_hdlr(void *clnt_hdl, 499 ibt_hca_hdl_t hca_hdl, ibt_async_code_t code, ibt_async_event_t *event) 500 { 501 ibdm_hca_list_t *hca_list; 502 ibdm_port_attr_t *port; 503 ibmf_saa_handle_t port_sa_hdl; 504 505 IBTF_DPRINTF_L4("ibdm", "\tevent_hdlr: async code 0x%x", code); 506 507 switch (code) { 508 case IBT_HCA_ATTACH_EVENT: /* New HCA registered with IBTF */ 509 ibdm_handle_hca_attach(event->ev_hca_guid); 510 break; 511 512 case IBT_HCA_DETACH_EVENT: /* HCA unregistered with IBTF */ 513 ibdm_handle_hca_detach(event->ev_hca_guid); 514 mutex_enter(&ibdm.ibdm_ibnex_mutex); 515 if (ibdm.ibdm_ibnex_callback != NULL) { 516 (*ibdm.ibdm_ibnex_callback)((void *) 517 &event->ev_hca_guid, IBDM_EVENT_HCA_REMOVED); 518 } 519 mutex_exit(&ibdm.ibdm_ibnex_mutex); 520 break; 521 522 case IBT_EVENT_PORT_UP: 523 IBTF_DPRINTF_L4("ibdm", "\tevent_hdlr: PORT_UP"); 524 mutex_enter(&ibdm.ibdm_hl_mutex); 525 port = ibdm_get_port_attr(event, &hca_list); 526 if (port == NULL) { 527 IBTF_DPRINTF_L2("ibdm", 528 "\tevent_hdlr: HCA not present"); 529 mutex_exit(&ibdm.ibdm_hl_mutex); 530 break; 531 } 532 ibdm_initialize_port(port); 533 hca_list->hl_nports_active++; 534 mutex_exit(&ibdm.ibdm_hl_mutex); 535 break; 536 537 case IBT_ERROR_PORT_DOWN: 538 IBTF_DPRINTF_L4("ibdm", "\tevent_hdlr: PORT_DOWN"); 539 mutex_enter(&ibdm.ibdm_hl_mutex); 540 port = ibdm_get_port_attr(event, &hca_list); 541 if (port == NULL) { 542 IBTF_DPRINTF_L2("ibdm", 543 "\tevent_hdlr: HCA not present"); 544 mutex_exit(&ibdm.ibdm_hl_mutex); 545 break; 546 } 547 hca_list->hl_nports_active--; 548 port_sa_hdl = port->pa_sa_hdl; 549 (void) ibdm_fini_port(port); 550 mutex_exit(&ibdm.ibdm_hl_mutex); 551 ibdm_reset_all_dgids(port_sa_hdl); 552 break; 553 554 default: /* Ignore all other events/errors */ 555 break; 556 } 557 } 558 559 560 /* 561 * ibdm_initialize_port() 562 * Register with IBMF 563 * Register with SA access 564 * Register a receive callback routine with IBMF. IBMF invokes 565 * this routine whenever a MAD arrives at this port. 566 * Update the port attributes 567 */ 568 static void 569 ibdm_initialize_port(ibdm_port_attr_t *port) 570 { 571 int ii; 572 uint_t nports, size; 573 uint_t pkey_idx; 574 ib_pkey_t pkey; 575 ibt_hca_portinfo_t *pinfop; 576 ibmf_register_info_t ibmf_reg; 577 ibmf_saa_subnet_event_args_t event_args; 578 579 IBTF_DPRINTF_L4("ibdm", "\tinitialize_port:"); 580 ASSERT(MUTEX_HELD(&ibdm.ibdm_hl_mutex)); 581 582 /* Check whether the port is active */ 583 if (ibt_get_port_state(port->pa_hca_hdl, port->pa_port_num, NULL, 584 NULL) != IBT_SUCCESS) 585 return; 586 587 if (port->pa_sa_hdl != NULL) 588 return; 589 590 if (ibt_query_hca_ports(port->pa_hca_hdl, port->pa_port_num, 591 &pinfop, &nports, &size) != IBT_SUCCESS) { 592 /* This should not occur */ 593 port->pa_npkeys = 0; 594 port->pa_pkey_tbl = NULL; 595 return; 596 } 597 port->pa_sn_prefix = pinfop->p_sgid_tbl[0].gid_prefix; 598 599 port->pa_state = pinfop->p_linkstate; 600 port->pa_npkeys = pinfop->p_pkey_tbl_sz; 601 port->pa_pkey_tbl = (ibdm_pkey_tbl_t *)kmem_zalloc( 602 port->pa_npkeys * sizeof (ibdm_pkey_tbl_t), KM_SLEEP); 603 604 for (pkey_idx = 0; pkey_idx < port->pa_npkeys; pkey_idx++) 605 port->pa_pkey_tbl[pkey_idx].pt_pkey = 606 pinfop->p_pkey_tbl[pkey_idx]; 607 608 ibt_free_portinfo(pinfop, size); 609 610 event_args.is_event_callback = ibdm_saa_event_cb; 611 event_args.is_event_callback_arg = port; 612 if (ibmf_sa_session_open(port->pa_port_guid, 0, &event_args, 613 IBMF_VERSION, 0, &port->pa_sa_hdl) != IBMF_SUCCESS) { 614 IBTF_DPRINTF_L2("ibdm", "\tinitialize_port: " 615 "sa access registration failed"); 616 return; 617 } 618 ibmf_reg.ir_ci_guid = port->pa_hca_guid; 619 ibmf_reg.ir_port_num = port->pa_port_num; 620 ibmf_reg.ir_client_class = DEV_MGT_MANAGER; 621 622 if (ibmf_register(&ibmf_reg, IBMF_VERSION, 0, NULL, NULL, 623 &port->pa_ibmf_hdl, &port->pa_ibmf_caps) != IBMF_SUCCESS) { 624 IBTF_DPRINTF_L2("ibdm", "\tinitialize_port: " 625 "IBMF registration failed"); 626 (void) ibdm_fini_port(port); 627 return; 628 } 629 if (ibmf_setup_async_cb(port->pa_ibmf_hdl, IBMF_QP_HANDLE_DEFAULT, 630 ibdm_ibmf_recv_cb, 0, 0) != IBMF_SUCCESS) { 631 IBTF_DPRINTF_L2("ibdm", "\tinitialize_port: " 632 "IBMF setup recv cb failed"); 633 (void) ibdm_fini_port(port); 634 return; 635 } 636 637 for (ii = 0; ii < port->pa_npkeys; ii++) { 638 pkey = port->pa_pkey_tbl[ii].pt_pkey; 639 if (IBDM_INVALID_PKEY(pkey)) { 640 port->pa_pkey_tbl[ii].pt_qp_hdl = NULL; 641 continue; 642 } 643 ibdm_port_attr_ibmf_init(port, pkey, ii); 644 } 645 } 646 647 648 /* 649 * ibdm_port_attr_ibmf_init: 650 * With IBMF - Alloc QP Handle and Setup Async callback 651 */ 652 static void 653 ibdm_port_attr_ibmf_init(ibdm_port_attr_t *port, ib_pkey_t pkey, int ii) 654 { 655 int ret; 656 657 if ((ret = ibmf_alloc_qp(port->pa_ibmf_hdl, pkey, IB_GSI_QKEY, 658 IBMF_ALT_QP_MAD_NO_RMPP, &port->pa_pkey_tbl[ii].pt_qp_hdl)) != 659 IBMF_SUCCESS) { 660 IBTF_DPRINTF_L2("ibdm", "\tport_attr_ibmf_init: " 661 "IBMF failed to alloc qp %d", ret); 662 port->pa_pkey_tbl[ii].pt_qp_hdl = NULL; 663 return; 664 } 665 666 IBTF_DPRINTF_L4("ibdm", "\tport_attr_ibmf_init: QP handle is %p", 667 port->pa_ibmf_hdl); 668 669 if ((ret = ibmf_setup_async_cb(port->pa_ibmf_hdl, 670 port->pa_pkey_tbl[ii].pt_qp_hdl, ibdm_ibmf_recv_cb, 0, 0)) != 671 IBMF_SUCCESS) { 672 IBTF_DPRINTF_L2("ibdm", "\tport_attr_ibmf_init: " 673 "IBMF setup recv cb failed %d", ret); 674 (void) ibmf_free_qp(port->pa_ibmf_hdl, 675 &port->pa_pkey_tbl[ii].pt_qp_hdl, 0); 676 port->pa_pkey_tbl[ii].pt_qp_hdl = NULL; 677 } 678 } 679 680 681 /* 682 * ibdm_get_port_attr() 683 * Get port attributes from HCA guid and port number 684 * Return pointer to ibdm_port_attr_t on Success 685 * and NULL on failure 686 */ 687 static ibdm_port_attr_t * 688 ibdm_get_port_attr(ibt_async_event_t *event, ibdm_hca_list_t **retval) 689 { 690 ibdm_hca_list_t *hca_list; 691 ibdm_port_attr_t *port_attr; 692 int ii; 693 694 IBTF_DPRINTF_L4("ibdm", "\tget_port_attr: port# %d", event->ev_port); 695 ASSERT(MUTEX_HELD(&ibdm.ibdm_hl_mutex)); 696 hca_list = ibdm.ibdm_hca_list_head; 697 while (hca_list) { 698 if (hca_list->hl_hca_guid == event->ev_hca_guid) { 699 for (ii = 0; ii < hca_list->hl_nports; ii++) { 700 port_attr = &hca_list->hl_port_attr[ii]; 701 if (port_attr->pa_port_num == event->ev_port) { 702 *retval = hca_list; 703 return (port_attr); 704 } 705 } 706 } 707 hca_list = hca_list->hl_next; 708 } 709 return (NULL); 710 } 711 712 713 /* 714 * ibdm_update_port_attr() 715 * Update the port attributes 716 */ 717 static void 718 ibdm_update_port_attr(ibdm_port_attr_t *port) 719 { 720 uint_t nports, size; 721 uint_t pkey_idx; 722 ibt_hca_portinfo_t *portinfop; 723 724 IBTF_DPRINTF_L4("ibdm", "\tupdate_port_attr: Begin"); 725 if (ibt_query_hca_ports(port->pa_hca_hdl, 726 port->pa_port_num, &portinfop, &nports, &size) != IBT_SUCCESS) { 727 /* This should not occur */ 728 port->pa_npkeys = 0; 729 port->pa_pkey_tbl = NULL; 730 return; 731 } 732 port->pa_sn_prefix = portinfop->p_sgid_tbl[0].gid_prefix; 733 734 port->pa_state = portinfop->p_linkstate; 735 736 /* 737 * PKey information in portinfo valid only if port is 738 * ACTIVE. Bail out if not. 739 */ 740 if (port->pa_state != IBT_PORT_ACTIVE) { 741 port->pa_npkeys = 0; 742 port->pa_pkey_tbl = NULL; 743 ibt_free_portinfo(portinfop, size); 744 return; 745 } 746 747 port->pa_npkeys = portinfop->p_pkey_tbl_sz; 748 port->pa_pkey_tbl = (ibdm_pkey_tbl_t *)kmem_zalloc( 749 port->pa_npkeys * sizeof (ibdm_pkey_tbl_t), KM_SLEEP); 750 751 for (pkey_idx = 0; pkey_idx < port->pa_npkeys; pkey_idx++) { 752 port->pa_pkey_tbl[pkey_idx].pt_pkey = 753 portinfop->p_pkey_tbl[pkey_idx]; 754 } 755 ibt_free_portinfo(portinfop, size); 756 } 757 758 759 /* 760 * ibdm_handle_hca_attach() 761 */ 762 static void 763 ibdm_handle_hca_attach(ib_guid_t hca_guid) 764 { 765 uint_t size; 766 uint_t ii, nports; 767 ibt_status_t status; 768 ibt_hca_hdl_t hca_hdl; 769 ibt_hca_attr_t *hca_attr; 770 ibdm_hca_list_t *hca_list, *temp; 771 ibdm_port_attr_t *port_attr; 772 ibt_hca_portinfo_t *portinfop; 773 774 IBTF_DPRINTF_L4("ibdm", 775 "\thandle_hca_attach: hca_guid = 0x%llX", hca_guid); 776 777 /* open the HCA first */ 778 if ((status = ibt_open_hca(ibdm.ibdm_ibt_clnt_hdl, hca_guid, 779 &hca_hdl)) != IBT_SUCCESS) { 780 IBTF_DPRINTF_L2("ibdm", "\thandle_hca_attach: " 781 "open_hca failed, status 0x%x", status); 782 return; 783 } 784 785 hca_attr = (ibt_hca_attr_t *) 786 kmem_alloc(sizeof (ibt_hca_attr_t), KM_SLEEP); 787 /* ibt_query_hca always returns IBT_SUCCESS */ 788 (void) ibt_query_hca(hca_hdl, hca_attr); 789 790 IBTF_DPRINTF_L4("ibdm", "\tvid: 0x%x, pid: 0x%x, ver: 0x%x," 791 " #ports: %d", hca_attr->hca_vendor_id, hca_attr->hca_device_id, 792 hca_attr->hca_version_id, hca_attr->hca_nports); 793 794 if ((status = ibt_query_hca_ports(hca_hdl, 0, &portinfop, &nports, 795 &size)) != IBT_SUCCESS) { 796 IBTF_DPRINTF_L2("ibdm", "\thandle_hca_attach: " 797 "ibt_query_hca_ports failed, status 0x%x", status); 798 kmem_free(hca_attr, sizeof (ibt_hca_attr_t)); 799 (void) ibt_close_hca(hca_hdl); 800 return; 801 } 802 hca_list = (ibdm_hca_list_t *) 803 kmem_zalloc((sizeof (ibdm_hca_list_t)), KM_SLEEP); 804 hca_list->hl_port_attr = (ibdm_port_attr_t *)kmem_zalloc( 805 (sizeof (ibdm_port_attr_t) * hca_attr->hca_nports), KM_SLEEP); 806 hca_list->hl_hca_guid = hca_attr->hca_node_guid; 807 hca_list->hl_nports = hca_attr->hca_nports; 808 hca_list->hl_attach_time = ddi_get_time(); 809 hca_list->hl_hca_hdl = hca_hdl; 810 811 /* 812 * Init a dummy port attribute for the HCA node 813 * This is for Per-HCA Node. Initialize port_attr : 814 * hca_guid & port_guid -> hca_guid 815 * npkeys, pkey_tbl is NULL 816 * port_num, sn_prefix is 0 817 * vendorid, product_id, dev_version from HCA 818 * pa_state is IBT_PORT_ACTIVE 819 */ 820 hca_list->hl_hca_port_attr = (ibdm_port_attr_t *)kmem_zalloc( 821 sizeof (ibdm_port_attr_t), KM_SLEEP); 822 port_attr = hca_list->hl_hca_port_attr; 823 port_attr->pa_vendorid = hca_attr->hca_vendor_id; 824 port_attr->pa_productid = hca_attr->hca_device_id; 825 port_attr->pa_dev_version = hca_attr->hca_version_id; 826 port_attr->pa_hca_guid = hca_attr->hca_node_guid; 827 port_attr->pa_hca_hdl = hca_list->hl_hca_hdl; 828 port_attr->pa_port_guid = hca_attr->hca_node_guid; 829 port_attr->pa_state = IBT_PORT_ACTIVE; 830 831 832 for (ii = 0; ii < nports; ii++) { 833 port_attr = &hca_list->hl_port_attr[ii]; 834 port_attr->pa_vendorid = hca_attr->hca_vendor_id; 835 port_attr->pa_productid = hca_attr->hca_device_id; 836 port_attr->pa_dev_version = hca_attr->hca_version_id; 837 port_attr->pa_hca_guid = hca_attr->hca_node_guid; 838 port_attr->pa_hca_hdl = hca_list->hl_hca_hdl; 839 port_attr->pa_port_guid = portinfop[ii].p_sgid_tbl->gid_guid; 840 port_attr->pa_sn_prefix = portinfop[ii].p_sgid_tbl->gid_prefix; 841 port_attr->pa_port_num = portinfop[ii].p_port_num; 842 port_attr->pa_state = portinfop[ii].p_linkstate; 843 844 /* 845 * Register with IBMF, SA access when the port is in 846 * ACTIVE state. Also register a callback routine 847 * with IBMF to receive incoming DM MAD's. 848 * The IBDM event handler takes care of registration of 849 * port which are not active. 850 */ 851 IBTF_DPRINTF_L4("ibdm", 852 "\thandle_hca_attach: port guid %llx Port state 0x%x", 853 port_attr->pa_port_guid, portinfop[ii].p_linkstate); 854 855 if (portinfop[ii].p_linkstate == IBT_PORT_ACTIVE) { 856 mutex_enter(&ibdm.ibdm_hl_mutex); 857 hca_list->hl_nports_active++; 858 ibdm_initialize_port(port_attr); 859 mutex_exit(&ibdm.ibdm_hl_mutex); 860 } 861 } 862 mutex_enter(&ibdm.ibdm_hl_mutex); 863 for (temp = ibdm.ibdm_hca_list_head; temp; temp = temp->hl_next) { 864 if (temp->hl_hca_guid == hca_guid) { 865 IBTF_DPRINTF_L2("ibdm", "hca_attach: HCA %llX " 866 "already seen by IBDM", hca_guid); 867 mutex_exit(&ibdm.ibdm_hl_mutex); 868 (void) ibdm_uninit_hca(hca_list); 869 return; 870 } 871 } 872 ibdm.ibdm_hca_count++; 873 if (ibdm.ibdm_hca_list_head == NULL) { 874 ibdm.ibdm_hca_list_head = hca_list; 875 ibdm.ibdm_hca_list_tail = hca_list; 876 } else { 877 ibdm.ibdm_hca_list_tail->hl_next = hca_list; 878 ibdm.ibdm_hca_list_tail = hca_list; 879 } 880 mutex_exit(&ibdm.ibdm_hl_mutex); 881 mutex_enter(&ibdm.ibdm_ibnex_mutex); 882 if (ibdm.ibdm_ibnex_callback != NULL) { 883 (*ibdm.ibdm_ibnex_callback)((void *) 884 &hca_guid, IBDM_EVENT_HCA_ADDED); 885 } 886 mutex_exit(&ibdm.ibdm_ibnex_mutex); 887 888 kmem_free(hca_attr, sizeof (ibt_hca_attr_t)); 889 ibt_free_portinfo(portinfop, size); 890 } 891 892 893 /* 894 * ibdm_handle_hca_detach() 895 */ 896 static void 897 ibdm_handle_hca_detach(ib_guid_t hca_guid) 898 { 899 ibdm_hca_list_t *head, *prev = NULL; 900 901 IBTF_DPRINTF_L4("ibdm", 902 "\thandle_hca_detach: hca_guid = 0x%llx", hca_guid); 903 904 /* Make sure no probes are running */ 905 mutex_enter(&ibdm.ibdm_mutex); 906 while (ibdm.ibdm_busy & IBDM_BUSY) 907 cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex); 908 ibdm.ibdm_busy |= IBDM_BUSY; 909 mutex_exit(&ibdm.ibdm_mutex); 910 911 mutex_enter(&ibdm.ibdm_hl_mutex); 912 head = ibdm.ibdm_hca_list_head; 913 while (head) { 914 if (head->hl_hca_guid == hca_guid) { 915 if (prev == NULL) 916 ibdm.ibdm_hca_list_head = head->hl_next; 917 else 918 prev->hl_next = head->hl_next; 919 ibdm.ibdm_hca_count--; 920 break; 921 } 922 prev = head; 923 head = head->hl_next; 924 } 925 mutex_exit(&ibdm.ibdm_hl_mutex); 926 if (ibdm_uninit_hca(head) != IBDM_SUCCESS) 927 (void) ibdm_handle_hca_attach(hca_guid); 928 929 mutex_enter(&ibdm.ibdm_mutex); 930 ibdm.ibdm_busy &= ~IBDM_BUSY; 931 cv_broadcast(&ibdm.ibdm_busy_cv); 932 mutex_exit(&ibdm.ibdm_mutex); 933 } 934 935 936 static int 937 ibdm_uninit_hca(ibdm_hca_list_t *head) 938 { 939 int ii; 940 ibdm_port_attr_t *port_attr; 941 942 for (ii = 0; ii < head->hl_nports; ii++) { 943 port_attr = &head->hl_port_attr[ii]; 944 if (ibdm_fini_port(port_attr) != IBDM_SUCCESS) { 945 IBTF_DPRINTF_L2("ibdm", "uninit_hca: HCA %p port 0x%x " 946 "ibdm_fini_port() failed", head, ii); 947 return (IBDM_FAILURE); 948 } 949 } 950 if (head->hl_hca_hdl) 951 if (ibt_close_hca(head->hl_hca_hdl) != IBT_SUCCESS) 952 return (IBDM_FAILURE); 953 kmem_free(head->hl_port_attr, 954 head->hl_nports * sizeof (ibdm_port_attr_t)); 955 kmem_free(head->hl_hca_port_attr, sizeof (ibdm_port_attr_t)); 956 kmem_free(head, sizeof (ibdm_hca_list_t)); 957 return (IBDM_SUCCESS); 958 } 959 960 961 /* 962 * For each port on the HCA, 963 * 1) Teardown IBMF receive callback function 964 * 2) Unregister with IBMF 965 * 3) Unregister with SA access 966 */ 967 static int 968 ibdm_fini_port(ibdm_port_attr_t *port_attr) 969 { 970 int ii, ibmf_status; 971 972 for (ii = 0; ii < port_attr->pa_npkeys; ii++) { 973 if (port_attr->pa_pkey_tbl == NULL) 974 break; 975 if (!port_attr->pa_pkey_tbl[ii].pt_qp_hdl) 976 continue; 977 if (ibdm_port_attr_ibmf_fini(port_attr, ii) != IBDM_SUCCESS) { 978 IBTF_DPRINTF_L4("ibdm", "\tfini_port: " 979 "ibdm_port_attr_ibmf_fini failed for " 980 "port pkey 0x%x", ii); 981 return (IBDM_FAILURE); 982 } 983 } 984 985 if (port_attr->pa_ibmf_hdl) { 986 ibmf_status = ibmf_tear_down_async_cb(port_attr->pa_ibmf_hdl, 987 IBMF_QP_HANDLE_DEFAULT, 0); 988 if (ibmf_status != IBMF_SUCCESS) { 989 IBTF_DPRINTF_L4("ibdm", "\tfini_port: " 990 "ibmf_tear_down_async_cb failed %d", ibmf_status); 991 return (IBDM_FAILURE); 992 } 993 994 ibmf_status = ibmf_unregister(&port_attr->pa_ibmf_hdl, 0); 995 if (ibmf_status != IBMF_SUCCESS) { 996 IBTF_DPRINTF_L4("ibdm", "\tfini_port: " 997 "ibmf_unregister failed %d", ibmf_status); 998 return (IBDM_FAILURE); 999 } 1000 1001 port_attr->pa_ibmf_hdl = NULL; 1002 } 1003 1004 if (port_attr->pa_sa_hdl) { 1005 ibmf_status = ibmf_sa_session_close(&port_attr->pa_sa_hdl, 0); 1006 if (ibmf_status != IBMF_SUCCESS) { 1007 IBTF_DPRINTF_L4("ibdm", "\tfini_port: " 1008 "ibmf_sa_session_close failed %d", ibmf_status); 1009 return (IBDM_FAILURE); 1010 } 1011 port_attr->pa_sa_hdl = NULL; 1012 } 1013 1014 if (port_attr->pa_pkey_tbl != NULL) { 1015 kmem_free(port_attr->pa_pkey_tbl, 1016 port_attr->pa_npkeys * sizeof (ibdm_pkey_tbl_t)); 1017 port_attr->pa_pkey_tbl = NULL; 1018 port_attr->pa_npkeys = 0; 1019 } 1020 1021 return (IBDM_SUCCESS); 1022 } 1023 1024 1025 /* 1026 * ibdm_port_attr_ibmf_fini: 1027 * With IBMF - Tear down Async callback and free QP Handle 1028 */ 1029 static int 1030 ibdm_port_attr_ibmf_fini(ibdm_port_attr_t *port_attr, int ii) 1031 { 1032 int ibmf_status; 1033 1034 IBTF_DPRINTF_L5("ibdm", "\tport_attr_ibmf_fini:"); 1035 1036 if (port_attr->pa_pkey_tbl[ii].pt_qp_hdl) { 1037 ibmf_status = ibmf_tear_down_async_cb(port_attr->pa_ibmf_hdl, 1038 port_attr->pa_pkey_tbl[ii].pt_qp_hdl, 0); 1039 if (ibmf_status != IBMF_SUCCESS) { 1040 IBTF_DPRINTF_L4("ibdm", "\tport_attr_ibmf_fini: " 1041 "ibmf_tear_down_async_cb failed %d", ibmf_status); 1042 return (IBDM_FAILURE); 1043 } 1044 ibmf_status = ibmf_free_qp(port_attr->pa_ibmf_hdl, 1045 &port_attr->pa_pkey_tbl[ii].pt_qp_hdl, 0); 1046 if (ibmf_status != IBMF_SUCCESS) { 1047 IBTF_DPRINTF_L4("ibdm", "\tport_attr_ibmf_fini: " 1048 "ibmf_free_qp failed %d", ibmf_status); 1049 return (IBDM_FAILURE); 1050 } 1051 port_attr->pa_pkey_tbl[ii].pt_qp_hdl = NULL; 1052 } 1053 return (IBDM_SUCCESS); 1054 } 1055 1056 1057 /* 1058 * ibdm_gid_decr_pending: 1059 * decrement gl_pending_cmds. If zero wakeup sleeping threads 1060 */ 1061 static void 1062 ibdm_gid_decr_pending(ibdm_dp_gidinfo_t *gidinfo) 1063 { 1064 mutex_enter(&ibdm.ibdm_mutex); 1065 mutex_enter(&gidinfo->gl_mutex); 1066 if (--gidinfo->gl_pending_cmds == 0) { 1067 /* 1068 * Handle DGID getting removed. 1069 */ 1070 if (gidinfo->gl_disconnected) { 1071 mutex_exit(&gidinfo->gl_mutex); 1072 mutex_exit(&ibdm.ibdm_mutex); 1073 1074 IBTF_DPRINTF_L3(ibdm_string, "\tgid_decr_pending: " 1075 "gidinfo %p hot removal", gidinfo); 1076 ibdm_delete_gidinfo(gidinfo); 1077 1078 mutex_enter(&ibdm.ibdm_mutex); 1079 ibdm.ibdm_ngid_probes_in_progress--; 1080 ibdm_wait_probe_completion(); 1081 mutex_exit(&ibdm.ibdm_mutex); 1082 return; 1083 } 1084 mutex_exit(&gidinfo->gl_mutex); 1085 mutex_exit(&ibdm.ibdm_mutex); 1086 ibdm_notify_newgid_iocs(gidinfo); 1087 mutex_enter(&ibdm.ibdm_mutex); 1088 mutex_enter(&gidinfo->gl_mutex); 1089 1090 ibdm.ibdm_ngid_probes_in_progress--; 1091 ibdm_wait_probe_completion(); 1092 } 1093 mutex_exit(&gidinfo->gl_mutex); 1094 mutex_exit(&ibdm.ibdm_mutex); 1095 } 1096 1097 1098 /* 1099 * ibdm_wait_probe_completion: 1100 * wait for probing to complete 1101 */ 1102 static void 1103 ibdm_wait_probe_completion(void) 1104 { 1105 ASSERT(MUTEX_HELD(&ibdm.ibdm_mutex)); 1106 if (ibdm.ibdm_ngid_probes_in_progress) { 1107 IBTF_DPRINTF_L4("ibdm", "\twait for probe complete"); 1108 ibdm.ibdm_busy |= IBDM_PROBE_IN_PROGRESS; 1109 while (ibdm.ibdm_busy & IBDM_PROBE_IN_PROGRESS) 1110 cv_wait(&ibdm.ibdm_probe_cv, &ibdm.ibdm_mutex); 1111 } 1112 } 1113 1114 1115 /* 1116 * ibdm_wakeup_probe_gid_cv: 1117 * wakeup waiting threads (based on ibdm_ngid_probes_in_progress) 1118 */ 1119 static void 1120 ibdm_wakeup_probe_gid_cv(void) 1121 { 1122 ASSERT(MUTEX_HELD(&ibdm.ibdm_mutex)); 1123 if (!ibdm.ibdm_ngid_probes_in_progress) { 1124 IBTF_DPRINTF_L4("ibdm", "wakeup_probe_gid_thread: Wakeup"); 1125 ibdm.ibdm_busy &= ~IBDM_PROBE_IN_PROGRESS; 1126 cv_broadcast(&ibdm.ibdm_probe_cv); 1127 } 1128 1129 } 1130 1131 1132 /* 1133 * ibdm_sweep_fabric(reprobe_flag) 1134 * Find all possible Managed IOU's and their IOC's that are visible 1135 * to the host. The algorithm used is as follows 1136 * 1137 * Send a "bus walk" request for each port on the host HCA to SA access 1138 * SA returns complete set of GID's that are reachable from 1139 * source port. This is done in parallel. 1140 * 1141 * Initialize GID state to IBDM_GID_PROBE_NOT_DONE 1142 * 1143 * Sort the GID list and eliminate duplicate GID's 1144 * 1) Use DGID for sorting 1145 * 2) use PortGuid for sorting 1146 * Send SA query to retrieve NodeRecord and 1147 * extract PortGuid from that. 1148 * 1149 * Set GID state to IBDM_GID_PROBE_FAILED to all the ports that dont 1150 * support DM MAD's 1151 * Send a "Portinfo" query to get the port capabilities and 1152 * then check for DM MAD's support 1153 * 1154 * Send "ClassPortInfo" request for all the GID's in parallel, 1155 * set the GID state to IBDM_GET_CLASSPORTINFO and wait on the 1156 * cv_signal to complete. 1157 * 1158 * When DM agent on the remote GID sends back the response, IBMF 1159 * invokes DM callback routine. 1160 * 1161 * If the response is proper, send "IOUnitInfo" request and set 1162 * GID state to IBDM_GET_IOUNITINFO. 1163 * 1164 * If the response is proper, send "IocProfileInfo" request to 1165 * all the IOC simultaneously and set GID state to IBDM_GET_IOC_DETAILS. 1166 * 1167 * Send request to get Service entries simultaneously 1168 * 1169 * Signal the waiting thread when received response for all the commands. 1170 * 1171 * Set the GID state to IBDM_GID_PROBE_FAILED when received a error 1172 * response during the probing period. 1173 * 1174 * Note: 1175 * ibdm.ibdm_ngid_probes_in_progress and ibdm_gid_list_t:gl_pending_cmds 1176 * keep track of number commands in progress at any point of time. 1177 * MAD transaction ID is used to identify a particular GID 1178 * TBD: Consider registering the IBMF receive callback on demand 1179 * 1180 * Note: This routine must be called with ibdm.ibdm_mutex held 1181 * TBD: Re probe the failure GID (for certain failures) when requested 1182 * for fabric sweep next time 1183 * 1184 * Parameters : If reprobe_flag is set, All IOCs will be reprobed. 1185 */ 1186 static void 1187 ibdm_sweep_fabric(int reprobe_flag) 1188 { 1189 int ii; 1190 int new_paths = 0; 1191 uint8_t niocs; 1192 taskqid_t tid; 1193 ibdm_ioc_info_t *ioc; 1194 ibdm_hca_list_t *hca_list = NULL; 1195 ibdm_port_attr_t *port = NULL; 1196 ibdm_dp_gidinfo_t *gid_info; 1197 1198 IBTF_DPRINTF_L4("ibdm", "\tsweep_fabric: Enter"); 1199 ASSERT(MUTEX_HELD(&ibdm.ibdm_mutex)); 1200 1201 /* 1202 * Check whether a sweep already in progress. If so, just 1203 * wait for the fabric sweep to complete 1204 */ 1205 while (ibdm.ibdm_busy & IBDM_BUSY) 1206 cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex); 1207 ibdm.ibdm_busy |= IBDM_BUSY; 1208 mutex_exit(&ibdm.ibdm_mutex); 1209 1210 ibdm_dump_sweep_fabric_timestamp(0); 1211 1212 /* Rescan the GID list for any removed GIDs for reprobe */ 1213 if (reprobe_flag) 1214 ibdm_rescan_gidlist(NULL); 1215 1216 /* 1217 * Get list of all the ports reachable from the local known HCA 1218 * ports which are active 1219 */ 1220 mutex_enter(&ibdm.ibdm_hl_mutex); 1221 for (ibdm_get_next_port(&hca_list, &port, 1); port; 1222 ibdm_get_next_port(&hca_list, &port, 1)) { 1223 /* 1224 * Get PATHS to all the reachable ports from 1225 * SGID and update the global ibdm structure. 1226 */ 1227 new_paths = ibdm_get_reachable_ports(port, hca_list); 1228 ibdm.ibdm_ngids += new_paths; 1229 } 1230 mutex_exit(&ibdm.ibdm_hl_mutex); 1231 1232 mutex_enter(&ibdm.ibdm_mutex); 1233 ibdm.ibdm_ngid_probes_in_progress += ibdm.ibdm_ngids; 1234 mutex_exit(&ibdm.ibdm_mutex); 1235 1236 /* Send a request to probe GIDs asynchronously. */ 1237 for (gid_info = ibdm.ibdm_dp_gidlist_head; gid_info; 1238 gid_info = gid_info->gl_next) { 1239 mutex_enter(&gid_info->gl_mutex); 1240 gid_info->gl_reprobe_flag = reprobe_flag; 1241 mutex_exit(&gid_info->gl_mutex); 1242 1243 /* process newly encountered GIDs */ 1244 tid = taskq_dispatch(system_taskq, ibdm_probe_gid_thread, 1245 (void *)gid_info, TQ_NOSLEEP); 1246 IBTF_DPRINTF_L4("ibdm", "\tsweep_fabric: gid_info = %p" 1247 " taskq_id = %x", gid_info, tid); 1248 /* taskq failed to dispatch call it directly */ 1249 if (tid == NULL) 1250 ibdm_probe_gid_thread((void *)gid_info); 1251 } 1252 1253 mutex_enter(&ibdm.ibdm_mutex); 1254 ibdm_wait_probe_completion(); 1255 1256 /* 1257 * Update the properties, if reprobe_flag is set 1258 * Skip if gl_reprobe_flag is set, this will be 1259 * a re-inserted / new GID, for which notifications 1260 * have already been send. 1261 */ 1262 if (reprobe_flag) { 1263 for (gid_info = ibdm.ibdm_dp_gidlist_head; gid_info; 1264 gid_info = gid_info->gl_next) { 1265 if (gid_info->gl_iou == NULL) 1266 continue; 1267 if (gid_info->gl_reprobe_flag) { 1268 gid_info->gl_reprobe_flag = 0; 1269 continue; 1270 } 1271 1272 niocs = gid_info->gl_iou->iou_info.iou_num_ctrl_slots; 1273 for (ii = 0; ii < niocs; ii++) { 1274 ioc = IBDM_GIDINFO2IOCINFO(gid_info, ii); 1275 if (ioc) 1276 ibdm_reprobe_update_port_srv(ioc, 1277 gid_info); 1278 } 1279 } 1280 } 1281 ibdm_dump_sweep_fabric_timestamp(1); 1282 1283 ibdm.ibdm_busy &= ~IBDM_BUSY; 1284 cv_broadcast(&ibdm.ibdm_busy_cv); 1285 IBTF_DPRINTF_L5("ibdm", "\tsweep_fabric: EXIT"); 1286 } 1287 1288 1289 /* 1290 * ibdm_probe_gid_thread: 1291 * thread that does the actual work for sweeping the fabric 1292 * for a given GID 1293 */ 1294 static void 1295 ibdm_probe_gid_thread(void *args) 1296 { 1297 int reprobe_flag; 1298 ib_guid_t node_guid; 1299 ib_guid_t port_guid; 1300 ibdm_dp_gidinfo_t *gid_info; 1301 1302 gid_info = (ibdm_dp_gidinfo_t *)args; 1303 reprobe_flag = gid_info->gl_reprobe_flag; 1304 IBTF_DPRINTF_L4("ibdm", "\tprobe_gid_thread: gid_info = %p, flag = %d", 1305 gid_info, reprobe_flag); 1306 ASSERT(gid_info != NULL); 1307 ASSERT(gid_info->gl_pending_cmds == 0); 1308 1309 if (gid_info->gl_state != IBDM_GID_PROBE_NOT_DONE && 1310 reprobe_flag == 0) { 1311 /* 1312 * This GID may have been already probed. Send 1313 * in a CLP to check if IOUnitInfo changed? 1314 * Explicitly set gl_reprobe_flag to 0 so that 1315 * IBnex is not notified on completion 1316 */ 1317 if (gid_info->gl_state == IBDM_GID_PROBING_COMPLETE) { 1318 IBTF_DPRINTF_L4("ibdm", "\tprobe_gid_thread: " 1319 "get new IOCs information"); 1320 mutex_enter(&gid_info->gl_mutex); 1321 gid_info->gl_pending_cmds++; 1322 gid_info->gl_state = IBDM_GET_IOUNITINFO; 1323 gid_info->gl_reprobe_flag = 0; 1324 mutex_exit(&gid_info->gl_mutex); 1325 if (ibdm_send_iounitinfo(gid_info) != IBDM_SUCCESS) { 1326 mutex_enter(&gid_info->gl_mutex); 1327 gid_info->gl_pending_cmds = 0; 1328 mutex_exit(&gid_info->gl_mutex); 1329 mutex_enter(&ibdm.ibdm_mutex); 1330 --ibdm.ibdm_ngid_probes_in_progress; 1331 ibdm_wakeup_probe_gid_cv(); 1332 mutex_exit(&ibdm.ibdm_mutex); 1333 } 1334 } else { 1335 mutex_enter(&ibdm.ibdm_mutex); 1336 --ibdm.ibdm_ngid_probes_in_progress; 1337 ibdm_wakeup_probe_gid_cv(); 1338 mutex_exit(&ibdm.ibdm_mutex); 1339 } 1340 return; 1341 } else if (reprobe_flag && gid_info->gl_state == 1342 IBDM_GID_PROBING_COMPLETE) { 1343 /* 1344 * Reprobe all IOCs for the GID which has completed 1345 * probe. Skip other port GIDs to same IOU. 1346 * Explicitly set gl_reprobe_flag to 0 so that 1347 * IBnex is not notified on completion 1348 */ 1349 ibdm_ioc_info_t *ioc_info; 1350 uint8_t niocs, ii; 1351 1352 mutex_enter(&gid_info->gl_mutex); 1353 niocs = gid_info->gl_iou->iou_info.iou_num_ctrl_slots; 1354 gid_info->gl_state = IBDM_GET_IOC_DETAILS; 1355 gid_info->gl_pending_cmds += niocs; 1356 gid_info->gl_reprobe_flag = 0; 1357 mutex_exit(&gid_info->gl_mutex); 1358 for (ii = 0; ii < niocs; ii++) { 1359 uchar_t slot_info; 1360 ib_dm_io_unitinfo_t *giou_info; 1361 1362 /* 1363 * Check whether IOC is present in the slot 1364 * Series of nibbles (in the field 1365 * iou_ctrl_list) represents a slot in the 1366 * IOU. 1367 * Byte format: 76543210 1368 * Bits 0-3 of first byte represent Slot 2 1369 * bits 4-7 of first byte represent slot 1, 1370 * bits 0-3 of second byte represent slot 4 1371 * and so on 1372 * Each 4-bit nibble has the following meaning 1373 * 0x0 : IOC not installed 1374 * 0x1 : IOC is present 1375 * 0xf : Slot does not exist 1376 * and all other values are reserved. 1377 */ 1378 ioc_info = IBDM_GIDINFO2IOCINFO(gid_info, ii); 1379 giou_info = &gid_info->gl_iou->iou_info; 1380 slot_info = giou_info->iou_ctrl_list[(ii/2)]; 1381 if ((ii % 2) == 0) 1382 slot_info = (slot_info >> 4); 1383 1384 if ((slot_info & 0xf) != 1) { 1385 ioc_info->ioc_state = 1386 IBDM_IOC_STATE_PROBE_FAILED; 1387 ibdm_gid_decr_pending(gid_info); 1388 continue; 1389 } 1390 1391 if (ibdm_send_ioc_profile(gid_info, ii) != 1392 IBDM_SUCCESS) { 1393 ibdm_gid_decr_pending(gid_info); 1394 } 1395 } 1396 1397 return; 1398 } else if (gid_info->gl_state != IBDM_GID_PROBE_NOT_DONE) { 1399 mutex_enter(&ibdm.ibdm_mutex); 1400 --ibdm.ibdm_ngid_probes_in_progress; 1401 ibdm_wakeup_probe_gid_cv(); 1402 mutex_exit(&ibdm.ibdm_mutex); 1403 return; 1404 } 1405 1406 mutex_enter(&gid_info->gl_mutex); 1407 gid_info->gl_pending_cmds++; 1408 gid_info->gl_state = IBDM_GET_CLASSPORTINFO; 1409 mutex_exit(&gid_info->gl_mutex); 1410 1411 /* 1412 * Check whether the destination GID supports DM agents. If 1413 * not, stop probing the GID and continue with the next GID 1414 * in the list. 1415 */ 1416 if (ibdm_is_dev_mgt_supported(gid_info) != IBDM_SUCCESS) { 1417 mutex_enter(&gid_info->gl_mutex); 1418 gid_info->gl_pending_cmds = 0; 1419 gid_info->gl_state = IBDM_GID_PROBING_FAILED; 1420 mutex_exit(&gid_info->gl_mutex); 1421 ibdm_delete_glhca_list(gid_info); 1422 mutex_enter(&ibdm.ibdm_mutex); 1423 --ibdm.ibdm_ngid_probes_in_progress; 1424 ibdm_wakeup_probe_gid_cv(); 1425 mutex_exit(&ibdm.ibdm_mutex); 1426 return; 1427 } 1428 1429 /* Get the nodeguid and portguid of the port */ 1430 if (ibdm_get_node_port_guids(gid_info->gl_sa_hdl, gid_info->gl_dlid, 1431 &node_guid, &port_guid) != IBDM_SUCCESS) { 1432 mutex_enter(&gid_info->gl_mutex); 1433 gid_info->gl_pending_cmds = 0; 1434 gid_info->gl_state = IBDM_GID_PROBING_FAILED; 1435 mutex_exit(&gid_info->gl_mutex); 1436 ibdm_delete_glhca_list(gid_info); 1437 mutex_enter(&ibdm.ibdm_mutex); 1438 --ibdm.ibdm_ngid_probes_in_progress; 1439 ibdm_wakeup_probe_gid_cv(); 1440 mutex_exit(&ibdm.ibdm_mutex); 1441 return; 1442 } 1443 1444 /* 1445 * Check whether we already knew about this NodeGuid 1446 * If so, do not probe the GID and continue with the 1447 * next GID in the gid list. Set the GID state to 1448 * probing done. 1449 */ 1450 mutex_enter(&ibdm.ibdm_mutex); 1451 gid_info->gl_nodeguid = node_guid; 1452 gid_info->gl_portguid = port_guid; 1453 if (ibdm_check_dest_nodeguid(gid_info) != NULL) { 1454 mutex_exit(&ibdm.ibdm_mutex); 1455 mutex_enter(&gid_info->gl_mutex); 1456 gid_info->gl_pending_cmds = 0; 1457 gid_info->gl_state = IBDM_GID_PROBING_SKIPPED; 1458 mutex_exit(&gid_info->gl_mutex); 1459 ibdm_delete_glhca_list(gid_info); 1460 mutex_enter(&ibdm.ibdm_mutex); 1461 --ibdm.ibdm_ngid_probes_in_progress; 1462 ibdm_wakeup_probe_gid_cv(); 1463 mutex_exit(&ibdm.ibdm_mutex); 1464 return; 1465 } 1466 ibdm_add_to_gl_gid(gid_info, gid_info); 1467 mutex_exit(&ibdm.ibdm_mutex); 1468 1469 /* 1470 * New or reinserted GID : Enable notification to IBnex 1471 */ 1472 mutex_enter(&gid_info->gl_mutex); 1473 gid_info->gl_reprobe_flag = 1; 1474 mutex_exit(&gid_info->gl_mutex); 1475 1476 /* 1477 * Send ClassPortInfo request to the GID asynchronously. 1478 */ 1479 if (ibdm_send_classportinfo(gid_info) != IBDM_SUCCESS) { 1480 mutex_enter(&gid_info->gl_mutex); 1481 gid_info->gl_state = IBDM_GID_PROBING_FAILED; 1482 gid_info->gl_pending_cmds = 0; 1483 mutex_exit(&gid_info->gl_mutex); 1484 ibdm_delete_glhca_list(gid_info); 1485 mutex_enter(&ibdm.ibdm_mutex); 1486 --ibdm.ibdm_ngid_probes_in_progress; 1487 ibdm_wakeup_probe_gid_cv(); 1488 mutex_exit(&ibdm.ibdm_mutex); 1489 return; 1490 } 1491 } 1492 1493 1494 /* 1495 * ibdm_check_dest_nodeguid 1496 * Searches for the NodeGuid in the GID list 1497 * Returns matching gid_info if found and otherwise NULL 1498 * 1499 * This function is called to handle new GIDs discovered 1500 * during device sweep / probe or for GID_AVAILABLE event. 1501 * 1502 * Parameter : 1503 * gid_info GID to check 1504 */ 1505 static ibdm_dp_gidinfo_t * 1506 ibdm_check_dest_nodeguid(ibdm_dp_gidinfo_t *gid_info) 1507 { 1508 ibdm_dp_gidinfo_t *gid_list; 1509 ibdm_gid_t *tmp; 1510 1511 IBTF_DPRINTF_L4("ibdm", "\tcheck_dest_nodeguid"); 1512 1513 gid_list = ibdm.ibdm_dp_gidlist_head; 1514 while (gid_list) { 1515 if ((gid_list != gid_info) && 1516 (gid_info->gl_nodeguid == gid_list->gl_nodeguid)) { 1517 IBTF_DPRINTF_L4("ibdm", 1518 "\tcheck_dest_nodeguid: NodeGuid is present"); 1519 1520 /* Add to gid_list */ 1521 tmp = kmem_zalloc(sizeof (ibdm_gid_t), 1522 KM_SLEEP); 1523 tmp->gid_dgid_hi = gid_info->gl_dgid_hi; 1524 tmp->gid_dgid_lo = gid_info->gl_dgid_lo; 1525 tmp->gid_next = gid_list->gl_gid; 1526 gid_list->gl_gid = tmp; 1527 gid_list->gl_ngids++; 1528 return (gid_list); 1529 } 1530 1531 gid_list = gid_list->gl_next; 1532 } 1533 1534 return (NULL); 1535 } 1536 1537 1538 /* 1539 * ibdm_is_dev_mgt_supported 1540 * Get the PortInfo attribute (SA Query) 1541 * Check "CompatabilityMask" field in the Portinfo. 1542 * Return IBDM_SUCCESS if DM MAD's supported (if bit 19 set) 1543 * by the port, otherwise IBDM_FAILURE 1544 */ 1545 static int 1546 ibdm_is_dev_mgt_supported(ibdm_dp_gidinfo_t *gid_info) 1547 { 1548 int ret; 1549 size_t length = 0; 1550 sa_portinfo_record_t req, *resp = NULL; 1551 ibmf_saa_access_args_t qargs; 1552 1553 bzero(&req, sizeof (sa_portinfo_record_t)); 1554 req.EndportLID = gid_info->gl_dlid; 1555 1556 qargs.sq_attr_id = SA_PORTINFORECORD_ATTRID; 1557 qargs.sq_access_type = IBMF_SAA_RETRIEVE; 1558 qargs.sq_component_mask = SA_PORTINFO_COMPMASK_PORTLID; 1559 qargs.sq_template = &req; 1560 qargs.sq_callback = NULL; 1561 qargs.sq_callback_arg = NULL; 1562 1563 ret = ibmf_sa_access(gid_info->gl_sa_hdl, 1564 &qargs, 0, &length, (void **)&resp); 1565 1566 if ((ret != IBMF_SUCCESS) || (length == 0) || (resp == NULL)) { 1567 IBTF_DPRINTF_L2("ibdm", "\tis_dev_mgt_supported:" 1568 "failed to get PORTINFO attribute %d", ret); 1569 return (IBDM_FAILURE); 1570 } 1571 1572 if (resp->PortInfo.CapabilityMask & SM_CAP_MASK_IS_DM_SUPPD) { 1573 IBTF_DPRINTF_L4("ibdm", "\tis_dev_mgt_supported: SUPPD !!"); 1574 ret = IBDM_SUCCESS; 1575 } else { 1576 IBTF_DPRINTF_L4("ibdm", "\tis_dev_mgt_supported: " 1577 "Not SUPPD !!, cap 0x%x", resp->PortInfo.CapabilityMask); 1578 ret = IBDM_FAILURE; 1579 } 1580 kmem_free(resp, length); 1581 return (ret); 1582 } 1583 1584 1585 /* 1586 * ibdm_get_node_port_guids() 1587 * Get the NodeInfoRecord of the port 1588 * Save NodeGuid and PortGUID values in the GID list structure. 1589 * Return IBDM_SUCCESS/IBDM_FAILURE 1590 */ 1591 static int 1592 ibdm_get_node_port_guids(ibmf_saa_handle_t sa_hdl, ib_lid_t dlid, 1593 ib_guid_t *node_guid, ib_guid_t *port_guid) 1594 { 1595 int ret; 1596 size_t length = 0; 1597 sa_node_record_t req, *resp = NULL; 1598 ibmf_saa_access_args_t qargs; 1599 1600 IBTF_DPRINTF_L4("ibdm", "\tget_node_port_guids"); 1601 1602 bzero(&req, sizeof (sa_node_record_t)); 1603 req.LID = dlid; 1604 1605 qargs.sq_attr_id = SA_NODERECORD_ATTRID; 1606 qargs.sq_access_type = IBMF_SAA_RETRIEVE; 1607 qargs.sq_component_mask = SA_NODEINFO_COMPMASK_NODELID; 1608 qargs.sq_template = &req; 1609 qargs.sq_callback = NULL; 1610 qargs.sq_callback_arg = NULL; 1611 1612 ret = ibmf_sa_access(sa_hdl, &qargs, 0, &length, (void **)&resp); 1613 if ((ret != IBMF_SUCCESS) || (length == 0) || (resp == NULL)) { 1614 IBTF_DPRINTF_L2("ibdm", "\tget_node_port_guids:" 1615 " SA Retrieve Failed: %d", ret); 1616 return (IBDM_FAILURE); 1617 } 1618 IBTF_DPRINTF_L4("ibdm", "\tget_node_port_guids: NodeGuid %llx Port" 1619 "GUID %llx", resp->NodeInfo.NodeGUID, resp->NodeInfo.NodeGUID); 1620 1621 *node_guid = resp->NodeInfo.NodeGUID; 1622 *port_guid = resp->NodeInfo.PortGUID; 1623 kmem_free(resp, length); 1624 return (IBDM_SUCCESS); 1625 } 1626 1627 1628 /* 1629 * ibdm_get_reachable_ports() 1630 * Get list of the destination GID (and its path records) by 1631 * querying the SA access. 1632 * 1633 * Returns Number paths 1634 */ 1635 static int 1636 ibdm_get_reachable_ports(ibdm_port_attr_t *portinfo, ibdm_hca_list_t *hca) 1637 { 1638 uint_t ii, jj, nrecs; 1639 uint_t npaths = 0; 1640 size_t length; 1641 ib_gid_t sgid; 1642 ibdm_pkey_tbl_t *pkey_tbl; 1643 sa_path_record_t *result; 1644 sa_path_record_t *precp; 1645 ibdm_dp_gidinfo_t *gid_info; 1646 1647 ASSERT(MUTEX_HELD(&ibdm.ibdm_hl_mutex)); 1648 IBTF_DPRINTF_L4("ibdm", "\tget_reachable_ports: portinfo %p", portinfo); 1649 1650 sgid.gid_prefix = portinfo->pa_sn_prefix; 1651 sgid.gid_guid = portinfo->pa_port_guid; 1652 1653 /* get reversible paths */ 1654 if (portinfo->pa_sa_hdl && ibmf_saa_paths_from_gid(portinfo->pa_sa_hdl, 1655 sgid, IBMF_SAA_PKEY_WC, B_TRUE, 0, &nrecs, &length, &result) 1656 != IBMF_SUCCESS) { 1657 IBTF_DPRINTF_L2("ibdm", 1658 "\tget_reachable_ports: Getting path records failed"); 1659 return (0); 1660 } 1661 1662 for (ii = 0; ii < nrecs; ii++) { 1663 precp = &result[ii]; 1664 if ((gid_info = ibdm_check_dgid(precp->DGID.gid_guid, 1665 precp->DGID.gid_prefix)) != NULL) { 1666 IBTF_DPRINTF_L2("ibdm", "\tget_reachable_ports: " 1667 "Already exists nrecs %d, ii %d", nrecs, ii); 1668 ibdm_addto_glhcalist(gid_info, hca); 1669 continue; 1670 } 1671 /* 1672 * This is a new GID. Allocate a GID structure and 1673 * initialize the structure 1674 * gl_state is initialized to IBDM_GID_PROBE_NOT_DONE (0) 1675 * by kmem_zalloc call 1676 */ 1677 gid_info = kmem_zalloc(sizeof (ibdm_dp_gidinfo_t), KM_SLEEP); 1678 mutex_init(&gid_info->gl_mutex, NULL, MUTEX_DEFAULT, NULL); 1679 gid_info->gl_dgid_hi = precp->DGID.gid_prefix; 1680 gid_info->gl_dgid_lo = precp->DGID.gid_guid; 1681 gid_info->gl_sgid_hi = precp->SGID.gid_prefix; 1682 gid_info->gl_sgid_lo = precp->SGID.gid_guid; 1683 gid_info->gl_p_key = precp->P_Key; 1684 gid_info->gl_sa_hdl = portinfo->pa_sa_hdl; 1685 gid_info->gl_ibmf_hdl = portinfo->pa_ibmf_hdl; 1686 gid_info->gl_slid = precp->SLID; 1687 gid_info->gl_dlid = precp->DLID; 1688 gid_info->gl_transactionID = (++ibdm.ibdm_transactionID) 1689 << IBDM_GID_TRANSACTIONID_SHIFT; 1690 gid_info->gl_min_transactionID = gid_info->gl_transactionID; 1691 gid_info->gl_max_transactionID = (ibdm.ibdm_transactionID +1) 1692 << IBDM_GID_TRANSACTIONID_SHIFT; 1693 ibdm_addto_glhcalist(gid_info, hca); 1694 1695 ibdm_dump_path_info(precp); 1696 1697 gid_info->gl_qp_hdl = NULL; 1698 ASSERT(portinfo->pa_pkey_tbl != NULL && 1699 portinfo->pa_npkeys != 0); 1700 1701 for (jj = 0; jj < portinfo->pa_npkeys; jj++) { 1702 pkey_tbl = &portinfo->pa_pkey_tbl[jj]; 1703 if ((gid_info->gl_p_key == pkey_tbl->pt_pkey) && 1704 (pkey_tbl->pt_qp_hdl != NULL)) { 1705 gid_info->gl_qp_hdl = pkey_tbl->pt_qp_hdl; 1706 break; 1707 } 1708 } 1709 1710 /* 1711 * QP handle for GID not initialized. No matching Pkey 1712 * was found!! ibdm should *not* hit this case. Flag an 1713 * error and drop the GID if ibdm does encounter this. 1714 */ 1715 if (gid_info->gl_qp_hdl == NULL) { 1716 IBTF_DPRINTF_L2(ibdm_string, 1717 "\tget_reachable_ports: No matching Pkey"); 1718 ibdm_delete_gidinfo(gid_info); 1719 continue; 1720 } 1721 if (ibdm.ibdm_dp_gidlist_head == NULL) { 1722 ibdm.ibdm_dp_gidlist_head = gid_info; 1723 ibdm.ibdm_dp_gidlist_tail = gid_info; 1724 } else { 1725 ibdm.ibdm_dp_gidlist_tail->gl_next = gid_info; 1726 gid_info->gl_prev = ibdm.ibdm_dp_gidlist_tail; 1727 ibdm.ibdm_dp_gidlist_tail = gid_info; 1728 } 1729 npaths++; 1730 } 1731 kmem_free(result, length); 1732 IBTF_DPRINTF_L4("ibdm", "\tget_reachable_ports: npaths = %d", npaths); 1733 return (npaths); 1734 } 1735 1736 1737 /* 1738 * ibdm_check_dgid() 1739 * Look in the global list to check whether we know this DGID already 1740 * Return IBDM_GID_PRESENT/IBDM_GID_NOT_PRESENT 1741 */ 1742 static ibdm_dp_gidinfo_t * 1743 ibdm_check_dgid(ib_guid_t guid, ib_sn_prefix_t prefix) 1744 { 1745 ibdm_dp_gidinfo_t *gid_list; 1746 1747 for (gid_list = ibdm.ibdm_dp_gidlist_head; gid_list; 1748 gid_list = gid_list->gl_next) { 1749 if ((guid == gid_list->gl_dgid_lo) && 1750 (prefix == gid_list->gl_dgid_hi)) { 1751 break; 1752 } 1753 } 1754 return (gid_list); 1755 } 1756 1757 1758 /* 1759 * ibdm_find_gid() 1760 * Look in the global list to find a GID entry with matching 1761 * port & node GUID. 1762 * Return pointer to gidinfo if found, else return NULL 1763 */ 1764 static ibdm_dp_gidinfo_t * 1765 ibdm_find_gid(ib_guid_t nodeguid, ib_guid_t portguid) 1766 { 1767 ibdm_dp_gidinfo_t *gid_list; 1768 1769 IBTF_DPRINTF_L4("ibdm", "ibdm_find_gid(%llx, %llx)\n", 1770 nodeguid, portguid); 1771 1772 for (gid_list = ibdm.ibdm_dp_gidlist_head; gid_list; 1773 gid_list = gid_list->gl_next) { 1774 if ((portguid == gid_list->gl_portguid) && 1775 (nodeguid == gid_list->gl_nodeguid)) { 1776 break; 1777 } 1778 } 1779 1780 IBTF_DPRINTF_L4("ibdm", "ibdm_find_gid : returned %p\n", 1781 gid_list); 1782 return (gid_list); 1783 } 1784 1785 1786 /* 1787 * ibdm_send_classportinfo() 1788 * Send classportinfo request. When the request is completed 1789 * IBMF calls ibdm_classportinfo_cb routine to inform about 1790 * the completion. 1791 * Returns IBDM_SUCCESS/IBDM_FAILURE 1792 */ 1793 static int 1794 ibdm_send_classportinfo(ibdm_dp_gidinfo_t *gid_info) 1795 { 1796 ibmf_msg_t *msg; 1797 ib_mad_hdr_t *hdr; 1798 ibdm_timeout_cb_args_t *cb_args; 1799 1800 IBTF_DPRINTF_L4("ibdm", 1801 "\tsend_classportinfo: gid info 0x%p", gid_info); 1802 1803 /* 1804 * Send command to get classportinfo attribute. Allocate a IBMF 1805 * packet and initialize the packet. 1806 */ 1807 if (ibmf_alloc_msg(gid_info->gl_ibmf_hdl, IBMF_ALLOC_SLEEP, 1808 &msg) != IBMF_SUCCESS) { 1809 IBTF_DPRINTF_L4("ibdm", "\tsend_classportinfo: pkt alloc fail"); 1810 return (IBDM_FAILURE); 1811 } 1812 1813 ibdm_alloc_send_buffers(msg); 1814 1815 msg->im_local_addr.ia_local_lid = gid_info->gl_slid; 1816 msg->im_local_addr.ia_remote_lid = gid_info->gl_dlid; 1817 msg->im_local_addr.ia_remote_qno = 1; 1818 msg->im_local_addr.ia_p_key = gid_info->gl_p_key; 1819 msg->im_local_addr.ia_q_key = IB_GSI_QKEY; 1820 1821 hdr = IBDM_OUT_IBMFMSG_MADHDR(msg); 1822 hdr->BaseVersion = MAD_CLASS_BASE_VERS_1; 1823 hdr->MgmtClass = MAD_MGMT_CLASS_DEV_MGT; 1824 hdr->ClassVersion = IB_DM_CLASS_VERSION_1; 1825 hdr->R_Method = IB_DM_DEVMGT_METHOD_GET; 1826 hdr->Status = 0; 1827 hdr->TransactionID = h2b64(gid_info->gl_transactionID); 1828 hdr->AttributeID = h2b16(IB_DM_ATTR_CLASSPORTINFO); 1829 hdr->AttributeModifier = 0; 1830 1831 cb_args = &gid_info->gl_cpi_cb_args; 1832 cb_args->cb_gid_info = gid_info; 1833 cb_args->cb_retry_count = ibdm_dft_retry_cnt; 1834 cb_args->cb_req_type = IBDM_REQ_TYPE_CLASSPORTINFO; 1835 1836 gid_info->gl_timeout_id = timeout(ibdm_pkt_timeout_hdlr, 1837 cb_args, IBDM_TIMEOUT_VALUE(ibdm_dft_timeout)); 1838 1839 IBTF_DPRINTF_L5("ibdm", "\tsend_classportinfo: " 1840 "timeout id %x", gid_info->gl_timeout_id); 1841 1842 if (ibmf_msg_transport(gid_info->gl_ibmf_hdl, gid_info->gl_qp_hdl, 1843 msg, NULL, ibdm_ibmf_send_cb, cb_args, 0) != IBMF_SUCCESS) { 1844 IBTF_DPRINTF_L2("ibdm", 1845 "\tsend_classportinfo: ibmf send failed"); 1846 ibdm_ibmf_send_cb(gid_info->gl_ibmf_hdl, msg, cb_args); 1847 } 1848 1849 return (IBDM_SUCCESS); 1850 } 1851 1852 1853 /* 1854 * ibdm_handle_classportinfo() 1855 * Invoked by the IBMF when the classportinfo request is completed. 1856 */ 1857 static void 1858 ibdm_handle_classportinfo(ibmf_handle_t ibmf_hdl, 1859 ibmf_msg_t *msg, ibdm_dp_gidinfo_t *gid_info, int *flag) 1860 { 1861 void *data; 1862 timeout_id_t timeout_id; 1863 ib_mad_hdr_t *hdr; 1864 ibdm_mad_classportinfo_t *cpi; 1865 1866 IBTF_DPRINTF_L4("ibdm", "\thandle_classportinfo:ibmf hdl " 1867 "%p msg %p gid info %p", ibmf_hdl, msg, gid_info); 1868 1869 if (IBDM_IN_IBMFMSG_ATTR(msg) != IB_DM_ATTR_CLASSPORTINFO) { 1870 IBTF_DPRINTF_L4("ibdm", "\thandle_classportinfo: " 1871 "Not a ClassPortInfo resp"); 1872 *flag |= IBDM_IBMF_PKT_UNEXP_RESP; 1873 return; 1874 } 1875 1876 /* 1877 * Verify whether timeout handler is created/active. 1878 * If created/ active, cancel the timeout handler 1879 */ 1880 mutex_enter(&gid_info->gl_mutex); 1881 ibdm_bump_transactionID(gid_info); 1882 if (gid_info->gl_state != IBDM_GET_CLASSPORTINFO) { 1883 IBTF_DPRINTF_L2("ibdm", "\thandle_classportinfo:DUP resp"); 1884 *flag |= IBDM_IBMF_PKT_DUP_RESP; 1885 mutex_exit(&gid_info->gl_mutex); 1886 return; 1887 } 1888 gid_info->gl_iou_cb_args.cb_req_type = 0; 1889 if (gid_info->gl_timeout_id) { 1890 timeout_id = gid_info->gl_timeout_id; 1891 mutex_exit(&gid_info->gl_mutex); 1892 IBTF_DPRINTF_L5("ibdm", "handle_ioclassportinfo: " 1893 "gl_timeout_id = 0x%x", timeout_id); 1894 if (untimeout(timeout_id) == -1) { 1895 IBTF_DPRINTF_L2("ibdm", "handle_classportinfo: " 1896 "untimeout gl_timeout_id failed"); 1897 } 1898 mutex_enter(&gid_info->gl_mutex); 1899 gid_info->gl_timeout_id = 0; 1900 } 1901 gid_info->gl_state = IBDM_GET_IOUNITINFO; 1902 gid_info->gl_pending_cmds++; 1903 mutex_exit(&gid_info->gl_mutex); 1904 1905 data = msg->im_msgbufs_recv.im_bufs_cl_data; 1906 cpi = (ibdm_mad_classportinfo_t *)data; 1907 1908 /* 1909 * Cache the "RespTimeValue" and redirection information in the 1910 * global gid list data structure. This cached information will 1911 * be used to send any further requests to the GID. 1912 */ 1913 gid_info->gl_resp_timeout = 1914 (b2h32(cpi->RespTimeValue) & 0x1F); 1915 1916 gid_info->gl_redirected = ((IBDM_IN_IBMFMSG_STATUS(msg) & 1917 MAD_STATUS_REDIRECT_REQUIRED) ? B_TRUE: B_FALSE); 1918 gid_info->gl_redirect_dlid = b2h16(cpi->RedirectLID); 1919 gid_info->gl_redirect_QP = (b2h32(cpi->RedirectQP) & 0xffffff); 1920 gid_info->gl_redirect_pkey = b2h16(cpi->RedirectP_Key); 1921 gid_info->gl_redirect_qkey = b2h32(cpi->RedirectQ_Key); 1922 gid_info->gl_redirectGID_hi = b2h64(cpi->RedirectGID_hi); 1923 gid_info->gl_redirectGID_lo = b2h64(cpi->RedirectGID_lo); 1924 1925 ibdm_dump_classportinfo(cpi); 1926 1927 /* 1928 * Send IOUnitInfo request 1929 * Reuse previously allocated IBMF packet for sending ClassPortInfo 1930 * Check whether DM agent on the remote node requested redirection 1931 * If so, send the request to the redirect DGID/DLID/PKEY/QP. 1932 */ 1933 ibdm_alloc_send_buffers(msg); 1934 msg->im_local_addr.ia_local_lid = gid_info->gl_slid; 1935 msg->im_local_addr.ia_remote_lid = gid_info->gl_dlid; 1936 1937 if (gid_info->gl_redirected == B_TRUE) { 1938 if (gid_info->gl_redirect_dlid != 0) { 1939 msg->im_local_addr.ia_remote_lid = 1940 gid_info->gl_redirect_dlid; 1941 } 1942 msg->im_local_addr.ia_remote_qno = gid_info->gl_redirect_QP; 1943 msg->im_local_addr.ia_p_key = gid_info->gl_redirect_pkey; 1944 msg->im_local_addr.ia_q_key = gid_info->gl_redirect_qkey; 1945 } else { 1946 msg->im_local_addr.ia_remote_qno = 1; 1947 msg->im_local_addr.ia_p_key = gid_info->gl_p_key; 1948 msg->im_local_addr.ia_q_key = IB_GSI_QKEY; 1949 } 1950 1951 hdr = IBDM_OUT_IBMFMSG_MADHDR(msg); 1952 hdr->BaseVersion = MAD_CLASS_BASE_VERS_1; 1953 hdr->MgmtClass = MAD_MGMT_CLASS_DEV_MGT; 1954 hdr->ClassVersion = IB_DM_CLASS_VERSION_1; 1955 hdr->R_Method = IB_DM_DEVMGT_METHOD_GET; 1956 hdr->Status = 0; 1957 hdr->TransactionID = h2b64(gid_info->gl_transactionID); 1958 hdr->AttributeID = h2b16(IB_DM_ATTR_IO_UNITINFO); 1959 hdr->AttributeModifier = 0; 1960 1961 gid_info->gl_iou_cb_args.cb_req_type = IBDM_REQ_TYPE_IOUINFO; 1962 gid_info->gl_iou_cb_args.cb_gid_info = gid_info; 1963 gid_info->gl_iou_cb_args.cb_retry_count = ibdm_dft_retry_cnt; 1964 1965 gid_info->gl_timeout_id = timeout(ibdm_pkt_timeout_hdlr, 1966 &gid_info->gl_iou_cb_args, IBDM_TIMEOUT_VALUE(ibdm_dft_timeout)); 1967 1968 IBTF_DPRINTF_L5("ibdm", "handle_classportinfo:" 1969 "timeout %x", gid_info->gl_timeout_id); 1970 1971 if (ibmf_msg_transport(ibmf_hdl, gid_info->gl_qp_hdl, msg, NULL, 1972 ibdm_ibmf_send_cb, &gid_info->gl_iou_cb_args, 0) != IBMF_SUCCESS) { 1973 IBTF_DPRINTF_L2("ibdm", 1974 "\thandle_classportinfo: msg transport failed"); 1975 ibdm_ibmf_send_cb(ibmf_hdl, msg, &gid_info->gl_iou_cb_args); 1976 } 1977 (*flag) |= IBDM_IBMF_PKT_REUSED; 1978 } 1979 1980 1981 /* 1982 * ibdm_send_iounitinfo: 1983 * Sends a DM request to get IOU unitinfo. 1984 */ 1985 static int 1986 ibdm_send_iounitinfo(ibdm_dp_gidinfo_t *gid_info) 1987 { 1988 ibmf_msg_t *msg; 1989 ib_mad_hdr_t *hdr; 1990 1991 IBTF_DPRINTF_L4("ibdm", "\tsend_iounitinfo: gid info 0x%p", gid_info); 1992 1993 /* 1994 * Send command to get iounitinfo attribute. Allocate a IBMF 1995 * packet and initialize the packet. 1996 */ 1997 if (ibmf_alloc_msg(gid_info->gl_ibmf_hdl, IBMF_ALLOC_SLEEP, &msg) != 1998 IBMF_SUCCESS) { 1999 IBTF_DPRINTF_L4("ibdm", "\tsend_iounitinfo: pkt alloc fail"); 2000 return (IBDM_FAILURE); 2001 } 2002 2003 mutex_enter(&gid_info->gl_mutex); 2004 ibdm_bump_transactionID(gid_info); 2005 mutex_exit(&gid_info->gl_mutex); 2006 2007 2008 ibdm_alloc_send_buffers(msg); 2009 msg->im_local_addr.ia_local_lid = gid_info->gl_slid; 2010 msg->im_local_addr.ia_remote_lid = gid_info->gl_dlid; 2011 msg->im_local_addr.ia_remote_qno = 1; 2012 msg->im_local_addr.ia_p_key = gid_info->gl_p_key; 2013 msg->im_local_addr.ia_q_key = IB_GSI_QKEY; 2014 2015 hdr = IBDM_OUT_IBMFMSG_MADHDR(msg); 2016 hdr->BaseVersion = MAD_CLASS_BASE_VERS_1; 2017 hdr->MgmtClass = MAD_MGMT_CLASS_DEV_MGT; 2018 hdr->ClassVersion = IB_DM_CLASS_VERSION_1; 2019 hdr->R_Method = IB_DM_DEVMGT_METHOD_GET; 2020 hdr->Status = 0; 2021 hdr->TransactionID = h2b64(gid_info->gl_transactionID); 2022 hdr->AttributeID = h2b16(IB_DM_ATTR_IO_UNITINFO); 2023 hdr->AttributeModifier = 0; 2024 2025 gid_info->gl_iou_cb_args.cb_gid_info = gid_info; 2026 gid_info->gl_iou_cb_args.cb_retry_count = ibdm_dft_retry_cnt; 2027 gid_info->gl_iou_cb_args.cb_req_type = IBDM_REQ_TYPE_IOUINFO; 2028 2029 gid_info->gl_timeout_id = timeout(ibdm_pkt_timeout_hdlr, 2030 &gid_info->gl_iou_cb_args, IBDM_TIMEOUT_VALUE(ibdm_dft_timeout)); 2031 2032 IBTF_DPRINTF_L5("ibdm", "send_iouunitinfo:" 2033 "timeout %x", gid_info->gl_timeout_id); 2034 2035 if (ibmf_msg_transport(gid_info->gl_ibmf_hdl, gid_info->gl_qp_hdl, msg, 2036 NULL, ibdm_ibmf_send_cb, &gid_info->gl_iou_cb_args, 0) != 2037 IBMF_SUCCESS) { 2038 IBTF_DPRINTF_L2("ibdm", "\tsend_iounitinfo: ibmf send failed"); 2039 ibdm_ibmf_send_cb(gid_info->gl_ibmf_hdl, 2040 msg, &gid_info->gl_iou_cb_args); 2041 } 2042 return (IBDM_SUCCESS); 2043 } 2044 2045 /* 2046 * ibdm_handle_iounitinfo() 2047 * Invoked by the IBMF when IO Unitinfo request is completed. 2048 */ 2049 static void 2050 ibdm_handle_iounitinfo(ibmf_handle_t ibmf_hdl, 2051 ibmf_msg_t *msg, ibdm_dp_gidinfo_t *gid_info, int *flag) 2052 { 2053 int ii, first = B_TRUE; 2054 int num_iocs; 2055 size_t size; 2056 uchar_t slot_info; 2057 timeout_id_t timeout_id; 2058 ib_mad_hdr_t *hdr; 2059 ibdm_ioc_info_t *ioc_info; 2060 ib_dm_io_unitinfo_t *iou_info; 2061 ib_dm_io_unitinfo_t *giou_info; 2062 ibdm_timeout_cb_args_t *cb_args; 2063 2064 IBTF_DPRINTF_L4("ibdm", "\thandle_iouintinfo:" 2065 " ibmf hdl %p pkt %p gid info %p", ibmf_hdl, msg, gid_info); 2066 2067 if (IBDM_IN_IBMFMSG_ATTR(msg) != IB_DM_ATTR_IO_UNITINFO) { 2068 IBTF_DPRINTF_L4("ibdm", "\thandle_iounitinfo: " 2069 "Unexpected response"); 2070 (*flag) |= IBDM_IBMF_PKT_UNEXP_RESP; 2071 return; 2072 } 2073 2074 mutex_enter(&gid_info->gl_mutex); 2075 if (gid_info->gl_state != IBDM_GET_IOUNITINFO) { 2076 IBTF_DPRINTF_L4("ibdm", 2077 "\thandle_iounitinfo: DUP resp"); 2078 mutex_exit(&gid_info->gl_mutex); 2079 (*flag) = IBDM_IBMF_PKT_DUP_RESP; 2080 return; 2081 } 2082 gid_info->gl_iou_cb_args.cb_req_type = 0; 2083 if (gid_info->gl_timeout_id) { 2084 timeout_id = gid_info->gl_timeout_id; 2085 mutex_exit(&gid_info->gl_mutex); 2086 IBTF_DPRINTF_L5("ibdm", "handle_iounitinfo: " 2087 "gl_timeout_id = 0x%x", timeout_id); 2088 if (untimeout(timeout_id) == -1) { 2089 IBTF_DPRINTF_L2("ibdm", "handle_iounitinfo: " 2090 "untimeout gl_timeout_id failed"); 2091 } 2092 mutex_enter(&gid_info->gl_mutex); 2093 gid_info->gl_timeout_id = 0; 2094 } 2095 gid_info->gl_state = IBDM_GET_IOC_DETAILS; 2096 2097 iou_info = IBDM_IN_IBMFMSG2IOU(msg); 2098 ibdm_dump_iounitinfo(iou_info); 2099 num_iocs = iou_info->iou_num_ctrl_slots; 2100 /* 2101 * check if number of IOCs reported is zero? if yes, return. 2102 * when num_iocs are reported zero internal IOC database needs 2103 * to be updated. To ensure that save the number of IOCs in 2104 * the new field "gl_num_iocs". Use a new field instead of 2105 * "giou_info->iou_num_ctrl_slots" as that would prevent 2106 * an unnecessary kmem_alloc/kmem_free when num_iocs is 0. 2107 */ 2108 if (num_iocs == 0 && gid_info->gl_num_iocs == 0) { 2109 IBTF_DPRINTF_L4("ibdm", "\thandle_iounitinfo: no IOC's"); 2110 mutex_exit(&gid_info->gl_mutex); 2111 return; 2112 } 2113 IBTF_DPRINTF_L4("ibdm", "\thandle_iounitinfo: num_iocs = %d", num_iocs); 2114 2115 /* 2116 * if there is an existing gl_iou (IOU has been probed before) 2117 * check if the "iou_changeid" is same as saved entry in 2118 * "giou_info->iou_changeid". 2119 * (note: this logic can prevent IOC enumeration if a given 2120 * vendor doesn't support setting iou_changeid field for its IOU) 2121 * 2122 * if there is an existing gl_iou and iou_changeid has changed : 2123 * free up existing gl_iou info and its related structures. 2124 * reallocate gl_iou info all over again. 2125 * if we donot free this up; then this leads to memory leaks 2126 */ 2127 if (gid_info->gl_iou) { 2128 giou_info = &gid_info->gl_iou->iou_info; 2129 if (iou_info->iou_changeid == giou_info->iou_changeid) { 2130 IBTF_DPRINTF_L3("ibdm", 2131 "\thandle_iounitinfo: no IOCs changed"); 2132 gid_info->gl_state = IBDM_GID_PROBING_COMPLETE; 2133 mutex_exit(&gid_info->gl_mutex); 2134 return; 2135 } 2136 if (ibdm_free_iou_info(gid_info)) { 2137 IBTF_DPRINTF_L3("ibdm", 2138 "\thandle_iounitinfo: failed to cleanup resources"); 2139 gid_info->gl_state = IBDM_GID_PROBING_COMPLETE; 2140 mutex_exit(&gid_info->gl_mutex); 2141 return; 2142 } 2143 } 2144 2145 size = sizeof (ibdm_iou_info_t) + num_iocs * sizeof (ibdm_ioc_info_t); 2146 gid_info->gl_iou = (ibdm_iou_info_t *)kmem_zalloc(size, KM_SLEEP); 2147 giou_info = &gid_info->gl_iou->iou_info; 2148 gid_info->gl_iou->iou_ioc_info = (ibdm_ioc_info_t *) 2149 ((char *)gid_info->gl_iou + sizeof (ibdm_iou_info_t)); 2150 2151 giou_info->iou_num_ctrl_slots = gid_info->gl_num_iocs = num_iocs; 2152 giou_info->iou_flag = iou_info->iou_flag; 2153 bcopy(iou_info->iou_ctrl_list, giou_info->iou_ctrl_list, 128); 2154 giou_info->iou_changeid = b2h16(iou_info->iou_changeid); 2155 gid_info->gl_pending_cmds += num_iocs; 2156 gid_info->gl_pending_cmds += 1; /* for diag code */ 2157 mutex_exit(&gid_info->gl_mutex); 2158 2159 if (ibdm_get_diagcode(gid_info, 0) != IBDM_SUCCESS) { 2160 mutex_enter(&gid_info->gl_mutex); 2161 gid_info->gl_pending_cmds--; 2162 mutex_exit(&gid_info->gl_mutex); 2163 } 2164 /* 2165 * Parallelize getting IOC controller profiles from here. 2166 * Allocate IBMF packets and send commands to get IOC profile for 2167 * each IOC present on the IOU. 2168 */ 2169 for (ii = 0; ii < num_iocs; ii++) { 2170 /* 2171 * Check whether IOC is present in the slot 2172 * Series of nibbles (in the field iou_ctrl_list) represents 2173 * a slot in the IOU. 2174 * Byte format: 76543210 2175 * Bits 0-3 of first byte represent Slot 2 2176 * bits 4-7 of first byte represent slot 1, 2177 * bits 0-3 of second byte represent slot 4 and so on 2178 * Each 4-bit nibble has the following meaning 2179 * 0x0 : IOC not installed 2180 * 0x1 : IOC is present 2181 * 0xf : Slot does not exist 2182 * and all other values are reserved. 2183 */ 2184 ioc_info = IBDM_GIDINFO2IOCINFO(gid_info, ii); 2185 slot_info = giou_info->iou_ctrl_list[(ii/2)]; 2186 if ((ii % 2) == 0) 2187 slot_info = (slot_info >> 4); 2188 2189 if ((slot_info & 0xf) != 1) { 2190 IBTF_DPRINTF_L4("ibdm", "\thandle_iouintinfo: " 2191 "No IOC is present in the slot = %d", ii); 2192 ioc_info->ioc_state = IBDM_IOC_STATE_PROBE_FAILED; 2193 mutex_enter(&gid_info->gl_mutex); 2194 gid_info->gl_pending_cmds--; 2195 mutex_exit(&gid_info->gl_mutex); 2196 continue; 2197 } 2198 2199 mutex_enter(&gid_info->gl_mutex); 2200 ibdm_bump_transactionID(gid_info); 2201 mutex_exit(&gid_info->gl_mutex); 2202 2203 /* 2204 * Re use the already allocated packet (for IOUnitinfo) to 2205 * send the first IOC controller attribute. Allocate new 2206 * IBMF packets for the rest of the IOC's 2207 */ 2208 if (first != B_TRUE) { 2209 msg = NULL; 2210 if (ibmf_alloc_msg(ibmf_hdl, IBMF_ALLOC_SLEEP, 2211 &msg) != IBMF_SUCCESS) { 2212 IBTF_DPRINTF_L4("ibdm", "\thandle_iouintinfo: " 2213 "IBMF packet allocation failed"); 2214 mutex_enter(&gid_info->gl_mutex); 2215 gid_info->gl_pending_cmds--; 2216 mutex_exit(&gid_info->gl_mutex); 2217 continue; 2218 } 2219 2220 } 2221 2222 /* allocate send buffers for all messages */ 2223 ibdm_alloc_send_buffers(msg); 2224 2225 msg->im_local_addr.ia_local_lid = gid_info->gl_slid; 2226 msg->im_local_addr.ia_remote_lid = gid_info->gl_dlid; 2227 if (gid_info->gl_redirected == B_TRUE) { 2228 if (gid_info->gl_redirect_dlid != 0) { 2229 msg->im_local_addr.ia_remote_lid = 2230 gid_info->gl_redirect_dlid; 2231 } 2232 msg->im_local_addr.ia_remote_qno = 2233 gid_info->gl_redirect_QP; 2234 msg->im_local_addr.ia_p_key = 2235 gid_info->gl_redirect_pkey; 2236 msg->im_local_addr.ia_q_key = 2237 gid_info->gl_redirect_qkey; 2238 } else { 2239 msg->im_local_addr.ia_remote_qno = 1; 2240 msg->im_local_addr.ia_p_key = gid_info->gl_p_key; 2241 msg->im_local_addr.ia_q_key = IB_GSI_QKEY; 2242 } 2243 2244 hdr = IBDM_OUT_IBMFMSG_MADHDR(msg); 2245 hdr->BaseVersion = MAD_CLASS_BASE_VERS_1; 2246 hdr->MgmtClass = MAD_MGMT_CLASS_DEV_MGT; 2247 hdr->ClassVersion = IB_DM_CLASS_VERSION_1; 2248 hdr->R_Method = IB_DM_DEVMGT_METHOD_GET; 2249 hdr->Status = 0; 2250 hdr->TransactionID = h2b64(gid_info->gl_transactionID); 2251 hdr->AttributeID = h2b16(IB_DM_ATTR_IOC_CTRL_PROFILE); 2252 hdr->AttributeModifier = h2b32(ii + 1); 2253 2254 ioc_info->ioc_state = IBDM_IOC_STATE_PROBE_INVALID; 2255 cb_args = &ioc_info->ioc_cb_args; 2256 cb_args->cb_gid_info = gid_info; 2257 cb_args->cb_retry_count = ibdm_dft_retry_cnt; 2258 cb_args->cb_req_type = IBDM_REQ_TYPE_IOCINFO; 2259 cb_args->cb_ioc_num = ii; 2260 2261 ioc_info->ioc_timeout_id = timeout(ibdm_pkt_timeout_hdlr, 2262 cb_args, IBDM_TIMEOUT_VALUE(ibdm_dft_timeout)); 2263 2264 IBTF_DPRINTF_L5("ibdm", "\thandle_iounitinfo:" 2265 "timeout 0x%x, ioc_num %d", ioc_info->ioc_timeout_id, ii); 2266 2267 if (ibmf_msg_transport(ibmf_hdl, gid_info->gl_qp_hdl, msg, 2268 NULL, ibdm_ibmf_send_cb, cb_args, 0) != IBMF_SUCCESS) { 2269 IBTF_DPRINTF_L2("ibdm", 2270 "\thandle_iounitinfo: msg transport failed"); 2271 ibdm_ibmf_send_cb(ibmf_hdl, msg, cb_args); 2272 } 2273 (*flag) |= IBDM_IBMF_PKT_REUSED; 2274 first = B_FALSE; 2275 gid_info->gl_iou->iou_niocs_probe_in_progress++; 2276 } 2277 } 2278 2279 2280 /* 2281 * ibdm_handle_ioc_profile() 2282 * Invoked by the IBMF when the IOCControllerProfile request 2283 * gets completed 2284 */ 2285 static void 2286 ibdm_handle_ioc_profile(ibmf_handle_t ibmf_hdl, 2287 ibmf_msg_t *msg, ibdm_dp_gidinfo_t *gid_info, int *flag) 2288 { 2289 int first = B_TRUE, reprobe = 0; 2290 uint_t ii, ioc_no, srv_start; 2291 uint_t nserv_entries; 2292 timeout_id_t timeout_id; 2293 ib_mad_hdr_t *hdr; 2294 ibdm_ioc_info_t *ioc_info; 2295 ibdm_timeout_cb_args_t *cb_args; 2296 ib_dm_ioc_ctrl_profile_t *ioc, *gioc; 2297 2298 IBTF_DPRINTF_L4("ibdm", "\thandle_ioc_profile:" 2299 " ibmf hdl %p msg %p gid info %p", ibmf_hdl, msg, gid_info); 2300 2301 ioc = IBDM_IN_IBMFMSG2IOC(msg); 2302 /* 2303 * Check whether we know this IOC already 2304 * This will return NULL if reprobe is in progress 2305 * IBDM_IOC_STATE_REPROBE_PROGRESS will be set. 2306 * Do not hold mutexes here. 2307 */ 2308 if (ibdm_is_ioc_present(ioc->ioc_guid, gid_info, flag) != NULL) { 2309 IBTF_DPRINTF_L4("ibdm", "\thandle_ioc_profile:" 2310 "IOC guid %llx is present", ioc->ioc_guid); 2311 return; 2312 } 2313 ioc_no = IBDM_IN_IBMFMSG_ATTRMOD(msg); 2314 IBTF_DPRINTF_L4("ibdm", "\thandle_ioc_profile: ioc_no = %d", ioc_no-1); 2315 2316 /* Make sure that IOC index is with the valid range */ 2317 if (IBDM_IS_IOC_NUM_INVALID(ioc_no, gid_info)) { 2318 IBTF_DPRINTF_L2("ibdm", "\thandle_ioc_profile: " 2319 "IOC index Out of range, index %d", ioc); 2320 (*flag) |= IBDM_IBMF_PKT_UNEXP_RESP; 2321 return; 2322 } 2323 ioc_info = &gid_info->gl_iou->iou_ioc_info[ioc_no - 1]; 2324 ioc_info->ioc_iou_info = gid_info->gl_iou; 2325 2326 mutex_enter(&gid_info->gl_mutex); 2327 if (ioc_info->ioc_state == IBDM_IOC_STATE_REPROBE_PROGRESS) { 2328 reprobe = 1; 2329 ioc_info->ioc_prev_serv = ioc_info->ioc_serv; 2330 ioc_info->ioc_serv = NULL; 2331 ioc_info->ioc_prev_serv_cnt = 2332 ioc_info->ioc_profile.ioc_service_entries; 2333 } else if (ioc_info->ioc_state != IBDM_IOC_STATE_PROBE_INVALID) { 2334 IBTF_DPRINTF_L2("ibdm", "\thandle_ioc_profile: DUP response" 2335 "ioc %d, ioc_state %x", ioc_no - 1, ioc_info->ioc_state); 2336 mutex_exit(&gid_info->gl_mutex); 2337 (*flag) |= IBDM_IBMF_PKT_DUP_RESP; 2338 return; 2339 } 2340 ioc_info->ioc_cb_args.cb_req_type = 0; 2341 if (ioc_info->ioc_timeout_id) { 2342 timeout_id = ioc_info->ioc_timeout_id; 2343 mutex_exit(&gid_info->gl_mutex); 2344 IBTF_DPRINTF_L5("ibdm", "handle_ioc_profile: " 2345 "ioc_timeout_id = 0x%x", timeout_id); 2346 if (untimeout(timeout_id) == -1) { 2347 IBTF_DPRINTF_L2("ibdm", "handle_ioc_profile: " 2348 "untimeout ioc_timeout_id failed"); 2349 } 2350 mutex_enter(&gid_info->gl_mutex); 2351 ioc_info->ioc_timeout_id = 0; 2352 } 2353 2354 ioc_info->ioc_state = IBDM_IOC_STATE_PROBE_SUCCESS; 2355 if (reprobe == 0) { 2356 ioc_info->ioc_iou_guid = gid_info->gl_nodeguid; 2357 ioc_info->ioc_nodeguid = gid_info->gl_nodeguid; 2358 } 2359 2360 /* 2361 * Save all the IOC information in the global structures. 2362 * Note the wire format is Big Endian and the Sparc process also 2363 * big endian. So, there is no need to convert the data fields 2364 * The conversion routines used below are ineffective on Sparc 2365 * machines where as they will be effective on little endian 2366 * machines such as Intel processors. 2367 */ 2368 gioc = (ib_dm_ioc_ctrl_profile_t *)&ioc_info->ioc_profile; 2369 2370 /* 2371 * Restrict updates to onlyport GIDs and service entries during reprobe 2372 */ 2373 if (reprobe == 0) { 2374 gioc->ioc_guid = b2h64(ioc->ioc_guid); 2375 gioc->ioc_vendorid = 2376 ((b2h32(ioc->ioc_vendorid) & IB_DM_VENDORID_MASK) 2377 >> IB_DM_VENDORID_SHIFT); 2378 gioc->ioc_deviceid = b2h32(ioc->ioc_deviceid); 2379 gioc->ioc_device_ver = b2h16(ioc->ioc_device_ver); 2380 gioc->ioc_subsys_vendorid = 2381 ((b2h32(ioc->ioc_subsys_vendorid) & IB_DM_VENDORID_MASK) 2382 >> IB_DM_VENDORID_SHIFT); 2383 gioc->ioc_subsys_id = b2h32(ioc->ioc_subsys_id); 2384 gioc->ioc_io_class = b2h16(ioc->ioc_io_class); 2385 gioc->ioc_io_subclass = b2h16(ioc->ioc_io_subclass); 2386 gioc->ioc_protocol = b2h16(ioc->ioc_protocol); 2387 gioc->ioc_protocol_ver = b2h16(ioc->ioc_protocol_ver); 2388 gioc->ioc_send_msg_qdepth = 2389 b2h16(ioc->ioc_send_msg_qdepth); 2390 gioc->ioc_rdma_read_qdepth = 2391 b2h16(ioc->ioc_rdma_read_qdepth); 2392 gioc->ioc_send_msg_sz = b2h32(ioc->ioc_send_msg_sz); 2393 gioc->ioc_rdma_xfer_sz = b2h32(ioc->ioc_rdma_xfer_sz); 2394 gioc->ioc_ctrl_opcap_mask = ioc->ioc_ctrl_opcap_mask; 2395 bcopy(ioc->ioc_id_string, gioc->ioc_id_string, 2396 IB_DM_IOC_ID_STRING_LEN); 2397 2398 ioc_info->ioc_iou_diagcode = gid_info->gl_iou->iou_diagcode; 2399 ioc_info->ioc_iou_dc_valid = gid_info->gl_iou->iou_dc_valid; 2400 ioc_info->ioc_diagdeviceid = (IB_DM_IOU_DEVICEID_MASK & 2401 gid_info->gl_iou->iou_info.iou_flag) ? B_TRUE : B_FALSE; 2402 2403 if (ioc_info->ioc_diagdeviceid == B_TRUE) 2404 gid_info->gl_pending_cmds++; 2405 } 2406 gioc->ioc_service_entries = ioc->ioc_service_entries; 2407 gid_info->gl_pending_cmds += (gioc->ioc_service_entries/4); 2408 if (gioc->ioc_service_entries % 4) 2409 gid_info->gl_pending_cmds++; 2410 2411 mutex_exit(&gid_info->gl_mutex); 2412 2413 ibdm_dump_ioc_profile(gioc); 2414 2415 if ((ioc_info->ioc_diagdeviceid == B_TRUE) && (reprobe == 0)) { 2416 if (ibdm_get_diagcode(gid_info, ioc_no) != IBDM_SUCCESS) { 2417 mutex_enter(&gid_info->gl_mutex); 2418 gid_info->gl_pending_cmds--; 2419 mutex_exit(&gid_info->gl_mutex); 2420 } 2421 } 2422 ioc_info->ioc_serv = (ibdm_srvents_info_t *)kmem_zalloc( 2423 (gioc->ioc_service_entries * sizeof (ibdm_srvents_info_t)), 2424 KM_SLEEP); 2425 2426 /* 2427 * In one single request, maximum number of requests that can be 2428 * obtained is 4. If number of service entries are more than four, 2429 * calculate number requests needed and send them parallelly. 2430 */ 2431 nserv_entries = ioc->ioc_service_entries; 2432 ii = 0; 2433 while (nserv_entries) { 2434 mutex_enter(&gid_info->gl_mutex); 2435 ibdm_bump_transactionID(gid_info); 2436 mutex_exit(&gid_info->gl_mutex); 2437 2438 if (first != B_TRUE) { 2439 if (ibmf_alloc_msg(ibmf_hdl, IBMF_ALLOC_SLEEP, 2440 &msg) != IBMF_SUCCESS) { 2441 continue; 2442 } 2443 2444 } 2445 ibdm_alloc_send_buffers(msg); 2446 msg->im_local_addr.ia_local_lid = gid_info->gl_slid; 2447 msg->im_local_addr.ia_remote_lid = gid_info->gl_dlid; 2448 if (gid_info->gl_redirected == B_TRUE) { 2449 if (gid_info->gl_redirect_dlid != 0) { 2450 msg->im_local_addr.ia_remote_lid = 2451 gid_info->gl_redirect_dlid; 2452 } 2453 msg->im_local_addr.ia_remote_qno = 2454 gid_info->gl_redirect_QP; 2455 msg->im_local_addr.ia_p_key = 2456 gid_info->gl_redirect_pkey; 2457 msg->im_local_addr.ia_q_key = 2458 gid_info->gl_redirect_qkey; 2459 } else { 2460 msg->im_local_addr.ia_remote_qno = 1; 2461 msg->im_local_addr.ia_p_key = gid_info->gl_p_key; 2462 msg->im_local_addr.ia_q_key = IB_GSI_QKEY; 2463 } 2464 2465 hdr = IBDM_OUT_IBMFMSG_MADHDR(msg); 2466 hdr->BaseVersion = MAD_CLASS_BASE_VERS_1; 2467 hdr->MgmtClass = MAD_MGMT_CLASS_DEV_MGT; 2468 hdr->ClassVersion = IB_DM_CLASS_VERSION_1; 2469 hdr->R_Method = IB_DM_DEVMGT_METHOD_GET; 2470 hdr->Status = 0; 2471 hdr->TransactionID = h2b64(gid_info->gl_transactionID); 2472 hdr->AttributeID = h2b16(IB_DM_ATTR_SERVICE_ENTRIES); 2473 2474 srv_start = ii * 4; 2475 cb_args = &ioc_info->ioc_serv[srv_start].se_cb_args; 2476 cb_args->cb_gid_info = gid_info; 2477 cb_args->cb_retry_count = ibdm_dft_retry_cnt; 2478 cb_args->cb_req_type = IBDM_REQ_TYPE_SRVENTS; 2479 cb_args->cb_srvents_start = srv_start; 2480 cb_args->cb_ioc_num = ioc_no - 1; 2481 2482 if (nserv_entries >= IBDM_MAX_SERV_ENTRIES_PER_REQ) { 2483 nserv_entries -= IBDM_MAX_SERV_ENTRIES_PER_REQ; 2484 cb_args->cb_srvents_end = (cb_args->cb_srvents_start + 2485 IBDM_MAX_SERV_ENTRIES_PER_REQ - 1); 2486 } else { 2487 cb_args->cb_srvents_end = 2488 (cb_args->cb_srvents_start + nserv_entries - 1); 2489 nserv_entries = 0; 2490 } 2491 ibdm_fill_srv_attr_mod(hdr, cb_args); 2492 2493 ioc_info->ioc_serv[srv_start].se_timeout_id = timeout( 2494 ibdm_pkt_timeout_hdlr, cb_args, 2495 IBDM_TIMEOUT_VALUE(ibdm_dft_timeout)); 2496 2497 IBTF_DPRINTF_L5("ibdm", "\thandle_ioc_profile:" 2498 "timeout %x, ioc %d srv %d", 2499 ioc_info->ioc_serv[srv_start].se_timeout_id, 2500 ioc_no - 1, srv_start); 2501 2502 if (ibmf_msg_transport(ibmf_hdl, gid_info->gl_qp_hdl, msg, 2503 NULL, ibdm_ibmf_send_cb, cb_args, 0) != IBMF_SUCCESS) { 2504 IBTF_DPRINTF_L2("ibdm", 2505 "\thandle_ioc_profile: msg send failed"); 2506 ibdm_ibmf_send_cb(ibmf_hdl, msg, cb_args); 2507 } 2508 (*flag) |= IBDM_IBMF_PKT_REUSED; 2509 first = B_FALSE; 2510 ii++; 2511 } 2512 } 2513 2514 2515 /* 2516 * ibdm_handle_srventry_mad() 2517 */ 2518 static void 2519 ibdm_handle_srventry_mad(ibmf_msg_t *msg, 2520 ibdm_dp_gidinfo_t *gid_info, int *flag) 2521 { 2522 uint_t ii, ioc_no, attrmod; 2523 uint_t nentries, start, end; 2524 timeout_id_t timeout_id; 2525 ib_dm_srv_t *srv_ents; 2526 ibdm_ioc_info_t *ioc_info; 2527 ibdm_srvents_info_t *gsrv_ents; 2528 2529 IBTF_DPRINTF_L4("ibdm", "\thandle_srventry_mad:" 2530 " IBMF msg %p gid info %p", msg, gid_info); 2531 2532 srv_ents = IBDM_IN_IBMFMSG2SRVENT(msg); 2533 /* 2534 * Get the start and end index of the service entries 2535 * Upper 16 bits identify the IOC 2536 * Lower 16 bits specify the range of service entries 2537 * LSB specifies (Big endian) end of the range 2538 * MSB specifies (Big endian) start of the range 2539 */ 2540 attrmod = IBDM_IN_IBMFMSG_ATTRMOD(msg); 2541 ioc_no = ((attrmod >> 16) & IBDM_16_BIT_MASK); 2542 end = ((attrmod >> 8) & IBDM_8_BIT_MASK); 2543 start = (attrmod & IBDM_8_BIT_MASK); 2544 2545 /* Make sure that IOC index is with the valid range */ 2546 if ((ioc_no < 1) | 2547 (ioc_no > gid_info->gl_iou->iou_info.iou_num_ctrl_slots)) { 2548 IBTF_DPRINTF_L2("ibdm", "\thandle_srventry_mad: " 2549 "IOC index Out of range, index %d", ioc_no); 2550 (*flag) |= IBDM_IBMF_PKT_UNEXP_RESP; 2551 return; 2552 } 2553 ioc_info = IBDM_GIDINFO2IOCINFO(gid_info, (ioc_no -1)); 2554 2555 /* 2556 * Make sure that the "start" and "end" service indexes are 2557 * with in the valid range 2558 */ 2559 nentries = ioc_info->ioc_profile.ioc_service_entries; 2560 if ((start > end) | (start >= nentries) | (end >= nentries)) { 2561 IBTF_DPRINTF_L2("ibdm", "\thandle_srventry_mad: " 2562 "Attr modifier 0x%x, #Serv entries %d", attrmod, nentries); 2563 (*flag) |= IBDM_IBMF_PKT_UNEXP_RESP; 2564 return; 2565 } 2566 gsrv_ents = &ioc_info->ioc_serv[start]; 2567 mutex_enter(&gid_info->gl_mutex); 2568 if (gsrv_ents->se_state != IBDM_SE_INVALID) { 2569 IBTF_DPRINTF_L2("ibdm", "\thandle_srventry_mad: " 2570 "already known, ioc %d, srv %d, se_state %x", 2571 ioc_no - 1, start, gsrv_ents->se_state); 2572 mutex_exit(&gid_info->gl_mutex); 2573 (*flag) |= IBDM_IBMF_PKT_DUP_RESP; 2574 return; 2575 } 2576 ioc_info->ioc_serv[start].se_cb_args.cb_req_type = 0; 2577 if (ioc_info->ioc_serv[start].se_timeout_id) { 2578 IBTF_DPRINTF_L2("ibdm", 2579 "\thandle_srventry_mad: ioc %d start %d", ioc_no, start); 2580 timeout_id = ioc_info->ioc_serv[start].se_timeout_id; 2581 mutex_exit(&gid_info->gl_mutex); 2582 IBTF_DPRINTF_L5("ibdm", "handle_srverntry_mad: " 2583 "se_timeout_id = 0x%x", timeout_id); 2584 if (untimeout(timeout_id) == -1) { 2585 IBTF_DPRINTF_L2("ibdm", "handle_srventry_mad: " 2586 "untimeout se_timeout_id failed"); 2587 } 2588 mutex_enter(&gid_info->gl_mutex); 2589 ioc_info->ioc_serv[start].se_timeout_id = 0; 2590 } 2591 2592 gsrv_ents->se_state = IBDM_SE_VALID; 2593 mutex_exit(&gid_info->gl_mutex); 2594 for (ii = start; ii <= end; ii++, srv_ents++, gsrv_ents++) { 2595 gsrv_ents->se_attr.srv_id = b2h64(srv_ents->srv_id); 2596 bcopy(srv_ents->srv_name, 2597 gsrv_ents->se_attr.srv_name, IB_DM_MAX_SVC_NAME_LEN); 2598 ibdm_dump_service_entries(&gsrv_ents->se_attr); 2599 } 2600 } 2601 2602 2603 /* 2604 * ibdm_get_diagcode: 2605 * Send request to get IOU/IOC diag code 2606 * Returns IBDM_SUCCESS/IBDM_FAILURE 2607 */ 2608 static int 2609 ibdm_get_diagcode(ibdm_dp_gidinfo_t *gid_info, int attr) 2610 { 2611 ibmf_msg_t *msg; 2612 ib_mad_hdr_t *hdr; 2613 ibdm_ioc_info_t *ioc; 2614 ibdm_timeout_cb_args_t *cb_args; 2615 timeout_id_t *timeout_id; 2616 2617 IBTF_DPRINTF_L4("ibdm", "\tget_diagcode: gid info %p, attr = %d", 2618 gid_info, attr); 2619 2620 if (ibmf_alloc_msg(gid_info->gl_ibmf_hdl, IBMF_ALLOC_SLEEP, 2621 &msg) != IBMF_SUCCESS) { 2622 IBTF_DPRINTF_L4("ibdm", "\tget_diagcode: pkt alloc fail"); 2623 return (IBDM_FAILURE); 2624 } 2625 2626 ibdm_alloc_send_buffers(msg); 2627 2628 mutex_enter(&gid_info->gl_mutex); 2629 ibdm_bump_transactionID(gid_info); 2630 mutex_exit(&gid_info->gl_mutex); 2631 2632 msg->im_local_addr.ia_local_lid = gid_info->gl_slid; 2633 msg->im_local_addr.ia_remote_lid = gid_info->gl_dlid; 2634 if (gid_info->gl_redirected == B_TRUE) { 2635 if (gid_info->gl_redirect_dlid != 0) { 2636 msg->im_local_addr.ia_remote_lid = 2637 gid_info->gl_redirect_dlid; 2638 } 2639 2640 msg->im_local_addr.ia_remote_qno = gid_info->gl_redirect_QP; 2641 msg->im_local_addr.ia_p_key = gid_info->gl_redirect_pkey; 2642 msg->im_local_addr.ia_q_key = gid_info->gl_redirect_qkey; 2643 } else { 2644 msg->im_local_addr.ia_remote_qno = 1; 2645 msg->im_local_addr.ia_p_key = gid_info->gl_p_key; 2646 msg->im_local_addr.ia_q_key = IB_GSI_QKEY; 2647 } 2648 2649 hdr = IBDM_OUT_IBMFMSG_MADHDR(msg); 2650 hdr->BaseVersion = MAD_CLASS_BASE_VERS_1; 2651 hdr->MgmtClass = MAD_MGMT_CLASS_DEV_MGT; 2652 hdr->ClassVersion = IB_DM_CLASS_VERSION_1; 2653 hdr->R_Method = IB_DM_DEVMGT_METHOD_GET; 2654 hdr->Status = 0; 2655 hdr->TransactionID = h2b64(gid_info->gl_transactionID); 2656 2657 hdr->AttributeID = h2b16(IB_DM_ATTR_DIAG_CODE); 2658 hdr->AttributeModifier = h2b32(attr); 2659 2660 if (attr == 0) { 2661 cb_args = &gid_info->gl_iou_cb_args; 2662 gid_info->gl_iou->iou_dc_valid = B_FALSE; 2663 cb_args->cb_ioc_num = 0; 2664 cb_args->cb_req_type = IBDM_REQ_TYPE_IOU_DIAGCODE; 2665 timeout_id = &gid_info->gl_timeout_id; 2666 } else { 2667 ioc = IBDM_GIDINFO2IOCINFO(gid_info, (attr - 1)); 2668 ioc->ioc_dc_valid = B_FALSE; 2669 cb_args = &ioc->ioc_dc_cb_args; 2670 cb_args->cb_ioc_num = attr - 1; 2671 cb_args->cb_req_type = IBDM_REQ_TYPE_IOC_DIAGCODE; 2672 timeout_id = &ioc->ioc_dc_timeout_id; 2673 } 2674 cb_args->cb_gid_info = gid_info; 2675 cb_args->cb_retry_count = ibdm_dft_retry_cnt; 2676 cb_args->cb_srvents_start = 0; 2677 2678 2679 *timeout_id = timeout(ibdm_pkt_timeout_hdlr, 2680 cb_args, IBDM_TIMEOUT_VALUE(ibdm_dft_timeout)); 2681 2682 IBTF_DPRINTF_L5("ibdm", "\tget_diagcode:" 2683 "timeout %x, ioc %d", *timeout_id, cb_args->cb_ioc_num); 2684 2685 if (ibmf_msg_transport(gid_info->gl_ibmf_hdl, gid_info->gl_qp_hdl, 2686 msg, NULL, ibdm_ibmf_send_cb, cb_args, 0) != IBMF_SUCCESS) { 2687 IBTF_DPRINTF_L2("ibdm", "\tget_diagcode: ibmf send failed"); 2688 ibdm_ibmf_send_cb(gid_info->gl_ibmf_hdl, msg, cb_args); 2689 } 2690 return (IBDM_SUCCESS); 2691 } 2692 2693 /* 2694 * ibdm_handle_diagcode: 2695 * Process the DiagCode MAD response and update local DM 2696 * data structure. 2697 */ 2698 static void 2699 ibdm_handle_diagcode(ibmf_msg_t *ibmf_msg, 2700 ibdm_dp_gidinfo_t *gid_info, int *flag) 2701 { 2702 uint16_t attrmod, *diagcode; 2703 ibdm_iou_info_t *iou; 2704 ibdm_ioc_info_t *ioc; 2705 timeout_id_t timeout_id; 2706 ibdm_timeout_cb_args_t *cb_args; 2707 2708 diagcode = (uint16_t *)ibmf_msg->im_msgbufs_recv.im_bufs_cl_data; 2709 2710 mutex_enter(&gid_info->gl_mutex); 2711 attrmod = IBDM_IN_IBMFMSG_ATTRMOD(ibmf_msg); 2712 iou = gid_info->gl_iou; 2713 if (attrmod == 0) { 2714 if (iou->iou_dc_valid != B_FALSE) { 2715 (*flag) |= IBDM_IBMF_PKT_DUP_RESP; 2716 IBTF_DPRINTF_L4("ibdm", 2717 "\thandle_diagcode: Duplicate IOU DiagCode"); 2718 mutex_exit(&gid_info->gl_mutex); 2719 return; 2720 } 2721 cb_args = &gid_info->gl_iou_cb_args; 2722 cb_args->cb_req_type = 0; 2723 iou->iou_diagcode = b2h16(*diagcode); 2724 iou->iou_dc_valid = B_TRUE; 2725 if (gid_info->gl_timeout_id) { 2726 timeout_id = gid_info->gl_timeout_id; 2727 mutex_exit(&gid_info->gl_mutex); 2728 IBTF_DPRINTF_L5("ibdm", "\thandle_diagcode: " 2729 "gl_timeout_id = 0x%x", timeout_id); 2730 if (untimeout(timeout_id) == -1) { 2731 IBTF_DPRINTF_L2("ibdm", "handle_diagcode: " 2732 "untimeout gl_timeout_id failed"); 2733 } 2734 mutex_enter(&gid_info->gl_mutex); 2735 gid_info->gl_timeout_id = 0; 2736 } 2737 } else { 2738 ioc = IBDM_GIDINFO2IOCINFO(gid_info, (attrmod - 1)); 2739 if (ioc->ioc_dc_valid != B_FALSE) { 2740 (*flag) |= IBDM_IBMF_PKT_DUP_RESP; 2741 IBTF_DPRINTF_L4("ibdm", 2742 "\thandle_diagcode: Duplicate IOC DiagCode"); 2743 mutex_exit(&gid_info->gl_mutex); 2744 return; 2745 } 2746 cb_args = &ioc->ioc_dc_cb_args; 2747 cb_args->cb_req_type = 0; 2748 ioc->ioc_diagcode = b2h16(*diagcode); 2749 ioc->ioc_dc_valid = B_TRUE; 2750 timeout_id = iou->iou_ioc_info[attrmod - 1].ioc_dc_timeout_id; 2751 if (timeout_id) { 2752 mutex_exit(&gid_info->gl_mutex); 2753 IBTF_DPRINTF_L5("ibdm", "handle_diagcode: " 2754 "timeout_id = 0x%x", timeout_id); 2755 if (untimeout(timeout_id) == -1) { 2756 IBTF_DPRINTF_L2("ibdm", "\thandle_diagcode: " 2757 "untimeout ioc_dc_timeout_id failed"); 2758 } 2759 mutex_enter(&gid_info->gl_mutex); 2760 iou->iou_ioc_info[attrmod - 1].ioc_dc_timeout_id = 0; 2761 } 2762 } 2763 mutex_exit(&gid_info->gl_mutex); 2764 2765 IBTF_DPRINTF_L4("ibdm", "\thandle_diagcode: DiagCode : 0x%x" 2766 "attrmod : 0x%x", b2h16(*diagcode), attrmod); 2767 } 2768 2769 2770 /* 2771 * ibdm_is_ioc_present() 2772 * Return ibdm_ioc_info_t if IOC guid is found in the global gid list 2773 */ 2774 static ibdm_ioc_info_t * 2775 ibdm_is_ioc_present(ib_guid_t ioc_guid, 2776 ibdm_dp_gidinfo_t *gid_info, int *flag) 2777 { 2778 int ii; 2779 ibdm_ioc_info_t *ioc; 2780 ibdm_dp_gidinfo_t *head; 2781 ib_dm_io_unitinfo_t *iou; 2782 2783 mutex_enter(&ibdm.ibdm_mutex); 2784 head = ibdm.ibdm_dp_gidlist_head; 2785 while (head) { 2786 mutex_enter(&head->gl_mutex); 2787 if (head->gl_iou == NULL) { 2788 mutex_exit(&head->gl_mutex); 2789 head = head->gl_next; 2790 continue; 2791 } 2792 iou = &head->gl_iou->iou_info; 2793 for (ii = 0; ii < iou->iou_num_ctrl_slots; ii++) { 2794 ioc = IBDM_GIDINFO2IOCINFO(head, ii); 2795 if ((ioc->ioc_state == IBDM_IOC_STATE_PROBE_SUCCESS) && 2796 (ioc->ioc_profile.ioc_guid == ioc_guid)) { 2797 if (gid_info == head) { 2798 *flag |= IBDM_IBMF_PKT_DUP_RESP; 2799 } else if (ibdm_check_dgid(head->gl_dgid_lo, 2800 head->gl_dgid_hi) != NULL) { 2801 IBTF_DPRINTF_L4("ibdm", "\tis_ioc_" 2802 "present: gid not present"); 2803 ibdm_add_to_gl_gid(gid_info, head); 2804 } 2805 mutex_exit(&head->gl_mutex); 2806 mutex_exit(&ibdm.ibdm_mutex); 2807 return (ioc); 2808 } 2809 } 2810 mutex_exit(&head->gl_mutex); 2811 head = head->gl_next; 2812 } 2813 mutex_exit(&ibdm.ibdm_mutex); 2814 return (NULL); 2815 } 2816 2817 2818 /* 2819 * ibdm_ibmf_send_cb() 2820 * IBMF invokes this callback routine after posting the DM MAD to 2821 * the HCA. 2822 */ 2823 /*ARGSUSED*/ 2824 static void 2825 ibdm_ibmf_send_cb(ibmf_handle_t ibmf_hdl, ibmf_msg_t *ibmf_msg, void *arg) 2826 { 2827 ibdm_dump_ibmf_msg(ibmf_msg, 1); 2828 ibdm_free_send_buffers(ibmf_msg); 2829 if (ibmf_free_msg(ibmf_hdl, &ibmf_msg) != IBMF_SUCCESS) { 2830 IBTF_DPRINTF_L4("ibdm", 2831 "\tibmf_send_cb: IBMF free msg failed"); 2832 } 2833 } 2834 2835 2836 /* 2837 * ibdm_ibmf_recv_cb() 2838 * Invoked by the IBMF when a response to the one of the DM requests 2839 * is received. 2840 */ 2841 /*ARGSUSED*/ 2842 static void 2843 ibdm_ibmf_recv_cb(ibmf_handle_t ibmf_hdl, ibmf_msg_t *msg, void *arg) 2844 { 2845 ibdm_taskq_args_t *taskq_args; 2846 2847 /* 2848 * If the taskq enable is set then dispatch a taskq to process 2849 * the MAD, otherwise just process it on this thread 2850 */ 2851 if (ibdm_taskq_enable != IBDM_ENABLE_TASKQ_HANDLING) { 2852 ibdm_process_incoming_mad(ibmf_hdl, msg, arg); 2853 return; 2854 } 2855 2856 /* 2857 * create a taskq and dispatch it to process the incoming MAD 2858 */ 2859 taskq_args = kmem_alloc(sizeof (ibdm_taskq_args_t), KM_NOSLEEP); 2860 if (taskq_args == NULL) { 2861 IBTF_DPRINTF_L2("ibdm", "ibmf_recv_cb: kmem_alloc failed for" 2862 "taskq_args"); 2863 if (ibmf_free_msg(ibmf_hdl, &msg) != IBMF_SUCCESS) { 2864 IBTF_DPRINTF_L4("ibmf_recv_cb", 2865 "\tibmf_recv_cb: IBMF free msg failed"); 2866 } 2867 return; 2868 } 2869 taskq_args->tq_ibmf_handle = ibmf_hdl; 2870 taskq_args->tq_ibmf_msg = msg; 2871 taskq_args->tq_args = arg; 2872 2873 if (taskq_dispatch(system_taskq, ibdm_recv_incoming_mad, taskq_args, 2874 TQ_NOSLEEP) == 0) { 2875 IBTF_DPRINTF_L2("ibdm", "ibmf_recv_cb: taskq_dispatch failed"); 2876 if (ibmf_free_msg(ibmf_hdl, &msg) != IBMF_SUCCESS) { 2877 IBTF_DPRINTF_L4("ibmf_recv_cb", 2878 "\tibmf_recv_cb: IBMF free msg failed"); 2879 } 2880 kmem_free(taskq_args, sizeof (ibdm_taskq_args_t)); 2881 return; 2882 } 2883 2884 /* taskq_args are deleted in ibdm_recv_incoming_mad() */ 2885 } 2886 2887 2888 void 2889 ibdm_recv_incoming_mad(void *args) 2890 { 2891 ibdm_taskq_args_t *taskq_args; 2892 2893 taskq_args = (ibdm_taskq_args_t *)args; 2894 2895 IBTF_DPRINTF_L4("ibdm", "\tibdm_recv_incoming_mad: " 2896 "Processing incoming MAD via taskq"); 2897 2898 ibdm_process_incoming_mad(taskq_args->tq_ibmf_handle, 2899 taskq_args->tq_ibmf_msg, taskq_args->tq_args); 2900 2901 kmem_free(taskq_args, sizeof (ibdm_taskq_args_t)); 2902 } 2903 2904 2905 /* 2906 * Calls ibdm_process_incoming_mad with all function arguments extracted 2907 * from args 2908 */ 2909 /*ARGSUSED*/ 2910 static void 2911 ibdm_process_incoming_mad(ibmf_handle_t ibmf_hdl, ibmf_msg_t *msg, void *arg) 2912 { 2913 int flag = 0; 2914 int ret; 2915 uint64_t transaction_id; 2916 ib_mad_hdr_t *hdr; 2917 ibdm_dp_gidinfo_t *gid_info = NULL; 2918 2919 IBTF_DPRINTF_L4("ibdm", 2920 "\tprocess_incoming_mad: ibmf hdl %p pkt %p", ibmf_hdl, msg); 2921 ibdm_dump_ibmf_msg(msg, 0); 2922 2923 /* 2924 * IBMF calls this routine for every DM MAD that arrives at this port. 2925 * But we handle only the responses for requests we sent. We drop all 2926 * the DM packets that does not have response bit set in the MAD 2927 * header(this eliminates all the requests sent to this port). 2928 * We handle only DM class version 1 MAD's 2929 */ 2930 hdr = IBDM_IN_IBMFMSG_MADHDR(msg); 2931 if (ibdm_verify_mad_status(hdr) != IBDM_SUCCESS) { 2932 if (ibmf_free_msg(ibmf_hdl, &msg) != IBMF_SUCCESS) { 2933 IBTF_DPRINTF_L2("ibdm", "\tprocess_incoming_mad: " 2934 "IBMF free msg failed DM request drop it"); 2935 } 2936 return; 2937 } 2938 2939 transaction_id = b2h64(hdr->TransactionID); 2940 2941 mutex_enter(&ibdm.ibdm_mutex); 2942 gid_info = ibdm.ibdm_dp_gidlist_head; 2943 while (gid_info) { 2944 if ((gid_info->gl_transactionID & 2945 IBDM_GID_TRANSACTIONID_MASK) == 2946 (transaction_id & IBDM_GID_TRANSACTIONID_MASK)) 2947 break; 2948 gid_info = gid_info->gl_next; 2949 } 2950 mutex_exit(&ibdm.ibdm_mutex); 2951 2952 if (gid_info == NULL) { 2953 /* Drop the packet */ 2954 IBTF_DPRINTF_L2("ibdm", "process_incoming_mad: transaction ID" 2955 " does not match: 0x%llx", transaction_id); 2956 if (ibmf_free_msg(ibmf_hdl, &msg) != IBMF_SUCCESS) { 2957 IBTF_DPRINTF_L2("ibdm", "process_incoming_mad: " 2958 "IBMF free msg failed DM request drop it"); 2959 } 2960 return; 2961 } 2962 2963 /* Handle redirection for all the MAD's, except ClassPortInfo */ 2964 if (((IBDM_IN_IBMFMSG_STATUS(msg) & MAD_STATUS_REDIRECT_REQUIRED)) && 2965 (IBDM_IN_IBMFMSG_ATTR(msg) != IB_DM_ATTR_CLASSPORTINFO)) { 2966 ret = ibdm_handle_redirection(msg, gid_info, &flag); 2967 if (ret == IBDM_SUCCESS) { 2968 return; 2969 } 2970 } else { 2971 uint_t gl_state; 2972 2973 mutex_enter(&gid_info->gl_mutex); 2974 gl_state = gid_info->gl_state; 2975 mutex_exit(&gid_info->gl_mutex); 2976 2977 switch (gl_state) { 2978 case IBDM_GET_CLASSPORTINFO: 2979 ibdm_handle_classportinfo( 2980 ibmf_hdl, msg, gid_info, &flag); 2981 break; 2982 2983 case IBDM_GET_IOUNITINFO: 2984 ibdm_handle_iounitinfo(ibmf_hdl, msg, gid_info, &flag); 2985 break; 2986 2987 case IBDM_GET_IOC_DETAILS: 2988 switch (IBDM_IN_IBMFMSG_ATTR(msg)) { 2989 2990 case IB_DM_ATTR_SERVICE_ENTRIES: 2991 ibdm_handle_srventry_mad(msg, gid_info, &flag); 2992 break; 2993 2994 case IB_DM_ATTR_IOC_CTRL_PROFILE: 2995 ibdm_handle_ioc_profile( 2996 ibmf_hdl, msg, gid_info, &flag); 2997 break; 2998 2999 case IB_DM_ATTR_DIAG_CODE: 3000 ibdm_handle_diagcode(msg, gid_info, &flag); 3001 break; 3002 3003 default: 3004 IBTF_DPRINTF_L2("ibdm", "process_incoming_mad: " 3005 "Error state, wrong attribute :-("); 3006 (void) ibmf_free_msg(ibmf_hdl, &msg); 3007 return; 3008 } 3009 break; 3010 default: 3011 IBTF_DPRINTF_L2("ibdm", 3012 "process_incoming_mad: Dropping the packet" 3013 " gl_state %x", gl_state); 3014 if (ibmf_free_msg(ibmf_hdl, &msg) != IBMF_SUCCESS) { 3015 IBTF_DPRINTF_L2("ibdm", "process_incoming_mad: " 3016 "IBMF free msg failed DM request drop it"); 3017 } 3018 return; 3019 } 3020 } 3021 3022 if ((flag & IBDM_IBMF_PKT_DUP_RESP) || 3023 (flag & IBDM_IBMF_PKT_UNEXP_RESP)) { 3024 IBTF_DPRINTF_L2("ibdm", 3025 "\tprocess_incoming_mad:Dup/unexp resp : 0x%x", flag); 3026 if (ibmf_free_msg(ibmf_hdl, &msg) != IBMF_SUCCESS) { 3027 IBTF_DPRINTF_L2("ibdm", "process_incoming_mad: " 3028 "IBMF free msg failed DM request drop it"); 3029 } 3030 return; 3031 } 3032 3033 mutex_enter(&gid_info->gl_mutex); 3034 if (gid_info->gl_pending_cmds < 1) { 3035 IBTF_DPRINTF_L2("ibdm", 3036 "\tprocess_incoming_mad: pending commands negative"); 3037 } 3038 if (--gid_info->gl_pending_cmds) { 3039 IBTF_DPRINTF_L4("ibdm", "\tprocess_incoming_mad: " 3040 "gid_info %p pending cmds %d", 3041 gid_info, gid_info->gl_pending_cmds); 3042 mutex_exit(&gid_info->gl_mutex); 3043 } else { 3044 IBTF_DPRINTF_L4("ibdm", "\tprocess_incoming_mad: Probing DONE"); 3045 gid_info->gl_state = IBDM_GID_PROBING_COMPLETE; 3046 mutex_exit(&gid_info->gl_mutex); 3047 ibdm_notify_newgid_iocs(gid_info); 3048 mutex_enter(&ibdm.ibdm_mutex); 3049 if (--ibdm.ibdm_ngid_probes_in_progress == 0) { 3050 IBTF_DPRINTF_L4("ibdm", 3051 "\tprocess_incoming_mad: Wakeup"); 3052 ibdm.ibdm_busy &= ~IBDM_PROBE_IN_PROGRESS; 3053 cv_broadcast(&ibdm.ibdm_probe_cv); 3054 } 3055 mutex_exit(&ibdm.ibdm_mutex); 3056 } 3057 3058 /* 3059 * Do not deallocate the IBMF packet if atleast one request 3060 * is posted. IBMF packet is reused. 3061 */ 3062 if (!(flag & IBDM_IBMF_PKT_REUSED)) { 3063 if (ibmf_free_msg(ibmf_hdl, &msg) != IBMF_SUCCESS) { 3064 IBTF_DPRINTF_L2("ibdm", "\tprocess_incoming_mad: " 3065 "IBMF free msg failed DM request drop it"); 3066 } 3067 } 3068 } 3069 3070 3071 /* 3072 * ibdm_verify_mad_status() 3073 * Verifies the MAD status 3074 * Returns IBDM_SUCCESS if status is correct 3075 * Returns IBDM_FAILURE for bogus MAD status 3076 */ 3077 static int 3078 ibdm_verify_mad_status(ib_mad_hdr_t *hdr) 3079 { 3080 int ret = 0; 3081 3082 if ((hdr->R_Method != IB_DM_DEVMGT_METHOD_GET_RESP) || 3083 (hdr->ClassVersion != IB_DM_CLASS_VERSION_1)) { 3084 return (IBDM_FAILURE); 3085 } 3086 3087 if (b2h16(hdr->Status) == 0) 3088 ret = IBDM_SUCCESS; 3089 else if ((b2h16(hdr->Status) & 0x1f) == MAD_STATUS_REDIRECT_REQUIRED) 3090 ret = IBDM_SUCCESS; 3091 else { 3092 IBTF_DPRINTF_L4("ibdm", 3093 "\tverify_mad_status: Stauts : 0x%x", b2h16(hdr->Status)); 3094 ret = IBDM_FAILURE; 3095 } 3096 return (ret); 3097 } 3098 3099 3100 3101 /* 3102 * ibdm_handle_redirection() 3103 * Returns IBDM_SUCCESS/IBDM_FAILURE 3104 */ 3105 static int 3106 ibdm_handle_redirection(ibmf_msg_t *msg, 3107 ibdm_dp_gidinfo_t *gid_info, int *flag) 3108 { 3109 int attrmod, ioc_no, start; 3110 void *data; 3111 timeout_id_t *timeout_id; 3112 ib_mad_hdr_t *hdr; 3113 ibdm_ioc_info_t *ioc = NULL; 3114 ibdm_timeout_cb_args_t *cb_args; 3115 ibdm_mad_classportinfo_t *cpi; 3116 3117 IBTF_DPRINTF_L4("ibdm", "\thandle_redirection: Enter"); 3118 mutex_enter(&gid_info->gl_mutex); 3119 switch (gid_info->gl_state) { 3120 case IBDM_GET_IOUNITINFO: 3121 cb_args = &gid_info->gl_iou_cb_args; 3122 timeout_id = &gid_info->gl_timeout_id; 3123 break; 3124 3125 case IBDM_GET_IOC_DETAILS: 3126 attrmod = IBDM_IN_IBMFMSG_ATTRMOD(msg); 3127 switch (IBDM_IN_IBMFMSG_ATTR(msg)) { 3128 3129 case IB_DM_ATTR_DIAG_CODE: 3130 if (attrmod == 0) { 3131 cb_args = &gid_info->gl_iou_cb_args; 3132 timeout_id = &gid_info->gl_timeout_id; 3133 break; 3134 } 3135 if (IBDM_IS_IOC_NUM_INVALID(attrmod, gid_info)) { 3136 IBTF_DPRINTF_L2("ibdm", "\thandle_redirction:" 3137 "IOC# Out of range %d", attrmod); 3138 (*flag) |= IBDM_IBMF_PKT_UNEXP_RESP; 3139 mutex_exit(&gid_info->gl_mutex); 3140 return (IBDM_FAILURE); 3141 } 3142 ioc = IBDM_GIDINFO2IOCINFO(gid_info, (attrmod -1)); 3143 cb_args = &ioc->ioc_dc_cb_args; 3144 timeout_id = &ioc->ioc_dc_timeout_id; 3145 break; 3146 3147 case IB_DM_ATTR_IOC_CTRL_PROFILE: 3148 if (IBDM_IS_IOC_NUM_INVALID(attrmod, gid_info)) { 3149 IBTF_DPRINTF_L2("ibdm", "\thandle_redirction:" 3150 "IOC# Out of range %d", attrmod); 3151 (*flag) |= IBDM_IBMF_PKT_UNEXP_RESP; 3152 mutex_exit(&gid_info->gl_mutex); 3153 return (IBDM_FAILURE); 3154 } 3155 ioc = IBDM_GIDINFO2IOCINFO(gid_info, (attrmod -1)); 3156 cb_args = &ioc->ioc_cb_args; 3157 timeout_id = &ioc->ioc_timeout_id; 3158 break; 3159 3160 case IB_DM_ATTR_SERVICE_ENTRIES: 3161 ioc_no = ((attrmod >> 16) & IBDM_16_BIT_MASK); 3162 if (IBDM_IS_IOC_NUM_INVALID(ioc_no, gid_info)) { 3163 IBTF_DPRINTF_L2("ibdm", "\thandle_redirction:" 3164 "IOC# Out of range %d", ioc_no); 3165 (*flag) |= IBDM_IBMF_PKT_UNEXP_RESP; 3166 mutex_exit(&gid_info->gl_mutex); 3167 return (IBDM_FAILURE); 3168 } 3169 start = (attrmod & IBDM_8_BIT_MASK); 3170 ioc = IBDM_GIDINFO2IOCINFO(gid_info, (ioc_no -1)); 3171 if (start > ioc->ioc_profile.ioc_service_entries) { 3172 IBTF_DPRINTF_L2("ibdm", "\thandle_redirction:" 3173 " SE index Out of range %d", start); 3174 (*flag) |= IBDM_IBMF_PKT_UNEXP_RESP; 3175 mutex_exit(&gid_info->gl_mutex); 3176 return (IBDM_FAILURE); 3177 } 3178 cb_args = &ioc->ioc_serv[start].se_cb_args; 3179 timeout_id = &ioc->ioc_serv[start].se_timeout_id; 3180 break; 3181 3182 default: 3183 /* ERROR State */ 3184 IBTF_DPRINTF_L2("ibdm", 3185 "\thandle_redirection: wrong attribute :-("); 3186 (*flag) |= IBDM_IBMF_PKT_UNEXP_RESP; 3187 mutex_exit(&gid_info->gl_mutex); 3188 return (IBDM_FAILURE); 3189 } 3190 break; 3191 default: 3192 /* ERROR State */ 3193 IBTF_DPRINTF_L2("ibdm", 3194 "\thandle_redirection: Error state :-("); 3195 (*flag) |= IBDM_IBMF_PKT_UNEXP_RESP; 3196 mutex_exit(&gid_info->gl_mutex); 3197 return (IBDM_FAILURE); 3198 } 3199 if ((*timeout_id) != 0) { 3200 mutex_exit(&gid_info->gl_mutex); 3201 if (untimeout(*timeout_id) == -1) { 3202 IBTF_DPRINTF_L2("ibdm", "\thandle_redirection: " 3203 "untimeout failed %x", *timeout_id); 3204 } else { 3205 IBTF_DPRINTF_L5("ibdm", 3206 "\thandle_redirection: timeout %x", *timeout_id); 3207 } 3208 mutex_enter(&gid_info->gl_mutex); 3209 *timeout_id = 0; 3210 } 3211 3212 data = msg->im_msgbufs_recv.im_bufs_cl_data; 3213 cpi = (ibdm_mad_classportinfo_t *)data; 3214 3215 gid_info->gl_resp_timeout = 3216 (b2h32(cpi->RespTimeValue) & 0x1F); 3217 3218 gid_info->gl_redirected = B_TRUE; 3219 gid_info->gl_redirect_dlid = b2h16(cpi->RedirectLID); 3220 gid_info->gl_redirect_QP = (b2h32(cpi->RedirectQP) & 0xffffff); 3221 gid_info->gl_redirect_pkey = b2h16(cpi->RedirectP_Key); 3222 gid_info->gl_redirect_qkey = b2h32(cpi->RedirectQ_Key); 3223 gid_info->gl_redirectGID_hi = b2h64(cpi->RedirectGID_hi); 3224 gid_info->gl_redirectGID_lo = b2h64(cpi->RedirectGID_lo); 3225 3226 if (gid_info->gl_redirect_dlid != 0) { 3227 msg->im_local_addr.ia_remote_lid = 3228 gid_info->gl_redirect_dlid; 3229 } 3230 ibdm_bump_transactionID(gid_info); 3231 mutex_exit(&gid_info->gl_mutex); 3232 3233 ibdm_alloc_send_buffers(msg); 3234 3235 hdr = IBDM_OUT_IBMFMSG_MADHDR(msg); 3236 hdr->BaseVersion = MAD_CLASS_BASE_VERS_1; 3237 hdr->MgmtClass = MAD_MGMT_CLASS_DEV_MGT; 3238 hdr->ClassVersion = IB_DM_CLASS_VERSION_1; 3239 hdr->R_Method = IB_DM_DEVMGT_METHOD_GET; 3240 hdr->Status = 0; 3241 hdr->TransactionID = h2b64(gid_info->gl_transactionID); 3242 hdr->AttributeID = 3243 msg->im_msgbufs_recv.im_bufs_mad_hdr->AttributeID; 3244 hdr->AttributeModifier = 3245 msg->im_msgbufs_recv.im_bufs_mad_hdr->AttributeModifier; 3246 3247 msg->im_local_addr.ia_remote_qno = gid_info->gl_redirect_QP; 3248 msg->im_local_addr.ia_p_key = gid_info->gl_redirect_pkey; 3249 msg->im_local_addr.ia_q_key = gid_info->gl_redirect_qkey; 3250 3251 *timeout_id = timeout(ibdm_pkt_timeout_hdlr, 3252 cb_args, IBDM_TIMEOUT_VALUE(ibdm_dft_timeout)); 3253 3254 IBTF_DPRINTF_L5("ibdm", "\thandle_redirect:" 3255 "timeout %x", *timeout_id); 3256 3257 if (ibmf_msg_transport(gid_info->gl_ibmf_hdl, gid_info->gl_qp_hdl, 3258 msg, NULL, ibdm_ibmf_send_cb, cb_args, 0) != IBMF_SUCCESS) { 3259 IBTF_DPRINTF_L4("ibdm", "\thandle_redirection:" 3260 "message transport failed"); 3261 ibdm_ibmf_send_cb(gid_info->gl_ibmf_hdl, msg, cb_args); 3262 } 3263 (*flag) |= IBDM_IBMF_PKT_REUSED; 3264 IBTF_DPRINTF_L4("ibdm", "\thandle_redirection: Exit"); 3265 return (IBDM_SUCCESS); 3266 } 3267 3268 3269 /* 3270 * ibdm_pkt_timeout_hdlr 3271 * This timeout handler is registed for every IBMF packet that is 3272 * sent through the IBMF. It gets called when no response is received 3273 * within the specified time for the packet. No retries for the failed 3274 * commands currently. Drops the failed IBMF packet and update the 3275 * pending list commands. 3276 */ 3277 static void 3278 ibdm_pkt_timeout_hdlr(void *arg) 3279 { 3280 int probe_done = B_FALSE; 3281 ibdm_iou_info_t *iou; 3282 ibdm_ioc_info_t *ioc; 3283 ibdm_timeout_cb_args_t *cb_args = arg; 3284 ibdm_dp_gidinfo_t *gid_info; 3285 int srv_ent; 3286 3287 IBTF_DPRINTF_L2("ibdm", "\tpkt_timeout_hdlr: gid_info: %p " 3288 "rtype 0x%x iocidx 0x%x srvidx %d", cb_args->cb_gid_info, 3289 cb_args->cb_req_type, cb_args->cb_ioc_num, 3290 cb_args->cb_srvents_start); 3291 3292 gid_info = cb_args->cb_gid_info; 3293 mutex_enter(&gid_info->gl_mutex); 3294 3295 if ((gid_info->gl_state == IBDM_GID_PROBING_COMPLETE) || 3296 (cb_args->cb_req_type == 0)) { 3297 3298 IBTF_DPRINTF_L2("ibdm", "\tpkt_timeout_hdlr: req completed" 3299 "rtype 0x%x iocidx 0x%x srvidx %d", cb_args->cb_req_type, 3300 cb_args->cb_ioc_num, cb_args->cb_srvents_start); 3301 3302 if (gid_info->gl_timeout_id) 3303 gid_info->gl_timeout_id = 0; 3304 mutex_exit(&gid_info->gl_mutex); 3305 return; 3306 } 3307 if (cb_args->cb_retry_count) { 3308 cb_args->cb_retry_count--; 3309 if (ibdm_retry_command(cb_args) == IBDM_SUCCESS) { 3310 if (gid_info->gl_timeout_id) 3311 gid_info->gl_timeout_id = 0; 3312 mutex_exit(&gid_info->gl_mutex); 3313 return; 3314 } 3315 cb_args->cb_retry_count = 0; 3316 } 3317 3318 IBTF_DPRINTF_L2("ibdm", "\tpkt_timeout_hdlr: command failed: gid %p" 3319 " rtype 0x%x iocidx 0x%x srvidx %d", cb_args->cb_gid_info, 3320 cb_args->cb_req_type, cb_args->cb_ioc_num, 3321 cb_args->cb_srvents_start); 3322 3323 switch (cb_args->cb_req_type) { 3324 3325 case IBDM_REQ_TYPE_CLASSPORTINFO: 3326 case IBDM_REQ_TYPE_IOUINFO: 3327 gid_info->gl_state = IBDM_GID_PROBING_FAILED; 3328 if (--gid_info->gl_pending_cmds == 0) 3329 probe_done = B_TRUE; 3330 if (gid_info->gl_timeout_id) 3331 gid_info->gl_timeout_id = 0; 3332 mutex_exit(&gid_info->gl_mutex); 3333 ibdm_delete_glhca_list(gid_info); 3334 mutex_enter(&gid_info->gl_mutex); 3335 break; 3336 case IBDM_REQ_TYPE_IOCINFO: 3337 iou = gid_info->gl_iou; 3338 ioc = &iou->iou_ioc_info[cb_args->cb_ioc_num]; 3339 ioc->ioc_state = IBDM_IOC_STATE_PROBE_FAILED; 3340 if (--gid_info->gl_pending_cmds == 0) 3341 probe_done = B_TRUE; 3342 #ifndef __lock_lint 3343 if (ioc->ioc_timeout_id) 3344 ioc->ioc_timeout_id = 0; 3345 #endif 3346 break; 3347 case IBDM_REQ_TYPE_SRVENTS: 3348 iou = gid_info->gl_iou; 3349 ioc = &iou->iou_ioc_info[cb_args->cb_ioc_num]; 3350 ioc->ioc_state = IBDM_IOC_STATE_PROBE_FAILED; 3351 if (--gid_info->gl_pending_cmds == 0) 3352 probe_done = B_TRUE; 3353 srv_ent = cb_args->cb_srvents_start; 3354 #ifndef __lock_lint 3355 if (ioc->ioc_serv[srv_ent].se_timeout_id) 3356 ioc->ioc_serv[srv_ent].se_timeout_id = 0; 3357 #endif 3358 break; 3359 case IBDM_REQ_TYPE_IOU_DIAGCODE: 3360 iou = gid_info->gl_iou; 3361 iou->iou_dc_valid = B_FALSE; 3362 if (--gid_info->gl_pending_cmds == 0) 3363 probe_done = B_TRUE; 3364 if (gid_info->gl_timeout_id) 3365 gid_info->gl_timeout_id = 0; 3366 break; 3367 case IBDM_REQ_TYPE_IOC_DIAGCODE: 3368 iou = gid_info->gl_iou; 3369 ioc = &iou->iou_ioc_info[cb_args->cb_ioc_num]; 3370 ioc->ioc_dc_valid = B_FALSE; 3371 if (--gid_info->gl_pending_cmds == 0) 3372 probe_done = B_TRUE; 3373 #ifndef __lock_lint 3374 if (ioc->ioc_dc_timeout_id) 3375 ioc->ioc_dc_timeout_id = 0; 3376 #endif 3377 break; 3378 } 3379 if (probe_done == B_TRUE) { 3380 gid_info->gl_state = IBDM_GID_PROBING_COMPLETE; 3381 mutex_exit(&gid_info->gl_mutex); 3382 ibdm_notify_newgid_iocs(gid_info); 3383 mutex_enter(&ibdm.ibdm_mutex); 3384 if (--ibdm.ibdm_ngid_probes_in_progress == 0) { 3385 IBTF_DPRINTF_L4("ibdm", "\tpkt_timeout_hdlr: Wakeup"); 3386 ibdm.ibdm_busy &= ~IBDM_PROBE_IN_PROGRESS; 3387 cv_broadcast(&ibdm.ibdm_probe_cv); 3388 } 3389 mutex_exit(&ibdm.ibdm_mutex); 3390 } else 3391 mutex_exit(&gid_info->gl_mutex); 3392 } 3393 3394 3395 /* 3396 * ibdm_retry_command() 3397 * Retries the failed command. 3398 * Returns IBDM_FAILURE/IBDM_SUCCESS 3399 */ 3400 static int 3401 ibdm_retry_command(ibdm_timeout_cb_args_t *cb_args) 3402 { 3403 int ret, rval = IBDM_SUCCESS; 3404 ibmf_msg_t *msg; 3405 ib_mad_hdr_t *hdr; 3406 ibdm_dp_gidinfo_t *gid_info = cb_args->cb_gid_info; 3407 timeout_id_t *timeout_id; 3408 ibdm_ioc_info_t *ioc; 3409 int ioc_no; 3410 3411 IBTF_DPRINTF_L2("ibdm", "\tretry_command: gid_info: %p " 3412 "rtype 0x%x iocidx 0x%x srvidx %d", cb_args->cb_gid_info, 3413 cb_args->cb_req_type, cb_args->cb_ioc_num, 3414 cb_args->cb_srvents_start); 3415 3416 ret = ibmf_alloc_msg(gid_info->gl_ibmf_hdl, IBMF_ALLOC_NOSLEEP, &msg); 3417 3418 3419 /* 3420 * Reset the gid if alloc_msg failed with BAD_HANDLE 3421 * ibdm_reset_gidinfo reinits the gid_info 3422 */ 3423 if (ret == IBMF_BAD_HANDLE) { 3424 IBTF_DPRINTF_L3(ibdm_string, "\tretry_command: gid %p hdl bad", 3425 gid_info); 3426 3427 mutex_exit(&gid_info->gl_mutex); 3428 ibdm_reset_gidinfo(gid_info); 3429 mutex_enter(&gid_info->gl_mutex); 3430 3431 /* Retry alloc */ 3432 ret = ibmf_alloc_msg(gid_info->gl_ibmf_hdl, IBMF_ALLOC_NOSLEEP, 3433 &msg); 3434 } 3435 3436 if (ret != IBDM_SUCCESS) { 3437 IBTF_DPRINTF_L2("ibdm", "\tretry_command: alloc failed: %p " 3438 "rtype 0x%x iocidx 0x%x srvidx %d", cb_args->cb_gid_info, 3439 cb_args->cb_req_type, cb_args->cb_ioc_num, 3440 cb_args->cb_srvents_start); 3441 return (IBDM_FAILURE); 3442 } 3443 3444 ibdm_alloc_send_buffers(msg); 3445 3446 ibdm_bump_transactionID(gid_info); 3447 3448 msg->im_local_addr.ia_local_lid = gid_info->gl_slid; 3449 msg->im_local_addr.ia_remote_lid = gid_info->gl_dlid; 3450 if (gid_info->gl_redirected == B_TRUE) { 3451 if (gid_info->gl_redirect_dlid != 0) { 3452 msg->im_local_addr.ia_remote_lid = 3453 gid_info->gl_redirect_dlid; 3454 } 3455 msg->im_local_addr.ia_remote_qno = gid_info->gl_redirect_QP; 3456 msg->im_local_addr.ia_p_key = gid_info->gl_redirect_pkey; 3457 msg->im_local_addr.ia_q_key = gid_info->gl_redirect_qkey; 3458 } else { 3459 msg->im_local_addr.ia_remote_qno = 1; 3460 msg->im_local_addr.ia_p_key = gid_info->gl_p_key; 3461 msg->im_local_addr.ia_q_key = IB_GSI_QKEY; 3462 } 3463 hdr = IBDM_OUT_IBMFMSG_MADHDR(msg); 3464 hdr->BaseVersion = MAD_CLASS_BASE_VERS_1; 3465 hdr->MgmtClass = MAD_MGMT_CLASS_DEV_MGT; 3466 hdr->ClassVersion = IB_DM_CLASS_VERSION_1; 3467 hdr->R_Method = IB_DM_DEVMGT_METHOD_GET; 3468 hdr->Status = 0; 3469 hdr->TransactionID = h2b64(gid_info->gl_transactionID); 3470 3471 switch (cb_args->cb_req_type) { 3472 case IBDM_REQ_TYPE_CLASSPORTINFO: 3473 hdr->AttributeID = h2b16(IB_DM_ATTR_CLASSPORTINFO); 3474 hdr->AttributeModifier = 0; 3475 timeout_id = &gid_info->gl_timeout_id; 3476 break; 3477 case IBDM_REQ_TYPE_IOUINFO: 3478 hdr->AttributeID = h2b16(IB_DM_ATTR_IO_UNITINFO); 3479 hdr->AttributeModifier = 0; 3480 timeout_id = &gid_info->gl_timeout_id; 3481 break; 3482 case IBDM_REQ_TYPE_IOCINFO: 3483 hdr->AttributeID = h2b16(IB_DM_ATTR_IOC_CTRL_PROFILE); 3484 hdr->AttributeModifier = h2b32(cb_args->cb_ioc_num + 1); 3485 ioc = IBDM_GIDINFO2IOCINFO(gid_info, cb_args->cb_ioc_num); 3486 timeout_id = &ioc->ioc_timeout_id; 3487 break; 3488 case IBDM_REQ_TYPE_SRVENTS: 3489 hdr->AttributeID = h2b16(IB_DM_ATTR_SERVICE_ENTRIES); 3490 ibdm_fill_srv_attr_mod(hdr, cb_args); 3491 ioc = IBDM_GIDINFO2IOCINFO(gid_info, cb_args->cb_ioc_num); 3492 timeout_id = 3493 &ioc->ioc_serv[cb_args->cb_srvents_start].se_timeout_id; 3494 break; 3495 case IBDM_REQ_TYPE_IOU_DIAGCODE: 3496 hdr->AttributeID = h2b16(IB_DM_ATTR_DIAG_CODE); 3497 hdr->AttributeModifier = 0; 3498 timeout_id = &gid_info->gl_timeout_id; 3499 break; 3500 case IBDM_REQ_TYPE_IOC_DIAGCODE: 3501 hdr->AttributeID = h2b16(IB_DM_ATTR_DIAG_CODE); 3502 hdr->AttributeModifier = h2b32(cb_args->cb_ioc_num + 1); 3503 ioc_no = cb_args->cb_ioc_num; 3504 ioc = &gid_info->gl_iou->iou_ioc_info[ioc_no]; 3505 timeout_id = &ioc->ioc_dc_timeout_id; 3506 break; 3507 } 3508 3509 mutex_exit(&gid_info->gl_mutex); 3510 3511 *timeout_id = timeout(ibdm_pkt_timeout_hdlr, 3512 cb_args, IBDM_TIMEOUT_VALUE(ibdm_dft_timeout)); 3513 3514 IBTF_DPRINTF_L5("ibdm", "\tretry_command: %p,%x,%d,%d:" 3515 "timeout %x", cb_args->cb_req_type, cb_args->cb_ioc_num, 3516 cb_args->cb_srvents_start, *timeout_id); 3517 3518 if ((rval = ibmf_msg_transport(gid_info->gl_ibmf_hdl, 3519 gid_info->gl_qp_hdl, msg, NULL, ibdm_ibmf_send_cb, 3520 cb_args, 0)) != IBMF_SUCCESS) { 3521 IBTF_DPRINTF_L2("ibdm", "\tretry_command: send failed: %p " 3522 "rtype 0x%x iocidx 0x%x srvidx %d", cb_args->cb_gid_info, 3523 cb_args->cb_req_type, cb_args->cb_ioc_num, 3524 cb_args->cb_srvents_start); 3525 ibdm_ibmf_send_cb(gid_info->gl_ibmf_hdl, msg, cb_args); 3526 rval = IBDM_FAILURE; 3527 } 3528 mutex_enter(&gid_info->gl_mutex); 3529 return (rval); 3530 } 3531 3532 3533 /* 3534 * ibdm_update_ioc_port_gidlist() 3535 */ 3536 static void 3537 ibdm_update_ioc_port_gidlist(ibdm_ioc_info_t *dest, 3538 ibdm_dp_gidinfo_t *gid_info) 3539 { 3540 int ii, ngid_ents; 3541 ibdm_gid_t *tmp; 3542 ibdm_hca_list_t *gid_hca_head, *temp; 3543 ibdm_hca_list_t *ioc_head = NULL; 3544 3545 IBTF_DPRINTF_L5("ibdm", "\tupdate_ioc_port_gidlist: Enter"); 3546 3547 ngid_ents = gid_info->gl_ngids; 3548 dest->ioc_nportgids = ngid_ents; 3549 dest->ioc_gid_list = kmem_zalloc(sizeof (ibdm_gid_t) * 3550 ngid_ents, KM_SLEEP); 3551 tmp = gid_info->gl_gid; 3552 for (ii = 0; (ii < ngid_ents) && (tmp); ii++) { 3553 dest->ioc_gid_list[ii].gid_dgid_hi = tmp->gid_dgid_hi; 3554 dest->ioc_gid_list[ii].gid_dgid_lo = tmp->gid_dgid_lo; 3555 tmp = tmp->gid_next; 3556 } 3557 3558 gid_hca_head = gid_info->gl_hca_list; 3559 while (gid_hca_head) { 3560 temp = ibdm_dup_hca_attr(gid_hca_head); 3561 temp->hl_next = ioc_head; 3562 ioc_head = temp; 3563 gid_hca_head = gid_hca_head->hl_next; 3564 } 3565 dest->ioc_hca_list = ioc_head; 3566 } 3567 3568 3569 /* 3570 * ibdm_alloc_send_buffers() 3571 * Allocates memory for the IBMF send buffer 3572 */ 3573 static void 3574 ibdm_alloc_send_buffers(ibmf_msg_t *msgp) 3575 { 3576 msgp->im_msgbufs_send.im_bufs_mad_hdr = 3577 kmem_zalloc(IBDM_MAD_SIZE, KM_SLEEP); 3578 msgp->im_msgbufs_send.im_bufs_cl_data = (uchar_t *) 3579 msgp->im_msgbufs_send.im_bufs_mad_hdr + sizeof (ib_mad_hdr_t); 3580 msgp->im_msgbufs_send.im_bufs_cl_data_len = 3581 IBDM_MAD_SIZE - sizeof (ib_mad_hdr_t); 3582 } 3583 3584 3585 /* 3586 * ibdm_alloc_send_buffers() 3587 * De-allocates memory for the IBMF send buffer 3588 */ 3589 static void 3590 ibdm_free_send_buffers(ibmf_msg_t *msgp) 3591 { 3592 if (msgp->im_msgbufs_send.im_bufs_mad_hdr != NULL) 3593 kmem_free(msgp->im_msgbufs_send.im_bufs_mad_hdr, IBDM_MAD_SIZE); 3594 } 3595 3596 /* 3597 * ibdm_probe_ioc() 3598 * 1. Gets the node records for the port GUID. This detects all the port 3599 * to the IOU. 3600 * 2. Selectively probes all the IOC, given it's node GUID 3601 * 3. In case of reprobe, only the IOC to be reprobed is send the IOC 3602 * Controller Profile asynchronously 3603 */ 3604 /*ARGSUSED*/ 3605 static void 3606 ibdm_probe_ioc(ib_guid_t nodeguid, ib_guid_t ioc_guid, int reprobe_flag) 3607 { 3608 int ii, nrecords; 3609 size_t nr_len = 0, pi_len = 0; 3610 ib_gid_t sgid, dgid; 3611 ibdm_hca_list_t *hca_list = NULL; 3612 sa_node_record_t *nr, *tmp; 3613 ibdm_port_attr_t *port = NULL; 3614 ibdm_dp_gidinfo_t *reprobe_gid, *new_gid, *node_gid; 3615 ibdm_dp_gidinfo_t *temp_gidinfo; 3616 ibdm_gid_t *temp_gid; 3617 sa_portinfo_record_t *pi; 3618 3619 IBTF_DPRINTF_L4("ibdm", "\tprobe_ioc(%x, %x, %x): Begin", 3620 nodeguid, ioc_guid, reprobe_flag); 3621 3622 /* Rescan the GID list for any removed GIDs for reprobe */ 3623 if (reprobe_flag) 3624 ibdm_rescan_gidlist(&ioc_guid); 3625 3626 mutex_enter(&ibdm.ibdm_hl_mutex); 3627 for (ibdm_get_next_port(&hca_list, &port, 1); port; 3628 ibdm_get_next_port(&hca_list, &port, 1)) { 3629 reprobe_gid = new_gid = node_gid = NULL; 3630 3631 nr = ibdm_get_node_records(port->pa_sa_hdl, &nr_len, nodeguid); 3632 if (nr == NULL) { 3633 IBTF_DPRINTF_L4("ibdm", "\tprobe_ioc: no records"); 3634 continue; 3635 } 3636 nrecords = (nr_len / sizeof (sa_node_record_t)); 3637 for (tmp = nr, ii = 0; (ii < nrecords); ii++, tmp++) { 3638 pi = ibdm_get_portinfo( 3639 port->pa_sa_hdl, &pi_len, tmp->LID); 3640 3641 if ((pi) && (pi->PortInfo.CapabilityMask & 3642 SM_CAP_MASK_IS_DM_SUPPD)) { 3643 /* 3644 * For reprobes: Check if GID, already in 3645 * the list. If so, set the state to SKIPPED 3646 */ 3647 if (((temp_gidinfo = ibdm_find_gid(nodeguid, 3648 tmp->NodeInfo.PortGUID)) != NULL) && 3649 temp_gidinfo->gl_state == 3650 IBDM_GID_PROBING_COMPLETE) { 3651 ASSERT(reprobe_gid == NULL); 3652 ibdm_addto_glhcalist(temp_gidinfo, 3653 hca_list); 3654 reprobe_gid = temp_gidinfo; 3655 kmem_free(pi, pi_len); 3656 continue; 3657 } else if (temp_gidinfo != NULL) { 3658 kmem_free(pi, pi_len); 3659 ibdm_addto_glhcalist(temp_gidinfo, 3660 hca_list); 3661 continue; 3662 } 3663 3664 IBTF_DPRINTF_L4("ibdm", "\tprobe_ioc : " 3665 "create_gid : prefix %llx, guid %llx\n", 3666 pi->PortInfo.GidPrefix, 3667 tmp->NodeInfo.PortGUID); 3668 3669 sgid.gid_prefix = port->pa_sn_prefix; 3670 sgid.gid_guid = port->pa_port_guid; 3671 dgid.gid_prefix = pi->PortInfo.GidPrefix; 3672 dgid.gid_guid = tmp->NodeInfo.PortGUID; 3673 new_gid = ibdm_create_gid_info(port, sgid, 3674 dgid); 3675 if (new_gid == NULL) { 3676 IBTF_DPRINTF_L2("ibdm", "\tprobe_ioc: " 3677 "create_gid_info failed\n"); 3678 kmem_free(pi, pi_len); 3679 continue; 3680 } 3681 if (node_gid == NULL) { 3682 node_gid = new_gid; 3683 ibdm_add_to_gl_gid(node_gid, node_gid); 3684 } else { 3685 IBTF_DPRINTF_L4("ibdm", 3686 "\tprobe_ioc: new gid"); 3687 temp_gid = kmem_zalloc( 3688 sizeof (ibdm_gid_t), KM_SLEEP); 3689 temp_gid->gid_dgid_hi = 3690 new_gid->gl_dgid_hi; 3691 temp_gid->gid_dgid_lo = 3692 new_gid->gl_dgid_lo; 3693 temp_gid->gid_next = node_gid->gl_gid; 3694 node_gid->gl_gid = temp_gid; 3695 node_gid->gl_ngids++; 3696 } 3697 new_gid->gl_nodeguid = nodeguid; 3698 new_gid->gl_portguid = dgid.gid_guid; 3699 ibdm_addto_glhcalist(new_gid, hca_list); 3700 3701 /* 3702 * Set the state to skipped as all these 3703 * gids point to the same node. 3704 * We (re)probe only one GID below and reset 3705 * state appropriately 3706 */ 3707 new_gid->gl_state = IBDM_GID_PROBING_SKIPPED; 3708 kmem_free(pi, pi_len); 3709 } 3710 } 3711 kmem_free(nr, nr_len); 3712 3713 IBTF_DPRINTF_L4("ibdm", "\tprobe_ioc : reprobe_flag %d " 3714 "reprobe_gid %p new_gid %p node_gid %p", 3715 reprobe_flag, reprobe_gid, new_gid, node_gid); 3716 3717 if (reprobe_flag != 0 && reprobe_gid != NULL) { 3718 int niocs, jj; 3719 ibdm_ioc_info_t *tmp_ioc; 3720 int ioc_matched = 0; 3721 3722 mutex_exit(&ibdm.ibdm_hl_mutex); 3723 mutex_enter(&reprobe_gid->gl_mutex); 3724 reprobe_gid->gl_state = IBDM_GET_IOC_DETAILS; 3725 niocs = 3726 reprobe_gid->gl_iou->iou_info.iou_num_ctrl_slots; 3727 reprobe_gid->gl_pending_cmds++; 3728 mutex_exit(&reprobe_gid->gl_mutex); 3729 3730 for (jj = 0; jj < niocs; jj++) { 3731 tmp_ioc = 3732 IBDM_GIDINFO2IOCINFO(reprobe_gid, jj); 3733 if (tmp_ioc->ioc_profile.ioc_guid != ioc_guid) 3734 continue; 3735 3736 ioc_matched = 1; 3737 3738 /* 3739 * Explicitly set gl_reprobe_flag to 0 so that 3740 * IBnex is not notified on completion 3741 */ 3742 mutex_enter(&reprobe_gid->gl_mutex); 3743 reprobe_gid->gl_reprobe_flag = 0; 3744 mutex_exit(&reprobe_gid->gl_mutex); 3745 3746 mutex_enter(&ibdm.ibdm_mutex); 3747 ibdm.ibdm_ngid_probes_in_progress++; 3748 mutex_exit(&ibdm.ibdm_mutex); 3749 if (ibdm_send_ioc_profile(reprobe_gid, jj) != 3750 IBDM_SUCCESS) { 3751 IBTF_DPRINTF_L4("ibdm", 3752 "\tprobe_ioc: " 3753 "send_ioc_profile failed " 3754 "for ioc %d", jj); 3755 ibdm_gid_decr_pending(reprobe_gid); 3756 break; 3757 } 3758 mutex_enter(&ibdm.ibdm_mutex); 3759 ibdm_wait_probe_completion(); 3760 mutex_exit(&ibdm.ibdm_mutex); 3761 break; 3762 } 3763 if (ioc_matched == 0) 3764 ibdm_gid_decr_pending(reprobe_gid); 3765 else { 3766 mutex_enter(&ibdm.ibdm_hl_mutex); 3767 break; 3768 } 3769 } else if (new_gid != NULL) { 3770 mutex_exit(&ibdm.ibdm_hl_mutex); 3771 node_gid = node_gid ? node_gid : new_gid; 3772 3773 /* 3774 * New or reinserted GID : Enable notification 3775 * to IBnex 3776 */ 3777 mutex_enter(&node_gid->gl_mutex); 3778 node_gid->gl_reprobe_flag = 1; 3779 mutex_exit(&node_gid->gl_mutex); 3780 3781 ibdm_probe_gid(node_gid); 3782 3783 mutex_enter(&ibdm.ibdm_hl_mutex); 3784 } else { 3785 IBTF_DPRINTF_L2("ibdm", "\tibdm_probe_ioc " 3786 "Invalid state!"); 3787 } 3788 3789 } 3790 mutex_exit(&ibdm.ibdm_hl_mutex); 3791 IBTF_DPRINTF_L4("ibdm", "\tprobe_ioc : End\n"); 3792 } 3793 3794 3795 /* 3796 * ibdm_probe_gid() 3797 * Selectively probes the GID 3798 */ 3799 static void 3800 ibdm_probe_gid(ibdm_dp_gidinfo_t *gid_info) 3801 { 3802 IBTF_DPRINTF_L4("ibdm", "\tprobe_gid:"); 3803 mutex_enter(&gid_info->gl_mutex); 3804 gid_info->gl_pending_cmds++; 3805 gid_info->gl_state = IBDM_GET_CLASSPORTINFO; 3806 mutex_exit(&gid_info->gl_mutex); 3807 if (ibdm_send_classportinfo(gid_info) != IBDM_SUCCESS) { 3808 mutex_enter(&gid_info->gl_mutex); 3809 gid_info->gl_state = IBDM_GID_PROBING_FAILED; 3810 --gid_info->gl_pending_cmds; 3811 mutex_exit(&gid_info->gl_mutex); 3812 ibdm_delete_glhca_list(gid_info); 3813 gid_info = gid_info->gl_next; 3814 return; 3815 } 3816 mutex_enter(&ibdm.ibdm_mutex); 3817 ibdm.ibdm_ngid_probes_in_progress++; 3818 gid_info = gid_info->gl_next; 3819 3820 ibdm_wait_probe_completion(); 3821 mutex_exit(&ibdm.ibdm_mutex); 3822 IBTF_DPRINTF_L4("ibdm", "\tprobe_gid: Wakeup signal received"); 3823 } 3824 3825 3826 /* 3827 * ibdm_create_gid_info() 3828 * Allocates a gid_info structure and initializes 3829 * Returns pointer to the structure on success 3830 * and NULL on failure 3831 */ 3832 static ibdm_dp_gidinfo_t * 3833 ibdm_create_gid_info(ibdm_port_attr_t *port, ib_gid_t sgid, ib_gid_t dgid) 3834 { 3835 uint8_t ii, npaths; 3836 sa_path_record_t *path; 3837 size_t len; 3838 ibdm_pkey_tbl_t *pkey_tbl; 3839 ibdm_dp_gidinfo_t *gid_info = NULL; 3840 int ret; 3841 3842 IBTF_DPRINTF_L4("ibdm", "\tcreate_gid_info: Begin"); 3843 npaths = 1; 3844 3845 /* query for reversible paths */ 3846 if (port->pa_sa_hdl) 3847 ret = ibmf_saa_gid_to_pathrecords(port->pa_sa_hdl, 3848 sgid, dgid, IBMF_SAA_PKEY_WC, 0, B_TRUE, &npaths, 0, 3849 &len, &path); 3850 else 3851 return (NULL); 3852 3853 if (ret == IBMF_SUCCESS && path) { 3854 ibdm_dump_path_info(path); 3855 3856 gid_info = kmem_zalloc( 3857 sizeof (ibdm_dp_gidinfo_t), KM_SLEEP); 3858 mutex_init(&gid_info->gl_mutex, NULL, MUTEX_DEFAULT, NULL); 3859 gid_info->gl_dgid_hi = path->DGID.gid_prefix; 3860 gid_info->gl_dgid_lo = path->DGID.gid_guid; 3861 gid_info->gl_sgid_hi = path->SGID.gid_prefix; 3862 gid_info->gl_sgid_lo = path->SGID.gid_guid; 3863 gid_info->gl_p_key = path->P_Key; 3864 gid_info->gl_sa_hdl = port->pa_sa_hdl; 3865 gid_info->gl_ibmf_hdl = port->pa_ibmf_hdl; 3866 gid_info->gl_slid = path->SLID; 3867 gid_info->gl_dlid = path->DLID; 3868 gid_info->gl_transactionID = (++ibdm.ibdm_transactionID) 3869 << IBDM_GID_TRANSACTIONID_SHIFT; 3870 gid_info->gl_min_transactionID = gid_info->gl_transactionID; 3871 gid_info->gl_max_transactionID = (ibdm.ibdm_transactionID +1) 3872 << IBDM_GID_TRANSACTIONID_SHIFT; 3873 3874 gid_info->gl_qp_hdl = IBMF_QP_HANDLE_DEFAULT; 3875 for (ii = 0; ii < port->pa_npkeys; ii++) { 3876 if (port->pa_pkey_tbl == NULL) 3877 break; 3878 3879 pkey_tbl = &port->pa_pkey_tbl[ii]; 3880 if ((gid_info->gl_p_key == pkey_tbl->pt_pkey) && 3881 (pkey_tbl->pt_qp_hdl != NULL)) { 3882 gid_info->gl_qp_hdl = pkey_tbl->pt_qp_hdl; 3883 break; 3884 } 3885 } 3886 kmem_free(path, len); 3887 3888 /* 3889 * QP handle for GID not initialized. No matching Pkey 3890 * was found!! ibdm should *not* hit this case. Flag an 3891 * error and drop the GID if ibdm does encounter this. 3892 */ 3893 if (gid_info->gl_qp_hdl == NULL) { 3894 IBTF_DPRINTF_L2(ibdm_string, 3895 "\tcreate_gid_info: No matching Pkey"); 3896 ibdm_delete_gidinfo(gid_info); 3897 return (NULL); 3898 } 3899 3900 ibdm.ibdm_ngids++; 3901 if (ibdm.ibdm_dp_gidlist_head == NULL) { 3902 ibdm.ibdm_dp_gidlist_head = gid_info; 3903 ibdm.ibdm_dp_gidlist_tail = gid_info; 3904 } else { 3905 ibdm.ibdm_dp_gidlist_tail->gl_next = gid_info; 3906 gid_info->gl_prev = ibdm.ibdm_dp_gidlist_tail; 3907 ibdm.ibdm_dp_gidlist_tail = gid_info; 3908 } 3909 } 3910 3911 return (gid_info); 3912 } 3913 3914 3915 /* 3916 * ibdm_get_node_records 3917 * Sends a SA query to get the NODE record 3918 * Returns pointer to the sa_node_record_t on success 3919 * and NULL on failure 3920 */ 3921 static sa_node_record_t * 3922 ibdm_get_node_records(ibmf_saa_handle_t sa_hdl, size_t *length, ib_guid_t guid) 3923 { 3924 sa_node_record_t req, *resp = NULL; 3925 ibmf_saa_access_args_t args; 3926 int ret; 3927 3928 IBTF_DPRINTF_L4("ibdm", "\tget_node_records: Begin"); 3929 3930 bzero(&req, sizeof (sa_node_record_t)); 3931 req.NodeInfo.NodeGUID = guid; 3932 3933 args.sq_attr_id = SA_NODERECORD_ATTRID; 3934 args.sq_access_type = IBMF_SAA_RETRIEVE; 3935 args.sq_component_mask = SA_NODEINFO_COMPMASK_NODEGUID; 3936 args.sq_template = &req; 3937 args.sq_callback = NULL; 3938 args.sq_callback_arg = NULL; 3939 3940 ret = ibmf_sa_access(sa_hdl, &args, 0, length, (void **) &resp); 3941 if (ret != IBMF_SUCCESS) { 3942 IBTF_DPRINTF_L2("ibdm", "\tget_node_records:" 3943 " SA Retrieve Failed: %d", ret); 3944 return (NULL); 3945 } 3946 if ((resp == NULL) || (*length == 0)) { 3947 IBTF_DPRINTF_L2("ibdm", "\tget_node_records: No records"); 3948 return (NULL); 3949 } 3950 3951 IBTF_DPRINTF_L4("ibdm", "\tget_node_records: NodeGuid %llx " 3952 "PortGUID %llx", resp->NodeInfo.NodeGUID, resp->NodeInfo.PortGUID); 3953 3954 return (resp); 3955 } 3956 3957 3958 /* 3959 * ibdm_get_portinfo() 3960 * Sends a SA query to get the PortInfo record 3961 * Returns pointer to the sa_portinfo_record_t on success 3962 * and NULL on failure 3963 */ 3964 static sa_portinfo_record_t * 3965 ibdm_get_portinfo(ibmf_saa_handle_t sa_hdl, size_t *length, ib_lid_t lid) 3966 { 3967 sa_portinfo_record_t req, *resp = NULL; 3968 ibmf_saa_access_args_t args; 3969 int ret; 3970 3971 IBTF_DPRINTF_L4("ibdm", "\tget_portinfo: Begin"); 3972 3973 bzero(&req, sizeof (sa_portinfo_record_t)); 3974 req.EndportLID = lid; 3975 3976 args.sq_attr_id = SA_PORTINFORECORD_ATTRID; 3977 args.sq_access_type = IBMF_SAA_RETRIEVE; 3978 args.sq_component_mask = SA_PORTINFO_COMPMASK_PORTLID; 3979 args.sq_template = &req; 3980 args.sq_callback = NULL; 3981 args.sq_callback_arg = NULL; 3982 3983 ret = ibmf_sa_access(sa_hdl, &args, 0, length, (void **) &resp); 3984 if (ret != IBMF_SUCCESS) { 3985 IBTF_DPRINTF_L2("ibdm", "\tget_portinfo:" 3986 " SA Retrieve Failed: 0x%X", ret); 3987 return (NULL); 3988 } 3989 if ((*length == 0) || (resp == NULL)) 3990 return (NULL); 3991 3992 IBTF_DPRINTF_L4("ibdm", "\tget_portinfo: GidPrefix %llx Cap 0x%x", 3993 resp->PortInfo.GidPrefix, resp->PortInfo.CapabilityMask); 3994 return (resp); 3995 } 3996 3997 3998 /* 3999 * ibdm_ibnex_register_callback 4000 * IB nexus callback routine for HCA attach and detach notification 4001 */ 4002 void 4003 ibdm_ibnex_register_callback(ibdm_callback_t ibnex_dm_callback) 4004 { 4005 IBTF_DPRINTF_L4("ibdm", "\tibnex_register_callbacks"); 4006 mutex_enter(&ibdm.ibdm_ibnex_mutex); 4007 ibdm.ibdm_ibnex_callback = ibnex_dm_callback; 4008 mutex_exit(&ibdm.ibdm_ibnex_mutex); 4009 } 4010 4011 4012 /* 4013 * ibdm_ibnex_unregister_callbacks 4014 */ 4015 void 4016 ibdm_ibnex_unregister_callback() 4017 { 4018 IBTF_DPRINTF_L4("ibdm", "\tibnex_unregister_callbacks"); 4019 mutex_enter(&ibdm.ibdm_ibnex_mutex); 4020 ibdm.ibdm_ibnex_callback = NULL; 4021 mutex_exit(&ibdm.ibdm_ibnex_mutex); 4022 } 4023 4024 4025 /* 4026 * ibdm_ibnex_get_waittime() 4027 * Calculates the wait time based on the last HCA attach time 4028 */ 4029 time_t 4030 ibdm_ibnex_get_waittime(ib_guid_t hca_guid, int *dft_wait) 4031 { 4032 int ii; 4033 time_t temp, wait_time = 0; 4034 ibdm_hca_list_t *hca; 4035 4036 IBTF_DPRINTF_L4("ibdm", "\tibnex_get_waittime hcaguid:%llx" 4037 "\tport settling time %d", hca_guid, *dft_wait); 4038 4039 mutex_enter(&ibdm.ibdm_hl_mutex); 4040 hca = ibdm.ibdm_hca_list_head; 4041 4042 if (hca_guid) { 4043 for (ii = 0; ii < ibdm.ibdm_hca_count; ii++) { 4044 if ((hca_guid == hca->hl_hca_guid) && 4045 (hca->hl_nports != hca->hl_nports_active)) { 4046 wait_time = 4047 ddi_get_time() - hca->hl_attach_time; 4048 wait_time = ((wait_time >= *dft_wait) ? 4049 0 : (*dft_wait - wait_time)); 4050 break; 4051 } 4052 hca = hca->hl_next; 4053 } 4054 mutex_exit(&ibdm.ibdm_hl_mutex); 4055 IBTF_DPRINTF_L4("ibdm", "\tibnex_get_waittime %llx", wait_time); 4056 return (wait_time); 4057 } 4058 4059 for (ii = 0; ii < ibdm.ibdm_hca_count; ii++) { 4060 if (hca->hl_nports != hca->hl_nports_active) { 4061 temp = ddi_get_time() - hca->hl_attach_time; 4062 temp = ((temp >= *dft_wait) ? 0 : (*dft_wait - temp)); 4063 wait_time = (temp > wait_time) ? temp : wait_time; 4064 } 4065 } 4066 mutex_exit(&ibdm.ibdm_hl_mutex); 4067 IBTF_DPRINTF_L4("ibdm", "\tibnex_get_waittime %llx", wait_time); 4068 return (wait_time); 4069 } 4070 4071 4072 /* 4073 * ibdm_ibnex_probe_hcaport 4074 * Probes the presence of HCA port (with HCA dip and port number) 4075 * Returns port attributes structure on SUCCESS 4076 */ 4077 ibdm_port_attr_t * 4078 ibdm_ibnex_probe_hcaport(ib_guid_t hca_guid, uint8_t port_num) 4079 { 4080 int ii, jj; 4081 ibdm_hca_list_t *hca_list; 4082 ibdm_port_attr_t *port_attr; 4083 4084 IBTF_DPRINTF_L4("ibdm", "\tibnex_probe_hcaport:"); 4085 4086 mutex_enter(&ibdm.ibdm_hl_mutex); 4087 hca_list = ibdm.ibdm_hca_list_head; 4088 for (ii = 0; ii < ibdm.ibdm_hca_count; ii++) { 4089 if (hca_list->hl_hca_guid == hca_guid) { 4090 for (jj = 0; jj < hca_list->hl_nports; jj++) { 4091 if (hca_list->hl_port_attr[jj].pa_port_num == 4092 port_num) { 4093 break; 4094 } 4095 } 4096 if (jj != hca_list->hl_nports) 4097 break; 4098 } 4099 hca_list = hca_list->hl_next; 4100 } 4101 if (ii == ibdm.ibdm_hca_count) { 4102 IBTF_DPRINTF_L2("ibdm", "\tibnex_probe_hcaport: not found"); 4103 mutex_exit(&ibdm.ibdm_hl_mutex); 4104 return (NULL); 4105 } 4106 port_attr = (ibdm_port_attr_t *)kmem_zalloc( 4107 sizeof (ibdm_port_attr_t), KM_SLEEP); 4108 bcopy((char *)&hca_list->hl_port_attr[jj], 4109 port_attr, sizeof (ibdm_port_attr_t)); 4110 ibdm_update_port_attr(port_attr); 4111 4112 mutex_exit(&ibdm.ibdm_hl_mutex); 4113 return (port_attr); 4114 } 4115 4116 4117 /* 4118 * ibdm_ibnex_get_port_attrs 4119 * Scan all HCAs for a matching port_guid. 4120 * Returns "port attributes" structure on success. 4121 */ 4122 ibdm_port_attr_t * 4123 ibdm_ibnex_get_port_attrs(ib_guid_t port_guid) 4124 { 4125 int ii, jj; 4126 ibdm_hca_list_t *hca_list; 4127 ibdm_port_attr_t *port_attr; 4128 4129 IBTF_DPRINTF_L4("ibdm", "\tibnex_get_port_attrs:"); 4130 4131 mutex_enter(&ibdm.ibdm_hl_mutex); 4132 hca_list = ibdm.ibdm_hca_list_head; 4133 4134 for (ii = 0; ii < ibdm.ibdm_hca_count; ii++) { 4135 for (jj = 0; jj < hca_list->hl_nports; jj++) { 4136 if (hca_list->hl_port_attr[jj].pa_port_guid == 4137 port_guid) { 4138 break; 4139 } 4140 } 4141 if (jj != hca_list->hl_nports) 4142 break; 4143 hca_list = hca_list->hl_next; 4144 } 4145 4146 if (ii == ibdm.ibdm_hca_count) { 4147 IBTF_DPRINTF_L2("ibdm", "\tibnex_get_port_attrs: not found"); 4148 mutex_exit(&ibdm.ibdm_hl_mutex); 4149 return (NULL); 4150 } 4151 4152 port_attr = (ibdm_port_attr_t *)kmem_alloc(sizeof (ibdm_port_attr_t), 4153 KM_SLEEP); 4154 bcopy((char *)&hca_list->hl_port_attr[jj], port_attr, 4155 sizeof (ibdm_port_attr_t)); 4156 ibdm_update_port_attr(port_attr); 4157 4158 mutex_exit(&ibdm.ibdm_hl_mutex); 4159 return (port_attr); 4160 } 4161 4162 4163 /* 4164 * ibdm_ibnex_free_port_attr() 4165 */ 4166 void 4167 ibdm_ibnex_free_port_attr(ibdm_port_attr_t *port_attr) 4168 { 4169 IBTF_DPRINTF_L4("ibdm", "\tibnex_free_port_attr:"); 4170 if (port_attr) { 4171 if (port_attr->pa_pkey_tbl != NULL) { 4172 kmem_free(port_attr->pa_pkey_tbl, 4173 (port_attr->pa_npkeys * sizeof (ibdm_pkey_tbl_t))); 4174 } 4175 kmem_free(port_attr, sizeof (ibdm_port_attr_t)); 4176 } 4177 } 4178 4179 4180 /* 4181 * ibdm_ibnex_get_hca_list() 4182 * Returns portinfo for all the port for all the HCA's 4183 */ 4184 void 4185 ibdm_ibnex_get_hca_list(ibdm_hca_list_t **hca, int *count) 4186 { 4187 ibdm_hca_list_t *head = NULL, *temp, *temp1; 4188 int ii; 4189 4190 IBTF_DPRINTF_L4("ibdm", "\tibnex_get_hca_list:"); 4191 4192 mutex_enter(&ibdm.ibdm_hl_mutex); 4193 temp = ibdm.ibdm_hca_list_head; 4194 for (ii = 0; ii < ibdm.ibdm_hca_count; ii++) { 4195 temp1 = ibdm_dup_hca_attr(temp); 4196 temp1->hl_next = head; 4197 head = temp1; 4198 temp = temp->hl_next; 4199 } 4200 *count = ibdm.ibdm_hca_count; 4201 *hca = head; 4202 mutex_exit(&ibdm.ibdm_hl_mutex); 4203 } 4204 4205 4206 /* 4207 * ibdm_ibnex_get_hca_info_by_guid() 4208 */ 4209 ibdm_hca_list_t * 4210 ibdm_ibnex_get_hca_info_by_guid(ib_guid_t hca_guid) 4211 { 4212 ibdm_hca_list_t *head = NULL, *hca = NULL; 4213 4214 IBTF_DPRINTF_L4("ibdm", "\tibnex_get_hca_info_by_dip"); 4215 4216 mutex_enter(&ibdm.ibdm_hl_mutex); 4217 head = ibdm.ibdm_hca_list_head; 4218 while (head) { 4219 if (head->hl_hca_guid == hca_guid) { 4220 hca = ibdm_dup_hca_attr(head); 4221 hca->hl_next = NULL; 4222 break; 4223 } 4224 head = head->hl_next; 4225 } 4226 mutex_exit(&ibdm.ibdm_hl_mutex); 4227 IBTF_DPRINTF_L4("ibdm", "\tibnex_get_hca_info_by_dip %p", hca); 4228 return (hca); 4229 } 4230 4231 4232 /* 4233 * ibdm_dup_hca_attr() 4234 * Allocate a new HCA attribute strucuture and initialize 4235 * hca attribute structure with the incoming HCA attributes 4236 * returned the allocated hca attributes. 4237 */ 4238 static ibdm_hca_list_t * 4239 ibdm_dup_hca_attr(ibdm_hca_list_t *in_hca) 4240 { 4241 int len; 4242 ibdm_hca_list_t *out_hca; 4243 4244 len = sizeof (ibdm_hca_list_t) + 4245 (in_hca->hl_nports * sizeof (ibdm_port_attr_t)); 4246 IBTF_DPRINTF_L4("ibdm", "\tdup_hca_attr len %d", len); 4247 out_hca = (ibdm_hca_list_t *)kmem_alloc(len, KM_SLEEP); 4248 bcopy((char *)in_hca, 4249 (char *)out_hca, sizeof (ibdm_hca_list_t)); 4250 if (in_hca->hl_nports) { 4251 out_hca->hl_port_attr = (ibdm_port_attr_t *) 4252 ((char *)out_hca + sizeof (ibdm_hca_list_t)); 4253 bcopy((char *)in_hca->hl_port_attr, 4254 (char *)out_hca->hl_port_attr, 4255 (in_hca->hl_nports * sizeof (ibdm_port_attr_t))); 4256 for (len = 0; len < out_hca->hl_nports; len++) 4257 ibdm_update_port_attr(&out_hca->hl_port_attr[len]); 4258 } 4259 return (out_hca); 4260 } 4261 4262 4263 /* 4264 * ibdm_ibnex_free_hca_list() 4265 * Free one/more HCA lists 4266 */ 4267 void 4268 ibdm_ibnex_free_hca_list(ibdm_hca_list_t *hca_list) 4269 { 4270 int ii, len; 4271 ibdm_hca_list_t *temp; 4272 ibdm_port_attr_t *port; 4273 4274 IBTF_DPRINTF_L4("ibdm", "\tibnex_free_hca_list:"); 4275 ASSERT(hca_list); 4276 while (hca_list) { 4277 temp = hca_list; 4278 hca_list = hca_list->hl_next; 4279 for (ii = 0; ii < temp->hl_nports; ii++) { 4280 port = &temp->hl_port_attr[ii]; 4281 len = (port->pa_npkeys * sizeof (ibdm_pkey_tbl_t)); 4282 if (len != 0) 4283 kmem_free(port->pa_pkey_tbl, len); 4284 } 4285 len = sizeof (ibdm_hca_list_t) + (temp->hl_nports * 4286 sizeof (ibdm_port_attr_t)); 4287 kmem_free(temp, len); 4288 } 4289 } 4290 4291 4292 /* 4293 * ibdm_ibnex_probe_iocguid() 4294 * Probes the IOC on the fabric and returns the IOC information 4295 * if present. Otherwise, NULL is returned 4296 */ 4297 /* ARGSUSED */ 4298 ibdm_ioc_info_t * 4299 ibdm_ibnex_probe_ioc(ib_guid_t iou, ib_guid_t ioc_guid, int reprobe_flag) 4300 { 4301 int k; 4302 ibdm_ioc_info_t *ioc_info; 4303 ibdm_dp_gidinfo_t *gid_info; 4304 4305 IBTF_DPRINTF_L4("ibdm", "\tibnex_probe_ioc: (%llX, %llX, %d) Begin", 4306 iou, ioc_guid, reprobe_flag); 4307 /* Check whether we know this already */ 4308 ioc_info = ibdm_ibnex_get_ioc_info(ioc_guid); 4309 if (ioc_info == NULL) { 4310 mutex_enter(&ibdm.ibdm_mutex); 4311 while (ibdm.ibdm_busy & IBDM_BUSY) 4312 cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex); 4313 ibdm.ibdm_busy |= IBDM_BUSY; 4314 mutex_exit(&ibdm.ibdm_mutex); 4315 ibdm_probe_ioc(iou, ioc_guid, 0); 4316 mutex_enter(&ibdm.ibdm_mutex); 4317 ibdm.ibdm_busy &= ~IBDM_BUSY; 4318 cv_broadcast(&ibdm.ibdm_busy_cv); 4319 mutex_exit(&ibdm.ibdm_mutex); 4320 ioc_info = ibdm_ibnex_get_ioc_info(ioc_guid); 4321 } else if (reprobe_flag) { /* Handle Reprobe for the IOC */ 4322 /* Free the ioc_list before reprobe; and cancel any timers */ 4323 mutex_enter(&ibdm.ibdm_mutex); 4324 if (ioc_info->ioc_timeout_id) { 4325 IBTF_DPRINTF_L5("ibdm", "\tprobe_ioc: " 4326 "ioc_timeout_id = 0x%x", 4327 ioc_info->ioc_timeout_id); 4328 if (untimeout(ioc_info->ioc_timeout_id) == -1) { 4329 IBTF_DPRINTF_L2("ibdm", "\tprobe_ioc: " 4330 "untimeout ioc_timeout_id failed"); 4331 } 4332 ioc_info->ioc_timeout_id = 0; 4333 } 4334 if (ioc_info->ioc_dc_timeout_id) { 4335 IBTF_DPRINTF_L5("ibdm", "\tprobe_ioc: " 4336 "ioc_dc_timeout_id = 0x%x", 4337 ioc_info->ioc_dc_timeout_id); 4338 if (untimeout(ioc_info->ioc_dc_timeout_id) == -1) { 4339 IBTF_DPRINTF_L2("ibdm", "\tprobe_ioc: " 4340 "untimeout ioc_dc_timeout_id failed"); 4341 } 4342 ioc_info->ioc_dc_timeout_id = 0; 4343 } 4344 for (k = 0; k < ioc_info->ioc_profile.ioc_service_entries; k++) 4345 if (ioc_info->ioc_serv[k].se_timeout_id) { 4346 IBTF_DPRINTF_L5("ibdm", "\tprobe_ioc: " 4347 "ioc_info->ioc_serv[k].se_timeout_id = %x", 4348 k, ioc_info->ioc_serv[k].se_timeout_id); 4349 if (untimeout(ioc_info->ioc_serv[k]. 4350 se_timeout_id) == -1) { 4351 IBTF_DPRINTF_L2("ibdm", "\tprobe_ioc: " 4352 "untimeout se_timeout_id %d " 4353 "failed", k); 4354 } 4355 ioc_info->ioc_serv[k].se_timeout_id = 0; 4356 } 4357 mutex_exit(&ibdm.ibdm_mutex); 4358 ibdm_ibnex_free_ioc_list(ioc_info); 4359 4360 mutex_enter(&ibdm.ibdm_mutex); 4361 while (ibdm.ibdm_busy & IBDM_BUSY) 4362 cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex); 4363 ibdm.ibdm_busy |= IBDM_BUSY; 4364 mutex_exit(&ibdm.ibdm_mutex); 4365 4366 ibdm_probe_ioc(iou, ioc_guid, 1); 4367 4368 /* 4369 * Skip if gl_reprobe_flag is set, this will be 4370 * a re-inserted / new GID, for which notifications 4371 * have already been send. 4372 */ 4373 for (gid_info = ibdm.ibdm_dp_gidlist_head; gid_info; 4374 gid_info = gid_info->gl_next) { 4375 uint8_t ii, niocs; 4376 ibdm_ioc_info_t *ioc; 4377 4378 if (gid_info->gl_iou == NULL) 4379 continue; 4380 4381 if (gid_info->gl_reprobe_flag) { 4382 gid_info->gl_reprobe_flag = 0; 4383 continue; 4384 } 4385 4386 niocs = gid_info->gl_iou->iou_info.iou_num_ctrl_slots; 4387 for (ii = 0; ii < niocs; ii++) { 4388 ioc = IBDM_GIDINFO2IOCINFO(gid_info, ii); 4389 if (ioc->ioc_profile.ioc_guid == ioc_guid) { 4390 mutex_enter(&ibdm.ibdm_mutex); 4391 ibdm_reprobe_update_port_srv(ioc, 4392 gid_info); 4393 mutex_exit(&ibdm.ibdm_mutex); 4394 } 4395 } 4396 } 4397 mutex_enter(&ibdm.ibdm_mutex); 4398 ibdm.ibdm_busy &= ~IBDM_BUSY; 4399 cv_broadcast(&ibdm.ibdm_busy_cv); 4400 mutex_exit(&ibdm.ibdm_mutex); 4401 4402 ioc_info = ibdm_ibnex_get_ioc_info(ioc_guid); 4403 } 4404 return (ioc_info); 4405 } 4406 4407 4408 /* 4409 * ibdm_ibnex_get_ioc_info() 4410 * Returns pointer to ibdm_ioc_info_t if it finds 4411 * matching record for the ioc_guid, otherwise NULL 4412 * is returned 4413 */ 4414 ibdm_ioc_info_t * 4415 ibdm_ibnex_get_ioc_info(ib_guid_t ioc_guid) 4416 { 4417 int ii; 4418 ibdm_ioc_info_t *ioc = NULL, *tmp = NULL; 4419 ibdm_dp_gidinfo_t *gid_list; 4420 ib_dm_io_unitinfo_t *iou; 4421 4422 IBTF_DPRINTF_L4("ibdm", "\tibnex_get_ioc_info: GUID %llx", ioc_guid); 4423 4424 mutex_enter(&ibdm.ibdm_mutex); 4425 while (ibdm.ibdm_busy & IBDM_BUSY) 4426 cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex); 4427 ibdm.ibdm_busy |= IBDM_BUSY; 4428 4429 gid_list = ibdm.ibdm_dp_gidlist_head; 4430 while (gid_list) { 4431 mutex_enter(&gid_list->gl_mutex); 4432 if (gid_list->gl_state != IBDM_GID_PROBING_COMPLETE) { 4433 mutex_exit(&gid_list->gl_mutex); 4434 gid_list = gid_list->gl_next; 4435 continue; 4436 } 4437 if (gid_list->gl_iou == NULL) { 4438 IBTF_DPRINTF_L2("ibdm", 4439 "\tget_ioc_info: No IOU info"); 4440 mutex_exit(&gid_list->gl_mutex); 4441 gid_list = gid_list->gl_next; 4442 continue; 4443 } 4444 iou = &gid_list->gl_iou->iou_info; 4445 for (ii = 0; ii < iou->iou_num_ctrl_slots; ii++) { 4446 tmp = IBDM_GIDINFO2IOCINFO(gid_list, ii); 4447 if ((tmp->ioc_profile.ioc_guid == ioc_guid) && 4448 (tmp->ioc_state == IBDM_IOC_STATE_PROBE_SUCCESS)) { 4449 ioc = ibdm_dup_ioc_info(tmp, gid_list); 4450 mutex_exit(&gid_list->gl_mutex); 4451 ibdm.ibdm_busy &= ~IBDM_BUSY; 4452 cv_broadcast(&ibdm.ibdm_busy_cv); 4453 mutex_exit(&ibdm.ibdm_mutex); 4454 IBTF_DPRINTF_L4("ibdm", "\tget_ioc_info: End"); 4455 return (ioc); 4456 } 4457 } 4458 if (ii == iou->iou_num_ctrl_slots) 4459 ioc = NULL; 4460 4461 mutex_exit(&gid_list->gl_mutex); 4462 gid_list = gid_list->gl_next; 4463 } 4464 4465 ibdm.ibdm_busy &= ~IBDM_BUSY; 4466 cv_broadcast(&ibdm.ibdm_busy_cv); 4467 mutex_exit(&ibdm.ibdm_mutex); 4468 IBTF_DPRINTF_L4("ibdm", "\tget_ioc_info: failure End"); 4469 return (ioc); 4470 } 4471 4472 4473 /* 4474 * ibdm_ibnex_get_ioc_count() 4475 * Returns number of ibdm_ioc_info_t it finds 4476 */ 4477 int 4478 ibdm_ibnex_get_ioc_count(void) 4479 { 4480 int count = 0, k; 4481 ibdm_ioc_info_t *ioc; 4482 ibdm_dp_gidinfo_t *gid_list; 4483 4484 mutex_enter(&ibdm.ibdm_mutex); 4485 ibdm_sweep_fabric(0); 4486 4487 while (ibdm.ibdm_busy & IBDM_BUSY) 4488 cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex); 4489 ibdm.ibdm_busy |= IBDM_BUSY; 4490 4491 for (gid_list = ibdm.ibdm_dp_gidlist_head; gid_list; 4492 gid_list = gid_list->gl_next) { 4493 mutex_enter(&gid_list->gl_mutex); 4494 if ((gid_list->gl_state != IBDM_GID_PROBING_COMPLETE) || 4495 (gid_list->gl_iou == NULL)) { 4496 mutex_exit(&gid_list->gl_mutex); 4497 continue; 4498 } 4499 for (k = 0; k < gid_list->gl_iou->iou_info.iou_num_ctrl_slots; 4500 k++) { 4501 ioc = IBDM_GIDINFO2IOCINFO(gid_list, k); 4502 if (ioc->ioc_state == IBDM_IOC_STATE_PROBE_SUCCESS) 4503 ++count; 4504 } 4505 mutex_exit(&gid_list->gl_mutex); 4506 } 4507 ibdm.ibdm_busy &= ~IBDM_BUSY; 4508 cv_broadcast(&ibdm.ibdm_busy_cv); 4509 mutex_exit(&ibdm.ibdm_mutex); 4510 4511 IBTF_DPRINTF_L4("ibdm", "\tget_ioc_count: count = %d", count); 4512 return (count); 4513 } 4514 4515 4516 /* 4517 * ibdm_ibnex_get_ioc_list() 4518 * Returns information about all the IOCs present on the fabric. 4519 * Reprobes the IOCs and the GID list if list_flag is set to REPROBE_ALL. 4520 * Does not sweep fabric if DONOT_PROBE is set 4521 */ 4522 ibdm_ioc_info_t * 4523 ibdm_ibnex_get_ioc_list(ibdm_ibnex_get_ioclist_mtd_t list_flag) 4524 { 4525 int ii; 4526 ibdm_ioc_info_t *ioc_list = NULL, *tmp, *ioc; 4527 ibdm_dp_gidinfo_t *gid_list; 4528 ib_dm_io_unitinfo_t *iou; 4529 4530 IBTF_DPRINTF_L4("ibdm", "\tget_ioc_list: Enter"); 4531 4532 mutex_enter(&ibdm.ibdm_mutex); 4533 if (list_flag != IBDM_IBNEX_DONOT_PROBE) 4534 ibdm_sweep_fabric(list_flag == IBDM_IBNEX_REPROBE_ALL); 4535 4536 while (ibdm.ibdm_busy & IBDM_BUSY) 4537 cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex); 4538 ibdm.ibdm_busy |= IBDM_BUSY; 4539 4540 gid_list = ibdm.ibdm_dp_gidlist_head; 4541 while (gid_list) { 4542 mutex_enter(&gid_list->gl_mutex); 4543 if (gid_list->gl_state != IBDM_GID_PROBING_COMPLETE) { 4544 mutex_exit(&gid_list->gl_mutex); 4545 gid_list = gid_list->gl_next; 4546 continue; 4547 } 4548 if (gid_list->gl_iou == NULL) { 4549 IBTF_DPRINTF_L2("ibdm", 4550 "\tget_ioc_list: No IOU info"); 4551 mutex_exit(&gid_list->gl_mutex); 4552 gid_list = gid_list->gl_next; 4553 continue; 4554 } 4555 iou = &gid_list->gl_iou->iou_info; 4556 for (ii = 0; ii < iou->iou_num_ctrl_slots; ii++) { 4557 ioc = IBDM_GIDINFO2IOCINFO(gid_list, ii); 4558 if (ioc->ioc_state == IBDM_IOC_STATE_PROBE_SUCCESS) { 4559 tmp = ibdm_dup_ioc_info(ioc, gid_list); 4560 tmp->ioc_next = ioc_list; 4561 ioc_list = tmp; 4562 } 4563 } 4564 mutex_exit(&gid_list->gl_mutex); 4565 gid_list = gid_list->gl_next; 4566 } 4567 ibdm.ibdm_busy &= ~IBDM_BUSY; 4568 cv_broadcast(&ibdm.ibdm_busy_cv); 4569 mutex_exit(&ibdm.ibdm_mutex); 4570 4571 IBTF_DPRINTF_L4("ibdm", "\tget_ioc_list: End"); 4572 return (ioc_list); 4573 } 4574 4575 /* 4576 * ibdm_dup_ioc_info() 4577 * Duplicate the IOC information and return the IOC 4578 * information. 4579 */ 4580 static ibdm_ioc_info_t * 4581 ibdm_dup_ioc_info(ibdm_ioc_info_t *in_ioc, ibdm_dp_gidinfo_t *gid_list) 4582 { 4583 ibdm_ioc_info_t *out_ioc; 4584 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*out_ioc)); 4585 4586 out_ioc = kmem_alloc(sizeof (ibdm_ioc_info_t), KM_SLEEP); 4587 bcopy(in_ioc, out_ioc, sizeof (ibdm_ioc_info_t)); 4588 ibdm_update_ioc_port_gidlist(out_ioc, gid_list); 4589 out_ioc->ioc_iou_dc_valid = gid_list->gl_iou->iou_dc_valid; 4590 out_ioc->ioc_iou_diagcode = gid_list->gl_iou->iou_diagcode; 4591 4592 return (out_ioc); 4593 } 4594 4595 4596 /* 4597 * ibdm_free_ioc_list() 4598 * Deallocate memory for IOC list structure 4599 */ 4600 void 4601 ibdm_ibnex_free_ioc_list(ibdm_ioc_info_t *ioc) 4602 { 4603 ibdm_ioc_info_t *temp; 4604 4605 IBTF_DPRINTF_L4("ibdm", "\tibnex_free_ioc_list:"); 4606 while (ioc) { 4607 temp = ioc; 4608 ioc = ioc->ioc_next; 4609 kmem_free(temp->ioc_gid_list, 4610 (sizeof (ibdm_gid_t) * temp->ioc_nportgids)); 4611 if (temp->ioc_hca_list) 4612 ibdm_ibnex_free_hca_list(temp->ioc_hca_list); 4613 kmem_free(temp, sizeof (ibdm_ioc_info_t)); 4614 } 4615 } 4616 4617 4618 /* 4619 * ibdm_ibnex_update_pkey_tbls 4620 * Updates the DM P_Key database. 4621 * NOTE: Two cases are handled here: P_Key being added or removed. 4622 * 4623 * Arguments : NONE 4624 * Return Values : NONE 4625 */ 4626 void 4627 ibdm_ibnex_update_pkey_tbls(void) 4628 { 4629 int h, pp, pidx; 4630 uint_t nports; 4631 uint_t size; 4632 ib_pkey_t new_pkey; 4633 ib_pkey_t *orig_pkey; 4634 ibdm_hca_list_t *hca_list; 4635 ibdm_port_attr_t *port; 4636 ibt_hca_portinfo_t *pinfop; 4637 4638 IBTF_DPRINTF_L4("ibdm", "\tibnex_update_pkey_tbls:"); 4639 4640 mutex_enter(&ibdm.ibdm_hl_mutex); 4641 hca_list = ibdm.ibdm_hca_list_head; 4642 4643 for (h = 0; h < ibdm.ibdm_hca_count; h++) { 4644 4645 /* This updates P_Key Tables for all ports of this HCA */ 4646 (void) ibt_query_hca_ports(hca_list->hl_hca_hdl, 0, &pinfop, 4647 &nports, &size); 4648 4649 /* number of ports shouldn't have changed */ 4650 ASSERT(nports == hca_list->hl_nports); 4651 4652 for (pp = 0; pp < hca_list->hl_nports; pp++) { 4653 port = &hca_list->hl_port_attr[pp]; 4654 4655 /* 4656 * First figure out the P_Keys from IBTL. 4657 * Three things could have happened: 4658 * New P_Keys added 4659 * Existing P_Keys removed 4660 * Both of the above two 4661 * 4662 * Loop through the P_Key Indices and check if a 4663 * give P_Key_Ix matches that of the one seen by 4664 * IBDM. If they match no action is needed. 4665 * 4666 * If they don't match: 4667 * 1. if orig_pkey is invalid and new_pkey is valid 4668 * ---> add new_pkey to DM database 4669 * 2. if orig_pkey is valid and new_pkey is invalid 4670 * ---> remove orig_pkey from DM database 4671 * 3. if orig_pkey and new_pkey are both valid: 4672 * ---> remov orig_pkey from DM database 4673 * ---> add new_pkey to DM database 4674 * 4. if orig_pkey and new_pkey are both invalid: 4675 * ---> do nothing. Updated DM database. 4676 */ 4677 4678 for (pidx = 0; pidx < port->pa_npkeys; pidx++) { 4679 new_pkey = pinfop[pp].p_pkey_tbl[pidx]; 4680 orig_pkey = &port->pa_pkey_tbl[pidx].pt_pkey; 4681 4682 /* keys match - do nothing */ 4683 if (*orig_pkey == new_pkey) 4684 continue; 4685 4686 if (IBDM_INVALID_PKEY(*orig_pkey) && 4687 !IBDM_INVALID_PKEY(new_pkey)) { 4688 /* P_Key was added */ 4689 IBTF_DPRINTF_L5("ibdm", 4690 "\tibnex_update_pkey_tbls: new " 4691 "P_Key added = 0x%x", new_pkey); 4692 *orig_pkey = new_pkey; 4693 ibdm_port_attr_ibmf_init(port, 4694 new_pkey, pp); 4695 } else if (!IBDM_INVALID_PKEY(*orig_pkey) && 4696 IBDM_INVALID_PKEY(new_pkey)) { 4697 /* P_Key was removed */ 4698 IBTF_DPRINTF_L5("ibdm", 4699 "\tibnex_update_pkey_tbls: P_Key " 4700 "removed = 0x%x", *orig_pkey); 4701 *orig_pkey = new_pkey; 4702 (void) ibdm_port_attr_ibmf_fini(port, 4703 pidx); 4704 } else if (!IBDM_INVALID_PKEY(*orig_pkey) && 4705 !IBDM_INVALID_PKEY(new_pkey)) { 4706 /* P_Key were replaced */ 4707 IBTF_DPRINTF_L5("ibdm", 4708 "\tibnex_update_pkey_tbls: P_Key " 4709 "replaced 0x%x with 0x%x", 4710 *orig_pkey, new_pkey); 4711 (void) ibdm_port_attr_ibmf_fini(port, 4712 pidx); 4713 *orig_pkey = new_pkey; 4714 ibdm_port_attr_ibmf_init(port, 4715 new_pkey, pp); 4716 } else { 4717 /* 4718 * P_Keys are invalid 4719 * set anyway to reflect if 4720 * INVALID_FULL was changed to 4721 * INVALID_LIMITED or vice-versa. 4722 */ 4723 *orig_pkey = new_pkey; 4724 } /* end of else */ 4725 4726 } /* loop of p_key index */ 4727 4728 } /* loop of #ports of HCA */ 4729 4730 ibt_free_portinfo(pinfop, size); 4731 hca_list = hca_list->hl_next; 4732 4733 } /* loop for all HCAs in the system */ 4734 4735 mutex_exit(&ibdm.ibdm_hl_mutex); 4736 } 4737 4738 4739 /* 4740 * ibdm_send_ioc_profile() 4741 * Send IOC Controller Profile request. When the request is completed 4742 * IBMF calls ibdm_process_incoming_mad routine to inform about 4743 * the completion. 4744 */ 4745 static int 4746 ibdm_send_ioc_profile(ibdm_dp_gidinfo_t *gid_info, uint8_t ioc_no) 4747 { 4748 ibmf_msg_t *msg; 4749 ib_mad_hdr_t *hdr; 4750 ibdm_ioc_info_t *ioc_info = &(gid_info->gl_iou->iou_ioc_info[ioc_no]); 4751 ibdm_timeout_cb_args_t *cb_args; 4752 4753 IBTF_DPRINTF_L4("ibdm", "\tsend_ioc_profile: " 4754 "gid info 0x%p, ioc_no = %d", gid_info, ioc_no); 4755 4756 /* 4757 * Send command to get IOC profile. 4758 * Allocate a IBMF packet and initialize the packet. 4759 */ 4760 if (ibmf_alloc_msg(gid_info->gl_ibmf_hdl, IBMF_ALLOC_SLEEP, 4761 &msg) != IBMF_SUCCESS) { 4762 IBTF_DPRINTF_L4("ibdm", "\tsend_ioc_profile: pkt alloc fail"); 4763 return (IBDM_FAILURE); 4764 } 4765 4766 ibdm_alloc_send_buffers(msg); 4767 4768 mutex_enter(&gid_info->gl_mutex); 4769 ibdm_bump_transactionID(gid_info); 4770 mutex_exit(&gid_info->gl_mutex); 4771 4772 msg->im_local_addr.ia_local_lid = gid_info->gl_slid; 4773 msg->im_local_addr.ia_remote_lid = gid_info->gl_dlid; 4774 if (gid_info->gl_redirected == B_TRUE) { 4775 if (gid_info->gl_redirect_dlid != 0) { 4776 msg->im_local_addr.ia_remote_lid = 4777 gid_info->gl_redirect_dlid; 4778 } 4779 msg->im_local_addr.ia_remote_qno = gid_info->gl_redirect_QP; 4780 msg->im_local_addr.ia_p_key = gid_info->gl_redirect_pkey; 4781 msg->im_local_addr.ia_q_key = gid_info->gl_redirect_qkey; 4782 } else { 4783 msg->im_local_addr.ia_remote_qno = 1; 4784 msg->im_local_addr.ia_p_key = gid_info->gl_p_key; 4785 msg->im_local_addr.ia_q_key = IB_GSI_QKEY; 4786 } 4787 4788 hdr = IBDM_OUT_IBMFMSG_MADHDR(msg); 4789 hdr->BaseVersion = MAD_CLASS_BASE_VERS_1; 4790 hdr->MgmtClass = MAD_MGMT_CLASS_DEV_MGT; 4791 hdr->ClassVersion = IB_DM_CLASS_VERSION_1; 4792 hdr->R_Method = IB_DM_DEVMGT_METHOD_GET; 4793 hdr->Status = 0; 4794 hdr->TransactionID = h2b64(gid_info->gl_transactionID); 4795 hdr->AttributeID = h2b16(IB_DM_ATTR_IOC_CTRL_PROFILE); 4796 hdr->AttributeModifier = h2b32(ioc_no + 1); 4797 4798 ioc_info->ioc_state = IBDM_IOC_STATE_REPROBE_PROGRESS; 4799 cb_args = &ioc_info->ioc_cb_args; 4800 cb_args->cb_gid_info = gid_info; 4801 cb_args->cb_retry_count = ibdm_dft_retry_cnt; 4802 cb_args->cb_req_type = IBDM_REQ_TYPE_IOCINFO; 4803 cb_args->cb_ioc_num = ioc_no; 4804 4805 ioc_info->ioc_timeout_id = timeout(ibdm_pkt_timeout_hdlr, 4806 cb_args, IBDM_TIMEOUT_VALUE(ibdm_dft_timeout)); 4807 4808 IBTF_DPRINTF_L5("ibdm", "\tsend_ioc_profile:" 4809 "timeout %x", ioc_info->ioc_timeout_id); 4810 4811 if (ibmf_msg_transport(gid_info->gl_ibmf_hdl, gid_info->gl_qp_hdl, msg, 4812 NULL, ibdm_ibmf_send_cb, cb_args, 0) != IBMF_SUCCESS) { 4813 IBTF_DPRINTF_L2("ibdm", 4814 "\tsend_ioc_profile: msg transport failed"); 4815 ibdm_ibmf_send_cb(gid_info->gl_ibmf_hdl, msg, cb_args); 4816 } 4817 ioc_info->ioc_state = IBDM_IOC_STATE_REPROBE_PROGRESS; 4818 return (IBDM_SUCCESS); 4819 } 4820 4821 4822 /* 4823 * ibdm_port_reachable 4824 * Sends a SA query to get the NODE record for port GUID 4825 * Returns IBDM_SUCCESS if the port GID is reachable 4826 */ 4827 static int 4828 ibdm_port_reachable(ibmf_saa_handle_t sa_hdl, ib_guid_t guid, 4829 ib_guid_t *node_guid) 4830 { 4831 sa_node_record_t req, *resp = NULL; 4832 ibmf_saa_access_args_t args; 4833 int ret; 4834 size_t length; 4835 4836 IBTF_DPRINTF_L4("ibdm", "\tport_reachable: port_guid %llx", 4837 guid); 4838 4839 bzero(&req, sizeof (sa_node_record_t)); 4840 req.NodeInfo.PortGUID = guid; 4841 4842 args.sq_attr_id = SA_NODERECORD_ATTRID; 4843 args.sq_access_type = IBMF_SAA_RETRIEVE; 4844 args.sq_component_mask = SA_NODEINFO_COMPMASK_PORTGUID; 4845 args.sq_template = &req; 4846 args.sq_callback = NULL; 4847 args.sq_callback_arg = NULL; 4848 4849 ret = ibmf_sa_access(sa_hdl, &args, 0, &length, (void **) &resp); 4850 if (ret != IBMF_SUCCESS) { 4851 IBTF_DPRINTF_L2("ibdm", "\tport_reachable:" 4852 " SA Retrieve Failed: %d", ret); 4853 return (IBDM_FAILURE); 4854 } 4855 4856 if ((resp == NULL) || (length == 0)) { 4857 IBTF_DPRINTF_L2("ibdm", "\tport_reachable: No records"); 4858 return (IBDM_FAILURE); 4859 } 4860 4861 if (node_guid != NULL) 4862 *node_guid = resp->NodeInfo.NodeGUID; 4863 4864 kmem_free(resp, length); 4865 4866 return (IBDM_SUCCESS); 4867 } 4868 4869 /* 4870 * Update the gidlist for all affected IOCs when GID becomes 4871 * available/unavailable. 4872 * 4873 * Parameters : 4874 * gidinfo - Incoming / Outgoing GID. 4875 * add_flag - 1 for GID added, 0 for GID removed. 4876 * - (-1) : IOC gid list updated, ioc_list required. 4877 * 4878 * This function gets the GID for the node GUID corresponding to the 4879 * port GID. Gets the IOU info 4880 */ 4881 static ibdm_ioc_info_t * 4882 ibdm_update_ioc_gidlist(ibdm_dp_gidinfo_t *gid_info, int avail_flag) 4883 { 4884 ibdm_dp_gidinfo_t *node_gid = NULL; 4885 uint8_t niocs, ii; 4886 ibdm_ioc_info_t *ioc, *ioc_list = NULL, *tmp; 4887 4888 IBTF_DPRINTF_L4("ibdm", "\tupdate_ioc_gidlist"); 4889 4890 switch (avail_flag) { 4891 case 1 : 4892 node_gid = ibdm_check_dest_nodeguid(gid_info); 4893 break; 4894 case 0 : 4895 node_gid = ibdm_handle_gid_rm(gid_info); 4896 break; 4897 case -1 : 4898 node_gid = gid_info; 4899 break; 4900 default : 4901 break; 4902 } 4903 4904 if (node_gid == NULL) { 4905 IBTF_DPRINTF_L4("ibdm", "\tupdate_ioc_gidlist: " 4906 "No node GID found, port gid 0x%p, avail_flag %d", 4907 gid_info, avail_flag); 4908 return (NULL); 4909 } 4910 4911 mutex_enter(&node_gid->gl_mutex); 4912 if ((node_gid->gl_state != IBDM_GID_PROBING_COMPLETE && 4913 node_gid->gl_state != IBDM_GID_PROBING_SKIPPED) || 4914 node_gid->gl_iou == NULL) { 4915 IBTF_DPRINTF_L4("ibdm", "\tupdate_ioc_gidlist " 4916 "gl_state %x, gl_iou %p", node_gid->gl_state, 4917 node_gid->gl_iou); 4918 mutex_exit(&node_gid->gl_mutex); 4919 return (NULL); 4920 } 4921 4922 niocs = node_gid->gl_iou->iou_info.iou_num_ctrl_slots; 4923 IBTF_DPRINTF_L4("ibdm", "\tupdate_ioc_gidlist : niocs %x", 4924 niocs); 4925 for (ii = 0; ii < niocs; ii++) { 4926 ioc = IBDM_GIDINFO2IOCINFO(node_gid, ii); 4927 /* 4928 * Skip IOCs for which probe is not complete or 4929 * reprobe is progress 4930 */ 4931 if (ioc->ioc_state == IBDM_IOC_STATE_PROBE_SUCCESS) { 4932 tmp = ibdm_dup_ioc_info(ioc, node_gid); 4933 tmp->ioc_info_updated.ib_gid_prop_updated = 1; 4934 tmp->ioc_next = ioc_list; 4935 ioc_list = tmp; 4936 } 4937 } 4938 mutex_exit(&node_gid->gl_mutex); 4939 4940 IBTF_DPRINTF_L4("ibdm", "\tupdate_ioc_gidlist : return %p", 4941 ioc_list); 4942 return (ioc_list); 4943 } 4944 4945 /* 4946 * ibdm_saa_event_cb : 4947 * Event handling which does *not* require ibdm_hl_mutex to be 4948 * held are executed in the same thread. This is to prevent 4949 * deadlocks with HCA port down notifications which hold the 4950 * ibdm_hl_mutex. 4951 * 4952 * GID_AVAILABLE event is handled here. A taskq is spawned to 4953 * handle GID_UNAVAILABLE. 4954 * 4955 * A new mutex ibdm_ibnex_mutex has been introduced to protect 4956 * ibnex_callback. This has been done to prevent any possible 4957 * deadlock (described above) while handling GID_AVAILABLE. 4958 * 4959 * IBMF calls the event callback for a HCA port. The SA handle 4960 * for this port would be valid, till the callback returns. 4961 * IBDM calling IBDM using the above SA handle should be valid. 4962 * 4963 * IBDM will additionally check (SA handle != NULL), before 4964 * calling IBMF. 4965 */ 4966 /*ARGSUSED*/ 4967 static void 4968 ibdm_saa_event_cb(ibmf_saa_handle_t ibmf_saa_handle, 4969 ibmf_saa_subnet_event_t ibmf_saa_event, 4970 ibmf_saa_event_details_t *event_details, void *callback_arg) 4971 { 4972 ibdm_saa_event_arg_t *event_arg; 4973 ib_gid_t sgid, dgid; 4974 ibdm_port_attr_t *hca_port; 4975 ibdm_dp_gidinfo_t *gid_info, *node_gid_info = NULL; 4976 ib_guid_t nodeguid; 4977 4978 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*event_arg)); 4979 4980 hca_port = (ibdm_port_attr_t *)callback_arg; 4981 4982 IBTF_DPRINTF_L4("ibdm", "\tsaa_event_cb(%x, %x, %x, %x)\n", 4983 ibmf_saa_handle, ibmf_saa_event, event_details, 4984 callback_arg); 4985 #ifdef DEBUG 4986 if (ibdm_ignore_saa_event) 4987 return; 4988 #endif 4989 4990 if (ibmf_saa_event == IBMF_SAA_EVENT_GID_AVAILABLE) { 4991 /* 4992 * Ensure no other probe / sweep fabric is in 4993 * progress. 4994 */ 4995 mutex_enter(&ibdm.ibdm_mutex); 4996 while (ibdm.ibdm_busy & IBDM_BUSY) 4997 cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex); 4998 ibdm.ibdm_busy |= IBDM_BUSY; 4999 mutex_exit(&ibdm.ibdm_mutex); 5000 5001 /* 5002 * If we already know about this GID, return. 5003 * GID_AVAILABLE may be reported for multiple HCA 5004 * ports. 5005 */ 5006 if ((ibdm_check_dgid(event_details->ie_gid.gid_guid, 5007 event_details->ie_gid.gid_prefix)) != NULL) { 5008 mutex_enter(&ibdm.ibdm_mutex); 5009 ibdm.ibdm_busy &= ~IBDM_BUSY; 5010 cv_broadcast(&ibdm.ibdm_busy_cv); 5011 mutex_exit(&ibdm.ibdm_mutex); 5012 return; 5013 } 5014 5015 IBTF_DPRINTF_L4("ibdm", "\tGID (prefix %x, guid %llx) " 5016 "Insertion notified", 5017 event_details->ie_gid.gid_prefix, 5018 event_details->ie_gid.gid_guid); 5019 5020 /* This is a new gid, insert it to GID list */ 5021 sgid.gid_prefix = hca_port->pa_sn_prefix; 5022 sgid.gid_guid = hca_port->pa_port_guid; 5023 dgid.gid_prefix = event_details->ie_gid.gid_prefix; 5024 dgid.gid_guid = event_details->ie_gid.gid_guid; 5025 gid_info = ibdm_create_gid_info(hca_port, sgid, dgid); 5026 if (gid_info == NULL) { 5027 IBTF_DPRINTF_L4("ibdm", "\tGID_AVAILABLE: " 5028 "create_gid_info returned NULL"); 5029 mutex_enter(&ibdm.ibdm_mutex); 5030 ibdm.ibdm_busy &= ~IBDM_BUSY; 5031 cv_broadcast(&ibdm.ibdm_busy_cv); 5032 mutex_exit(&ibdm.ibdm_mutex); 5033 return; 5034 } 5035 mutex_enter(&gid_info->gl_mutex); 5036 gid_info->gl_state = IBDM_GID_PROBING_SKIPPED; 5037 mutex_exit(&gid_info->gl_mutex); 5038 5039 /* Get the node GUID */ 5040 if (ibdm_port_reachable(ibmf_saa_handle, dgid.gid_guid, 5041 &nodeguid) != IBDM_SUCCESS) { 5042 /* 5043 * Set the state to PROBE_NOT_DONE for the 5044 * next sweep to probe it 5045 */ 5046 IBTF_DPRINTF_L2("ibdm", "\tsaa_event_taskq: " 5047 "Skipping GID : port GUID not found"); 5048 mutex_enter(&gid_info->gl_mutex); 5049 gid_info->gl_state = IBDM_GID_PROBE_NOT_DONE; 5050 mutex_exit(&gid_info->gl_mutex); 5051 mutex_enter(&ibdm.ibdm_mutex); 5052 ibdm.ibdm_busy &= ~IBDM_BUSY; 5053 cv_broadcast(&ibdm.ibdm_busy_cv); 5054 mutex_exit(&ibdm.ibdm_mutex); 5055 return; 5056 } 5057 5058 gid_info->gl_nodeguid = nodeguid; 5059 gid_info->gl_portguid = dgid.gid_guid; 5060 5061 /* 5062 * Get the gid info with the same node GUID. 5063 */ 5064 mutex_enter(&ibdm.ibdm_mutex); 5065 node_gid_info = ibdm.ibdm_dp_gidlist_head; 5066 while (node_gid_info) { 5067 if (node_gid_info->gl_nodeguid == 5068 gid_info->gl_nodeguid && 5069 node_gid_info->gl_iou != NULL) { 5070 break; 5071 } 5072 node_gid_info = node_gid_info->gl_next; 5073 } 5074 mutex_exit(&ibdm.ibdm_mutex); 5075 5076 /* 5077 * Handling a new GID requires filling of gl_hca_list. 5078 * This require ibdm hca_list to be parsed and hence 5079 * holding the ibdm_hl_mutex. Spawning a new thread to 5080 * handle this. 5081 */ 5082 if (node_gid_info == NULL) { 5083 if (taskq_dispatch(system_taskq, 5084 ibdm_saa_handle_new_gid, (void *)gid_info, 5085 TQ_NOSLEEP) == NULL) { 5086 IBTF_DPRINTF_L2("ibdm", "\tsaa_event_cb: " 5087 "new_gid taskq_dispatch failed"); 5088 return; 5089 } 5090 } 5091 5092 mutex_enter(&ibdm.ibdm_mutex); 5093 ibdm.ibdm_busy &= ~IBDM_BUSY; 5094 cv_broadcast(&ibdm.ibdm_busy_cv); 5095 mutex_exit(&ibdm.ibdm_mutex); 5096 return; 5097 } 5098 5099 if (ibmf_saa_event != IBMF_SAA_EVENT_GID_UNAVAILABLE) 5100 return; 5101 5102 event_arg = (ibdm_saa_event_arg_t *)kmem_alloc( 5103 sizeof (ibdm_saa_event_arg_t), KM_SLEEP); 5104 event_arg->ibmf_saa_handle = ibmf_saa_handle; 5105 event_arg->ibmf_saa_event = ibmf_saa_event; 5106 bcopy(event_details, &event_arg->event_details, 5107 sizeof (ibmf_saa_event_details_t)); 5108 event_arg->callback_arg = callback_arg; 5109 5110 if (taskq_dispatch(system_taskq, ibdm_saa_event_taskq, 5111 (void *)event_arg, TQ_NOSLEEP) == NULL) { 5112 IBTF_DPRINTF_L2("ibdm", "\tsaa_event_cb: " 5113 "taskq_dispatch failed"); 5114 ibdm_free_saa_event_arg(event_arg); 5115 return; 5116 } 5117 } 5118 5119 /* 5120 * Handle a new GID discovered by GID_AVAILABLE saa event. 5121 */ 5122 void 5123 ibdm_saa_handle_new_gid(void *arg) 5124 { 5125 ibdm_dp_gidinfo_t *gid_info; 5126 ibdm_hca_list_t *hca_list = NULL; 5127 ibdm_port_attr_t *port = NULL; 5128 ibdm_ioc_info_t *ioc_list = NULL; 5129 5130 IBTF_DPRINTF_L4(ibdm_string, "\tsaa_handle_new_gid(%p)", arg); 5131 5132 gid_info = (ibdm_dp_gidinfo_t *)arg; 5133 5134 /* 5135 * Ensure that no other sweep / probe has completed 5136 * probing this gid. 5137 */ 5138 mutex_enter(&gid_info->gl_mutex); 5139 if (gid_info->gl_state != IBDM_GID_PROBE_NOT_DONE) { 5140 mutex_exit(&gid_info->gl_mutex); 5141 return; 5142 } 5143 mutex_exit(&gid_info->gl_mutex); 5144 5145 /* 5146 * Parse HCAs to fill gl_hca_list 5147 */ 5148 mutex_enter(&ibdm.ibdm_hl_mutex); 5149 for (ibdm_get_next_port(&hca_list, &port, 1); port; 5150 ibdm_get_next_port(&hca_list, &port, 1)) { 5151 if (ibdm_port_reachable(port->pa_sa_hdl, 5152 gid_info->gl_portguid, NULL) == 5153 IBDM_SUCCESS) { 5154 ibdm_addto_glhcalist(gid_info, hca_list); 5155 } 5156 } 5157 mutex_exit(&ibdm.ibdm_hl_mutex); 5158 5159 /* 5160 * Ensure no other probe / sweep fabric is in 5161 * progress. 5162 */ 5163 mutex_enter(&ibdm.ibdm_mutex); 5164 while (ibdm.ibdm_busy & IBDM_BUSY) 5165 cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex); 5166 ibdm.ibdm_busy |= IBDM_BUSY; 5167 mutex_exit(&ibdm.ibdm_mutex); 5168 5169 /* 5170 * New IOU probe it, to check if new IOCs 5171 */ 5172 IBTF_DPRINTF_L4(ibdm_string, "\tsaa_handle_new_gid: " 5173 "new GID : probing"); 5174 mutex_enter(&ibdm.ibdm_mutex); 5175 ibdm.ibdm_ngid_probes_in_progress++; 5176 mutex_exit(&ibdm.ibdm_mutex); 5177 mutex_enter(&gid_info->gl_mutex); 5178 gid_info->gl_reprobe_flag = 0; 5179 gid_info->gl_state = IBDM_GID_PROBE_NOT_DONE; 5180 mutex_exit(&gid_info->gl_mutex); 5181 ibdm_probe_gid_thread((void *)gid_info); 5182 5183 mutex_enter(&ibdm.ibdm_mutex); 5184 ibdm_wait_probe_completion(); 5185 mutex_exit(&ibdm.ibdm_mutex); 5186 5187 if (gid_info->gl_iou == NULL) { 5188 mutex_enter(&ibdm.ibdm_mutex); 5189 ibdm.ibdm_busy &= ~IBDM_BUSY; 5190 cv_broadcast(&ibdm.ibdm_busy_cv); 5191 mutex_exit(&ibdm.ibdm_mutex); 5192 return; 5193 } 5194 5195 /* 5196 * Update GID list in all IOCs affected by this 5197 */ 5198 ioc_list = ibdm_update_ioc_gidlist(gid_info, 1); 5199 5200 /* 5201 * Pass on the IOCs with updated GIDs to IBnexus 5202 */ 5203 if (ioc_list) { 5204 mutex_enter(&ibdm.ibdm_ibnex_mutex); 5205 if (ibdm.ibdm_ibnex_callback != NULL) { 5206 (*ibdm.ibdm_ibnex_callback)((void *)ioc_list, 5207 IBDM_EVENT_IOC_PROP_UPDATE); 5208 } 5209 mutex_exit(&ibdm.ibdm_ibnex_mutex); 5210 } 5211 5212 mutex_enter(&ibdm.ibdm_mutex); 5213 ibdm.ibdm_busy &= ~IBDM_BUSY; 5214 cv_broadcast(&ibdm.ibdm_busy_cv); 5215 mutex_exit(&ibdm.ibdm_mutex); 5216 } 5217 5218 /* 5219 * ibdm_saa_event_taskq : 5220 * GID_UNAVAILABLE Event handling requires ibdm_hl_mutex to be 5221 * held. The GID_UNAVAILABLE handling is done in a taskq to 5222 * prevent deadlocks with HCA port down notifications which hold 5223 * ibdm_hl_mutex. 5224 */ 5225 void 5226 ibdm_saa_event_taskq(void *arg) 5227 { 5228 ibdm_saa_event_arg_t *event_arg; 5229 ibmf_saa_handle_t ibmf_saa_handle; 5230 ibmf_saa_subnet_event_t ibmf_saa_event; 5231 ibmf_saa_event_details_t *event_details; 5232 void *callback_arg; 5233 5234 ibdm_dp_gidinfo_t *gid_info; 5235 ibdm_port_attr_t *hca_port, *port = NULL; 5236 ibdm_hca_list_t *hca_list = NULL; 5237 int sa_handle_valid = 0; 5238 ibdm_ioc_info_t *ioc_list = NULL; 5239 5240 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*event_arg)); 5241 5242 event_arg = (ibdm_saa_event_arg_t *)arg; 5243 ibmf_saa_handle = event_arg->ibmf_saa_handle; 5244 ibmf_saa_event = event_arg->ibmf_saa_event; 5245 event_details = &event_arg->event_details; 5246 callback_arg = event_arg->callback_arg; 5247 5248 ASSERT(callback_arg != NULL); 5249 ASSERT(ibmf_saa_event == IBMF_SAA_EVENT_GID_UNAVAILABLE); 5250 IBTF_DPRINTF_L4("ibdm", "\tsaa_event_taskq(%x, %x, %x, %x)", 5251 ibmf_saa_handle, ibmf_saa_event, event_details, 5252 callback_arg); 5253 5254 hca_port = (ibdm_port_attr_t *)callback_arg; 5255 5256 /* Check if the port_attr is still valid */ 5257 mutex_enter(&ibdm.ibdm_hl_mutex); 5258 for (ibdm_get_next_port(&hca_list, &port, 0); port; 5259 ibdm_get_next_port(&hca_list, &port, 0)) { 5260 if (port == hca_port && port->pa_port_guid == 5261 hca_port->pa_port_guid) { 5262 if (ibmf_saa_handle == hca_port->pa_sa_hdl) 5263 sa_handle_valid = 1; 5264 break; 5265 } 5266 } 5267 mutex_exit(&ibdm.ibdm_hl_mutex); 5268 if (sa_handle_valid == 0) { 5269 ibdm_free_saa_event_arg(event_arg); 5270 return; 5271 } 5272 5273 if (hca_port && (hca_port->pa_sa_hdl == NULL || 5274 ibmf_saa_handle != hca_port->pa_sa_hdl)) { 5275 ibdm_free_saa_event_arg(event_arg); 5276 return; 5277 } 5278 hca_list = NULL; 5279 port = NULL; 5280 5281 /* 5282 * Check if the GID is visible to other HCA ports. 5283 * Return if so. 5284 */ 5285 mutex_enter(&ibdm.ibdm_hl_mutex); 5286 for (ibdm_get_next_port(&hca_list, &port, 1); port; 5287 ibdm_get_next_port(&hca_list, &port, 1)) { 5288 if (ibdm_port_reachable(port->pa_sa_hdl, 5289 event_details->ie_gid.gid_guid, NULL) == 5290 IBDM_SUCCESS) { 5291 mutex_exit(&ibdm.ibdm_hl_mutex); 5292 ibdm_free_saa_event_arg(event_arg); 5293 return; 5294 } 5295 } 5296 mutex_exit(&ibdm.ibdm_hl_mutex); 5297 5298 /* 5299 * Ensure no other probe / sweep fabric is in 5300 * progress. 5301 */ 5302 mutex_enter(&ibdm.ibdm_mutex); 5303 while (ibdm.ibdm_busy & IBDM_BUSY) 5304 cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex); 5305 ibdm.ibdm_busy |= IBDM_BUSY; 5306 mutex_exit(&ibdm.ibdm_mutex); 5307 5308 /* 5309 * If this GID is no longer in GID list, return 5310 * GID_UNAVAILABLE may be reported for multiple HCA 5311 * ports. 5312 */ 5313 mutex_enter(&ibdm.ibdm_mutex); 5314 gid_info = ibdm.ibdm_dp_gidlist_head; 5315 while (gid_info) { 5316 if (gid_info->gl_portguid == 5317 event_details->ie_gid.gid_guid) { 5318 break; 5319 } 5320 gid_info = gid_info->gl_next; 5321 } 5322 mutex_exit(&ibdm.ibdm_mutex); 5323 if (gid_info == NULL) { 5324 mutex_enter(&ibdm.ibdm_mutex); 5325 ibdm.ibdm_busy &= ~IBDM_BUSY; 5326 cv_broadcast(&ibdm.ibdm_busy_cv); 5327 mutex_exit(&ibdm.ibdm_mutex); 5328 ibdm_free_saa_event_arg(event_arg); 5329 return; 5330 } 5331 5332 IBTF_DPRINTF_L4("ibdm", "\tGID (prefix %x, guid %llx) " 5333 "Unavailable notification", 5334 event_details->ie_gid.gid_prefix, 5335 event_details->ie_gid.gid_guid); 5336 5337 /* 5338 * Update GID list in all IOCs affected by this 5339 */ 5340 if (gid_info->gl_state == IBDM_GID_PROBING_SKIPPED || 5341 gid_info->gl_state == IBDM_GID_PROBING_COMPLETE) 5342 ioc_list = ibdm_update_ioc_gidlist(gid_info, 0); 5343 5344 /* 5345 * Remove GID from the global GID list 5346 * Handle the case where all port GIDs for an 5347 * IOU have been hot-removed. Check both gid_info 5348 * & ioc_info for checking ngids. 5349 */ 5350 mutex_enter(&ibdm.ibdm_mutex); 5351 if (gid_info->gl_iou != NULL && gid_info->gl_ngids == 0) { 5352 mutex_enter(&gid_info->gl_mutex); 5353 (void) ibdm_free_iou_info(gid_info); 5354 mutex_exit(&gid_info->gl_mutex); 5355 } 5356 if (gid_info->gl_prev != NULL) 5357 gid_info->gl_prev->gl_next = gid_info->gl_next; 5358 if (gid_info->gl_next != NULL) 5359 gid_info->gl_next->gl_prev = gid_info->gl_prev; 5360 5361 if (gid_info == ibdm.ibdm_dp_gidlist_head) 5362 ibdm.ibdm_dp_gidlist_head = gid_info->gl_next; 5363 if (gid_info == ibdm.ibdm_dp_gidlist_tail) 5364 ibdm.ibdm_dp_gidlist_tail = gid_info->gl_prev; 5365 ibdm.ibdm_ngids--; 5366 5367 ibdm.ibdm_busy &= ~IBDM_BUSY; 5368 cv_broadcast(&ibdm.ibdm_busy_cv); 5369 mutex_exit(&ibdm.ibdm_mutex); 5370 5371 mutex_destroy(&gid_info->gl_mutex); 5372 kmem_free(gid_info, sizeof (ibdm_dp_gidinfo_t)); 5373 5374 /* 5375 * Pass on the IOCs with updated GIDs to IBnexus 5376 */ 5377 if (ioc_list) { 5378 IBTF_DPRINTF_L4("ibdm", "\tGID_UNAVAILABLE " 5379 "IOC_PROP_UPDATE for %p\n", ioc_list); 5380 mutex_enter(&ibdm.ibdm_ibnex_mutex); 5381 if (ibdm.ibdm_ibnex_callback != NULL) { 5382 (*ibdm.ibdm_ibnex_callback)((void *) 5383 ioc_list, IBDM_EVENT_IOC_PROP_UPDATE); 5384 } 5385 mutex_exit(&ibdm.ibdm_ibnex_mutex); 5386 } 5387 5388 ibdm_free_saa_event_arg(event_arg); 5389 } 5390 5391 5392 static int 5393 ibdm_cmp_gid_list(ibdm_gid_t *new, ibdm_gid_t *prev) 5394 { 5395 ibdm_gid_t *scan_new, *scan_prev; 5396 int cmp_failed = 0; 5397 5398 ASSERT(new != NULL); 5399 ASSERT(prev != NULL); 5400 5401 /* 5402 * Search for each new gid anywhere in the prev GID list. 5403 * Note that the gid list could have been re-ordered. 5404 */ 5405 for (scan_new = new; scan_new; scan_new = scan_new->gid_next) { 5406 for (scan_prev = prev, cmp_failed = 1; scan_prev; 5407 scan_prev = scan_prev->gid_next) { 5408 if (scan_prev->gid_dgid_hi == scan_new->gid_dgid_hi && 5409 scan_prev->gid_dgid_lo == scan_new->gid_dgid_lo) { 5410 cmp_failed = 0; 5411 break; 5412 } 5413 } 5414 5415 if (cmp_failed) 5416 return (1); 5417 } 5418 return (0); 5419 } 5420 5421 /* 5422 * This is always called in a single thread 5423 * This function updates the gid_list and serv_list of IOC 5424 * The current gid_list is in ioc_info_t(contains only port 5425 * guids for which probe is done) & gidinfo_t(other port gids) 5426 * The gids in both locations are used for comparision. 5427 */ 5428 static void 5429 ibdm_reprobe_update_port_srv(ibdm_ioc_info_t *ioc, ibdm_dp_gidinfo_t *gidinfo) 5430 { 5431 ibdm_gid_t *cur_gid_list; 5432 uint_t cur_nportgids; 5433 5434 ASSERT(MUTEX_HELD(&ibdm.ibdm_mutex)); 5435 5436 ioc->ioc_info_updated.ib_prop_updated = 0; 5437 5438 5439 /* Current GID list in gid_info only */ 5440 cur_gid_list = gidinfo->gl_gid; 5441 cur_nportgids = gidinfo->gl_ngids; 5442 5443 /* 5444 * Service entry names and IDs are not compared currently. 5445 * This may require change. 5446 */ 5447 if (ioc->ioc_prev_serv_cnt != ioc->ioc_profile.ioc_service_entries) 5448 ioc->ioc_info_updated.ib_srv_prop_updated = 1; 5449 5450 if (ioc->ioc_prev_nportgids != cur_nportgids || 5451 ioc->ioc_prev_gid_list == NULL || cur_gid_list == NULL) { 5452 ioc->ioc_info_updated.ib_gid_prop_updated = 1; 5453 } else if (ibdm_cmp_gid_list(ioc->ioc_prev_gid_list, cur_gid_list)) { 5454 ioc->ioc_info_updated.ib_gid_prop_updated = 1; 5455 } 5456 5457 /* Zero out previous entries */ 5458 ibdm_free_gid_list(ioc->ioc_prev_gid_list); 5459 if (ioc->ioc_prev_serv) 5460 kmem_free(ioc->ioc_prev_serv, ioc->ioc_prev_serv_cnt * 5461 sizeof (ibdm_srvents_info_t)); 5462 ioc->ioc_prev_serv_cnt = 0; 5463 ioc->ioc_prev_nportgids = 0; 5464 ioc->ioc_prev_serv = NULL; 5465 ioc->ioc_prev_gid_list = NULL; 5466 } 5467 5468 /* 5469 * Handle GID removal. This returns gid_info of an GID for the same 5470 * node GUID, if found. For an GID with IOU information, the same 5471 * gid_info is returned if no gid_info with same node_guid is found. 5472 */ 5473 static ibdm_dp_gidinfo_t * 5474 ibdm_handle_gid_rm(ibdm_dp_gidinfo_t *rm_gid) 5475 { 5476 ibdm_dp_gidinfo_t *gid_list; 5477 5478 IBTF_DPRINTF_L4("ibdm", "\thandle_gid_rm(0x%p)", rm_gid); 5479 5480 if (rm_gid->gl_iou == NULL) { 5481 IBTF_DPRINTF_L4("ibdm", "\thandle_gid_rm NO iou"); 5482 /* 5483 * Search for a GID with same node_guid and 5484 * gl_iou != NULL 5485 */ 5486 for (gid_list = ibdm.ibdm_dp_gidlist_head; gid_list; 5487 gid_list = gid_list->gl_next) { 5488 if (gid_list->gl_iou != NULL && (gid_list->gl_nodeguid 5489 == rm_gid->gl_nodeguid)) 5490 break; 5491 } 5492 5493 if (gid_list) 5494 ibdm_rmfrom_glgid_list(gid_list, rm_gid); 5495 5496 IBTF_DPRINTF_L4("ibdm", "\thandle_gid_rm ret %p", gid_list); 5497 return (gid_list); 5498 } else { 5499 /* 5500 * Search for a GID with same node_guid and 5501 * gl_iou == NULL 5502 */ 5503 IBTF_DPRINTF_L4("ibdm", "\thandle_gid_rm with iou"); 5504 for (gid_list = ibdm.ibdm_dp_gidlist_head; gid_list; 5505 gid_list = gid_list->gl_next) { 5506 if (gid_list->gl_iou == NULL && (gid_list->gl_nodeguid 5507 == rm_gid->gl_nodeguid)) 5508 break; 5509 } 5510 5511 if (gid_list) { 5512 /* 5513 * Copy the following fields from rm_gid : 5514 * 1. gl_state 5515 * 2. gl_iou 5516 * 3. gl_gid & gl_ngids 5517 * 5518 * Note : Function is synchronized by 5519 * ibdm_busy flag. 5520 * 5521 * Note : Redirect info is initialized if 5522 * any MADs for the GID fail 5523 */ 5524 IBTF_DPRINTF_L4("ibdm", "\thandle_gid_rm " 5525 "copying info to GID with gl_iou != NULl"); 5526 gid_list->gl_state = rm_gid->gl_state; 5527 gid_list->gl_iou = rm_gid->gl_iou; 5528 gid_list->gl_gid = rm_gid->gl_gid; 5529 gid_list->gl_ngids = rm_gid->gl_ngids; 5530 5531 /* Remove the GID from gl_gid list */ 5532 ibdm_rmfrom_glgid_list(gid_list, rm_gid); 5533 } else { 5534 /* 5535 * Handle a case where all GIDs to the IOU have 5536 * been removed. 5537 */ 5538 IBTF_DPRINTF_L4("ibdm", "\thandle_gid_rm 0 GID " 5539 "to IOU"); 5540 5541 ibdm_rmfrom_glgid_list(rm_gid, rm_gid); 5542 return (rm_gid); 5543 } 5544 IBTF_DPRINTF_L4("ibdm", "\thandle_gid_rm ret %p", gid_list); 5545 return (gid_list); 5546 } 5547 } 5548 5549 static void 5550 ibdm_rmfrom_glgid_list(ibdm_dp_gidinfo_t *gid_info, 5551 ibdm_dp_gidinfo_t *rm_gid) 5552 { 5553 ibdm_gid_t *tmp, *prev; 5554 5555 IBTF_DPRINTF_L4("ibdm", "\trmfrom_glgid (%p, %p)", 5556 gid_info, rm_gid); 5557 5558 for (tmp = gid_info->gl_gid, prev = NULL; tmp; ) { 5559 if (tmp->gid_dgid_hi == rm_gid->gl_dgid_hi && 5560 tmp->gid_dgid_lo == rm_gid->gl_dgid_lo) { 5561 if (prev == NULL) 5562 gid_info->gl_gid = tmp->gid_next; 5563 else 5564 prev->gid_next = tmp->gid_next; 5565 5566 kmem_free(tmp, sizeof (ibdm_gid_t)); 5567 gid_info->gl_ngids--; 5568 break; 5569 } else { 5570 prev = tmp; 5571 tmp = tmp->gid_next; 5572 } 5573 } 5574 } 5575 5576 static void 5577 ibdm_addto_gidlist(ibdm_gid_t **src_ptr, ibdm_gid_t *dest) 5578 { 5579 ibdm_gid_t *head = NULL, *new, *tail; 5580 5581 /* First copy the destination */ 5582 for (; dest; dest = dest->gid_next) { 5583 new = kmem_zalloc(sizeof (ibdm_gid_t), KM_SLEEP); 5584 new->gid_dgid_hi = dest->gid_dgid_hi; 5585 new->gid_dgid_lo = dest->gid_dgid_lo; 5586 new->gid_next = head; 5587 head = new; 5588 } 5589 5590 /* Insert this to the source */ 5591 if (*src_ptr == NULL) 5592 *src_ptr = head; 5593 else { 5594 for (tail = *src_ptr; tail->gid_next != NULL; 5595 tail = tail->gid_next) 5596 ; 5597 5598 tail->gid_next = head; 5599 } 5600 } 5601 5602 static void 5603 ibdm_free_gid_list(ibdm_gid_t *head) 5604 { 5605 ibdm_gid_t *delete; 5606 5607 for (delete = head; delete; ) { 5608 head = delete->gid_next; 5609 kmem_free(delete, sizeof (ibdm_gid_t)); 5610 delete = head; 5611 } 5612 } 5613 5614 /* 5615 * This function rescans the DM capable GIDs (gl_state is 5616 * GID_PROBE_COMPLETE or IBDM_GID_PROBING_SKIPPED.This 5617 * basically checks if the DM capable GID is reachable. If 5618 * not this is handled the same way as GID_UNAVAILABLE, 5619 * except that notifications are not send to IBnexus. 5620 * 5621 * This function also initializes the ioc_prev_list for 5622 * a particular IOC (when called from probe_ioc, with 5623 * ioc_guidp != NULL) or all IOCs for the gid (called from 5624 * sweep_fabric, ioc_guidp == NULL). 5625 */ 5626 static void 5627 ibdm_rescan_gidlist(ib_guid_t *ioc_guidp) 5628 { 5629 ibdm_dp_gidinfo_t *gid_info, *tmp; 5630 int ii, niocs, found; 5631 ibdm_hca_list_t *hca_list = NULL; 5632 ibdm_port_attr_t *port = NULL; 5633 ibdm_ioc_info_t *ioc_list; 5634 5635 for (gid_info = ibdm.ibdm_dp_gidlist_head; gid_info; ) { 5636 found = 0; 5637 if (gid_info->gl_state != IBDM_GID_PROBING_SKIPPED && 5638 gid_info->gl_state != IBDM_GID_PROBING_COMPLETE) { 5639 gid_info = gid_info->gl_next; 5640 continue; 5641 } 5642 5643 /* 5644 * Check if the GID is visible to any HCA ports. 5645 * Return if so. 5646 */ 5647 mutex_enter(&ibdm.ibdm_hl_mutex); 5648 for (ibdm_get_next_port(&hca_list, &port, 1); port; 5649 ibdm_get_next_port(&hca_list, &port, 1)) { 5650 if (ibdm_port_reachable(port->pa_sa_hdl, 5651 gid_info->gl_dgid_lo, NULL) == IBDM_SUCCESS) { 5652 found = 1; 5653 break; 5654 } 5655 } 5656 mutex_exit(&ibdm.ibdm_hl_mutex); 5657 5658 if (found) { 5659 if (gid_info->gl_iou == NULL) { 5660 gid_info = gid_info->gl_next; 5661 continue; 5662 } 5663 5664 /* Intialize the ioc_prev_gid_list */ 5665 niocs = 5666 gid_info->gl_iou->iou_info.iou_num_ctrl_slots; 5667 for (ii = 0; ii < niocs; ii++) { 5668 ioc_list = IBDM_GIDINFO2IOCINFO(gid_info, ii); 5669 5670 if (ioc_guidp == NULL || (*ioc_guidp == 5671 ioc_list->ioc_profile.ioc_guid)) { 5672 /* Add info of GIDs in gid_info also */ 5673 ibdm_addto_gidlist( 5674 &ioc_list->ioc_prev_gid_list, 5675 gid_info->gl_gid); 5676 ioc_list->ioc_prev_nportgids = 5677 gid_info->gl_ngids; 5678 } 5679 } 5680 gid_info = gid_info->gl_next; 5681 continue; 5682 } 5683 5684 IBTF_DPRINTF_L4("ibdm", "\trescan_gidlist " 5685 "deleted port GUID %llx", 5686 gid_info->gl_dgid_lo); 5687 5688 /* 5689 * Update GID list in all IOCs affected by this 5690 */ 5691 ioc_list = ibdm_update_ioc_gidlist(gid_info, 0); 5692 5693 /* 5694 * Remove GID from the global GID list 5695 * Handle the case where all port GIDs for an 5696 * IOU have been hot-removed. 5697 */ 5698 mutex_enter(&ibdm.ibdm_mutex); 5699 if (gid_info->gl_iou != NULL && gid_info->gl_ngids == 0) { 5700 mutex_enter(&gid_info->gl_mutex); 5701 (void) ibdm_free_iou_info(gid_info); 5702 mutex_exit(&gid_info->gl_mutex); 5703 } 5704 tmp = gid_info->gl_next; 5705 if (gid_info->gl_prev != NULL) 5706 gid_info->gl_prev->gl_next = gid_info->gl_next; 5707 if (gid_info->gl_next != NULL) 5708 gid_info->gl_next->gl_prev = gid_info->gl_prev; 5709 5710 if (gid_info == ibdm.ibdm_dp_gidlist_head) 5711 ibdm.ibdm_dp_gidlist_head = gid_info->gl_next; 5712 if (gid_info == ibdm.ibdm_dp_gidlist_tail) 5713 ibdm.ibdm_dp_gidlist_tail = gid_info->gl_prev; 5714 ibdm.ibdm_ngids--; 5715 5716 mutex_destroy(&gid_info->gl_mutex); 5717 kmem_free(gid_info, sizeof (ibdm_dp_gidinfo_t)); 5718 gid_info = tmp; 5719 5720 mutex_exit(&ibdm.ibdm_mutex); 5721 5722 /* 5723 * Pass on the IOCs with updated GIDs to IBnexus 5724 */ 5725 if (ioc_list) { 5726 IBTF_DPRINTF_L4("ibdm", "\trescan_gidlist " 5727 "IOC_PROP_UPDATE for %p\n", ioc_list); 5728 mutex_enter(&ibdm.ibdm_ibnex_mutex); 5729 if (ibdm.ibdm_ibnex_callback != NULL) { 5730 (*ibdm.ibdm_ibnex_callback)((void *) 5731 ioc_list, IBDM_EVENT_IOC_PROP_UPDATE); 5732 } 5733 mutex_exit(&ibdm.ibdm_ibnex_mutex); 5734 } 5735 } 5736 } 5737 5738 /* 5739 * This function notifies IBnex of IOCs on this GID. 5740 * Notification is for GIDs with gl_reprobe_flag set. 5741 * The flag is set when IOC probe / fabric sweep 5742 * probes a GID starting from CLASS port info. 5743 * 5744 * IBnexus will have information of a reconnected IOC 5745 * if it had probed it before. If this is a new IOC, 5746 * IBnexus ignores the notification. 5747 * 5748 * This function should be called with no locks held. 5749 */ 5750 static void 5751 ibdm_notify_newgid_iocs(ibdm_dp_gidinfo_t *gid_info) 5752 { 5753 ibdm_ioc_info_t *ioc_list; 5754 5755 if (gid_info->gl_reprobe_flag == 0 || 5756 gid_info->gl_iou == NULL) 5757 return; 5758 5759 ioc_list = ibdm_update_ioc_gidlist(gid_info, -1); 5760 5761 /* 5762 * Pass on the IOCs with updated GIDs to IBnexus 5763 */ 5764 if (ioc_list) { 5765 mutex_enter(&ibdm.ibdm_ibnex_mutex); 5766 if (ibdm.ibdm_ibnex_callback != NULL) { 5767 (*ibdm.ibdm_ibnex_callback)((void *)ioc_list, 5768 IBDM_EVENT_IOC_PROP_UPDATE); 5769 } 5770 mutex_exit(&ibdm.ibdm_ibnex_mutex); 5771 } 5772 } 5773 5774 5775 static void 5776 ibdm_free_saa_event_arg(ibdm_saa_event_arg_t *arg) 5777 { 5778 if (arg != NULL) 5779 kmem_free(arg, sizeof (ibdm_saa_event_arg_t)); 5780 } 5781 5782 /* 5783 * This function parses the list of HCAs and HCA ports 5784 * to return the port_attr of the next HCA port. A port 5785 * connected to IB fabric (port_state active) is returned, 5786 * if connected_flag is set. 5787 */ 5788 static void 5789 ibdm_get_next_port(ibdm_hca_list_t **inp_hcap, 5790 ibdm_port_attr_t **inp_portp, int connect_flag) 5791 { 5792 int ii; 5793 ibdm_port_attr_t *port, *next_port = NULL; 5794 ibdm_port_attr_t *inp_port; 5795 ibdm_hca_list_t *hca_list; 5796 int found = 0; 5797 5798 ASSERT(MUTEX_HELD(&ibdm.ibdm_hl_mutex)); 5799 IBTF_DPRINTF_L4(ibdm_string, "\tget_next_port(%p, %p, %x)", 5800 inp_hcap, inp_portp, connect_flag); 5801 5802 hca_list = *inp_hcap; 5803 inp_port = *inp_portp; 5804 5805 if (hca_list == NULL) 5806 hca_list = ibdm.ibdm_hca_list_head; 5807 5808 for (; hca_list; hca_list = hca_list->hl_next) { 5809 for (ii = 0; ii < hca_list->hl_nports; ii++) { 5810 port = &hca_list->hl_port_attr[ii]; 5811 5812 /* 5813 * inp_port != NULL; 5814 * Skip till we find the matching port 5815 */ 5816 if (inp_port && !found) { 5817 if (inp_port == port) 5818 found = 1; 5819 continue; 5820 } 5821 5822 if (!connect_flag) { 5823 next_port = port; 5824 break; 5825 } 5826 5827 if (port->pa_sa_hdl == NULL) 5828 ibdm_initialize_port(port); 5829 if (port->pa_sa_hdl == NULL) 5830 (void) ibdm_fini_port(port); 5831 else if (next_port == NULL && 5832 port->pa_sa_hdl != NULL && 5833 port->pa_state == IBT_PORT_ACTIVE) { 5834 next_port = port; 5835 break; 5836 } 5837 } 5838 5839 if (next_port) 5840 break; 5841 } 5842 5843 IBTF_DPRINTF_L4(ibdm_string, "\tget_next_port : " 5844 "returns hca_list %p port %p", hca_list, next_port); 5845 *inp_hcap = hca_list; 5846 *inp_portp = next_port; 5847 } 5848 5849 static void 5850 ibdm_add_to_gl_gid(ibdm_dp_gidinfo_t *nodegid, ibdm_dp_gidinfo_t *addgid) 5851 { 5852 ibdm_gid_t *tmp; 5853 5854 tmp = kmem_zalloc(sizeof (ibdm_gid_t), KM_SLEEP); 5855 tmp->gid_dgid_hi = addgid->gl_dgid_hi; 5856 tmp->gid_dgid_lo = addgid->gl_dgid_lo; 5857 5858 mutex_enter(&nodegid->gl_mutex); 5859 tmp->gid_next = nodegid->gl_gid; 5860 nodegid->gl_gid = tmp; 5861 nodegid->gl_ngids++; 5862 mutex_exit(&nodegid->gl_mutex); 5863 } 5864 5865 static void 5866 ibdm_addto_glhcalist(ibdm_dp_gidinfo_t *gid_info, 5867 ibdm_hca_list_t *hca) 5868 { 5869 ibdm_hca_list_t *head, *prev = NULL, *temp; 5870 5871 IBTF_DPRINTF_L4(ibdm_string, "\taddto_glhcalist(%p, %p) " 5872 ": gl_hca_list %p", gid_info, hca, gid_info->gl_hca_list); 5873 ASSERT(!MUTEX_HELD(&gid_info->gl_mutex)); 5874 mutex_enter(&gid_info->gl_mutex); 5875 head = gid_info->gl_hca_list; 5876 if (head == NULL) { 5877 head = ibdm_dup_hca_attr(hca); 5878 head->hl_next = NULL; 5879 gid_info->gl_hca_list = head; 5880 mutex_exit(&gid_info->gl_mutex); 5881 IBTF_DPRINTF_L4(ibdm_string, "\tadd_to_glhcalist: " 5882 "gid %p, gl_hca_list %p", gid_info, 5883 gid_info->gl_hca_list); 5884 return; 5885 } 5886 5887 /* Check if already in the list */ 5888 while (head) { 5889 if (head->hl_hca_guid == hca->hl_hca_guid) { 5890 mutex_exit(&gid_info->gl_mutex); 5891 IBTF_DPRINTF_L4(ibdm_string, 5892 "\taddto_glhcalist : gid %x hca %x dup", 5893 gid_info, hca); 5894 return; 5895 } 5896 prev = head; 5897 head = head->hl_next; 5898 } 5899 5900 /* Add this HCA to gl_hca_list */ 5901 temp = ibdm_dup_hca_attr(hca); 5902 temp->hl_next = NULL; 5903 prev->hl_next = temp; 5904 5905 mutex_exit(&gid_info->gl_mutex); 5906 5907 IBTF_DPRINTF_L4(ibdm_string, "\tadd_to_glhcalist: " 5908 "gid %p, gl_hca_list %p", gid_info, gid_info->gl_hca_list); 5909 } 5910 5911 static void 5912 ibdm_delete_glhca_list(ibdm_dp_gidinfo_t *gid_info) 5913 { 5914 ASSERT(!MUTEX_HELD(&gid_info->gl_mutex)); 5915 ASSERT(!MUTEX_HELD(&ibdm.ibdm_mutex)); 5916 5917 mutex_enter(&gid_info->gl_mutex); 5918 if (gid_info->gl_hca_list) 5919 ibdm_ibnex_free_hca_list(gid_info->gl_hca_list); 5920 gid_info->gl_hca_list = NULL; 5921 mutex_exit(&gid_info->gl_mutex); 5922 } 5923 5924 5925 static void 5926 ibdm_reset_all_dgids(ibmf_saa_handle_t port_sa_hdl) 5927 { 5928 IBTF_DPRINTF_L4(ibdm_string, "\treset_all_dgids(%X)", 5929 port_sa_hdl); 5930 5931 ASSERT(!MUTEX_HELD(&ibdm.ibdm_mutex)); 5932 ASSERT(!MUTEX_HELD(&ibdm.ibdm_hl_mutex)); 5933 5934 /* Check : Not busy in another probe / sweep */ 5935 mutex_enter(&ibdm.ibdm_mutex); 5936 if ((ibdm.ibdm_busy & IBDM_BUSY) == 0) { 5937 ibdm_dp_gidinfo_t *gid_info; 5938 5939 ibdm.ibdm_busy |= IBDM_BUSY; 5940 mutex_exit(&ibdm.ibdm_mutex); 5941 5942 /* 5943 * Check if any GID is using the SA & IBMF handle 5944 * of HCA port going down. Reset ibdm_dp_gidinfo_t 5945 * using another HCA port which can reach the GID. 5946 * This is for DM capable GIDs only, no need to do 5947 * this for others 5948 * 5949 * Delete the GID if no alternate HCA port to reach 5950 * it is found. 5951 */ 5952 for (gid_info = ibdm.ibdm_dp_gidlist_head; gid_info; ) { 5953 ibdm_dp_gidinfo_t *tmp; 5954 5955 IBTF_DPRINTF_L4(ibdm_string, "\tevent_hdlr " 5956 "checking gidinfo %p", gid_info); 5957 5958 if (gid_info->gl_nodeguid != 0 && 5959 gid_info->gl_sa_hdl == port_sa_hdl) { 5960 IBTF_DPRINTF_L3(ibdm_string, 5961 "\tevent_hdlr: down HCA port hdl " 5962 "matches gid %p", gid_info); 5963 5964 ibdm_reset_gidinfo(gid_info); 5965 if (gid_info->gl_disconnected) { 5966 IBTF_DPRINTF_L3(ibdm_string, 5967 "\tevent_hdlr: deleting" 5968 " gid %p", gid_info); 5969 tmp = gid_info; 5970 gid_info = gid_info->gl_next; 5971 ibdm_delete_gidinfo(tmp); 5972 } else 5973 gid_info = gid_info->gl_next; 5974 } else 5975 gid_info = gid_info->gl_next; 5976 } 5977 5978 mutex_enter(&ibdm.ibdm_mutex); 5979 ibdm.ibdm_busy &= ~IBDM_BUSY; 5980 cv_signal(&ibdm.ibdm_busy_cv); 5981 } 5982 mutex_exit(&ibdm.ibdm_mutex); 5983 } 5984 5985 static void 5986 ibdm_reset_gidinfo(ibdm_dp_gidinfo_t *gidinfo) 5987 { 5988 ibdm_hca_list_t *hca_list = NULL; 5989 ibdm_port_attr_t *port = NULL; 5990 int gid_reinited = 0; 5991 sa_node_record_t *nr, *tmp; 5992 sa_portinfo_record_t *pi; 5993 size_t nr_len = 0, pi_len = 0; 5994 size_t path_len; 5995 ib_gid_t sgid, dgid; 5996 int ret, ii, nrecords; 5997 sa_path_record_t *path; 5998 uint8_t npaths = 1; 5999 6000 IBTF_DPRINTF_L4(ibdm_string, "\treset_gidinfo(%p)", gidinfo); 6001 6002 /* 6003 * Get list of all the ports reachable from the local known HCA 6004 * ports which are active 6005 */ 6006 mutex_enter(&ibdm.ibdm_hl_mutex); 6007 for (ibdm_get_next_port(&hca_list, &port, 1); port; 6008 ibdm_get_next_port(&hca_list, &port, 1)) { 6009 6010 6011 /* 6012 * Get the path and re-populate the gidinfo. 6013 * Getting the path is the same probe_ioc 6014 * Init the gid info as in ibdm_create_gidinfo() 6015 */ 6016 nr = ibdm_get_node_records(port->pa_sa_hdl, &nr_len, 6017 gidinfo->gl_nodeguid); 6018 if (nr == NULL) { 6019 IBTF_DPRINTF_L4(ibdm_string, 6020 "\treset_gidinfo : no records"); 6021 continue; 6022 } 6023 6024 nrecords = (nr_len / sizeof (sa_node_record_t)); 6025 for (tmp = nr, ii = 0; (ii < nrecords); ii++, tmp++) { 6026 if (tmp->NodeInfo.PortGUID == gidinfo->gl_portguid) 6027 break; 6028 } 6029 6030 if (ii == nrecords) { 6031 IBTF_DPRINTF_L4(ibdm_string, 6032 "\treset_gidinfo : no record for portguid"); 6033 kmem_free(nr, nr_len); 6034 continue; 6035 } 6036 6037 pi = ibdm_get_portinfo(port->pa_sa_hdl, &pi_len, tmp->LID); 6038 if (pi == NULL) { 6039 IBTF_DPRINTF_L4(ibdm_string, 6040 "\treset_gidinfo : no portinfo"); 6041 kmem_free(nr, nr_len); 6042 continue; 6043 } 6044 6045 sgid.gid_prefix = port->pa_sn_prefix; 6046 sgid.gid_guid = port->pa_port_guid; 6047 dgid.gid_prefix = pi->PortInfo.GidPrefix; 6048 dgid.gid_guid = tmp->NodeInfo.PortGUID; 6049 6050 ret = ibmf_saa_gid_to_pathrecords(port->pa_sa_hdl, sgid, dgid, 6051 IBMF_SAA_PKEY_WC, 0, B_TRUE, &npaths, 0, &path_len, &path); 6052 6053 if ((ret != IBMF_SUCCESS) && path) { 6054 IBTF_DPRINTF_L4(ibdm_string, 6055 "\treset_gidinfo : no paths"); 6056 kmem_free(pi, pi_len); 6057 kmem_free(nr, nr_len); 6058 continue; 6059 } 6060 6061 gidinfo->gl_dgid_hi = path->DGID.gid_prefix; 6062 gidinfo->gl_dgid_lo = path->DGID.gid_guid; 6063 gidinfo->gl_sgid_hi = path->SGID.gid_prefix; 6064 gidinfo->gl_sgid_lo = path->SGID.gid_guid; 6065 gidinfo->gl_p_key = path->P_Key; 6066 gidinfo->gl_sa_hdl = port->pa_sa_hdl; 6067 gidinfo->gl_ibmf_hdl = port->pa_ibmf_hdl; 6068 gidinfo->gl_slid = path->SLID; 6069 gidinfo->gl_dlid = path->DLID; 6070 6071 /* Reset redirect info, next MAD will set if redirected */ 6072 gidinfo->gl_redirected = 0; 6073 6074 gid_reinited = 1; 6075 6076 kmem_free(path, path_len); 6077 kmem_free(pi, pi_len); 6078 kmem_free(nr, nr_len); 6079 break; 6080 } 6081 mutex_exit(&ibdm.ibdm_hl_mutex); 6082 6083 if (!gid_reinited) 6084 gidinfo->gl_disconnected = 1; 6085 } 6086 6087 static void 6088 ibdm_delete_gidinfo(ibdm_dp_gidinfo_t *gidinfo) 6089 { 6090 ibdm_ioc_info_t *ioc_list; 6091 6092 ioc_list = ibdm_update_ioc_gidlist(gidinfo, 0); 6093 6094 /* 6095 * Remove GID from the global GID list 6096 * Handle the case where all port GIDs for an 6097 * IOU have been hot-removed. 6098 */ 6099 mutex_enter(&ibdm.ibdm_mutex); 6100 if (gidinfo->gl_iou != NULL && gidinfo->gl_ngids == 0) { 6101 mutex_enter(&gidinfo->gl_mutex); 6102 (void) ibdm_free_iou_info(gidinfo); 6103 mutex_exit(&gidinfo->gl_mutex); 6104 } 6105 if (gidinfo->gl_prev != NULL) 6106 gidinfo->gl_prev->gl_next = gidinfo->gl_next; 6107 if (gidinfo->gl_next != NULL) 6108 gidinfo->gl_next->gl_prev = gidinfo->gl_prev; 6109 6110 if (gidinfo == ibdm.ibdm_dp_gidlist_head) 6111 ibdm.ibdm_dp_gidlist_head = gidinfo->gl_next; 6112 if (gidinfo == ibdm.ibdm_dp_gidlist_tail) 6113 ibdm.ibdm_dp_gidlist_tail = gidinfo->gl_prev; 6114 ibdm.ibdm_ngids--; 6115 mutex_exit(&ibdm.ibdm_mutex); 6116 6117 mutex_destroy(&gidinfo->gl_mutex); 6118 kmem_free(gidinfo, sizeof (ibdm_dp_gidinfo_t)); 6119 6120 /* 6121 * Pass on the IOCs with updated GIDs to IBnexus 6122 */ 6123 if (ioc_list) { 6124 IBTF_DPRINTF_L4("ibdm", "\tdelete_gidinfo " 6125 "IOC_PROP_UPDATE for %p\n", ioc_list); 6126 mutex_enter(&ibdm.ibdm_ibnex_mutex); 6127 if (ibdm.ibdm_ibnex_callback != NULL) { 6128 (*ibdm.ibdm_ibnex_callback)((void *) 6129 ioc_list, IBDM_EVENT_IOC_PROP_UPDATE); 6130 } 6131 mutex_exit(&ibdm.ibdm_ibnex_mutex); 6132 } 6133 } 6134 6135 6136 static void 6137 ibdm_fill_srv_attr_mod(ib_mad_hdr_t *hdr, ibdm_timeout_cb_args_t *cb_args) 6138 { 6139 uint32_t attr_mod; 6140 6141 attr_mod = (cb_args->cb_ioc_num + 1) << 16; 6142 attr_mod |= cb_args->cb_srvents_start; 6143 attr_mod |= (cb_args->cb_srvents_end) << 8; 6144 hdr->AttributeModifier = h2b32(attr_mod); 6145 } 6146 6147 static void 6148 ibdm_bump_transactionID(ibdm_dp_gidinfo_t *gid_info) 6149 { 6150 ASSERT(MUTEX_HELD(&gid_info->gl_mutex)); 6151 gid_info->gl_transactionID++; 6152 if (gid_info->gl_transactionID == gid_info->gl_max_transactionID) { 6153 IBTF_DPRINTF_L4(ibdm_string, 6154 "\tbump_transactionID(%p), wrapup", gid_info); 6155 gid_info->gl_transactionID = gid_info->gl_min_transactionID; 6156 } 6157 } 6158 6159 /* For debugging purpose only */ 6160 #ifdef DEBUG 6161 void 6162 ibdm_dump_ibmf_msg(ibmf_msg_t *ibmf_msg, int flag) 6163 { 6164 ib_mad_hdr_t *mad_hdr; 6165 6166 IBTF_DPRINTF_L4("ibdm", "\t\t(IBMF_PKT): Local address info"); 6167 IBTF_DPRINTF_L4("ibdm", "\t\t ------------------"); 6168 6169 IBTF_DPRINTF_L4("ibdm", "\tLocal Lid : 0x%x\tRemote Lid : 0x%x" 6170 " Remote Qp : 0x%x", ibmf_msg->im_local_addr.ia_local_lid, 6171 ibmf_msg->im_local_addr.ia_remote_lid, 6172 ibmf_msg->im_local_addr.ia_remote_qno); 6173 IBTF_DPRINTF_L4("ibdm", "\tP_key : 0x%x\tQ_key : 0x%x", 6174 ibmf_msg->im_local_addr.ia_p_key, ibmf_msg->im_local_addr.ia_q_key); 6175 6176 if (flag) 6177 mad_hdr = (ib_mad_hdr_t *)IBDM_OUT_IBMFMSG_MADHDR(ibmf_msg); 6178 else 6179 mad_hdr = IBDM_IN_IBMFMSG_MADHDR(ibmf_msg); 6180 6181 IBTF_DPRINTF_L4("ibdm", "\t\t MAD Header info"); 6182 IBTF_DPRINTF_L4("ibdm", "\t\t ---------------"); 6183 6184 IBTF_DPRINTF_L4("ibdm", "\tBase version : 0x%x" 6185 "\tMgmt Class : 0x%x", mad_hdr->BaseVersion, mad_hdr->MgmtClass); 6186 IBTF_DPRINTF_L4("ibdm", "\tClass version : 0x%x" 6187 "\tR Method : 0x%x", 6188 mad_hdr->ClassVersion, mad_hdr->R_Method); 6189 IBTF_DPRINTF_L4("ibdm", "\tMAD Status : 0x%x" 6190 "\tTransaction ID : 0x%llx", 6191 mad_hdr->Status, mad_hdr->TransactionID); 6192 IBTF_DPRINTF_L4("ibdm", "\t Attribute ID : 0x%x" 6193 "\tAttribute Modified : 0x%lx", 6194 mad_hdr->AttributeID, mad_hdr->AttributeModifier); 6195 } 6196 6197 void 6198 ibdm_dump_path_info(sa_path_record_t *path) 6199 { 6200 IBTF_DPRINTF_L4("ibdm", "\t\t Path information"); 6201 IBTF_DPRINTF_L4("ibdm", "\t\t ----------------"); 6202 6203 IBTF_DPRINTF_L4("ibdm", "\t DGID hi : %llx\tDGID lo : %llx", 6204 path->DGID.gid_prefix, path->DGID.gid_guid); 6205 IBTF_DPRINTF_L4("ibdm", "\t SGID hi : %llx\tSGID lo : %llx", 6206 path->SGID.gid_prefix, path->SGID.gid_guid); 6207 IBTF_DPRINTF_L4("ibdm", "\t SLID : %x\tDlID : %x", 6208 path->SLID, path->DLID); 6209 IBTF_DPRINTF_L4("ibdm", "\t P Key : %x", path->P_Key); 6210 } 6211 6212 6213 void 6214 ibdm_dump_classportinfo(ibdm_mad_classportinfo_t *classportinfo) 6215 { 6216 IBTF_DPRINTF_L4("ibdm", "\t\t CLASSPORT INFO"); 6217 IBTF_DPRINTF_L4("ibdm", "\t\t --------------"); 6218 6219 IBTF_DPRINTF_L4("ibdm", "\t Response Time Value : 0x%x", 6220 ((b2h32(classportinfo->RespTimeValue)) & 0x1F)); 6221 6222 IBTF_DPRINTF_L4("ibdm", "\t Redirected QP : 0x%x", 6223 (b2h32(classportinfo->RedirectQP))); 6224 IBTF_DPRINTF_L4("ibdm", "\t Redirected P KEY : 0x%x", 6225 b2h16(classportinfo->RedirectP_Key)); 6226 IBTF_DPRINTF_L4("ibdm", "\t Redirected Q KEY : 0x%x", 6227 b2h16(classportinfo->RedirectQ_Key)); 6228 IBTF_DPRINTF_L4("ibdm", "\t Redirected GID hi : 0x%x", 6229 b2h64(classportinfo->RedirectGID_hi)); 6230 IBTF_DPRINTF_L4("ibdm", "\t Redirected GID lo : 0x%x", 6231 b2h64(classportinfo->RedirectGID_lo)); 6232 } 6233 6234 6235 void 6236 ibdm_dump_iounitinfo(ib_dm_io_unitinfo_t *iou_info) 6237 { 6238 IBTF_DPRINTF_L4("ibdm", "\t\t I/O UnitInfo"); 6239 IBTF_DPRINTF_L4("ibdm", "\t\t ------------"); 6240 6241 IBTF_DPRINTF_L4("ibdm", "\tChange ID : 0x%x", 6242 b2h16(iou_info->iou_changeid)); 6243 IBTF_DPRINTF_L4("ibdm", "\t#of ctrl slots : %d", 6244 iou_info->iou_num_ctrl_slots); 6245 IBTF_DPRINTF_L4("ibdm", "\tIOU flag : 0x%x", 6246 iou_info->iou_flag); 6247 IBTF_DPRINTF_L4("ibdm", "\tContrl list byte 0 : 0x%x", 6248 iou_info->iou_ctrl_list[0]); 6249 IBTF_DPRINTF_L4("ibdm", "\tContrl list byte 1 : 0x%x", 6250 iou_info->iou_ctrl_list[1]); 6251 IBTF_DPRINTF_L4("ibdm", "\tContrl list byte 2 : 0x%x", 6252 iou_info->iou_ctrl_list[2]); 6253 } 6254 6255 6256 void 6257 ibdm_dump_ioc_profile(ib_dm_ioc_ctrl_profile_t *ioc) 6258 { 6259 IBTF_DPRINTF_L4("ibdm", "\t\t IOC Controller Profile"); 6260 IBTF_DPRINTF_L4("ibdm", "\t\t ----------------------"); 6261 6262 IBTF_DPRINTF_L4("ibdm", "\tIOC Guid : %llx", ioc->ioc_guid); 6263 IBTF_DPRINTF_L4("ibdm", "\tVendorID : 0x%x", ioc->ioc_vendorid); 6264 IBTF_DPRINTF_L4("ibdm", "\tDevice Id : 0x%x", ioc->ioc_deviceid); 6265 IBTF_DPRINTF_L4("ibdm", "\tDevice Ver : 0x%x", ioc->ioc_device_ver); 6266 IBTF_DPRINTF_L4("ibdm", "\tSubsys ID : 0x%x", ioc->ioc_subsys_id); 6267 IBTF_DPRINTF_L4("ibdm", "\tIO class : 0x%x", ioc->ioc_io_class); 6268 IBTF_DPRINTF_L4("ibdm", "\tIO subclass : 0x%x", ioc->ioc_io_subclass); 6269 IBTF_DPRINTF_L4("ibdm", "\tProtocol : 0x%x", ioc->ioc_protocol); 6270 IBTF_DPRINTF_L4("ibdm", "\tProtocolV : 0x%x", ioc->ioc_protocol_ver); 6271 IBTF_DPRINTF_L4("ibdm", "\tmsg qdepth : %d", ioc->ioc_send_msg_qdepth); 6272 IBTF_DPRINTF_L4("ibdm", "\trdma qdepth : %d", 6273 ioc->ioc_rdma_read_qdepth); 6274 IBTF_DPRINTF_L4("ibdm", "\tsndmsg sz : %d", ioc->ioc_send_msg_sz); 6275 IBTF_DPRINTF_L4("ibdm", "\trdma xfersz : %d", ioc->ioc_rdma_xfer_sz); 6276 IBTF_DPRINTF_L4("ibdm", "\topcal mask : 0x%x", 6277 ioc->ioc_ctrl_opcap_mask); 6278 IBTF_DPRINTF_L4("ibdm", "\tsrventries : %x", ioc->ioc_service_entries); 6279 } 6280 6281 6282 void 6283 ibdm_dump_service_entries(ib_dm_srv_t *srv_ents) 6284 { 6285 IBTF_DPRINTF_L4("ibdm", 6286 "\thandle_srventry_mad: service id : %llx", srv_ents->srv_id); 6287 6288 IBTF_DPRINTF_L4("ibdm", "\thandle_srventry_mad: " 6289 "Service Name : %s", srv_ents->srv_name); 6290 } 6291 6292 int ibdm_allow_sweep_fabric_timestamp = 1; 6293 6294 void 6295 ibdm_dump_sweep_fabric_timestamp(int flag) 6296 { 6297 static hrtime_t x; 6298 if (flag) { 6299 if (ibdm_allow_sweep_fabric_timestamp) { 6300 IBTF_DPRINTF_L4("ibdm", "\tTime taken to complete " 6301 "sweep %lld ms", ((gethrtime() - x)/ 1000000)); 6302 } 6303 x = 0; 6304 } else 6305 x = gethrtime(); 6306 } 6307 #endif 6308