xref: /illumos-gate/usr/src/uts/common/io/ib/mgt/ibdm/ibdm.c (revision 0250c53ad267726f2438e3c6556199a0bbf588a2)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
23  */
24 /*
25  * Copyright 2014 Nexenta Systems, Inc.  All rights reserved.
26  */
27 
28 /*
29  * ibdm.c
30  *
31  * This file contains the InifiniBand Device Manager (IBDM) support functions.
32  * IB nexus driver will only be the client for the IBDM module.
33  *
34  * IBDM registers with IBTF for HCA arrival/removal notification.
35  * IBDM registers with SA access to send DM MADs to discover the IOC's behind
36  * the IOU's.
37  *
38  * IB nexus driver registers with IBDM to find the information about the
39  * HCA's and IOC's (behind the IOU) present on the IB fabric.
40  */
41 
42 #include <sys/sysmacros.h>
43 #include <sys/systm.h>
44 #include <sys/taskq.h>
45 #include <sys/ib/mgt/ibdm/ibdm_impl.h>
46 #include <sys/ib/mgt/ibmf/ibmf_impl.h>
47 #include <sys/ib/ibtl/impl/ibtl_ibnex.h>
48 #include <sys/modctl.h>
49 
50 /* Function Prototype declarations */
51 static int	ibdm_free_iou_info(ibdm_dp_gidinfo_t *, ibdm_iou_info_t **);
52 static int	ibdm_fini(void);
53 static int	ibdm_init(void);
54 static int	ibdm_get_reachable_ports(ibdm_port_attr_t *,
55 			ibdm_hca_list_t *);
56 static ibdm_dp_gidinfo_t *ibdm_check_dgid(ib_guid_t, ib_sn_prefix_t);
57 static ibdm_dp_gidinfo_t *ibdm_check_dest_nodeguid(ibdm_dp_gidinfo_t *);
58 static boolean_t ibdm_is_cisco(ib_guid_t);
59 static boolean_t ibdm_is_cisco_switch(ibdm_dp_gidinfo_t *);
60 static void	ibdm_wait_cisco_probe_completion(ibdm_dp_gidinfo_t *);
61 static int	ibdm_set_classportinfo(ibdm_dp_gidinfo_t *);
62 static int	ibdm_send_classportinfo(ibdm_dp_gidinfo_t *);
63 static int	ibdm_send_iounitinfo(ibdm_dp_gidinfo_t *);
64 static int	ibdm_is_dev_mgt_supported(ibdm_dp_gidinfo_t *);
65 static int	ibdm_get_node_port_guids(ibmf_saa_handle_t, ib_lid_t,
66 		    ib_guid_t *, ib_guid_t *);
67 static int	ibdm_retry_command(ibdm_timeout_cb_args_t *);
68 static int	ibdm_get_diagcode(ibdm_dp_gidinfo_t *, int);
69 static int	ibdm_verify_mad_status(ib_mad_hdr_t *);
70 static int	ibdm_handle_redirection(ibmf_msg_t *,
71 		    ibdm_dp_gidinfo_t *, int *);
72 static void	ibdm_wait_probe_completion(void);
73 static void	ibdm_sweep_fabric(int);
74 static void	ibdm_probe_gid_thread(void *);
75 static void	ibdm_wakeup_probe_gid_cv(void);
76 static void	ibdm_port_attr_ibmf_init(ibdm_port_attr_t *, ib_pkey_t, int);
77 static int	ibdm_port_attr_ibmf_fini(ibdm_port_attr_t *, int);
78 static void	ibdm_update_port_attr(ibdm_port_attr_t *);
79 static void	ibdm_handle_hca_attach(ib_guid_t);
80 static void	ibdm_handle_srventry_mad(ibmf_msg_t *,
81 		    ibdm_dp_gidinfo_t *, int *);
82 static void	ibdm_ibmf_recv_cb(ibmf_handle_t, ibmf_msg_t *, void *);
83 static void	ibdm_recv_incoming_mad(void *);
84 static void	ibdm_process_incoming_mad(ibmf_handle_t, ibmf_msg_t *, void *);
85 static void	ibdm_ibmf_send_cb(ibmf_handle_t, ibmf_msg_t *, void *);
86 static void	ibdm_pkt_timeout_hdlr(void *arg);
87 static void	ibdm_initialize_port(ibdm_port_attr_t *);
88 static void	ibdm_update_port_pkeys(ibdm_port_attr_t *port);
89 static void	ibdm_handle_diagcode(ibmf_msg_t *, ibdm_dp_gidinfo_t *, int *);
90 static void	ibdm_probe_gid(ibdm_dp_gidinfo_t *);
91 static void	ibdm_alloc_send_buffers(ibmf_msg_t *);
92 static void	ibdm_free_send_buffers(ibmf_msg_t *);
93 static void	ibdm_handle_hca_detach(ib_guid_t);
94 static void	ibdm_handle_port_change_event(ibt_async_event_t *);
95 static int	ibdm_fini_port(ibdm_port_attr_t *);
96 static int	ibdm_uninit_hca(ibdm_hca_list_t *);
97 static void	ibdm_handle_setclassportinfo(ibmf_handle_t, ibmf_msg_t *,
98 		    ibdm_dp_gidinfo_t *, int *);
99 static void	ibdm_handle_iounitinfo(ibmf_handle_t,
100 		    ibmf_msg_t *, ibdm_dp_gidinfo_t *, int *);
101 static void	ibdm_handle_ioc_profile(ibmf_handle_t,
102 		    ibmf_msg_t *, ibdm_dp_gidinfo_t *, int *);
103 static void	ibdm_event_hdlr(void *, ibt_hca_hdl_t,
104 		    ibt_async_code_t, ibt_async_event_t *);
105 static void	ibdm_handle_classportinfo(ibmf_handle_t,
106 		    ibmf_msg_t *, ibdm_dp_gidinfo_t *, int *);
107 static void	ibdm_update_ioc_port_gidlist(ibdm_ioc_info_t *,
108 		    ibdm_dp_gidinfo_t *);
109 
110 static ibdm_hca_list_t		*ibdm_dup_hca_attr(ibdm_hca_list_t *);
111 static ibdm_ioc_info_t		*ibdm_dup_ioc_info(ibdm_ioc_info_t *,
112 				    ibdm_dp_gidinfo_t *gid_list);
113 static void			ibdm_probe_ioc(ib_guid_t, ib_guid_t, int);
114 static ibdm_ioc_info_t		*ibdm_is_ioc_present(ib_guid_t,
115 				    ibdm_dp_gidinfo_t *, int *);
116 static ibdm_port_attr_t		*ibdm_get_port_attr(ibt_async_event_t *,
117 				    ibdm_hca_list_t **);
118 static sa_node_record_t		*ibdm_get_node_records(ibmf_saa_handle_t,
119 				    size_t *, ib_guid_t);
120 static int			ibdm_get_node_record_by_port(ibmf_saa_handle_t,
121 				    ib_guid_t, sa_node_record_t **, size_t *);
122 static sa_portinfo_record_t	*ibdm_get_portinfo(ibmf_saa_handle_t, size_t *,
123 				    ib_lid_t);
124 static ibdm_dp_gidinfo_t	*ibdm_create_gid_info(ibdm_port_attr_t *,
125 				    ib_gid_t, ib_gid_t);
126 static ibdm_dp_gidinfo_t	*ibdm_find_gid(ib_guid_t, ib_guid_t);
127 static int	ibdm_send_ioc_profile(ibdm_dp_gidinfo_t *, uint8_t);
128 static ibdm_ioc_info_t	*ibdm_update_ioc_gidlist(ibdm_dp_gidinfo_t *, int);
129 static void	ibdm_saa_event_cb(ibmf_saa_handle_t, ibmf_saa_subnet_event_t,
130 		    ibmf_saa_event_details_t *, void *);
131 static void	ibdm_reprobe_update_port_srv(ibdm_ioc_info_t *,
132     ibdm_dp_gidinfo_t *);
133 static ibdm_dp_gidinfo_t *ibdm_handle_gid_rm(ibdm_dp_gidinfo_t *);
134 static void ibdm_rmfrom_glgid_list(ibdm_dp_gidinfo_t *,
135     ibdm_dp_gidinfo_t *);
136 static void ibdm_addto_gidlist(ibdm_gid_t **, ibdm_gid_t *);
137 static void ibdm_free_gid_list(ibdm_gid_t *);
138 static void ibdm_rescan_gidlist(ib_guid_t *ioc_guid);
139 static void ibdm_notify_newgid_iocs(ibdm_dp_gidinfo_t *);
140 static void ibdm_saa_event_taskq(void *);
141 static void ibdm_free_saa_event_arg(ibdm_saa_event_arg_t *);
142 static void ibdm_get_next_port(ibdm_hca_list_t **,
143     ibdm_port_attr_t **, int);
144 static void ibdm_add_to_gl_gid(ibdm_dp_gidinfo_t *,
145     ibdm_dp_gidinfo_t *);
146 static void ibdm_addto_glhcalist(ibdm_dp_gidinfo_t *,
147     ibdm_hca_list_t *);
148 static void ibdm_delete_glhca_list(ibdm_dp_gidinfo_t *);
149 static void ibdm_saa_handle_new_gid(void *);
150 static void ibdm_reset_all_dgids(ibmf_saa_handle_t);
151 static void ibdm_reset_gidinfo(ibdm_dp_gidinfo_t *);
152 static void ibdm_delete_gidinfo(ibdm_dp_gidinfo_t *);
153 static void ibdm_fill_srv_attr_mod(ib_mad_hdr_t *, ibdm_timeout_cb_args_t *);
154 static void ibdm_bump_transactionID(ibdm_dp_gidinfo_t *);
155 static ibdm_ioc_info_t	*ibdm_handle_prev_iou();
156 static int ibdm_serv_cmp(ibdm_srvents_info_t *, ibdm_srvents_info_t *,
157     int);
158 static ibdm_ioc_info_t *ibdm_get_ioc_info_with_gid(ib_guid_t,
159     ibdm_dp_gidinfo_t **);
160 
161 int	ibdm_dft_timeout	= IBDM_DFT_TIMEOUT;
162 int	ibdm_dft_retry_cnt	= IBDM_DFT_NRETRIES;
163 #ifdef DEBUG
164 int	ibdm_ignore_saa_event = 0;
165 #endif
166 int	ibdm_enumerate_iocs = 0;
167 
168 /* Modload support */
169 static struct modlmisc ibdm_modlmisc	= {
170 	&mod_miscops,
171 	"InfiniBand Device Manager"
172 };
173 
174 struct modlinkage ibdm_modlinkage = {
175 	MODREV_1,
176 	(void *)&ibdm_modlmisc,
177 	NULL
178 };
179 
180 static ibt_clnt_modinfo_t ibdm_ibt_modinfo = {
181 	IBTI_V_CURR,
182 	IBT_DM,
183 	ibdm_event_hdlr,
184 	NULL,
185 	"ibdm"
186 };
187 
188 /* Global variables */
189 ibdm_t	ibdm;
190 int	ibdm_taskq_enable = IBDM_ENABLE_TASKQ_HANDLING;
191 char	*ibdm_string = "ibdm";
192 
193 _NOTE(SCHEME_PROTECTS_DATA("Serialized access by cv",
194     ibdm.ibdm_dp_gidlist_head))
195 
196 /*
197  * _init
198  *	Loadable module init, called before any other module.
199  *	Initialize mutex
200  *	Register with IBTF
201  */
202 int
203 _init(void)
204 {
205 	int		err;
206 
207 	IBTF_DPRINTF_L4("ibdm", "\t_init: addr of ibdm %p", &ibdm);
208 
209 	if ((err = ibdm_init()) != IBDM_SUCCESS) {
210 		IBTF_DPRINTF_L2("ibdm", "_init: ibdm_init failed 0x%x", err);
211 		(void) ibdm_fini();
212 		return (DDI_FAILURE);
213 	}
214 
215 	if ((err = mod_install(&ibdm_modlinkage)) != 0) {
216 		IBTF_DPRINTF_L2("ibdm", "_init: mod_install failed 0x%x", err);
217 		(void) ibdm_fini();
218 	}
219 	return (err);
220 }
221 
222 
223 int
224 _fini(void)
225 {
226 	int err;
227 
228 	if ((err = ibdm_fini()) != IBDM_SUCCESS) {
229 		IBTF_DPRINTF_L2("ibdm", "_fini: ibdm_fini failed 0x%x", err);
230 		(void) ibdm_init();
231 		return (EBUSY);
232 	}
233 
234 	if ((err = mod_remove(&ibdm_modlinkage)) != 0) {
235 		IBTF_DPRINTF_L2("ibdm", "_fini: mod_remove failed 0x%x", err);
236 		(void) ibdm_init();
237 	}
238 	return (err);
239 }
240 
241 
242 int
243 _info(struct modinfo *modinfop)
244 {
245 	return (mod_info(&ibdm_modlinkage, modinfop));
246 }
247 
248 
249 /*
250  * ibdm_init():
251  * 	Register with IBTF
252  *	Allocate memory for the HCAs
253  *	Allocate minor-nodes for the HCAs
254  */
255 static int
256 ibdm_init(void)
257 {
258 	int			i, hca_count;
259 	ib_guid_t		*hca_guids;
260 	ibt_status_t		status;
261 
262 	IBTF_DPRINTF_L4("ibdm", "\tibdm_init:");
263 	if (!(ibdm.ibdm_state & IBDM_LOCKS_ALLOCED)) {
264 		mutex_init(&ibdm.ibdm_mutex, NULL, MUTEX_DEFAULT, NULL);
265 		mutex_init(&ibdm.ibdm_hl_mutex, NULL, MUTEX_DEFAULT, NULL);
266 		mutex_init(&ibdm.ibdm_ibnex_mutex, NULL, MUTEX_DEFAULT, NULL);
267 		cv_init(&ibdm.ibdm_port_settle_cv, NULL, CV_DRIVER, NULL);
268 		mutex_enter(&ibdm.ibdm_mutex);
269 		ibdm.ibdm_state |= IBDM_LOCKS_ALLOCED;
270 	}
271 
272 	if (!(ibdm.ibdm_state & IBDM_IBT_ATTACHED)) {
273 		if ((status = ibt_attach(&ibdm_ibt_modinfo, NULL, NULL,
274 		    (void *)&ibdm.ibdm_ibt_clnt_hdl)) != IBT_SUCCESS) {
275 			IBTF_DPRINTF_L2("ibdm", "ibdm_init: ibt_attach "
276 			    "failed %x", status);
277 			mutex_exit(&ibdm.ibdm_mutex);
278 			return (IBDM_FAILURE);
279 		}
280 
281 		ibdm.ibdm_state |= IBDM_IBT_ATTACHED;
282 		mutex_exit(&ibdm.ibdm_mutex);
283 	}
284 
285 
286 	if (!(ibdm.ibdm_state & IBDM_HCA_ATTACHED)) {
287 		hca_count = ibt_get_hca_list(&hca_guids);
288 		IBTF_DPRINTF_L4("ibdm", "ibdm_init: num_hcas = %d", hca_count);
289 		for (i = 0; i < hca_count; i++)
290 			(void) ibdm_handle_hca_attach(hca_guids[i]);
291 		if (hca_count)
292 			ibt_free_hca_list(hca_guids, hca_count);
293 
294 		mutex_enter(&ibdm.ibdm_mutex);
295 		ibdm.ibdm_state |= IBDM_HCA_ATTACHED;
296 		mutex_exit(&ibdm.ibdm_mutex);
297 	}
298 
299 	if (!(ibdm.ibdm_state & IBDM_CVS_ALLOCED)) {
300 		cv_init(&ibdm.ibdm_probe_cv, NULL, CV_DRIVER, NULL);
301 		cv_init(&ibdm.ibdm_busy_cv, NULL, CV_DRIVER, NULL);
302 		mutex_enter(&ibdm.ibdm_mutex);
303 		ibdm.ibdm_state |= IBDM_CVS_ALLOCED;
304 		mutex_exit(&ibdm.ibdm_mutex);
305 	}
306 	return (IBDM_SUCCESS);
307 }
308 
309 
310 static int
311 ibdm_free_iou_info(ibdm_dp_gidinfo_t *gid_info, ibdm_iou_info_t **ioup)
312 {
313 	int			ii, k, niocs;
314 	size_t			size;
315 	ibdm_gid_t		*delete, *head;
316 	timeout_id_t		timeout_id;
317 	ibdm_ioc_info_t		*ioc;
318 	ibdm_iou_info_t		*gl_iou = *ioup;
319 
320 	ASSERT(mutex_owned(&gid_info->gl_mutex));
321 	if (gl_iou == NULL) {
322 		IBTF_DPRINTF_L4("ibdm", "\tibdm_free_iou_info: No IOU");
323 		return (0);
324 	}
325 
326 	niocs = gl_iou->iou_info.iou_num_ctrl_slots;
327 	IBTF_DPRINTF_L4("ibdm", "\tfree_iou_info: gid_info = %p, niocs %d",
328 	    gid_info, niocs);
329 
330 	for (ii = 0; ii < niocs; ii++) {
331 		ioc = (ibdm_ioc_info_t *)&gl_iou->iou_ioc_info[ii];
332 
333 		/* handle the case where an ioc_timeout_id is scheduled */
334 		if (ioc->ioc_timeout_id) {
335 			timeout_id = ioc->ioc_timeout_id;
336 			ioc->ioc_timeout_id = 0;
337 			mutex_exit(&gid_info->gl_mutex);
338 			IBTF_DPRINTF_L5("ibdm", "free_iou_info: "
339 			    "ioc_timeout_id = 0x%x", timeout_id);
340 			if (untimeout(timeout_id) == -1) {
341 				IBTF_DPRINTF_L2("ibdm", "free_iou_info: "
342 				    "untimeout ioc_timeout_id failed");
343 				mutex_enter(&gid_info->gl_mutex);
344 				return (-1);
345 			}
346 			mutex_enter(&gid_info->gl_mutex);
347 		}
348 
349 		/* handle the case where an ioc_dc_timeout_id is scheduled */
350 		if (ioc->ioc_dc_timeout_id) {
351 			timeout_id = ioc->ioc_dc_timeout_id;
352 			ioc->ioc_dc_timeout_id = 0;
353 			mutex_exit(&gid_info->gl_mutex);
354 			IBTF_DPRINTF_L5("ibdm", "free_iou_info: "
355 			    "ioc_dc_timeout_id = 0x%x", timeout_id);
356 			if (untimeout(timeout_id) == -1) {
357 				IBTF_DPRINTF_L2("ibdm", "free_iou_info: "
358 				    "untimeout ioc_dc_timeout_id failed");
359 				mutex_enter(&gid_info->gl_mutex);
360 				return (-1);
361 			}
362 			mutex_enter(&gid_info->gl_mutex);
363 		}
364 
365 		/* handle the case where serv[k].se_timeout_id is scheduled */
366 		for (k = 0; k < ioc->ioc_profile.ioc_service_entries; k++) {
367 			if (ioc->ioc_serv[k].se_timeout_id) {
368 				timeout_id = ioc->ioc_serv[k].se_timeout_id;
369 				ioc->ioc_serv[k].se_timeout_id = 0;
370 				mutex_exit(&gid_info->gl_mutex);
371 				IBTF_DPRINTF_L5("ibdm", "free_iou_info: "
372 				    "ioc->ioc_serv[%d].se_timeout_id = 0x%x",
373 				    k, timeout_id);
374 				if (untimeout(timeout_id) == -1) {
375 					IBTF_DPRINTF_L2("ibdm", "free_iou_info:"
376 					    " untimeout se_timeout_id failed");
377 					mutex_enter(&gid_info->gl_mutex);
378 					return (-1);
379 				}
380 				mutex_enter(&gid_info->gl_mutex);
381 			}
382 		}
383 
384 		/* delete GID list in IOC */
385 		head = ioc->ioc_gid_list;
386 		while (head) {
387 			IBTF_DPRINTF_L4("ibdm", "\tibdm_free_iou_info: "
388 			    "Deleting gid_list struct %p", head);
389 			delete = head;
390 			head = head->gid_next;
391 			kmem_free(delete, sizeof (ibdm_gid_t));
392 		}
393 		ioc->ioc_gid_list = NULL;
394 
395 		/* delete ioc_serv */
396 		size = ioc->ioc_profile.ioc_service_entries *
397 		    sizeof (ibdm_srvents_info_t);
398 		if (ioc->ioc_serv && size) {
399 			kmem_free(ioc->ioc_serv, size);
400 			ioc->ioc_serv = NULL;
401 		}
402 	}
403 	/*
404 	 * Clear the IBDM_CISCO_PROBE_DONE flag to get the IO Unit information
405 	 * via the switch during the probe process.
406 	 */
407 	gid_info->gl_flag &= ~IBDM_CISCO_PROBE_DONE;
408 
409 	IBTF_DPRINTF_L4("ibdm", "\tibdm_free_iou_info: deleting IOU & IOC");
410 	size = sizeof (ibdm_iou_info_t) + niocs * sizeof (ibdm_ioc_info_t);
411 	kmem_free(gl_iou, size);
412 	*ioup = NULL;
413 	return (0);
414 }
415 
416 
417 /*
418  * ibdm_fini():
419  * 	Un-register with IBTF
420  *	De allocate memory for the GID info
421  */
422 static int
423 ibdm_fini()
424 {
425 	int			ii;
426 	ibdm_hca_list_t		*hca_list, *temp;
427 	ibdm_dp_gidinfo_t	*gid_info, *tmp;
428 	ibdm_gid_t		*head, *delete;
429 
430 	IBTF_DPRINTF_L4("ibdm", "\tibdm_fini");
431 
432 	mutex_enter(&ibdm.ibdm_hl_mutex);
433 	if (ibdm.ibdm_state & IBDM_IBT_ATTACHED) {
434 		if (ibt_detach(ibdm.ibdm_ibt_clnt_hdl) != IBT_SUCCESS) {
435 			IBTF_DPRINTF_L2("ibdm", "\t_fini: ibt_detach failed");
436 			mutex_exit(&ibdm.ibdm_hl_mutex);
437 			return (IBDM_FAILURE);
438 		}
439 		ibdm.ibdm_state &= ~IBDM_IBT_ATTACHED;
440 		ibdm.ibdm_ibt_clnt_hdl = NULL;
441 	}
442 
443 	hca_list = ibdm.ibdm_hca_list_head;
444 	IBTF_DPRINTF_L4("ibdm", "\tibdm_fini: nhcas %d", ibdm.ibdm_hca_count);
445 	for (ii = 0; ii < ibdm.ibdm_hca_count; ii++) {
446 		temp = hca_list;
447 		hca_list = hca_list->hl_next;
448 		IBTF_DPRINTF_L4("ibdm", "\tibdm_fini: hca %p", temp);
449 		if (ibdm_uninit_hca(temp) != IBDM_SUCCESS) {
450 			IBTF_DPRINTF_L2("ibdm", "\tibdm_fini: "
451 			    "uninit_hca %p failed", temp);
452 			mutex_exit(&ibdm.ibdm_hl_mutex);
453 			return (IBDM_FAILURE);
454 		}
455 	}
456 	mutex_exit(&ibdm.ibdm_hl_mutex);
457 
458 	mutex_enter(&ibdm.ibdm_mutex);
459 	if (ibdm.ibdm_state & IBDM_HCA_ATTACHED)
460 		ibdm.ibdm_state &= ~IBDM_HCA_ATTACHED;
461 
462 	gid_info = ibdm.ibdm_dp_gidlist_head;
463 	while (gid_info) {
464 		mutex_enter(&gid_info->gl_mutex);
465 		(void) ibdm_free_iou_info(gid_info, &gid_info->gl_iou);
466 		mutex_exit(&gid_info->gl_mutex);
467 		ibdm_delete_glhca_list(gid_info);
468 
469 		tmp = gid_info;
470 		gid_info = gid_info->gl_next;
471 		mutex_destroy(&tmp->gl_mutex);
472 		head = tmp->gl_gid;
473 		while (head) {
474 			IBTF_DPRINTF_L4("ibdm",
475 			    "\tibdm_fini: Deleting gid structs");
476 			delete = head;
477 			head = head->gid_next;
478 			kmem_free(delete, sizeof (ibdm_gid_t));
479 		}
480 		kmem_free(tmp, sizeof (ibdm_dp_gidinfo_t));
481 	}
482 	mutex_exit(&ibdm.ibdm_mutex);
483 
484 	if (ibdm.ibdm_state & IBDM_LOCKS_ALLOCED) {
485 		ibdm.ibdm_state &= ~IBDM_LOCKS_ALLOCED;
486 		mutex_destroy(&ibdm.ibdm_mutex);
487 		mutex_destroy(&ibdm.ibdm_hl_mutex);
488 		mutex_destroy(&ibdm.ibdm_ibnex_mutex);
489 		cv_destroy(&ibdm.ibdm_port_settle_cv);
490 	}
491 	if (ibdm.ibdm_state & IBDM_CVS_ALLOCED) {
492 		ibdm.ibdm_state &= ~IBDM_CVS_ALLOCED;
493 		cv_destroy(&ibdm.ibdm_probe_cv);
494 		cv_destroy(&ibdm.ibdm_busy_cv);
495 	}
496 	return (IBDM_SUCCESS);
497 }
498 
499 
500 /*
501  * ibdm_event_hdlr()
502  *
503  *	IBDM registers  this asynchronous event handler at the time of
504  *	ibt_attach. IBDM support the following async events. For other
505  *	event, simply returns success.
506  *	IBT_HCA_ATTACH_EVENT:
507  *		Retrieves the  information about all the port that are
508  *		present on this HCA,  allocates  the  port  attributes
509  *		structure  and calls IB  nexus  callback  routine with
510  *		the port attributes structure as an input argument.
511  *	IBT_HCA_DETACH_EVENT:
512  *		Retrieves the information about all the ports that are
513  *		present on  this HCA and  calls IB nexus callback with
514  *		port guid as an argument
515  *	IBT_EVENT_PORT_UP:
516  *		Register with IBMF and SA access
517  *		Setup IBMF receive callback routine
518  *	IBT_EVENT_PORT_DOWN:
519  *		Un-Register with IBMF and SA access
520  *		Teardown IBMF receive callback routine
521  */
522 /*ARGSUSED*/
523 static void
524 ibdm_event_hdlr(void *clnt_hdl,
525     ibt_hca_hdl_t hca_hdl, ibt_async_code_t code, ibt_async_event_t *event)
526 {
527 	ibdm_hca_list_t		*hca_list;
528 	ibdm_port_attr_t	*port;
529 	ibmf_saa_handle_t	port_sa_hdl;
530 
531 	IBTF_DPRINTF_L4("ibdm", "\tevent_hdlr: async code 0x%x", code);
532 
533 	switch (code) {
534 	case IBT_HCA_ATTACH_EVENT:	/* New HCA registered with IBTF */
535 		ibdm_handle_hca_attach(event->ev_hca_guid);
536 		break;
537 
538 	case IBT_HCA_DETACH_EVENT:	/* HCA unregistered with IBTF */
539 		ibdm_handle_hca_detach(event->ev_hca_guid);
540 		mutex_enter(&ibdm.ibdm_ibnex_mutex);
541 		if (ibdm.ibdm_ibnex_callback != NULL) {
542 			(*ibdm.ibdm_ibnex_callback)((void *)
543 			    &event->ev_hca_guid, IBDM_EVENT_HCA_REMOVED);
544 		}
545 		mutex_exit(&ibdm.ibdm_ibnex_mutex);
546 		break;
547 
548 	case IBT_EVENT_PORT_UP:
549 		IBTF_DPRINTF_L4("ibdm", "\tevent_hdlr: PORT_UP");
550 		mutex_enter(&ibdm.ibdm_hl_mutex);
551 		port = ibdm_get_port_attr(event, &hca_list);
552 		if (port == NULL) {
553 			IBTF_DPRINTF_L2("ibdm",
554 			    "\tevent_hdlr: HCA not present");
555 			mutex_exit(&ibdm.ibdm_hl_mutex);
556 			break;
557 		}
558 		ibdm_initialize_port(port);
559 		hca_list->hl_nports_active++;
560 		cv_broadcast(&ibdm.ibdm_port_settle_cv);
561 		mutex_exit(&ibdm.ibdm_hl_mutex);
562 
563 		/* Inform IB nexus driver */
564 		mutex_enter(&ibdm.ibdm_ibnex_mutex);
565 		if (ibdm.ibdm_ibnex_callback != NULL) {
566 			(*ibdm.ibdm_ibnex_callback)((void *)
567 			    &event->ev_hca_guid, IBDM_EVENT_PORT_UP);
568 		}
569 		mutex_exit(&ibdm.ibdm_ibnex_mutex);
570 		break;
571 
572 	case IBT_ERROR_PORT_DOWN:
573 		IBTF_DPRINTF_L4("ibdm", "\tevent_hdlr: PORT_DOWN");
574 		mutex_enter(&ibdm.ibdm_hl_mutex);
575 		port = ibdm_get_port_attr(event, &hca_list);
576 		if (port == NULL) {
577 			IBTF_DPRINTF_L2("ibdm",
578 			    "\tevent_hdlr: HCA not present");
579 			mutex_exit(&ibdm.ibdm_hl_mutex);
580 			break;
581 		}
582 		hca_list->hl_nports_active--;
583 		port_sa_hdl = port->pa_sa_hdl;
584 		(void) ibdm_fini_port(port);
585 		port->pa_state = IBT_PORT_DOWN;
586 		cv_broadcast(&ibdm.ibdm_port_settle_cv);
587 		mutex_exit(&ibdm.ibdm_hl_mutex);
588 		ibdm_reset_all_dgids(port_sa_hdl);
589 		break;
590 
591 	case IBT_PORT_CHANGE_EVENT:
592 		IBTF_DPRINTF_L4("ibdm", "\tevent_hdlr: PORT_CHANGE");
593 		if (event->ev_port_flags & IBT_PORT_CHANGE_PKEY)
594 			ibdm_handle_port_change_event(event);
595 		break;
596 
597 	default:		/* Ignore all other events/errors */
598 		break;
599 	}
600 }
601 
602 static void
603 ibdm_handle_port_change_event(ibt_async_event_t *event)
604 {
605 	ibdm_port_attr_t	*port;
606 	ibdm_hca_list_t		*hca_list;
607 
608 	IBTF_DPRINTF_L2("ibdm", "\tibdm_handle_port_change_event:"
609 	    " HCA guid  %llx", event->ev_hca_guid);
610 	mutex_enter(&ibdm.ibdm_hl_mutex);
611 	port = ibdm_get_port_attr(event, &hca_list);
612 	if (port == NULL) {
613 		IBTF_DPRINTF_L2("ibdm", "\tevent_hdlr: HCA not present");
614 		mutex_exit(&ibdm.ibdm_hl_mutex);
615 		return;
616 	}
617 	ibdm_update_port_pkeys(port);
618 	cv_broadcast(&ibdm.ibdm_port_settle_cv);
619 	mutex_exit(&ibdm.ibdm_hl_mutex);
620 
621 	/* Inform IB nexus driver */
622 	mutex_enter(&ibdm.ibdm_ibnex_mutex);
623 	if (ibdm.ibdm_ibnex_callback != NULL) {
624 		(*ibdm.ibdm_ibnex_callback)((void *)
625 		    &event->ev_hca_guid, IBDM_EVENT_PORT_PKEY_CHANGE);
626 	}
627 	mutex_exit(&ibdm.ibdm_ibnex_mutex);
628 }
629 
630 /*
631  * ibdm_update_port_pkeys()
632  *	Update the pkey table
633  *	Update the port attributes
634  */
635 static void
636 ibdm_update_port_pkeys(ibdm_port_attr_t *port)
637 {
638 	uint_t				nports, size;
639 	uint_t				pkey_idx, opkey_idx;
640 	uint16_t			npkeys;
641 	ibt_hca_portinfo_t		*pinfop;
642 	ib_pkey_t			pkey;
643 	ibdm_pkey_tbl_t			*pkey_tbl;
644 	ibdm_port_attr_t		newport;
645 
646 	IBTF_DPRINTF_L4("ibdm", "\tupdate_port_pkeys:");
647 	ASSERT(MUTEX_HELD(&ibdm.ibdm_hl_mutex));
648 
649 	/* Check whether the port is active */
650 	if (ibt_get_port_state(port->pa_hca_hdl, port->pa_port_num, NULL,
651 	    NULL) != IBT_SUCCESS)
652 		return;
653 
654 	if (ibt_query_hca_ports(port->pa_hca_hdl, port->pa_port_num,
655 	    &pinfop, &nports, &size) != IBT_SUCCESS) {
656 		/* This should not occur */
657 		port->pa_npkeys = 0;
658 		port->pa_pkey_tbl = NULL;
659 		return;
660 	}
661 
662 	npkeys = pinfop->p_pkey_tbl_sz;
663 	pkey_tbl = kmem_zalloc(npkeys * sizeof (ibdm_pkey_tbl_t), KM_SLEEP);
664 	newport.pa_pkey_tbl = pkey_tbl;
665 	newport.pa_ibmf_hdl = port->pa_ibmf_hdl;
666 
667 	for (pkey_idx = 0; pkey_idx < npkeys; pkey_idx++) {
668 		pkey = pkey_tbl[pkey_idx].pt_pkey =
669 		    pinfop->p_pkey_tbl[pkey_idx];
670 		/*
671 		 * Is this pkey present in the current table ?
672 		 */
673 		for (opkey_idx = 0; opkey_idx < port->pa_npkeys; opkey_idx++) {
674 			if (pkey == port->pa_pkey_tbl[opkey_idx].pt_pkey) {
675 				pkey_tbl[pkey_idx].pt_qp_hdl =
676 				    port->pa_pkey_tbl[opkey_idx].pt_qp_hdl;
677 				port->pa_pkey_tbl[opkey_idx].pt_qp_hdl = NULL;
678 				break;
679 			}
680 		}
681 
682 		if (opkey_idx == port->pa_npkeys) {
683 			pkey = pkey_tbl[pkey_idx].pt_pkey;
684 			if (IBDM_INVALID_PKEY(pkey)) {
685 				pkey_tbl[pkey_idx].pt_qp_hdl = NULL;
686 				continue;
687 			}
688 			ibdm_port_attr_ibmf_init(&newport, pkey, pkey_idx);
689 		}
690 	}
691 
692 	for (opkey_idx = 0; opkey_idx < port->pa_npkeys; opkey_idx++) {
693 		if (port->pa_pkey_tbl[opkey_idx].pt_qp_hdl != NULL) {
694 			if (ibdm_port_attr_ibmf_fini(port, opkey_idx) !=
695 			    IBDM_SUCCESS) {
696 				IBTF_DPRINTF_L2("ibdm", "\tupdate_port_pkeys: "
697 				    "ibdm_port_attr_ibmf_fini failed for "
698 				    "port pkey 0x%x",
699 				    port->pa_pkey_tbl[opkey_idx].pt_pkey);
700 			}
701 		}
702 	}
703 
704 	if (port->pa_pkey_tbl != NULL) {
705 		kmem_free(port->pa_pkey_tbl,
706 		    port->pa_npkeys * sizeof (ibdm_pkey_tbl_t));
707 	}
708 
709 	port->pa_npkeys = npkeys;
710 	port->pa_pkey_tbl = pkey_tbl;
711 	port->pa_sn_prefix = pinfop->p_sgid_tbl[0].gid_prefix;
712 	port->pa_state = pinfop->p_linkstate;
713 	ibt_free_portinfo(pinfop, size);
714 }
715 
716 /*
717  * ibdm_initialize_port()
718  *	Register with IBMF
719  *	Register with SA access
720  *	Register a receive callback routine with IBMF. IBMF invokes
721  *	this routine whenever a MAD arrives at this port.
722  *	Update the port attributes
723  */
724 static void
725 ibdm_initialize_port(ibdm_port_attr_t *port)
726 {
727 	int				ii;
728 	uint_t				nports, size;
729 	uint_t				pkey_idx;
730 	ib_pkey_t			pkey;
731 	ibt_hca_portinfo_t		*pinfop;
732 	ibmf_register_info_t		ibmf_reg;
733 	ibmf_saa_subnet_event_args_t	event_args;
734 
735 	IBTF_DPRINTF_L4("ibdm", "\tinitialize_port:");
736 	ASSERT(MUTEX_HELD(&ibdm.ibdm_hl_mutex));
737 
738 	/* Check whether the port is active */
739 	if (ibt_get_port_state(port->pa_hca_hdl, port->pa_port_num, NULL,
740 	    NULL) != IBT_SUCCESS)
741 		return;
742 
743 	if (port->pa_sa_hdl != NULL || port->pa_pkey_tbl != NULL)
744 		return;
745 
746 	if (ibt_query_hca_ports(port->pa_hca_hdl, port->pa_port_num,
747 	    &pinfop, &nports, &size) != IBT_SUCCESS) {
748 		/* This should not occur */
749 		port->pa_npkeys		= 0;
750 		port->pa_pkey_tbl	= NULL;
751 		return;
752 	}
753 	port->pa_sn_prefix = pinfop->p_sgid_tbl[0].gid_prefix;
754 
755 	port->pa_state		= pinfop->p_linkstate;
756 	port->pa_npkeys		= pinfop->p_pkey_tbl_sz;
757 	port->pa_pkey_tbl	= (ibdm_pkey_tbl_t *)kmem_zalloc(
758 	    port->pa_npkeys * sizeof (ibdm_pkey_tbl_t), KM_SLEEP);
759 
760 	for (pkey_idx = 0; pkey_idx < port->pa_npkeys; pkey_idx++)
761 		port->pa_pkey_tbl[pkey_idx].pt_pkey =
762 		    pinfop->p_pkey_tbl[pkey_idx];
763 
764 	ibt_free_portinfo(pinfop, size);
765 
766 	if (ibdm_enumerate_iocs) {
767 		event_args.is_event_callback = ibdm_saa_event_cb;
768 		event_args.is_event_callback_arg = port;
769 		if (ibmf_sa_session_open(port->pa_port_guid, 0, &event_args,
770 		    IBMF_VERSION, 0, &port->pa_sa_hdl) != IBMF_SUCCESS) {
771 			IBTF_DPRINTF_L2("ibdm", "\tinitialize_port: "
772 			    "sa access registration failed");
773 			(void) ibdm_fini_port(port);
774 			return;
775 		}
776 
777 		ibmf_reg.ir_ci_guid		= port->pa_hca_guid;
778 		ibmf_reg.ir_port_num		= port->pa_port_num;
779 		ibmf_reg.ir_client_class	= DEV_MGT_MANAGER;
780 
781 		if (ibmf_register(&ibmf_reg, IBMF_VERSION, 0, NULL, NULL,
782 		    &port->pa_ibmf_hdl, &port->pa_ibmf_caps) != IBMF_SUCCESS) {
783 			IBTF_DPRINTF_L2("ibdm", "\tinitialize_port: "
784 			    "IBMF registration failed");
785 			(void) ibdm_fini_port(port);
786 			return;
787 		}
788 
789 		if (ibmf_setup_async_cb(port->pa_ibmf_hdl,
790 		    IBMF_QP_HANDLE_DEFAULT,
791 		    ibdm_ibmf_recv_cb, 0, 0) != IBMF_SUCCESS) {
792 			IBTF_DPRINTF_L2("ibdm", "\tinitialize_port: "
793 			    "IBMF setup recv cb failed");
794 			(void) ibdm_fini_port(port);
795 			return;
796 		}
797 	} else {
798 		port->pa_sa_hdl = NULL;
799 		port->pa_ibmf_hdl = NULL;
800 	}
801 
802 	for (ii = 0; ii < port->pa_npkeys; ii++) {
803 		pkey = port->pa_pkey_tbl[ii].pt_pkey;
804 		if (IBDM_INVALID_PKEY(pkey)) {
805 			port->pa_pkey_tbl[ii].pt_qp_hdl = NULL;
806 			continue;
807 		}
808 		ibdm_port_attr_ibmf_init(port, pkey, ii);
809 	}
810 }
811 
812 
813 /*
814  * ibdm_port_attr_ibmf_init:
815  *	With IBMF - Alloc QP Handle and Setup Async callback
816  */
817 static void
818 ibdm_port_attr_ibmf_init(ibdm_port_attr_t *port, ib_pkey_t pkey, int ii)
819 {
820 	int ret;
821 
822 	if (ibdm_enumerate_iocs == 0) {
823 		port->pa_pkey_tbl[ii].pt_qp_hdl = NULL;
824 		return;
825 	}
826 
827 	if ((ret = ibmf_alloc_qp(port->pa_ibmf_hdl, pkey, IB_GSI_QKEY,
828 	    IBMF_ALT_QP_MAD_NO_RMPP, &port->pa_pkey_tbl[ii].pt_qp_hdl)) !=
829 	    IBMF_SUCCESS) {
830 		IBTF_DPRINTF_L2("ibdm", "\tport_attr_ibmf_init: "
831 		    "IBMF failed to alloc qp %d", ret);
832 		port->pa_pkey_tbl[ii].pt_qp_hdl = NULL;
833 		return;
834 	}
835 
836 	IBTF_DPRINTF_L4("ibdm", "\tport_attr_ibmf_init: QP handle is %p",
837 	    port->pa_ibmf_hdl);
838 
839 	if ((ret = ibmf_setup_async_cb(port->pa_ibmf_hdl,
840 	    port->pa_pkey_tbl[ii].pt_qp_hdl, ibdm_ibmf_recv_cb, 0, 0)) !=
841 	    IBMF_SUCCESS) {
842 		IBTF_DPRINTF_L2("ibdm", "\tport_attr_ibmf_init: "
843 		    "IBMF setup recv cb failed %d", ret);
844 		(void) ibmf_free_qp(port->pa_ibmf_hdl,
845 		    &port->pa_pkey_tbl[ii].pt_qp_hdl, 0);
846 		port->pa_pkey_tbl[ii].pt_qp_hdl = NULL;
847 	}
848 }
849 
850 
851 /*
852  * ibdm_get_port_attr()
853  *	Get port attributes from HCA guid and port number
854  *	Return pointer to ibdm_port_attr_t on Success
855  *	and NULL on failure
856  */
857 static ibdm_port_attr_t *
858 ibdm_get_port_attr(ibt_async_event_t *event, ibdm_hca_list_t **retval)
859 {
860 	ibdm_hca_list_t		*hca_list;
861 	ibdm_port_attr_t	*port_attr;
862 	int			ii;
863 
864 	IBTF_DPRINTF_L4("ibdm", "\tget_port_attr: port# %d", event->ev_port);
865 	ASSERT(MUTEX_HELD(&ibdm.ibdm_hl_mutex));
866 	hca_list = ibdm.ibdm_hca_list_head;
867 	while (hca_list) {
868 		if (hca_list->hl_hca_guid == event->ev_hca_guid) {
869 			for (ii = 0; ii < hca_list->hl_nports; ii++) {
870 				port_attr = &hca_list->hl_port_attr[ii];
871 				if (port_attr->pa_port_num == event->ev_port) {
872 					*retval = hca_list;
873 					return (port_attr);
874 				}
875 			}
876 		}
877 		hca_list = hca_list->hl_next;
878 	}
879 	return (NULL);
880 }
881 
882 
883 /*
884  * ibdm_update_port_attr()
885  *	Update the port attributes
886  */
887 static void
888 ibdm_update_port_attr(ibdm_port_attr_t *port)
889 {
890 	uint_t			nports, size;
891 	uint_t			pkey_idx;
892 	ibt_hca_portinfo_t	*portinfop;
893 
894 	IBTF_DPRINTF_L4("ibdm", "\tupdate_port_attr: Begin");
895 	if (ibt_query_hca_ports(port->pa_hca_hdl,
896 	    port->pa_port_num, &portinfop, &nports, &size) != IBT_SUCCESS) {
897 		/* This should not occur */
898 		port->pa_npkeys		= 0;
899 		port->pa_pkey_tbl	= NULL;
900 		return;
901 	}
902 	port->pa_sn_prefix = portinfop->p_sgid_tbl[0].gid_prefix;
903 
904 	port->pa_state		= portinfop->p_linkstate;
905 
906 	/*
907 	 * PKey information in portinfo valid only if port is
908 	 * ACTIVE. Bail out if not.
909 	 */
910 	if (port->pa_state != IBT_PORT_ACTIVE) {
911 		port->pa_npkeys		= 0;
912 		port->pa_pkey_tbl	= NULL;
913 		ibt_free_portinfo(portinfop, size);
914 		return;
915 	}
916 
917 	port->pa_npkeys		= portinfop->p_pkey_tbl_sz;
918 	port->pa_pkey_tbl	= (ibdm_pkey_tbl_t *)kmem_zalloc(
919 	    port->pa_npkeys * sizeof (ibdm_pkey_tbl_t), KM_SLEEP);
920 
921 	for (pkey_idx = 0; pkey_idx < port->pa_npkeys; pkey_idx++) {
922 		port->pa_pkey_tbl[pkey_idx].pt_pkey =
923 		    portinfop->p_pkey_tbl[pkey_idx];
924 	}
925 	ibt_free_portinfo(portinfop, size);
926 }
927 
928 
929 /*
930  * ibdm_handle_hca_attach()
931  */
932 static void
933 ibdm_handle_hca_attach(ib_guid_t hca_guid)
934 {
935 	uint_t			size;
936 	uint_t			ii, nports;
937 	ibt_status_t		status;
938 	ibt_hca_hdl_t		hca_hdl;
939 	ibt_hca_attr_t		*hca_attr;
940 	ibdm_hca_list_t		*hca_list, *temp;
941 	ibdm_port_attr_t	*port_attr;
942 	ibt_hca_portinfo_t	*portinfop;
943 
944 	IBTF_DPRINTF_L4("ibdm",
945 	    "\thandle_hca_attach: hca_guid = 0x%llX", hca_guid);
946 
947 	/* open the HCA first */
948 	if ((status = ibt_open_hca(ibdm.ibdm_ibt_clnt_hdl, hca_guid,
949 	    &hca_hdl)) != IBT_SUCCESS) {
950 		IBTF_DPRINTF_L2("ibdm", "\thandle_hca_attach: "
951 		    "open_hca failed, status 0x%x", status);
952 		return;
953 	}
954 
955 	hca_attr = (ibt_hca_attr_t *)
956 	    kmem_alloc(sizeof (ibt_hca_attr_t), KM_SLEEP);
957 	/* ibt_query_hca always returns IBT_SUCCESS */
958 	(void) ibt_query_hca(hca_hdl, hca_attr);
959 
960 	IBTF_DPRINTF_L4("ibdm", "\tvid: 0x%x, pid: 0x%x, ver: 0x%x,"
961 	    " #ports: %d", hca_attr->hca_vendor_id, hca_attr->hca_device_id,
962 	    hca_attr->hca_version_id, hca_attr->hca_nports);
963 
964 	if ((status = ibt_query_hca_ports(hca_hdl, 0, &portinfop, &nports,
965 	    &size)) != IBT_SUCCESS) {
966 		IBTF_DPRINTF_L2("ibdm", "\thandle_hca_attach: "
967 		    "ibt_query_hca_ports failed, status 0x%x", status);
968 		kmem_free(hca_attr, sizeof (ibt_hca_attr_t));
969 		(void) ibt_close_hca(hca_hdl);
970 		return;
971 	}
972 	hca_list = (ibdm_hca_list_t *)
973 	    kmem_zalloc((sizeof (ibdm_hca_list_t)), KM_SLEEP);
974 	hca_list->hl_port_attr = (ibdm_port_attr_t *)kmem_zalloc(
975 	    (sizeof (ibdm_port_attr_t) * hca_attr->hca_nports), KM_SLEEP);
976 	hca_list->hl_hca_guid = hca_attr->hca_node_guid;
977 	hca_list->hl_nports = hca_attr->hca_nports;
978 	hca_list->hl_attach_time = gethrtime();
979 	hca_list->hl_hca_hdl = hca_hdl;
980 
981 	/*
982 	 * Init a dummy port attribute for the HCA node
983 	 * This is for Per-HCA Node. Initialize port_attr :
984 	 * 	hca_guid & port_guid -> hca_guid
985 	 *	npkeys, pkey_tbl is NULL
986 	 *	port_num, sn_prefix is 0
987 	 *	vendorid, product_id, dev_version from HCA
988 	 *	pa_state is IBT_PORT_ACTIVE
989 	 */
990 	hca_list->hl_hca_port_attr = (ibdm_port_attr_t *)kmem_zalloc(
991 	    sizeof (ibdm_port_attr_t), KM_SLEEP);
992 	port_attr = hca_list->hl_hca_port_attr;
993 	port_attr->pa_vendorid  = hca_attr->hca_vendor_id;
994 	port_attr->pa_productid	= hca_attr->hca_device_id;
995 	port_attr->pa_dev_version = hca_attr->hca_version_id;
996 	port_attr->pa_hca_guid	= hca_attr->hca_node_guid;
997 	port_attr->pa_hca_hdl	= hca_list->hl_hca_hdl;
998 	port_attr->pa_port_guid	= hca_attr->hca_node_guid;
999 	port_attr->pa_state	= IBT_PORT_ACTIVE;
1000 
1001 
1002 	for (ii = 0; ii < nports; ii++) {
1003 		port_attr		= &hca_list->hl_port_attr[ii];
1004 		port_attr->pa_vendorid	= hca_attr->hca_vendor_id;
1005 		port_attr->pa_productid	= hca_attr->hca_device_id;
1006 		port_attr->pa_dev_version = hca_attr->hca_version_id;
1007 		port_attr->pa_hca_guid	= hca_attr->hca_node_guid;
1008 		port_attr->pa_hca_hdl	= hca_list->hl_hca_hdl;
1009 		port_attr->pa_port_guid	= portinfop[ii].p_sgid_tbl->gid_guid;
1010 		port_attr->pa_sn_prefix	= portinfop[ii].p_sgid_tbl->gid_prefix;
1011 		port_attr->pa_port_num	= portinfop[ii].p_port_num;
1012 		port_attr->pa_state	= portinfop[ii].p_linkstate;
1013 
1014 		/*
1015 		 * Register with IBMF, SA access when the port is in
1016 		 * ACTIVE state. Also register a callback routine
1017 		 * with IBMF to receive incoming DM MAD's.
1018 		 * The IBDM event handler takes care of registration of
1019 		 * port which are not active.
1020 		 */
1021 		IBTF_DPRINTF_L4("ibdm",
1022 		    "\thandle_hca_attach: port guid %llx Port state 0x%x",
1023 		    port_attr->pa_port_guid, portinfop[ii].p_linkstate);
1024 
1025 		if (portinfop[ii].p_linkstate == IBT_PORT_ACTIVE) {
1026 			mutex_enter(&ibdm.ibdm_hl_mutex);
1027 			hca_list->hl_nports_active++;
1028 			ibdm_initialize_port(port_attr);
1029 			cv_broadcast(&ibdm.ibdm_port_settle_cv);
1030 			mutex_exit(&ibdm.ibdm_hl_mutex);
1031 		}
1032 	}
1033 	mutex_enter(&ibdm.ibdm_hl_mutex);
1034 	for (temp = ibdm.ibdm_hca_list_head; temp; temp = temp->hl_next) {
1035 		if (temp->hl_hca_guid == hca_guid) {
1036 			IBTF_DPRINTF_L2("ibdm", "hca_attach: HCA %llX "
1037 			    "already seen by IBDM", hca_guid);
1038 			mutex_exit(&ibdm.ibdm_hl_mutex);
1039 			(void) ibdm_uninit_hca(hca_list);
1040 			return;
1041 		}
1042 	}
1043 	ibdm.ibdm_hca_count++;
1044 	if (ibdm.ibdm_hca_list_head == NULL) {
1045 		ibdm.ibdm_hca_list_head = hca_list;
1046 		ibdm.ibdm_hca_list_tail = hca_list;
1047 	} else {
1048 		ibdm.ibdm_hca_list_tail->hl_next = hca_list;
1049 		ibdm.ibdm_hca_list_tail = hca_list;
1050 	}
1051 	mutex_exit(&ibdm.ibdm_hl_mutex);
1052 	mutex_enter(&ibdm.ibdm_ibnex_mutex);
1053 	if (ibdm.ibdm_ibnex_callback != NULL) {
1054 		(*ibdm.ibdm_ibnex_callback)((void *)
1055 		    &hca_guid, IBDM_EVENT_HCA_ADDED);
1056 	}
1057 	mutex_exit(&ibdm.ibdm_ibnex_mutex);
1058 
1059 	kmem_free(hca_attr, sizeof (ibt_hca_attr_t));
1060 	ibt_free_portinfo(portinfop, size);
1061 }
1062 
1063 
1064 /*
1065  * ibdm_handle_hca_detach()
1066  */
1067 static void
1068 ibdm_handle_hca_detach(ib_guid_t hca_guid)
1069 {
1070 	ibdm_hca_list_t		*head, *prev = NULL;
1071 	size_t			len;
1072 	ibdm_dp_gidinfo_t	*gidinfo;
1073 	ibdm_port_attr_t	*port_attr;
1074 	int			i;
1075 
1076 	IBTF_DPRINTF_L4("ibdm",
1077 	    "\thandle_hca_detach: hca_guid = 0x%llx", hca_guid);
1078 
1079 	/* Make sure no probes are running */
1080 	mutex_enter(&ibdm.ibdm_mutex);
1081 	while (ibdm.ibdm_busy & IBDM_BUSY)
1082 		cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex);
1083 	ibdm.ibdm_busy |= IBDM_BUSY;
1084 	mutex_exit(&ibdm.ibdm_mutex);
1085 
1086 	mutex_enter(&ibdm.ibdm_hl_mutex);
1087 	head = ibdm.ibdm_hca_list_head;
1088 	while (head) {
1089 		if (head->hl_hca_guid == hca_guid) {
1090 			if (prev == NULL)
1091 				ibdm.ibdm_hca_list_head = head->hl_next;
1092 			else
1093 				prev->hl_next = head->hl_next;
1094 			if (ibdm.ibdm_hca_list_tail == head)
1095 				ibdm.ibdm_hca_list_tail = prev;
1096 			ibdm.ibdm_hca_count--;
1097 			break;
1098 		}
1099 		prev = head;
1100 		head = head->hl_next;
1101 	}
1102 	mutex_exit(&ibdm.ibdm_hl_mutex);
1103 	if (ibdm_uninit_hca(head) != IBDM_SUCCESS)
1104 		(void) ibdm_handle_hca_attach(hca_guid);
1105 
1106 #ifdef DEBUG
1107 	if (ibdm_enumerate_iocs == 0) {
1108 		ASSERT(ibdm.ibdm_dp_gidlist_head == NULL);
1109 	}
1110 #endif
1111 
1112 	/*
1113 	 * Now clean up the HCA lists in the gidlist.
1114 	 */
1115 	for (gidinfo = ibdm.ibdm_dp_gidlist_head; gidinfo; gidinfo =
1116 	    gidinfo->gl_next) {
1117 		prev = NULL;
1118 		head = gidinfo->gl_hca_list;
1119 		while (head) {
1120 			if (head->hl_hca_guid == hca_guid) {
1121 				if (prev == NULL)
1122 					gidinfo->gl_hca_list =
1123 					    head->hl_next;
1124 				else
1125 					prev->hl_next = head->hl_next;
1126 				for (i = 0; i < head->hl_nports; i++) {
1127 					port_attr = &head->hl_port_attr[i];
1128 					if (port_attr->pa_pkey_tbl != NULL)
1129 						kmem_free(
1130 						    port_attr->pa_pkey_tbl,
1131 						    port_attr->pa_npkeys *
1132 						    sizeof (ibdm_pkey_tbl_t));
1133 				}
1134 				len = sizeof (ibdm_hca_list_t) +
1135 				    (head->hl_nports *
1136 				    sizeof (ibdm_port_attr_t));
1137 				kmem_free(head, len);
1138 
1139 				break;
1140 			}
1141 			prev = head;
1142 			head = head->hl_next;
1143 		}
1144 	}
1145 
1146 	mutex_enter(&ibdm.ibdm_mutex);
1147 	ibdm.ibdm_busy &= ~IBDM_BUSY;
1148 	cv_broadcast(&ibdm.ibdm_busy_cv);
1149 	mutex_exit(&ibdm.ibdm_mutex);
1150 }
1151 
1152 
1153 static int
1154 ibdm_uninit_hca(ibdm_hca_list_t *head)
1155 {
1156 	int			ii;
1157 	ibdm_port_attr_t	*port_attr;
1158 
1159 	for (ii = 0; ii < head->hl_nports; ii++) {
1160 		port_attr = &head->hl_port_attr[ii];
1161 		if (ibdm_fini_port(port_attr) != IBDM_SUCCESS) {
1162 			IBTF_DPRINTF_L2("ibdm", "uninit_hca: HCA %p port 0x%x "
1163 			    "ibdm_fini_port() failed", head, ii);
1164 			return (IBDM_FAILURE);
1165 		}
1166 	}
1167 	if (head->hl_hca_hdl)
1168 		if (ibt_close_hca(head->hl_hca_hdl) != IBT_SUCCESS) {
1169 			IBTF_DPRINTF_L2("ibdm", "uninit_hca: "
1170 			    "ibt_close_hca() failed");
1171 			return (IBDM_FAILURE);
1172 		}
1173 	kmem_free(head->hl_port_attr,
1174 	    head->hl_nports * sizeof (ibdm_port_attr_t));
1175 	kmem_free(head->hl_hca_port_attr, sizeof (ibdm_port_attr_t));
1176 	kmem_free(head, sizeof (ibdm_hca_list_t));
1177 	return (IBDM_SUCCESS);
1178 }
1179 
1180 
1181 /*
1182  * For each port on the HCA,
1183  *	1) Teardown IBMF receive callback function
1184  *	2) Unregister with IBMF
1185  *	3) Unregister with SA access
1186  */
1187 static int
1188 ibdm_fini_port(ibdm_port_attr_t *port_attr)
1189 {
1190 	int	ii, ibmf_status;
1191 
1192 	for (ii = 0; ii < port_attr->pa_npkeys; ii++) {
1193 		if (port_attr->pa_pkey_tbl == NULL)
1194 			break;
1195 		if (!port_attr->pa_pkey_tbl[ii].pt_qp_hdl)
1196 			continue;
1197 		if (ibdm_port_attr_ibmf_fini(port_attr, ii) != IBDM_SUCCESS) {
1198 			IBTF_DPRINTF_L4("ibdm", "\tfini_port: "
1199 			    "ibdm_port_attr_ibmf_fini failed for "
1200 			    "port pkey 0x%x", ii);
1201 			return (IBDM_FAILURE);
1202 		}
1203 	}
1204 
1205 	if (port_attr->pa_ibmf_hdl) {
1206 		ibmf_status = ibmf_tear_down_async_cb(port_attr->pa_ibmf_hdl,
1207 		    IBMF_QP_HANDLE_DEFAULT, 0);
1208 		if (ibmf_status != IBMF_SUCCESS) {
1209 			IBTF_DPRINTF_L4("ibdm", "\tfini_port: "
1210 			    "ibmf_tear_down_async_cb failed %d", ibmf_status);
1211 			return (IBDM_FAILURE);
1212 		}
1213 
1214 		ibmf_status = ibmf_unregister(&port_attr->pa_ibmf_hdl, 0);
1215 		if (ibmf_status != IBMF_SUCCESS) {
1216 			IBTF_DPRINTF_L2("ibdm", "\tfini_port: "
1217 			    "ibmf_unregister failed %d", ibmf_status);
1218 			return (IBDM_FAILURE);
1219 		}
1220 
1221 		port_attr->pa_ibmf_hdl = NULL;
1222 	}
1223 
1224 	if (port_attr->pa_sa_hdl) {
1225 		ibmf_status = ibmf_sa_session_close(&port_attr->pa_sa_hdl, 0);
1226 		if (ibmf_status != IBMF_SUCCESS) {
1227 			IBTF_DPRINTF_L2("ibdm", "\tfini_port: "
1228 			    "ibmf_sa_session_close failed %d", ibmf_status);
1229 			return (IBDM_FAILURE);
1230 		}
1231 		port_attr->pa_sa_hdl = NULL;
1232 	}
1233 
1234 	if (port_attr->pa_pkey_tbl != NULL) {
1235 		kmem_free(port_attr->pa_pkey_tbl,
1236 		    port_attr->pa_npkeys * sizeof (ibdm_pkey_tbl_t));
1237 		port_attr->pa_pkey_tbl = NULL;
1238 		port_attr->pa_npkeys = 0;
1239 	}
1240 
1241 	return (IBDM_SUCCESS);
1242 }
1243 
1244 
1245 /*
1246  * ibdm_port_attr_ibmf_fini:
1247  *	With IBMF - Tear down Async callback and free QP Handle
1248  */
1249 static int
1250 ibdm_port_attr_ibmf_fini(ibdm_port_attr_t *port_attr, int ii)
1251 {
1252 	int ibmf_status;
1253 
1254 	IBTF_DPRINTF_L5("ibdm", "\tport_attr_ibmf_fini:");
1255 
1256 	if (ibdm_enumerate_iocs == 0) {
1257 		ASSERT(port_attr->pa_pkey_tbl[ii].pt_qp_hdl == NULL);
1258 		return (IBDM_SUCCESS);
1259 	}
1260 
1261 	if (port_attr->pa_pkey_tbl[ii].pt_qp_hdl) {
1262 		ibmf_status = ibmf_tear_down_async_cb(port_attr->pa_ibmf_hdl,
1263 		    port_attr->pa_pkey_tbl[ii].pt_qp_hdl, 0);
1264 		if (ibmf_status != IBMF_SUCCESS) {
1265 			IBTF_DPRINTF_L4("ibdm", "\tport_attr_ibmf_fini: "
1266 			    "ibmf_tear_down_async_cb failed %d", ibmf_status);
1267 			return (IBDM_FAILURE);
1268 		}
1269 		ibmf_status = ibmf_free_qp(port_attr->pa_ibmf_hdl,
1270 		    &port_attr->pa_pkey_tbl[ii].pt_qp_hdl, 0);
1271 		if (ibmf_status != IBMF_SUCCESS) {
1272 			IBTF_DPRINTF_L4("ibdm", "\tport_attr_ibmf_fini: "
1273 			    "ibmf_free_qp failed %d", ibmf_status);
1274 			return (IBDM_FAILURE);
1275 		}
1276 		port_attr->pa_pkey_tbl[ii].pt_qp_hdl = NULL;
1277 	}
1278 	return (IBDM_SUCCESS);
1279 }
1280 
1281 
1282 /*
1283  * ibdm_gid_decr_pending:
1284  *	decrement gl_pending_cmds. If zero wakeup sleeping threads
1285  */
1286 static void
1287 ibdm_gid_decr_pending(ibdm_dp_gidinfo_t *gidinfo)
1288 {
1289 	mutex_enter(&ibdm.ibdm_mutex);
1290 	mutex_enter(&gidinfo->gl_mutex);
1291 	if (--gidinfo->gl_pending_cmds == 0) {
1292 		/*
1293 		 * Handle DGID getting removed.
1294 		 */
1295 		if (gidinfo->gl_disconnected) {
1296 			mutex_exit(&gidinfo->gl_mutex);
1297 			mutex_exit(&ibdm.ibdm_mutex);
1298 
1299 			IBTF_DPRINTF_L3(ibdm_string, "\tgid_decr_pending: "
1300 			    "gidinfo %p hot removal", gidinfo);
1301 			ibdm_delete_gidinfo(gidinfo);
1302 
1303 			mutex_enter(&ibdm.ibdm_mutex);
1304 			ibdm.ibdm_ngid_probes_in_progress--;
1305 			ibdm_wait_probe_completion();
1306 			mutex_exit(&ibdm.ibdm_mutex);
1307 			return;
1308 		}
1309 		mutex_exit(&gidinfo->gl_mutex);
1310 		mutex_exit(&ibdm.ibdm_mutex);
1311 		ibdm_notify_newgid_iocs(gidinfo);
1312 		mutex_enter(&ibdm.ibdm_mutex);
1313 		mutex_enter(&gidinfo->gl_mutex);
1314 
1315 		ibdm.ibdm_ngid_probes_in_progress--;
1316 		ibdm_wait_probe_completion();
1317 	}
1318 	mutex_exit(&gidinfo->gl_mutex);
1319 	mutex_exit(&ibdm.ibdm_mutex);
1320 }
1321 
1322 
1323 /*
1324  * ibdm_wait_probe_completion:
1325  *	wait for probing to complete
1326  */
1327 static void
1328 ibdm_wait_probe_completion(void)
1329 {
1330 	ASSERT(MUTEX_HELD(&ibdm.ibdm_mutex));
1331 	if (ibdm.ibdm_ngid_probes_in_progress) {
1332 		IBTF_DPRINTF_L4("ibdm",	"\twait for probe complete");
1333 		ibdm.ibdm_busy |= IBDM_PROBE_IN_PROGRESS;
1334 		while (ibdm.ibdm_busy & IBDM_PROBE_IN_PROGRESS)
1335 			cv_wait(&ibdm.ibdm_probe_cv, &ibdm.ibdm_mutex);
1336 	}
1337 }
1338 
1339 
1340 /*
1341  * ibdm_wait_cisco_probe_completion:
1342  *	wait for the reply from the Cisco FC GW switch after a setclassportinfo
1343  *	request is sent. This wait can be achieved on each gid.
1344  */
1345 static void
1346 ibdm_wait_cisco_probe_completion(ibdm_dp_gidinfo_t *gidinfo)
1347 {
1348 	ASSERT(MUTEX_HELD(&gidinfo->gl_mutex));
1349 	IBTF_DPRINTF_L4("ibdm",	"\twait for cisco probe complete");
1350 	gidinfo->gl_flag |= IBDM_CISCO_PROBE;
1351 	while (gidinfo->gl_flag & IBDM_CISCO_PROBE)
1352 		cv_wait(&gidinfo->gl_probe_cv, &gidinfo->gl_mutex);
1353 }
1354 
1355 
1356 /*
1357  * ibdm_wakeup_probe_gid_cv:
1358  *	wakeup waiting threads (based on ibdm_ngid_probes_in_progress)
1359  */
1360 static void
1361 ibdm_wakeup_probe_gid_cv(void)
1362 {
1363 	ASSERT(MUTEX_HELD(&ibdm.ibdm_mutex));
1364 	if (!ibdm.ibdm_ngid_probes_in_progress) {
1365 		IBTF_DPRINTF_L4("ibdm", "wakeup_probe_gid_thread: Wakeup");
1366 		ibdm.ibdm_busy &= ~IBDM_PROBE_IN_PROGRESS;
1367 		cv_broadcast(&ibdm.ibdm_probe_cv);
1368 	}
1369 
1370 }
1371 
1372 
1373 /*
1374  * ibdm_sweep_fabric(reprobe_flag)
1375  *	Find all possible Managed IOU's and their IOC's that are visible
1376  *	to the host. The algorithm used is as follows
1377  *
1378  *	Send a "bus walk" request for each port on the host HCA to SA access
1379  *	SA returns complete set of GID's that are reachable from
1380  *	source port. This is done in parallel.
1381  *
1382  *	Initialize GID state to IBDM_GID_PROBE_NOT_DONE
1383  *
1384  *	Sort the GID list and eliminate duplicate GID's
1385  *		1) Use DGID for sorting
1386  *		2) use PortGuid for sorting
1387  *			Send SA query to retrieve NodeRecord and
1388  *			extract PortGuid from that.
1389  *
1390  *	Set GID state to IBDM_GID_PROBE_FAILED to all the ports that dont
1391  *	support DM MAD's
1392  *		Send a "Portinfo" query to get the port capabilities and
1393  *		then check for DM MAD's support
1394  *
1395  *	Send "ClassPortInfo" request for all the GID's in parallel,
1396  *	set the GID state to IBDM_GET_CLASSPORTINFO and wait on the
1397  *	cv_signal to complete.
1398  *
1399  *	When DM agent on the remote GID sends back the response, IBMF
1400  *	invokes DM callback routine.
1401  *
1402  *	If the response is proper, send "IOUnitInfo" request and set
1403  *	GID state to IBDM_GET_IOUNITINFO.
1404  *
1405  *	If the response is proper, send "IocProfileInfo" request to
1406  *	all the IOC simultaneously and set GID state to IBDM_GET_IOC_DETAILS.
1407  *
1408  *	Send request to get Service entries simultaneously
1409  *
1410  *	Signal the waiting thread when received response for all the commands.
1411  *
1412  *	Set the GID state to IBDM_GID_PROBE_FAILED when received a error
1413  *	response during the probing period.
1414  *
1415  *	Note:
1416  *	ibdm.ibdm_ngid_probes_in_progress and ibdm_gid_list_t:gl_pending_cmds
1417  *	keep track of number commands in progress at any point of time.
1418  *	MAD transaction ID is used to identify a particular GID
1419  *	TBD: Consider registering the IBMF receive callback on demand
1420  *
1421  *	Note: This routine must be called with ibdm.ibdm_mutex held
1422  *	TBD: Re probe the failure GID (for certain failures) when requested
1423  *	     for fabric sweep next time
1424  *
1425  *	Parameters : If reprobe_flag is set, All IOCs will be reprobed.
1426  */
1427 static void
1428 ibdm_sweep_fabric(int reprobe_flag)
1429 {
1430 	int			ii;
1431 	int			new_paths = 0;
1432 	uint8_t			niocs;
1433 	taskqid_t		tid;
1434 	ibdm_ioc_info_t		*ioc;
1435 	ibdm_hca_list_t		*hca_list = NULL;
1436 	ibdm_port_attr_t	*port = NULL;
1437 	ibdm_dp_gidinfo_t 	*gid_info;
1438 
1439 	IBTF_DPRINTF_L4("ibdm", "\tsweep_fabric: Enter");
1440 	ASSERT(MUTEX_HELD(&ibdm.ibdm_mutex));
1441 
1442 	/*
1443 	 * Check whether a sweep already in progress. If so, just
1444 	 * wait for the fabric sweep to complete
1445 	 */
1446 	while (ibdm.ibdm_busy & IBDM_BUSY)
1447 		cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex);
1448 	ibdm.ibdm_busy |= IBDM_BUSY;
1449 	mutex_exit(&ibdm.ibdm_mutex);
1450 
1451 	ibdm_dump_sweep_fabric_timestamp(0);
1452 
1453 	/* Rescan the GID list for any removed GIDs for reprobe */
1454 	if (reprobe_flag)
1455 		ibdm_rescan_gidlist(NULL);
1456 
1457 	/*
1458 	 * Get list of all the ports reachable from the local known HCA
1459 	 * ports which are active
1460 	 */
1461 	mutex_enter(&ibdm.ibdm_hl_mutex);
1462 	for (ibdm_get_next_port(&hca_list, &port, 1); port;
1463 	    ibdm_get_next_port(&hca_list, &port, 1)) {
1464 		/*
1465 		 * Get PATHS to all the reachable ports from
1466 		 * SGID and update the global ibdm structure.
1467 		 */
1468 		new_paths = ibdm_get_reachable_ports(port, hca_list);
1469 		ibdm.ibdm_ngids += new_paths;
1470 	}
1471 	mutex_exit(&ibdm.ibdm_hl_mutex);
1472 
1473 	mutex_enter(&ibdm.ibdm_mutex);
1474 	ibdm.ibdm_ngid_probes_in_progress += ibdm.ibdm_ngids;
1475 	mutex_exit(&ibdm.ibdm_mutex);
1476 
1477 	/* Send a request to probe GIDs asynchronously. */
1478 	for (gid_info = ibdm.ibdm_dp_gidlist_head; gid_info;
1479 	    gid_info = gid_info->gl_next) {
1480 		mutex_enter(&gid_info->gl_mutex);
1481 		gid_info->gl_reprobe_flag = reprobe_flag;
1482 		mutex_exit(&gid_info->gl_mutex);
1483 
1484 		/* process newly encountered GIDs */
1485 		tid = taskq_dispatch(system_taskq, ibdm_probe_gid_thread,
1486 		    (void *)gid_info, TQ_NOSLEEP);
1487 		IBTF_DPRINTF_L4("ibdm", "\tsweep_fabric: gid_info = %p"
1488 		    " taskq_id = %x", gid_info, tid);
1489 		/* taskq failed to dispatch call it directly */
1490 		if (tid == TASKQID_INVALID)
1491 			ibdm_probe_gid_thread((void *)gid_info);
1492 	}
1493 
1494 	mutex_enter(&ibdm.ibdm_mutex);
1495 	ibdm_wait_probe_completion();
1496 
1497 	/*
1498 	 * Update the properties, if reprobe_flag is set
1499 	 * Skip if gl_reprobe_flag is set, this will be
1500 	 * a re-inserted / new GID, for which notifications
1501 	 * have already been send.
1502 	 */
1503 	if (reprobe_flag) {
1504 		for (gid_info = ibdm.ibdm_dp_gidlist_head; gid_info;
1505 		    gid_info = gid_info->gl_next) {
1506 			if (gid_info->gl_iou == NULL)
1507 				continue;
1508 			if (gid_info->gl_reprobe_flag) {
1509 				gid_info->gl_reprobe_flag = 0;
1510 				continue;
1511 			}
1512 
1513 			niocs = gid_info->gl_iou->iou_info.iou_num_ctrl_slots;
1514 			for (ii = 0; ii < niocs; ii++) {
1515 				ioc = IBDM_GIDINFO2IOCINFO(gid_info, ii);
1516 				if (ioc)
1517 					ibdm_reprobe_update_port_srv(ioc,
1518 					    gid_info);
1519 			}
1520 		}
1521 	} else if (ibdm.ibdm_prev_iou) {
1522 		ibdm_ioc_info_t	*ioc_list;
1523 
1524 		/*
1525 		 * Get the list of IOCs which have changed.
1526 		 * If any IOCs have changed, Notify IBNexus
1527 		 */
1528 		ibdm.ibdm_prev_iou = 0;
1529 		ioc_list = ibdm_handle_prev_iou();
1530 		if (ioc_list) {
1531 			if (ibdm.ibdm_ibnex_callback != NULL) {
1532 				(*ibdm.ibdm_ibnex_callback)(
1533 				    (void *)ioc_list,
1534 				    IBDM_EVENT_IOC_PROP_UPDATE);
1535 			}
1536 		}
1537 	}
1538 
1539 	ibdm_dump_sweep_fabric_timestamp(1);
1540 
1541 	ibdm.ibdm_busy &= ~IBDM_BUSY;
1542 	cv_broadcast(&ibdm.ibdm_busy_cv);
1543 	IBTF_DPRINTF_L5("ibdm", "\tsweep_fabric: EXIT");
1544 }
1545 
1546 
1547 /*
1548  * ibdm_is_cisco:
1549  * 	Check if this is a Cisco device or not.
1550  */
1551 static boolean_t
1552 ibdm_is_cisco(ib_guid_t guid)
1553 {
1554 	if ((guid >> IBDM_OUI_GUID_SHIFT) == IBDM_CISCO_COMPANY_ID)
1555 		return (B_TRUE);
1556 	return (B_FALSE);
1557 }
1558 
1559 
1560 /*
1561  * ibdm_is_cisco_switch:
1562  * 	Check if this switch is a CISCO switch or not.
1563  * 	Note that if this switch is already activated, ibdm_is_cisco_switch()
1564  * 	returns B_FALSE not to re-activate it again.
1565  */
1566 static boolean_t
1567 ibdm_is_cisco_switch(ibdm_dp_gidinfo_t *gid_info)
1568 {
1569 	int company_id, device_id;
1570 	ASSERT(gid_info != 0);
1571 	ASSERT(MUTEX_HELD(&gid_info->gl_mutex));
1572 
1573 	/*
1574 	 * If this switch is already activated, don't re-activate it.
1575 	 */
1576 	if (gid_info->gl_flag & IBDM_CISCO_PROBE_DONE)
1577 		return (B_FALSE);
1578 
1579 	/*
1580 	 * Check if this switch is a Cisco FC GW or not.
1581 	 * Use the node guid (the OUI part) instead of the vendor id
1582 	 * since the vendor id is zero in practice.
1583 	 */
1584 	company_id = gid_info->gl_nodeguid >> IBDM_OUI_GUID_SHIFT;
1585 	device_id = gid_info->gl_devid;
1586 
1587 	if (company_id == IBDM_CISCO_COMPANY_ID &&
1588 	    device_id == IBDM_CISCO_DEVICE_ID)
1589 		return (B_TRUE);
1590 	return (B_FALSE);
1591 }
1592 
1593 
1594 /*
1595  * ibdm_probe_gid_thread:
1596  *	thread that does the actual work for sweeping the fabric
1597  *	for a given GID
1598  */
1599 static void
1600 ibdm_probe_gid_thread(void *args)
1601 {
1602 	int			reprobe_flag;
1603 	ib_guid_t		node_guid;
1604 	ib_guid_t		port_guid;
1605 	ibdm_dp_gidinfo_t	*gid_info;
1606 
1607 	gid_info = (ibdm_dp_gidinfo_t *)args;
1608 	reprobe_flag = gid_info->gl_reprobe_flag;
1609 	IBTF_DPRINTF_L4("ibdm", "\tprobe_gid_thread: gid_info = %p, flag = %d",
1610 	    gid_info, reprobe_flag);
1611 	ASSERT(gid_info != NULL);
1612 	ASSERT(gid_info->gl_pending_cmds == 0);
1613 
1614 	if (gid_info->gl_state != IBDM_GID_PROBE_NOT_DONE &&
1615 	    reprobe_flag == 0) {
1616 		/*
1617 		 * This GID may have been already probed. Send
1618 		 * in a CLP to check if IOUnitInfo changed?
1619 		 * Explicitly set gl_reprobe_flag to 0 so that
1620 		 * IBnex is not notified on completion
1621 		 */
1622 		if (gid_info->gl_state == IBDM_GID_PROBING_COMPLETE) {
1623 			IBTF_DPRINTF_L4("ibdm", "\tprobe_gid_thread: "
1624 			    "get new IOCs information");
1625 			mutex_enter(&gid_info->gl_mutex);
1626 			gid_info->gl_pending_cmds++;
1627 			gid_info->gl_state = IBDM_GET_IOUNITINFO;
1628 			gid_info->gl_reprobe_flag = 0;
1629 			mutex_exit(&gid_info->gl_mutex);
1630 			if (ibdm_send_iounitinfo(gid_info) != IBDM_SUCCESS) {
1631 				mutex_enter(&gid_info->gl_mutex);
1632 				--gid_info->gl_pending_cmds;
1633 				mutex_exit(&gid_info->gl_mutex);
1634 				mutex_enter(&ibdm.ibdm_mutex);
1635 				--ibdm.ibdm_ngid_probes_in_progress;
1636 				ibdm_wakeup_probe_gid_cv();
1637 				mutex_exit(&ibdm.ibdm_mutex);
1638 			}
1639 		} else {
1640 			mutex_enter(&ibdm.ibdm_mutex);
1641 			--ibdm.ibdm_ngid_probes_in_progress;
1642 			ibdm_wakeup_probe_gid_cv();
1643 			mutex_exit(&ibdm.ibdm_mutex);
1644 		}
1645 		return;
1646 	} else if (reprobe_flag && gid_info->gl_state ==
1647 	    IBDM_GID_PROBING_COMPLETE) {
1648 		/*
1649 		 * Reprobe all IOCs for the GID which has completed
1650 		 * probe. Skip other port GIDs to same IOU.
1651 		 * Explicitly set gl_reprobe_flag to 0 so that
1652 		 * IBnex is not notified on completion
1653 		 */
1654 		ibdm_ioc_info_t *ioc_info;
1655 		uint8_t		niocs, ii;
1656 
1657 		ASSERT(gid_info->gl_iou);
1658 		mutex_enter(&gid_info->gl_mutex);
1659 		niocs = gid_info->gl_iou->iou_info.iou_num_ctrl_slots;
1660 		gid_info->gl_state = IBDM_GET_IOC_DETAILS;
1661 		gid_info->gl_pending_cmds += niocs;
1662 		gid_info->gl_reprobe_flag = 0;
1663 		mutex_exit(&gid_info->gl_mutex);
1664 		for (ii = 0; ii < niocs; ii++) {
1665 			uchar_t			slot_info;
1666 			ib_dm_io_unitinfo_t	*giou_info;
1667 
1668 			/*
1669 			 * Check whether IOC is present in the slot
1670 			 * Series of nibbles (in the field
1671 			 * iou_ctrl_list) represents a slot in the
1672 			 * IOU.
1673 			 * Byte format: 76543210
1674 			 * Bits 0-3 of first byte represent Slot 2
1675 			 * bits 4-7 of first byte represent slot 1,
1676 			 * bits 0-3 of second byte represent slot 4
1677 			 * and so on
1678 			 * Each 4-bit nibble has the following meaning
1679 			 * 0x0 : IOC not installed
1680 			 * 0x1 : IOC is present
1681 			 * 0xf : Slot does not exist
1682 			 * and all other values are reserved.
1683 			 */
1684 			ioc_info = IBDM_GIDINFO2IOCINFO(gid_info, ii);
1685 			giou_info = &gid_info->gl_iou->iou_info;
1686 			slot_info = giou_info->iou_ctrl_list[(ii/2)];
1687 			if ((ii % 2) == 0)
1688 				slot_info = (slot_info >> 4);
1689 
1690 			if ((slot_info & 0xf) != 1) {
1691 				ioc_info->ioc_state =
1692 				    IBDM_IOC_STATE_PROBE_FAILED;
1693 				ibdm_gid_decr_pending(gid_info);
1694 				continue;
1695 			}
1696 
1697 			if (ibdm_send_ioc_profile(gid_info, ii) !=
1698 			    IBDM_SUCCESS) {
1699 				ibdm_gid_decr_pending(gid_info);
1700 			}
1701 		}
1702 
1703 		return;
1704 	} else if (gid_info->gl_state != IBDM_GID_PROBE_NOT_DONE) {
1705 		mutex_enter(&ibdm.ibdm_mutex);
1706 		--ibdm.ibdm_ngid_probes_in_progress;
1707 		ibdm_wakeup_probe_gid_cv();
1708 		mutex_exit(&ibdm.ibdm_mutex);
1709 		return;
1710 	}
1711 
1712 	/*
1713 	 * Check whether the destination GID supports DM agents. If
1714 	 * not, stop probing the GID and continue with the next GID
1715 	 * in the list.
1716 	 */
1717 	if (ibdm_is_dev_mgt_supported(gid_info) != IBDM_SUCCESS) {
1718 		mutex_enter(&gid_info->gl_mutex);
1719 		gid_info->gl_state = IBDM_GID_PROBING_FAILED;
1720 		gid_info->gl_is_dm_capable = B_FALSE;
1721 		mutex_exit(&gid_info->gl_mutex);
1722 		ibdm_delete_glhca_list(gid_info);
1723 		mutex_enter(&ibdm.ibdm_mutex);
1724 		--ibdm.ibdm_ngid_probes_in_progress;
1725 		ibdm_wakeup_probe_gid_cv();
1726 		mutex_exit(&ibdm.ibdm_mutex);
1727 		return;
1728 	}
1729 
1730 	/*
1731 	 * This GID is Device management capable
1732 	 */
1733 	mutex_enter(&gid_info->gl_mutex);
1734 	gid_info->gl_is_dm_capable = B_TRUE;
1735 	mutex_exit(&gid_info->gl_mutex);
1736 
1737 	/* Get the nodeguid and portguid of the port */
1738 	if (ibdm_get_node_port_guids(gid_info->gl_sa_hdl, gid_info->gl_dlid,
1739 	    &node_guid, &port_guid) != IBDM_SUCCESS) {
1740 		mutex_enter(&gid_info->gl_mutex);
1741 		gid_info->gl_state = IBDM_GID_PROBING_FAILED;
1742 		mutex_exit(&gid_info->gl_mutex);
1743 		ibdm_delete_glhca_list(gid_info);
1744 		mutex_enter(&ibdm.ibdm_mutex);
1745 		--ibdm.ibdm_ngid_probes_in_progress;
1746 		ibdm_wakeup_probe_gid_cv();
1747 		mutex_exit(&ibdm.ibdm_mutex);
1748 		return;
1749 	}
1750 
1751 	/*
1752 	 * Check whether we already knew about this NodeGuid
1753 	 * If so, do not probe the GID and continue with the
1754 	 * next  GID  in the gid  list. Set the GID state to
1755 	 * probing done.
1756 	 */
1757 	mutex_enter(&ibdm.ibdm_mutex);
1758 	gid_info->gl_nodeguid = node_guid;
1759 	gid_info->gl_portguid = port_guid;
1760 	if (ibdm_check_dest_nodeguid(gid_info) != NULL) {
1761 		mutex_exit(&ibdm.ibdm_mutex);
1762 		mutex_enter(&gid_info->gl_mutex);
1763 		gid_info->gl_state = IBDM_GID_PROBING_SKIPPED;
1764 		mutex_exit(&gid_info->gl_mutex);
1765 		ibdm_delete_glhca_list(gid_info);
1766 		mutex_enter(&ibdm.ibdm_mutex);
1767 		--ibdm.ibdm_ngid_probes_in_progress;
1768 		ibdm_wakeup_probe_gid_cv();
1769 		mutex_exit(&ibdm.ibdm_mutex);
1770 		return;
1771 	}
1772 	ibdm_add_to_gl_gid(gid_info, gid_info);
1773 	mutex_exit(&ibdm.ibdm_mutex);
1774 
1775 	/*
1776 	 * New or reinserted GID : Enable notification to IBnex
1777 	 */
1778 	mutex_enter(&gid_info->gl_mutex);
1779 	gid_info->gl_reprobe_flag = 1;
1780 
1781 	/*
1782 	 * A Cisco FC GW needs the special handling to get IOUnitInfo.
1783 	 */
1784 	if (ibdm_is_cisco_switch(gid_info)) {
1785 		gid_info->gl_pending_cmds++;
1786 		gid_info->gl_state = IBDM_SET_CLASSPORTINFO;
1787 		mutex_exit(&gid_info->gl_mutex);
1788 
1789 		if (ibdm_set_classportinfo(gid_info) != IBDM_SUCCESS) {
1790 			mutex_enter(&gid_info->gl_mutex);
1791 			gid_info->gl_state = IBDM_GID_PROBING_FAILED;
1792 			--gid_info->gl_pending_cmds;
1793 			mutex_exit(&gid_info->gl_mutex);
1794 
1795 			/* free the hca_list on this gid_info */
1796 			ibdm_delete_glhca_list(gid_info);
1797 
1798 			mutex_enter(&ibdm.ibdm_mutex);
1799 			--ibdm.ibdm_ngid_probes_in_progress;
1800 			ibdm_wakeup_probe_gid_cv();
1801 			mutex_exit(&ibdm.ibdm_mutex);
1802 
1803 			return;
1804 		}
1805 
1806 		mutex_enter(&gid_info->gl_mutex);
1807 		ibdm_wait_cisco_probe_completion(gid_info);
1808 
1809 		IBTF_DPRINTF_L4("ibdm", "\tibdm_probe_gid_thread: "
1810 		    "CISCO Wakeup signal received");
1811 	}
1812 
1813 	/* move on to the 'GET_CLASSPORTINFO' stage */
1814 	gid_info->gl_pending_cmds++;
1815 	gid_info->gl_state = IBDM_GET_CLASSPORTINFO;
1816 	mutex_exit(&gid_info->gl_mutex);
1817 
1818 	IBTF_DPRINTF_L3(ibdm_string, "\tibdm_probe_gid_thread: "
1819 	    "%d: gid_info %p gl_state %d pending_cmds %d",
1820 	    __LINE__, gid_info, gid_info->gl_state,
1821 	    gid_info->gl_pending_cmds);
1822 
1823 	/*
1824 	 * Send ClassPortInfo request to the GID asynchronously.
1825 	 */
1826 	if (ibdm_send_classportinfo(gid_info) != IBDM_SUCCESS) {
1827 
1828 		mutex_enter(&gid_info->gl_mutex);
1829 		gid_info->gl_state = IBDM_GID_PROBING_FAILED;
1830 		--gid_info->gl_pending_cmds;
1831 		mutex_exit(&gid_info->gl_mutex);
1832 
1833 		/* free the hca_list on this gid_info */
1834 		ibdm_delete_glhca_list(gid_info);
1835 
1836 		mutex_enter(&ibdm.ibdm_mutex);
1837 		--ibdm.ibdm_ngid_probes_in_progress;
1838 		ibdm_wakeup_probe_gid_cv();
1839 		mutex_exit(&ibdm.ibdm_mutex);
1840 
1841 		return;
1842 	}
1843 }
1844 
1845 
1846 /*
1847  * ibdm_check_dest_nodeguid
1848  *	Searches for the NodeGuid in the GID list
1849  *	Returns matching gid_info if found and otherwise NULL
1850  *
1851  *	This function is called to handle new GIDs discovered
1852  *	during device sweep / probe or for GID_AVAILABLE event.
1853  *
1854  *	Parameter :
1855  *		gid_info	GID to check
1856  */
1857 static ibdm_dp_gidinfo_t *
1858 ibdm_check_dest_nodeguid(ibdm_dp_gidinfo_t *gid_info)
1859 {
1860 	ibdm_dp_gidinfo_t	*gid_list;
1861 	ibdm_gid_t		*tmp;
1862 
1863 	IBTF_DPRINTF_L4("ibdm", "\tcheck_dest_nodeguid");
1864 
1865 	gid_list = ibdm.ibdm_dp_gidlist_head;
1866 	while (gid_list) {
1867 		if ((gid_list != gid_info) &&
1868 		    (gid_info->gl_nodeguid == gid_list->gl_nodeguid)) {
1869 			IBTF_DPRINTF_L4("ibdm",
1870 			    "\tcheck_dest_nodeguid: NodeGuid is present");
1871 
1872 			/* Add to gid_list */
1873 			tmp = kmem_zalloc(sizeof (ibdm_gid_t),
1874 			    KM_SLEEP);
1875 			tmp->gid_dgid_hi = gid_info->gl_dgid_hi;
1876 			tmp->gid_dgid_lo = gid_info->gl_dgid_lo;
1877 			tmp->gid_next = gid_list->gl_gid;
1878 			gid_list->gl_gid = tmp;
1879 			gid_list->gl_ngids++;
1880 			return (gid_list);
1881 		}
1882 
1883 		gid_list = gid_list->gl_next;
1884 	}
1885 
1886 	return (NULL);
1887 }
1888 
1889 
1890 /*
1891  * ibdm_is_dev_mgt_supported
1892  *	Get the PortInfo attribute (SA Query)
1893  *	Check "CompatabilityMask" field in the Portinfo.
1894  *	Return IBDM_SUCCESS if DM MAD's supported (if bit 19 set)
1895  *	by the port, otherwise IBDM_FAILURE
1896  */
1897 static int
1898 ibdm_is_dev_mgt_supported(ibdm_dp_gidinfo_t *gid_info)
1899 {
1900 	int			ret;
1901 	size_t			length = 0;
1902 	sa_portinfo_record_t	req, *resp = NULL;
1903 	ibmf_saa_access_args_t	qargs;
1904 
1905 	bzero(&req, sizeof (sa_portinfo_record_t));
1906 	req.EndportLID	= gid_info->gl_dlid;
1907 
1908 	qargs.sq_attr_id	= SA_PORTINFORECORD_ATTRID;
1909 	qargs.sq_access_type	= IBMF_SAA_RETRIEVE;
1910 	qargs.sq_component_mask = SA_PORTINFO_COMPMASK_PORTLID;
1911 	qargs.sq_template	= &req;
1912 	qargs.sq_callback	= NULL;
1913 	qargs.sq_callback_arg	= NULL;
1914 
1915 	ret = ibmf_sa_access(gid_info->gl_sa_hdl,
1916 	    &qargs, 0, &length, (void **)&resp);
1917 
1918 	if ((ret != IBMF_SUCCESS) || (length == 0) || (resp == NULL)) {
1919 		IBTF_DPRINTF_L2("ibdm", "\tis_dev_mgt_supported:"
1920 		    "failed to get PORTINFO attribute %d", ret);
1921 		return (IBDM_FAILURE);
1922 	}
1923 
1924 	if (resp->PortInfo.CapabilityMask & SM_CAP_MASK_IS_DM_SUPPD) {
1925 		IBTF_DPRINTF_L4("ibdm", "\tis_dev_mgt_supported: SUPPD !!");
1926 		ret = IBDM_SUCCESS;
1927 	} else {
1928 		IBTF_DPRINTF_L4("ibdm", "\tis_dev_mgt_supported: "
1929 		    "Not SUPPD !!, cap 0x%x", resp->PortInfo.CapabilityMask);
1930 		ret = IBDM_FAILURE;
1931 	}
1932 	kmem_free(resp, length);
1933 	return (ret);
1934 }
1935 
1936 
1937 /*
1938  * ibdm_get_node_port_guids()
1939  *	Get the NodeInfoRecord of the port
1940  *	Save NodeGuid and PortGUID values in the GID list structure.
1941  *	Return IBDM_SUCCESS/IBDM_FAILURE
1942  */
1943 static int
1944 ibdm_get_node_port_guids(ibmf_saa_handle_t sa_hdl, ib_lid_t dlid,
1945     ib_guid_t *node_guid, ib_guid_t *port_guid)
1946 {
1947 	int			ret;
1948 	size_t			length = 0;
1949 	sa_node_record_t	req, *resp = NULL;
1950 	ibmf_saa_access_args_t	qargs;
1951 
1952 	IBTF_DPRINTF_L4("ibdm", "\tget_node_port_guids");
1953 
1954 	bzero(&req, sizeof (sa_node_record_t));
1955 	req.LID = dlid;
1956 
1957 	qargs.sq_attr_id	= SA_NODERECORD_ATTRID;
1958 	qargs.sq_access_type	= IBMF_SAA_RETRIEVE;
1959 	qargs.sq_component_mask = SA_NODEINFO_COMPMASK_NODELID;
1960 	qargs.sq_template	= &req;
1961 	qargs.sq_callback	= NULL;
1962 	qargs.sq_callback_arg	= NULL;
1963 
1964 	ret = ibmf_sa_access(sa_hdl, &qargs, 0, &length, (void **)&resp);
1965 	if ((ret != IBMF_SUCCESS) || (length == 0) || (resp == NULL)) {
1966 		IBTF_DPRINTF_L2("ibdm", "\tget_node_port_guids:"
1967 		    " SA Retrieve Failed: %d", ret);
1968 		return (IBDM_FAILURE);
1969 	}
1970 	IBTF_DPRINTF_L4("ibdm", "\tget_node_port_guids: NodeGuid %llx Port"
1971 	    "GUID %llx", resp->NodeInfo.NodeGUID, resp->NodeInfo.NodeGUID);
1972 
1973 	*node_guid = resp->NodeInfo.NodeGUID;
1974 	*port_guid = resp->NodeInfo.PortGUID;
1975 	kmem_free(resp, length);
1976 	return (IBDM_SUCCESS);
1977 }
1978 
1979 
1980 /*
1981  * ibdm_get_reachable_ports()
1982  *	Get list of the destination GID (and its path  records) by
1983  *	querying the SA access.
1984  *
1985  *	Returns Number paths
1986  */
1987 static int
1988 ibdm_get_reachable_ports(ibdm_port_attr_t *portinfo, ibdm_hca_list_t *hca)
1989 {
1990 	uint_t			ii, jj, nrecs;
1991 	uint_t			npaths = 0;
1992 	size_t			length;
1993 	ib_gid_t		sgid;
1994 	ibdm_pkey_tbl_t		*pkey_tbl;
1995 	sa_path_record_t	*result;
1996 	sa_path_record_t	*precp;
1997 	ibdm_dp_gidinfo_t	*gid_info;
1998 
1999 	ASSERT(MUTEX_HELD(&ibdm.ibdm_hl_mutex));
2000 	IBTF_DPRINTF_L4("ibdm", "\tget_reachable_ports: portinfo %p", portinfo);
2001 
2002 	sgid.gid_prefix = portinfo->pa_sn_prefix;
2003 	sgid.gid_guid	= portinfo->pa_port_guid;
2004 
2005 	/* get reversible paths */
2006 	if (portinfo->pa_sa_hdl && ibmf_saa_paths_from_gid(portinfo->pa_sa_hdl,
2007 	    sgid, IBMF_SAA_PKEY_WC, B_TRUE, 0, &nrecs, &length, &result)
2008 	    != IBMF_SUCCESS) {
2009 		IBTF_DPRINTF_L2("ibdm",
2010 		    "\tget_reachable_ports: Getting path records failed");
2011 		return (0);
2012 	}
2013 
2014 	for (ii = 0; ii < nrecs; ii++) {
2015 		sa_node_record_t *nrec;
2016 		size_t length;
2017 
2018 		precp = &result[ii];
2019 		if ((gid_info = ibdm_check_dgid(precp->DGID.gid_guid,
2020 		    precp->DGID.gid_prefix)) != NULL) {
2021 			IBTF_DPRINTF_L5("ibdm", "\tget_reachable_ports: "
2022 			    "Already exists nrecs %d, ii %d", nrecs, ii);
2023 			ibdm_addto_glhcalist(gid_info, hca);
2024 			continue;
2025 		}
2026 		/*
2027 		 * This is a new GID. Allocate a GID structure and
2028 		 * initialize the structure
2029 		 * gl_state is initialized to IBDM_GID_PROBE_NOT_DONE (0)
2030 		 * by kmem_zalloc call
2031 		 */
2032 		gid_info = kmem_zalloc(sizeof (ibdm_dp_gidinfo_t), KM_SLEEP);
2033 		mutex_init(&gid_info->gl_mutex, NULL, MUTEX_DEFAULT, NULL);
2034 		cv_init(&gid_info->gl_probe_cv, NULL, CV_DRIVER, NULL);
2035 		gid_info->gl_dgid_hi		= precp->DGID.gid_prefix;
2036 		gid_info->gl_dgid_lo		= precp->DGID.gid_guid;
2037 		gid_info->gl_sgid_hi		= precp->SGID.gid_prefix;
2038 		gid_info->gl_sgid_lo		= precp->SGID.gid_guid;
2039 		gid_info->gl_p_key		= precp->P_Key;
2040 		gid_info->gl_sa_hdl		= portinfo->pa_sa_hdl;
2041 		gid_info->gl_ibmf_hdl		= portinfo->pa_ibmf_hdl;
2042 		gid_info->gl_slid		= precp->SLID;
2043 		gid_info->gl_dlid		= precp->DLID;
2044 		gid_info->gl_transactionID	= (++ibdm.ibdm_transactionID)
2045 		    << IBDM_GID_TRANSACTIONID_SHIFT;
2046 		gid_info->gl_min_transactionID  = gid_info->gl_transactionID;
2047 		gid_info->gl_max_transactionID  = (ibdm.ibdm_transactionID +1)
2048 		    << IBDM_GID_TRANSACTIONID_SHIFT;
2049 		gid_info->gl_SL			= precp->SL;
2050 
2051 		/*
2052 		 * get the node record with this guid if the destination
2053 		 * device is a Cisco one.
2054 		 */
2055 		if (ibdm_is_cisco(precp->DGID.gid_guid) &&
2056 		    (gid_info->gl_nodeguid == 0 || gid_info->gl_devid == 0) &&
2057 		    ibdm_get_node_record_by_port(portinfo->pa_sa_hdl,
2058 		    precp->DGID.gid_guid, &nrec, &length) == IBDM_SUCCESS) {
2059 			gid_info->gl_nodeguid = nrec->NodeInfo.NodeGUID;
2060 			gid_info->gl_devid = nrec->NodeInfo.DeviceID;
2061 			kmem_free(nrec, length);
2062 		}
2063 
2064 		ibdm_addto_glhcalist(gid_info,  hca);
2065 
2066 		ibdm_dump_path_info(precp);
2067 
2068 		gid_info->gl_qp_hdl = NULL;
2069 		ASSERT(portinfo->pa_pkey_tbl != NULL &&
2070 		    portinfo->pa_npkeys != 0);
2071 
2072 		for (jj = 0; jj < portinfo->pa_npkeys; jj++) {
2073 			pkey_tbl = &portinfo->pa_pkey_tbl[jj];
2074 			if ((gid_info->gl_p_key == pkey_tbl->pt_pkey) &&
2075 			    (pkey_tbl->pt_qp_hdl != NULL)) {
2076 				gid_info->gl_qp_hdl = pkey_tbl->pt_qp_hdl;
2077 				break;
2078 			}
2079 		}
2080 
2081 		/*
2082 		 * QP handle for GID not initialized. No matching Pkey
2083 		 * was found!! ibdm should *not* hit this case. Flag an
2084 		 * error and drop the GID if ibdm does encounter this.
2085 		 */
2086 		if (gid_info->gl_qp_hdl == NULL) {
2087 			IBTF_DPRINTF_L2(ibdm_string,
2088 			    "\tget_reachable_ports: No matching Pkey");
2089 			ibdm_delete_gidinfo(gid_info);
2090 			continue;
2091 		}
2092 		if (ibdm.ibdm_dp_gidlist_head == NULL) {
2093 			ibdm.ibdm_dp_gidlist_head = gid_info;
2094 			ibdm.ibdm_dp_gidlist_tail = gid_info;
2095 		} else {
2096 			ibdm.ibdm_dp_gidlist_tail->gl_next = gid_info;
2097 			gid_info->gl_prev = ibdm.ibdm_dp_gidlist_tail;
2098 			ibdm.ibdm_dp_gidlist_tail = gid_info;
2099 		}
2100 		npaths++;
2101 	}
2102 	kmem_free(result, length);
2103 	IBTF_DPRINTF_L4("ibdm", "\tget_reachable_ports: npaths = %d", npaths);
2104 	return (npaths);
2105 }
2106 
2107 
2108 /*
2109  * ibdm_check_dgid()
2110  *	Look in the global list to check whether we know this DGID already
2111  *	Return IBDM_GID_PRESENT/IBDM_GID_NOT_PRESENT
2112  */
2113 static ibdm_dp_gidinfo_t *
2114 ibdm_check_dgid(ib_guid_t guid, ib_sn_prefix_t prefix)
2115 {
2116 	ibdm_dp_gidinfo_t	*gid_list;
2117 
2118 	for (gid_list = ibdm.ibdm_dp_gidlist_head; gid_list;
2119 	    gid_list = gid_list->gl_next) {
2120 		if ((guid == gid_list->gl_dgid_lo) &&
2121 		    (prefix == gid_list->gl_dgid_hi)) {
2122 			break;
2123 		}
2124 	}
2125 	return (gid_list);
2126 }
2127 
2128 
2129 /*
2130  * ibdm_find_gid()
2131  *	Look in the global list to find a GID entry with matching
2132  *	port & node GUID.
2133  *	Return pointer to gidinfo if found, else return NULL
2134  */
2135 static ibdm_dp_gidinfo_t *
2136 ibdm_find_gid(ib_guid_t nodeguid, ib_guid_t portguid)
2137 {
2138 	ibdm_dp_gidinfo_t	*gid_list;
2139 
2140 	IBTF_DPRINTF_L4("ibdm", "ibdm_find_gid(%llx, %llx)\n",
2141 	    nodeguid, portguid);
2142 
2143 	for (gid_list = ibdm.ibdm_dp_gidlist_head; gid_list;
2144 	    gid_list = gid_list->gl_next) {
2145 		if ((portguid == gid_list->gl_portguid) &&
2146 		    (nodeguid == gid_list->gl_nodeguid)) {
2147 			break;
2148 		}
2149 	}
2150 
2151 	IBTF_DPRINTF_L4("ibdm", "ibdm_find_gid : returned %p\n",
2152 	    gid_list);
2153 	return (gid_list);
2154 }
2155 
2156 
2157 /*
2158  * ibdm_set_classportinfo()
2159  *	ibdm_set_classportinfo() is a function to activate a Cisco FC GW
2160  *	by sending the setClassPortInfo request with the trapLID, trapGID
2161  *	and etc. to the gateway since the gateway doesn't provide the IO
2162  *	Unit Information othewise. This behavior is the Cisco specific one,
2163  *	and this function is called to a Cisco FC GW only.
2164  *	Returns IBDM_SUCCESS/IBDM_FAILURE
2165  */
2166 static int
2167 ibdm_set_classportinfo(ibdm_dp_gidinfo_t *gid_info)
2168 {
2169 	ibmf_msg_t		*msg;
2170 	ib_mad_hdr_t		*hdr;
2171 	ibdm_timeout_cb_args_t	*cb_args;
2172 	void			*data;
2173 	ib_mad_classportinfo_t *cpi;
2174 
2175 	IBTF_DPRINTF_L4("ibdm",
2176 	    "\tset_classportinfo: gid info 0x%p", gid_info);
2177 
2178 	/*
2179 	 * Send command to set classportinfo attribute. Allocate a IBMF
2180 	 * packet and initialize the packet.
2181 	 */
2182 	if (ibmf_alloc_msg(gid_info->gl_ibmf_hdl, IBMF_ALLOC_SLEEP,
2183 	    &msg) != IBMF_SUCCESS) {
2184 		IBTF_DPRINTF_L4("ibdm", "\tset_classportinfo: pkt alloc fail");
2185 		return (IBDM_FAILURE);
2186 	}
2187 
2188 	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*msg))
2189 	ibdm_alloc_send_buffers(msg);
2190 	_NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*msg))
2191 
2192 	msg->im_local_addr.ia_local_lid		= gid_info->gl_slid;
2193 	msg->im_local_addr.ia_remote_lid	= gid_info->gl_dlid;
2194 	msg->im_local_addr.ia_remote_qno	= 1;
2195 	msg->im_local_addr.ia_p_key		= gid_info->gl_p_key;
2196 	msg->im_local_addr.ia_q_key		= IB_GSI_QKEY;
2197 	msg->im_local_addr.ia_service_level	= gid_info->gl_SL;
2198 
2199 	hdr			= IBDM_OUT_IBMFMSG_MADHDR(msg);
2200 	hdr->BaseVersion	= MAD_CLASS_BASE_VERS_1;
2201 	hdr->MgmtClass		= MAD_MGMT_CLASS_DEV_MGT;
2202 	hdr->ClassVersion	= IB_DM_CLASS_VERSION_1;
2203 	hdr->R_Method		= IB_DM_DEVMGT_METHOD_SET;
2204 	hdr->Status		= 0;
2205 	hdr->TransactionID	= h2b64(gid_info->gl_transactionID);
2206 	hdr->AttributeID	= h2b16(IB_DM_ATTR_CLASSPORTINFO);
2207 	hdr->AttributeModifier	= 0;
2208 
2209 	data = msg->im_msgbufs_send.im_bufs_cl_data;
2210 	cpi = (ib_mad_classportinfo_t *)data;
2211 
2212 	/*
2213 	 * Set the classportinfo values to activate this Cisco FC GW.
2214 	 */
2215 	cpi->TrapGID_hi = h2b64(gid_info->gl_sgid_hi);
2216 	cpi->TrapGID_lo = h2b64(gid_info->gl_sgid_lo);
2217 	cpi->TrapLID = h2b16(gid_info->gl_slid);
2218 	cpi->TrapSL = gid_info->gl_SL;
2219 	cpi->TrapP_Key = h2b16(gid_info->gl_p_key);
2220 	cpi->TrapQP = h2b32((((ibmf_alt_qp_t *)gid_info->gl_qp_hdl)->isq_qpn));
2221 	cpi->TrapQ_Key = h2b32((((ibmf_alt_qp_t *)
2222 	    gid_info->gl_qp_hdl)->isq_qkey));
2223 
2224 	cb_args = &gid_info->gl_cpi_cb_args;
2225 	cb_args->cb_gid_info = gid_info;
2226 	cb_args->cb_retry_count	= ibdm_dft_retry_cnt;
2227 	cb_args->cb_req_type = IBDM_REQ_TYPE_CLASSPORTINFO;
2228 
2229 	mutex_enter(&gid_info->gl_mutex);
2230 	gid_info->gl_timeout_id = timeout(ibdm_pkt_timeout_hdlr,
2231 	    cb_args, IBDM_TIMEOUT_VALUE(ibdm_dft_timeout));
2232 	mutex_exit(&gid_info->gl_mutex);
2233 
2234 	IBTF_DPRINTF_L5("ibdm", "\tset_classportinfo: "
2235 	    "timeout id %x", gid_info->gl_timeout_id);
2236 
2237 	if (ibmf_msg_transport(gid_info->gl_ibmf_hdl, gid_info->gl_qp_hdl,
2238 	    msg, NULL, ibdm_ibmf_send_cb, cb_args, 0) != IBMF_SUCCESS) {
2239 		IBTF_DPRINTF_L2("ibdm",
2240 		    "\tset_classportinfo: ibmf send failed");
2241 		ibdm_ibmf_send_cb(gid_info->gl_ibmf_hdl, msg, cb_args);
2242 	}
2243 
2244 	return (IBDM_SUCCESS);
2245 }
2246 
2247 
2248 /*
2249  * ibdm_send_classportinfo()
2250  *	Send classportinfo request. When the request is completed
2251  *	IBMF calls ibdm_classportinfo_cb routine to inform about
2252  *	the completion.
2253  *	Returns IBDM_SUCCESS/IBDM_FAILURE
2254  */
2255 static int
2256 ibdm_send_classportinfo(ibdm_dp_gidinfo_t *gid_info)
2257 {
2258 	ibmf_msg_t		*msg;
2259 	ib_mad_hdr_t		*hdr;
2260 	ibdm_timeout_cb_args_t	*cb_args;
2261 
2262 	IBTF_DPRINTF_L4("ibdm",
2263 	    "\tsend_classportinfo: gid info 0x%p", gid_info);
2264 
2265 	/*
2266 	 * Send command to get classportinfo attribute. Allocate a IBMF
2267 	 * packet and initialize the packet.
2268 	 */
2269 	if (ibmf_alloc_msg(gid_info->gl_ibmf_hdl, IBMF_ALLOC_SLEEP,
2270 	    &msg) != IBMF_SUCCESS) {
2271 		IBTF_DPRINTF_L4("ibdm", "\tsend_classportinfo: pkt alloc fail");
2272 		return (IBDM_FAILURE);
2273 	}
2274 
2275 	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*msg))
2276 	ibdm_alloc_send_buffers(msg);
2277 	_NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*msg))
2278 
2279 	msg->im_local_addr.ia_local_lid		= gid_info->gl_slid;
2280 	msg->im_local_addr.ia_remote_lid	= gid_info->gl_dlid;
2281 	msg->im_local_addr.ia_remote_qno	= 1;
2282 	msg->im_local_addr.ia_p_key		= gid_info->gl_p_key;
2283 	msg->im_local_addr.ia_q_key		= IB_GSI_QKEY;
2284 	msg->im_local_addr.ia_service_level	= gid_info->gl_SL;
2285 
2286 	hdr			= IBDM_OUT_IBMFMSG_MADHDR(msg);
2287 	hdr->BaseVersion	= MAD_CLASS_BASE_VERS_1;
2288 	hdr->MgmtClass		= MAD_MGMT_CLASS_DEV_MGT;
2289 	hdr->ClassVersion	= IB_DM_CLASS_VERSION_1;
2290 	hdr->R_Method		= IB_DM_DEVMGT_METHOD_GET;
2291 	hdr->Status		= 0;
2292 	hdr->TransactionID	= h2b64(gid_info->gl_transactionID);
2293 	hdr->AttributeID	= h2b16(IB_DM_ATTR_CLASSPORTINFO);
2294 	hdr->AttributeModifier	= 0;
2295 
2296 	cb_args = &gid_info->gl_cpi_cb_args;
2297 	cb_args->cb_gid_info = gid_info;
2298 	cb_args->cb_retry_count	= ibdm_dft_retry_cnt;
2299 	cb_args->cb_req_type = IBDM_REQ_TYPE_CLASSPORTINFO;
2300 
2301 	mutex_enter(&gid_info->gl_mutex);
2302 	gid_info->gl_timeout_id = timeout(ibdm_pkt_timeout_hdlr,
2303 	    cb_args, IBDM_TIMEOUT_VALUE(ibdm_dft_timeout));
2304 	mutex_exit(&gid_info->gl_mutex);
2305 
2306 	IBTF_DPRINTF_L5("ibdm", "\tsend_classportinfo: "
2307 	    "timeout id %x", gid_info->gl_timeout_id);
2308 
2309 	if (ibmf_msg_transport(gid_info->gl_ibmf_hdl, gid_info->gl_qp_hdl,
2310 	    msg, NULL, ibdm_ibmf_send_cb, cb_args, 0) != IBMF_SUCCESS) {
2311 		IBTF_DPRINTF_L2("ibdm",
2312 		    "\tsend_classportinfo: ibmf send failed");
2313 		ibdm_ibmf_send_cb(gid_info->gl_ibmf_hdl, msg, cb_args);
2314 	}
2315 
2316 	return (IBDM_SUCCESS);
2317 }
2318 
2319 
2320 /*
2321  * ibdm_handle_setclassportinfo()
2322  *	Invoked by the IBMF when setClassPortInfo request is completed.
2323  */
2324 static void
2325 ibdm_handle_setclassportinfo(ibmf_handle_t ibmf_hdl,
2326     ibmf_msg_t *msg, ibdm_dp_gidinfo_t *gid_info, int *flag)
2327 {
2328 	void			*data;
2329 	timeout_id_t		timeout_id;
2330 	ib_mad_classportinfo_t *cpi;
2331 
2332 	IBTF_DPRINTF_L4("ibdm", "\thandle_setclassportinfo:ibmf hdl "
2333 	    "%p msg %p gid info %p", ibmf_hdl, msg, gid_info);
2334 
2335 	if (IBDM_IN_IBMFMSG_ATTR(msg) != IB_DM_ATTR_CLASSPORTINFO) {
2336 		IBTF_DPRINTF_L4("ibdm", "\thandle_setclassportinfo: "
2337 		    "Not a ClassPortInfo resp");
2338 		*flag |= IBDM_IBMF_PKT_UNEXP_RESP;
2339 		return;
2340 	}
2341 
2342 	/*
2343 	 * Verify whether timeout handler is created/active.
2344 	 * If created/ active,  cancel the timeout  handler
2345 	 */
2346 	mutex_enter(&gid_info->gl_mutex);
2347 	if (gid_info->gl_state != IBDM_SET_CLASSPORTINFO) {
2348 		IBTF_DPRINTF_L2("ibdm", "\thandle_setclassportinfo:DUP resp");
2349 		*flag |= IBDM_IBMF_PKT_DUP_RESP;
2350 		mutex_exit(&gid_info->gl_mutex);
2351 		return;
2352 	}
2353 	ibdm_bump_transactionID(gid_info);
2354 
2355 	gid_info->gl_iou_cb_args.cb_req_type = 0;
2356 	if (gid_info->gl_timeout_id) {
2357 		timeout_id = gid_info->gl_timeout_id;
2358 		mutex_exit(&gid_info->gl_mutex);
2359 		IBTF_DPRINTF_L5("ibdm", "handle_setlassportinfo: "
2360 		    "gl_timeout_id = 0x%x", timeout_id);
2361 		if (untimeout(timeout_id) == -1) {
2362 			IBTF_DPRINTF_L2("ibdm", "handle_setclassportinfo: "
2363 			    "untimeout gl_timeout_id failed");
2364 		}
2365 		mutex_enter(&gid_info->gl_mutex);
2366 		gid_info->gl_timeout_id = 0;
2367 	}
2368 	mutex_exit(&gid_info->gl_mutex);
2369 
2370 	data = msg->im_msgbufs_recv.im_bufs_cl_data;
2371 	cpi = (ib_mad_classportinfo_t *)data;
2372 
2373 	ibdm_dump_classportinfo(cpi);
2374 }
2375 
2376 
2377 /*
2378  * ibdm_handle_classportinfo()
2379  *	Invoked by the IBMF when the classportinfo request is completed.
2380  */
2381 static void
2382 ibdm_handle_classportinfo(ibmf_handle_t ibmf_hdl,
2383     ibmf_msg_t *msg, ibdm_dp_gidinfo_t *gid_info, int *flag)
2384 {
2385 	void			*data;
2386 	timeout_id_t		timeout_id;
2387 	ib_mad_hdr_t		*hdr;
2388 	ib_mad_classportinfo_t *cpi;
2389 
2390 	IBTF_DPRINTF_L4("ibdm", "\thandle_classportinfo:ibmf hdl "
2391 	    "%p msg %p gid info %p", ibmf_hdl, msg, gid_info);
2392 
2393 	if (IBDM_IN_IBMFMSG_ATTR(msg) != IB_DM_ATTR_CLASSPORTINFO) {
2394 		IBTF_DPRINTF_L4("ibdm", "\thandle_classportinfo: "
2395 		    "Not a ClassPortInfo resp");
2396 		*flag |= IBDM_IBMF_PKT_UNEXP_RESP;
2397 		return;
2398 	}
2399 
2400 	/*
2401 	 * Verify whether timeout handler is created/active.
2402 	 * If created/ active,  cancel the timeout  handler
2403 	 */
2404 	mutex_enter(&gid_info->gl_mutex);
2405 	ibdm_bump_transactionID(gid_info);
2406 	if (gid_info->gl_state != IBDM_GET_CLASSPORTINFO) {
2407 		IBTF_DPRINTF_L2("ibdm", "\thandle_classportinfo:DUP resp");
2408 		*flag |= IBDM_IBMF_PKT_DUP_RESP;
2409 		mutex_exit(&gid_info->gl_mutex);
2410 		return;
2411 	}
2412 	gid_info->gl_iou_cb_args.cb_req_type = 0;
2413 	if (gid_info->gl_timeout_id) {
2414 		timeout_id = gid_info->gl_timeout_id;
2415 		mutex_exit(&gid_info->gl_mutex);
2416 		IBTF_DPRINTF_L5("ibdm", "handle_ioclassportinfo: "
2417 		    "gl_timeout_id = 0x%x", timeout_id);
2418 		if (untimeout(timeout_id) == -1) {
2419 			IBTF_DPRINTF_L2("ibdm", "handle_classportinfo: "
2420 			    "untimeout gl_timeout_id failed");
2421 		}
2422 		mutex_enter(&gid_info->gl_mutex);
2423 		gid_info->gl_timeout_id = 0;
2424 	}
2425 	gid_info->gl_state = IBDM_GET_IOUNITINFO;
2426 	gid_info->gl_pending_cmds++;
2427 	mutex_exit(&gid_info->gl_mutex);
2428 
2429 	data = msg->im_msgbufs_recv.im_bufs_cl_data;
2430 	cpi = (ib_mad_classportinfo_t *)data;
2431 
2432 	/*
2433 	 * Cache the "RespTimeValue" and redirection information in the
2434 	 * global gid list data structure. This cached information will
2435 	 * be used to send any further requests to the GID.
2436 	 */
2437 	gid_info->gl_resp_timeout	=
2438 	    (b2h32(cpi->RespTimeValue) & 0x1F);
2439 
2440 	gid_info->gl_redirected		= ((IBDM_IN_IBMFMSG_STATUS(msg) &
2441 	    MAD_STATUS_REDIRECT_REQUIRED) ? B_TRUE: B_FALSE);
2442 	gid_info->gl_redirect_dlid	= b2h16(cpi->RedirectLID);
2443 	gid_info->gl_redirect_QP	= (b2h32(cpi->RedirectQP) & 0xffffff);
2444 	gid_info->gl_redirect_pkey	= b2h16(cpi->RedirectP_Key);
2445 	gid_info->gl_redirect_qkey	= b2h32(cpi->RedirectQ_Key);
2446 	gid_info->gl_redirectGID_hi	= b2h64(cpi->RedirectGID_hi);
2447 	gid_info->gl_redirectGID_lo	= b2h64(cpi->RedirectGID_lo);
2448 	gid_info->gl_redirectSL		= cpi->RedirectSL;
2449 
2450 	ibdm_dump_classportinfo(cpi);
2451 
2452 	/*
2453 	 * Send IOUnitInfo request
2454 	 * Reuse previously allocated IBMF packet for sending ClassPortInfo
2455 	 * Check whether DM agent on the remote node requested redirection
2456 	 * If so, send the request to the redirect DGID/DLID/PKEY/QP.
2457 	 */
2458 	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*msg))
2459 	ibdm_alloc_send_buffers(msg);
2460 	_NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*msg))
2461 	msg->im_local_addr.ia_local_lid	= gid_info->gl_slid;
2462 	msg->im_local_addr.ia_remote_lid	= gid_info->gl_dlid;
2463 
2464 	if (gid_info->gl_redirected == B_TRUE) {
2465 		if (gid_info->gl_redirect_dlid != 0) {
2466 			msg->im_local_addr.ia_remote_lid =
2467 			    gid_info->gl_redirect_dlid;
2468 		}
2469 		msg->im_local_addr.ia_remote_qno = gid_info->gl_redirect_QP;
2470 		msg->im_local_addr.ia_p_key = gid_info->gl_redirect_pkey;
2471 		msg->im_local_addr.ia_q_key = gid_info->gl_redirect_qkey;
2472 		msg->im_local_addr.ia_service_level = gid_info->gl_redirectSL;
2473 	} else {
2474 		msg->im_local_addr.ia_remote_qno = 1;
2475 		msg->im_local_addr.ia_p_key = gid_info->gl_p_key;
2476 		msg->im_local_addr.ia_q_key = IB_GSI_QKEY;
2477 		msg->im_local_addr.ia_service_level = gid_info->gl_SL;
2478 	}
2479 
2480 	hdr			= IBDM_OUT_IBMFMSG_MADHDR(msg);
2481 	hdr->BaseVersion	= MAD_CLASS_BASE_VERS_1;
2482 	hdr->MgmtClass		= MAD_MGMT_CLASS_DEV_MGT;
2483 	hdr->ClassVersion	= IB_DM_CLASS_VERSION_1;
2484 	hdr->R_Method		= IB_DM_DEVMGT_METHOD_GET;
2485 	hdr->Status		= 0;
2486 	hdr->TransactionID	= h2b64(gid_info->gl_transactionID);
2487 	hdr->AttributeID	= h2b16(IB_DM_ATTR_IO_UNITINFO);
2488 	hdr->AttributeModifier	= 0;
2489 
2490 	gid_info->gl_iou_cb_args.cb_req_type = IBDM_REQ_TYPE_IOUINFO;
2491 	gid_info->gl_iou_cb_args.cb_gid_info = gid_info;
2492 	gid_info->gl_iou_cb_args.cb_retry_count = ibdm_dft_retry_cnt;
2493 
2494 	mutex_enter(&gid_info->gl_mutex);
2495 	gid_info->gl_timeout_id = timeout(ibdm_pkt_timeout_hdlr,
2496 	    &gid_info->gl_iou_cb_args, IBDM_TIMEOUT_VALUE(ibdm_dft_timeout));
2497 	mutex_exit(&gid_info->gl_mutex);
2498 
2499 	IBTF_DPRINTF_L5("ibdm", "handle_classportinfo:"
2500 	    "timeout %x", gid_info->gl_timeout_id);
2501 
2502 	if (ibmf_msg_transport(ibmf_hdl, gid_info->gl_qp_hdl, msg, NULL,
2503 	    ibdm_ibmf_send_cb, &gid_info->gl_iou_cb_args, 0) != IBMF_SUCCESS) {
2504 		IBTF_DPRINTF_L2("ibdm",
2505 		    "\thandle_classportinfo: msg transport failed");
2506 		ibdm_ibmf_send_cb(ibmf_hdl, msg, &gid_info->gl_iou_cb_args);
2507 	}
2508 	(*flag) |= IBDM_IBMF_PKT_REUSED;
2509 }
2510 
2511 
2512 /*
2513  * ibdm_send_iounitinfo:
2514  *	Sends a DM request to get IOU unitinfo.
2515  */
2516 static int
2517 ibdm_send_iounitinfo(ibdm_dp_gidinfo_t *gid_info)
2518 {
2519 	ibmf_msg_t	*msg;
2520 	ib_mad_hdr_t	*hdr;
2521 
2522 	IBTF_DPRINTF_L4("ibdm", "\tsend_iounitinfo: gid info 0x%p", gid_info);
2523 
2524 	/*
2525 	 * Send command to get iounitinfo attribute. Allocate a IBMF
2526 	 * packet and initialize the packet.
2527 	 */
2528 	if (ibmf_alloc_msg(gid_info->gl_ibmf_hdl, IBMF_ALLOC_SLEEP, &msg) !=
2529 	    IBMF_SUCCESS) {
2530 		IBTF_DPRINTF_L4("ibdm", "\tsend_iounitinfo: pkt alloc fail");
2531 		return (IBDM_FAILURE);
2532 	}
2533 
2534 	mutex_enter(&gid_info->gl_mutex);
2535 	ibdm_bump_transactionID(gid_info);
2536 	mutex_exit(&gid_info->gl_mutex);
2537 
2538 
2539 	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*msg))
2540 	ibdm_alloc_send_buffers(msg);
2541 	_NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*msg))
2542 	msg->im_local_addr.ia_local_lid		= gid_info->gl_slid;
2543 	msg->im_local_addr.ia_remote_lid	= gid_info->gl_dlid;
2544 	msg->im_local_addr.ia_remote_qno	= 1;
2545 	msg->im_local_addr.ia_p_key		= gid_info->gl_p_key;
2546 	msg->im_local_addr.ia_q_key		= IB_GSI_QKEY;
2547 	msg->im_local_addr.ia_service_level	= gid_info->gl_SL;
2548 
2549 	hdr			= IBDM_OUT_IBMFMSG_MADHDR(msg);
2550 	hdr->BaseVersion	= MAD_CLASS_BASE_VERS_1;
2551 	hdr->MgmtClass		= MAD_MGMT_CLASS_DEV_MGT;
2552 	hdr->ClassVersion	= IB_DM_CLASS_VERSION_1;
2553 	hdr->R_Method		= IB_DM_DEVMGT_METHOD_GET;
2554 	hdr->Status		= 0;
2555 	hdr->TransactionID	= h2b64(gid_info->gl_transactionID);
2556 	hdr->AttributeID	= h2b16(IB_DM_ATTR_IO_UNITINFO);
2557 	hdr->AttributeModifier	= 0;
2558 
2559 	gid_info->gl_iou_cb_args.cb_gid_info = gid_info;
2560 	gid_info->gl_iou_cb_args.cb_retry_count = ibdm_dft_retry_cnt;
2561 	gid_info->gl_iou_cb_args.cb_req_type = IBDM_REQ_TYPE_IOUINFO;
2562 
2563 	mutex_enter(&gid_info->gl_mutex);
2564 	gid_info->gl_timeout_id = timeout(ibdm_pkt_timeout_hdlr,
2565 	    &gid_info->gl_iou_cb_args, IBDM_TIMEOUT_VALUE(ibdm_dft_timeout));
2566 	mutex_exit(&gid_info->gl_mutex);
2567 
2568 	IBTF_DPRINTF_L5("ibdm", "send_iouunitinfo:"
2569 	    "timeout %x", gid_info->gl_timeout_id);
2570 
2571 	if (ibmf_msg_transport(gid_info->gl_ibmf_hdl, gid_info->gl_qp_hdl, msg,
2572 	    NULL, ibdm_ibmf_send_cb, &gid_info->gl_iou_cb_args, 0) !=
2573 	    IBMF_SUCCESS) {
2574 		IBTF_DPRINTF_L2("ibdm", "\tsend_iounitinfo: ibmf send failed");
2575 		ibdm_ibmf_send_cb(gid_info->gl_ibmf_hdl,
2576 		    msg, &gid_info->gl_iou_cb_args);
2577 	}
2578 	return (IBDM_SUCCESS);
2579 }
2580 
2581 /*
2582  * ibdm_handle_iounitinfo()
2583  *	Invoked by the IBMF when IO Unitinfo request is completed.
2584  */
2585 static void
2586 ibdm_handle_iounitinfo(ibmf_handle_t ibmf_hdl,
2587     ibmf_msg_t *msg, ibdm_dp_gidinfo_t *gid_info, int *flag)
2588 {
2589 	int			ii, first = B_TRUE;
2590 	int			num_iocs;
2591 	size_t			size;
2592 	uchar_t			slot_info;
2593 	timeout_id_t		timeout_id;
2594 	ib_mad_hdr_t		*hdr;
2595 	ibdm_ioc_info_t		*ioc_info;
2596 	ib_dm_io_unitinfo_t	*iou_info;
2597 	ib_dm_io_unitinfo_t	*giou_info;
2598 	ibdm_timeout_cb_args_t	*cb_args;
2599 
2600 	IBTF_DPRINTF_L4("ibdm", "\thandle_iouintinfo:"
2601 	    " ibmf hdl %p pkt %p gid info %p", ibmf_hdl, msg, gid_info);
2602 
2603 	if (IBDM_IN_IBMFMSG_ATTR(msg) != IB_DM_ATTR_IO_UNITINFO) {
2604 		IBTF_DPRINTF_L4("ibdm", "\thandle_iounitinfo: "
2605 		    "Unexpected response");
2606 		(*flag) |= IBDM_IBMF_PKT_UNEXP_RESP;
2607 		return;
2608 	}
2609 
2610 	mutex_enter(&gid_info->gl_mutex);
2611 	if (gid_info->gl_state != IBDM_GET_IOUNITINFO) {
2612 		IBTF_DPRINTF_L4("ibdm",
2613 		    "\thandle_iounitinfo: DUP resp");
2614 		mutex_exit(&gid_info->gl_mutex);
2615 		(*flag) = IBDM_IBMF_PKT_DUP_RESP;
2616 		return;
2617 	}
2618 	gid_info->gl_iou_cb_args.cb_req_type = 0;
2619 	if (gid_info->gl_timeout_id) {
2620 		timeout_id = gid_info->gl_timeout_id;
2621 		mutex_exit(&gid_info->gl_mutex);
2622 		IBTF_DPRINTF_L5("ibdm", "handle_iounitinfo: "
2623 		    "gl_timeout_id = 0x%x", timeout_id);
2624 		if (untimeout(timeout_id) == -1) {
2625 			IBTF_DPRINTF_L2("ibdm", "handle_iounitinfo: "
2626 			    "untimeout gl_timeout_id failed");
2627 		}
2628 		mutex_enter(&gid_info->gl_mutex);
2629 		gid_info->gl_timeout_id = 0;
2630 	}
2631 	gid_info->gl_state = IBDM_GET_IOC_DETAILS;
2632 
2633 	iou_info = IBDM_IN_IBMFMSG2IOU(msg);
2634 	ibdm_dump_iounitinfo(iou_info);
2635 	num_iocs = iou_info->iou_num_ctrl_slots;
2636 	/*
2637 	 * check if number of IOCs reported is zero? if yes, return.
2638 	 * when num_iocs are reported zero internal IOC database needs
2639 	 * to be updated. To ensure that save the number of IOCs in
2640 	 * the new field "gl_num_iocs". Use a new field instead of
2641 	 * "giou_info->iou_num_ctrl_slots" as that would prevent
2642 	 * an unnecessary kmem_alloc/kmem_free when num_iocs is 0.
2643 	 */
2644 	if (num_iocs == 0 && gid_info->gl_num_iocs == 0) {
2645 		IBTF_DPRINTF_L4("ibdm", "\thandle_iounitinfo: no IOC's");
2646 		mutex_exit(&gid_info->gl_mutex);
2647 		return;
2648 	}
2649 	IBTF_DPRINTF_L4("ibdm", "\thandle_iounitinfo: num_iocs = %d", num_iocs);
2650 
2651 	/*
2652 	 * if there is an existing gl_iou (IOU has been probed before)
2653 	 * check if the "iou_changeid" is same as saved entry in
2654 	 * "giou_info->iou_changeid".
2655 	 * (note: this logic can prevent IOC enumeration if a given
2656 	 * vendor doesn't support setting iou_changeid field for its IOU)
2657 	 *
2658 	 * if there is an existing gl_iou and iou_changeid has changed :
2659 	 * free up existing gl_iou info and its related structures.
2660 	 * reallocate gl_iou info all over again.
2661 	 * if we donot free this up; then this leads to memory leaks
2662 	 */
2663 	if (gid_info->gl_iou) {
2664 		giou_info = &gid_info->gl_iou->iou_info;
2665 		if (b2h16(iou_info->iou_changeid) ==
2666 		    giou_info->iou_changeid) {
2667 			IBTF_DPRINTF_L3("ibdm",
2668 			    "\thandle_iounitinfo: no IOCs changed");
2669 			gid_info->gl_state = IBDM_GID_PROBING_COMPLETE;
2670 			mutex_exit(&gid_info->gl_mutex);
2671 			return;
2672 		}
2673 
2674 		/*
2675 		 * Store the iou info as prev_iou to be used after
2676 		 * sweep is done.
2677 		 */
2678 		ASSERT(gid_info->gl_prev_iou == NULL);
2679 		IBTF_DPRINTF_L4(ibdm_string,
2680 		    "\thandle_iounitinfo: setting gl_prev_iou %p",
2681 		    gid_info->gl_prev_iou);
2682 		gid_info->gl_prev_iou = gid_info->gl_iou;
2683 		ibdm.ibdm_prev_iou = 1;
2684 		gid_info->gl_iou = NULL;
2685 	}
2686 
2687 	size = sizeof (ibdm_iou_info_t) + num_iocs * sizeof (ibdm_ioc_info_t);
2688 	gid_info->gl_iou = (ibdm_iou_info_t *)kmem_zalloc(size, KM_SLEEP);
2689 	giou_info = &gid_info->gl_iou->iou_info;
2690 	gid_info->gl_iou->iou_ioc_info = (ibdm_ioc_info_t *)
2691 	    ((char *)gid_info->gl_iou + sizeof (ibdm_iou_info_t));
2692 
2693 	giou_info->iou_num_ctrl_slots	= gid_info->gl_num_iocs	= num_iocs;
2694 	giou_info->iou_flag		= iou_info->iou_flag;
2695 	bcopy(iou_info->iou_ctrl_list, giou_info->iou_ctrl_list, 128);
2696 	giou_info->iou_changeid	= b2h16(iou_info->iou_changeid);
2697 	gid_info->gl_pending_cmds++; /* for diag code */
2698 	mutex_exit(&gid_info->gl_mutex);
2699 
2700 	if (ibdm_get_diagcode(gid_info, 0) != IBDM_SUCCESS) {
2701 		mutex_enter(&gid_info->gl_mutex);
2702 		gid_info->gl_pending_cmds--;
2703 		mutex_exit(&gid_info->gl_mutex);
2704 	}
2705 	/*
2706 	 * Parallelize getting IOC controller profiles from here.
2707 	 * Allocate IBMF packets and send commands to get IOC profile for
2708 	 * each IOC present on the IOU.
2709 	 */
2710 	for (ii = 0; ii < num_iocs; ii++) {
2711 		/*
2712 		 * Check whether IOC is present in the slot
2713 		 * Series of nibbles (in the field iou_ctrl_list) represents
2714 		 * a slot in the IOU.
2715 		 * Byte format: 76543210
2716 		 * Bits 0-3 of first byte represent Slot 2
2717 		 * bits 4-7 of first byte represent slot 1,
2718 		 * bits 0-3 of second byte represent slot 4 and so on
2719 		 * Each 4-bit nibble has the following meaning
2720 		 * 0x0 : IOC not installed
2721 		 * 0x1 : IOC is present
2722 		 * 0xf : Slot does not exist
2723 		 * and all other values are reserved.
2724 		 */
2725 		ioc_info = IBDM_GIDINFO2IOCINFO(gid_info, ii);
2726 		slot_info = giou_info->iou_ctrl_list[(ii/2)];
2727 		if ((ii % 2) == 0)
2728 			slot_info = (slot_info >> 4);
2729 
2730 		if ((slot_info & 0xf) != 1) {
2731 			IBTF_DPRINTF_L4("ibdm", "\thandle_iouintinfo: "
2732 			    "No IOC is present in the slot = %d", ii);
2733 			ioc_info->ioc_state = IBDM_IOC_STATE_PROBE_FAILED;
2734 			continue;
2735 		}
2736 
2737 		mutex_enter(&gid_info->gl_mutex);
2738 		ibdm_bump_transactionID(gid_info);
2739 		mutex_exit(&gid_info->gl_mutex);
2740 
2741 		/*
2742 		 * Re use the already allocated packet (for IOUnitinfo) to
2743 		 * send the first IOC controller attribute. Allocate new
2744 		 * IBMF packets for the rest of the IOC's
2745 		 */
2746 		if (first != B_TRUE) {
2747 			msg = NULL;
2748 			if (ibmf_alloc_msg(ibmf_hdl, IBMF_ALLOC_SLEEP,
2749 			    &msg) != IBMF_SUCCESS) {
2750 				IBTF_DPRINTF_L4("ibdm", "\thandle_iouintinfo: "
2751 				    "IBMF packet allocation failed");
2752 				continue;
2753 			}
2754 
2755 		}
2756 
2757 		/* allocate send buffers for all messages */
2758 		_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*msg))
2759 		ibdm_alloc_send_buffers(msg);
2760 		_NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*msg))
2761 
2762 		msg->im_local_addr.ia_local_lid	= gid_info->gl_slid;
2763 		msg->im_local_addr.ia_remote_lid	= gid_info->gl_dlid;
2764 		if (gid_info->gl_redirected == B_TRUE) {
2765 			if (gid_info->gl_redirect_dlid != 0) {
2766 				msg->im_local_addr.ia_remote_lid =
2767 				    gid_info->gl_redirect_dlid;
2768 			}
2769 			msg->im_local_addr.ia_remote_qno =
2770 			    gid_info->gl_redirect_QP;
2771 			msg->im_local_addr.ia_p_key =
2772 			    gid_info->gl_redirect_pkey;
2773 			msg->im_local_addr.ia_q_key =
2774 			    gid_info->gl_redirect_qkey;
2775 			msg->im_local_addr.ia_service_level =
2776 			    gid_info->gl_redirectSL;
2777 		} else {
2778 			msg->im_local_addr.ia_remote_qno = 1;
2779 			msg->im_local_addr.ia_p_key = gid_info->gl_p_key;
2780 			msg->im_local_addr.ia_q_key = IB_GSI_QKEY;
2781 			msg->im_local_addr.ia_service_level = gid_info->gl_SL;
2782 		}
2783 
2784 		hdr			= IBDM_OUT_IBMFMSG_MADHDR(msg);
2785 		hdr->BaseVersion	= MAD_CLASS_BASE_VERS_1;
2786 		hdr->MgmtClass		= MAD_MGMT_CLASS_DEV_MGT;
2787 		hdr->ClassVersion	= IB_DM_CLASS_VERSION_1;
2788 		hdr->R_Method		= IB_DM_DEVMGT_METHOD_GET;
2789 		hdr->Status		= 0;
2790 		hdr->TransactionID	= h2b64(gid_info->gl_transactionID);
2791 		hdr->AttributeID	= h2b16(IB_DM_ATTR_IOC_CTRL_PROFILE);
2792 		hdr->AttributeModifier 	= h2b32(ii + 1);
2793 
2794 		ioc_info->ioc_state	= IBDM_IOC_STATE_PROBE_INVALID;
2795 		cb_args			= &ioc_info->ioc_cb_args;
2796 		cb_args->cb_gid_info	= gid_info;
2797 		cb_args->cb_retry_count	= ibdm_dft_retry_cnt;
2798 		cb_args->cb_req_type	= IBDM_REQ_TYPE_IOCINFO;
2799 		cb_args->cb_ioc_num	= ii;
2800 
2801 		mutex_enter(&gid_info->gl_mutex);
2802 		gid_info->gl_pending_cmds++; /* for diag code */
2803 
2804 		ioc_info->ioc_timeout_id = timeout(ibdm_pkt_timeout_hdlr,
2805 		    cb_args, IBDM_TIMEOUT_VALUE(ibdm_dft_timeout));
2806 		mutex_exit(&gid_info->gl_mutex);
2807 
2808 		IBTF_DPRINTF_L5("ibdm", "\thandle_iounitinfo:"
2809 		    "timeout 0x%x, ioc_num %d", ioc_info->ioc_timeout_id, ii);
2810 
2811 		if (ibmf_msg_transport(ibmf_hdl, gid_info->gl_qp_hdl, msg,
2812 		    NULL, ibdm_ibmf_send_cb, cb_args, 0) != IBMF_SUCCESS) {
2813 			IBTF_DPRINTF_L2("ibdm",
2814 			    "\thandle_iounitinfo: msg transport failed");
2815 			ibdm_ibmf_send_cb(ibmf_hdl, msg, cb_args);
2816 		}
2817 		(*flag) |= IBDM_IBMF_PKT_REUSED;
2818 		first = B_FALSE;
2819 		gid_info->gl_iou->iou_niocs_probe_in_progress++;
2820 	}
2821 }
2822 
2823 
2824 /*
2825  * ibdm_handle_ioc_profile()
2826  *	Invoked by the IBMF when the IOCControllerProfile request
2827  *	gets completed
2828  */
2829 static void
2830 ibdm_handle_ioc_profile(ibmf_handle_t ibmf_hdl,
2831     ibmf_msg_t *msg, ibdm_dp_gidinfo_t *gid_info, int *flag)
2832 {
2833 	int				first = B_TRUE, reprobe = 0;
2834 	uint_t				ii, ioc_no, srv_start;
2835 	uint_t				nserv_entries;
2836 	timeout_id_t			timeout_id;
2837 	ib_mad_hdr_t			*hdr;
2838 	ibdm_ioc_info_t			*ioc_info;
2839 	ibdm_timeout_cb_args_t		*cb_args;
2840 	ib_dm_ioc_ctrl_profile_t	*ioc, *gioc;
2841 
2842 	IBTF_DPRINTF_L4("ibdm", "\thandle_ioc_profile:"
2843 	    " ibmf hdl %p msg %p gid info %p", ibmf_hdl, msg, gid_info);
2844 
2845 	ioc = IBDM_IN_IBMFMSG2IOC(msg);
2846 	/*
2847 	 * Check whether we know this IOC already
2848 	 * This will return NULL if reprobe is in progress
2849 	 * IBDM_IOC_STATE_REPROBE_PROGRESS will be set.
2850 	 * Do not hold mutexes here.
2851 	 */
2852 	if (ibdm_is_ioc_present(ioc->ioc_guid, gid_info, flag) != NULL) {
2853 		IBTF_DPRINTF_L4("ibdm", "\thandle_ioc_profile:"
2854 		    "IOC guid %llx is present", ioc->ioc_guid);
2855 		return;
2856 	}
2857 	ioc_no = IBDM_IN_IBMFMSG_ATTRMOD(msg);
2858 	IBTF_DPRINTF_L4("ibdm", "\thandle_ioc_profile: ioc_no = %d", ioc_no-1);
2859 
2860 	/* Make sure that IOC index is with the valid range */
2861 	if (IBDM_IS_IOC_NUM_INVALID(ioc_no, gid_info)) {
2862 		IBTF_DPRINTF_L2("ibdm", "\thandle_ioc_profile: "
2863 		    "IOC index Out of range, index %d", ioc);
2864 		(*flag) |= IBDM_IBMF_PKT_UNEXP_RESP;
2865 		return;
2866 	}
2867 	ioc_info = &gid_info->gl_iou->iou_ioc_info[ioc_no - 1];
2868 	ioc_info->ioc_iou_info = gid_info->gl_iou;
2869 
2870 	mutex_enter(&gid_info->gl_mutex);
2871 	if (ioc_info->ioc_state == IBDM_IOC_STATE_REPROBE_PROGRESS) {
2872 		reprobe = 1;
2873 		ioc_info->ioc_prev_serv = ioc_info->ioc_serv;
2874 		ioc_info->ioc_serv = NULL;
2875 		ioc_info->ioc_prev_serv_cnt =
2876 		    ioc_info->ioc_profile.ioc_service_entries;
2877 	} else if (ioc_info->ioc_state != IBDM_IOC_STATE_PROBE_INVALID) {
2878 		IBTF_DPRINTF_L2("ibdm", "\thandle_ioc_profile: DUP response"
2879 		    "ioc %d, ioc_state %x", ioc_no - 1, ioc_info->ioc_state);
2880 		mutex_exit(&gid_info->gl_mutex);
2881 		(*flag) |= IBDM_IBMF_PKT_DUP_RESP;
2882 		return;
2883 	}
2884 	ioc_info->ioc_cb_args.cb_req_type = 0;
2885 	if (ioc_info->ioc_timeout_id) {
2886 		timeout_id = ioc_info->ioc_timeout_id;
2887 		ioc_info->ioc_timeout_id = 0;
2888 		mutex_exit(&gid_info->gl_mutex);
2889 		IBTF_DPRINTF_L5("ibdm", "handle_ioc_profile: "
2890 		    "ioc_timeout_id = 0x%x", timeout_id);
2891 		if (untimeout(timeout_id) == -1) {
2892 			IBTF_DPRINTF_L2("ibdm", "handle_ioc_profile: "
2893 			    "untimeout ioc_timeout_id failed");
2894 		}
2895 		mutex_enter(&gid_info->gl_mutex);
2896 	}
2897 
2898 	ioc_info->ioc_state = IBDM_IOC_STATE_PROBE_SUCCESS;
2899 	if (reprobe == 0) {
2900 		ioc_info->ioc_iou_guid = gid_info->gl_nodeguid;
2901 		ioc_info->ioc_nodeguid = gid_info->gl_nodeguid;
2902 	}
2903 
2904 	/*
2905 	 * Save all the IOC information in the global structures.
2906 	 * Note the wire format is Big Endian and the Sparc process also
2907 	 * big endian. So, there is no need to convert the data fields
2908 	 * The conversion routines used below are ineffective on Sparc
2909 	 * machines where as they will be effective on little endian
2910 	 * machines such as Intel processors.
2911 	 */
2912 	gioc = (ib_dm_ioc_ctrl_profile_t *)&ioc_info->ioc_profile;
2913 
2914 	/*
2915 	 * Restrict updates to onlyport GIDs and service entries during reprobe
2916 	 */
2917 	if (reprobe == 0) {
2918 		gioc->ioc_guid			= b2h64(ioc->ioc_guid);
2919 		gioc->ioc_vendorid		=
2920 		    ((b2h32(ioc->ioc_vendorid) & IB_DM_VENDORID_MASK)
2921 		    >> IB_DM_VENDORID_SHIFT);
2922 		gioc->ioc_deviceid		= b2h32(ioc->ioc_deviceid);
2923 		gioc->ioc_device_ver		= b2h16(ioc->ioc_device_ver);
2924 		gioc->ioc_subsys_vendorid	=
2925 		    ((b2h32(ioc->ioc_subsys_vendorid) & IB_DM_VENDORID_MASK)
2926 		    >> IB_DM_VENDORID_SHIFT);
2927 		gioc->ioc_subsys_id		= b2h32(ioc->ioc_subsys_id);
2928 		gioc->ioc_io_class		= b2h16(ioc->ioc_io_class);
2929 		gioc->ioc_io_subclass		= b2h16(ioc->ioc_io_subclass);
2930 		gioc->ioc_protocol		= b2h16(ioc->ioc_protocol);
2931 		gioc->ioc_protocol_ver		= b2h16(ioc->ioc_protocol_ver);
2932 		gioc->ioc_send_msg_qdepth	=
2933 		    b2h16(ioc->ioc_send_msg_qdepth);
2934 		gioc->ioc_rdma_read_qdepth	=
2935 		    b2h16(ioc->ioc_rdma_read_qdepth);
2936 		gioc->ioc_send_msg_sz		= b2h32(ioc->ioc_send_msg_sz);
2937 		gioc->ioc_rdma_xfer_sz		= b2h32(ioc->ioc_rdma_xfer_sz);
2938 		gioc->ioc_ctrl_opcap_mask	= ioc->ioc_ctrl_opcap_mask;
2939 		bcopy(ioc->ioc_id_string, gioc->ioc_id_string,
2940 		    IB_DM_IOC_ID_STRING_LEN);
2941 
2942 		ioc_info->ioc_iou_diagcode = gid_info->gl_iou->iou_diagcode;
2943 		ioc_info->ioc_iou_dc_valid = gid_info->gl_iou->iou_dc_valid;
2944 		ioc_info->ioc_diagdeviceid = (IB_DM_IOU_DEVICEID_MASK &
2945 		    gid_info->gl_iou->iou_info.iou_flag) ? B_TRUE : B_FALSE;
2946 
2947 		if (ioc_info->ioc_diagdeviceid == B_TRUE) {
2948 			gid_info->gl_pending_cmds++;
2949 			IBTF_DPRINTF_L3(ibdm_string,
2950 			    "\tibdm_handle_ioc_profile: "
2951 			    "%d: gid_info %p gl_state %d pending_cmds %d",
2952 			    __LINE__, gid_info, gid_info->gl_state,
2953 			    gid_info->gl_pending_cmds);
2954 		}
2955 	}
2956 	gioc->ioc_service_entries	= ioc->ioc_service_entries;
2957 	mutex_exit(&gid_info->gl_mutex);
2958 
2959 	ibdm_dump_ioc_profile(gioc);
2960 
2961 	if ((ioc_info->ioc_diagdeviceid == B_TRUE) && (reprobe == 0)) {
2962 		if (ibdm_get_diagcode(gid_info, ioc_no) != IBDM_SUCCESS) {
2963 			mutex_enter(&gid_info->gl_mutex);
2964 			gid_info->gl_pending_cmds--;
2965 			mutex_exit(&gid_info->gl_mutex);
2966 		}
2967 	}
2968 	ioc_info->ioc_serv = (ibdm_srvents_info_t *)kmem_zalloc(
2969 	    (gioc->ioc_service_entries * sizeof (ibdm_srvents_info_t)),
2970 	    KM_SLEEP);
2971 
2972 	/*
2973 	 * In one single request, maximum number of requests that can be
2974 	 * obtained is 4. If number of service entries are more than four,
2975 	 * calculate number requests needed and send them parallelly.
2976 	 */
2977 	nserv_entries = ioc->ioc_service_entries;
2978 	ii = 0;
2979 	while (nserv_entries) {
2980 		mutex_enter(&gid_info->gl_mutex);
2981 		gid_info->gl_pending_cmds++;
2982 		ibdm_bump_transactionID(gid_info);
2983 		mutex_exit(&gid_info->gl_mutex);
2984 
2985 		if (first != B_TRUE) {
2986 			if (ibmf_alloc_msg(ibmf_hdl, IBMF_ALLOC_SLEEP,
2987 			    &msg) != IBMF_SUCCESS) {
2988 				continue;
2989 			}
2990 
2991 		}
2992 		_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*msg))
2993 		ibdm_alloc_send_buffers(msg);
2994 		_NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*msg))
2995 		msg->im_local_addr.ia_local_lid	= gid_info->gl_slid;
2996 		msg->im_local_addr.ia_remote_lid	= gid_info->gl_dlid;
2997 		if (gid_info->gl_redirected == B_TRUE) {
2998 			if (gid_info->gl_redirect_dlid != 0) {
2999 				msg->im_local_addr.ia_remote_lid =
3000 				    gid_info->gl_redirect_dlid;
3001 			}
3002 			msg->im_local_addr.ia_remote_qno =
3003 			    gid_info->gl_redirect_QP;
3004 			msg->im_local_addr.ia_p_key =
3005 			    gid_info->gl_redirect_pkey;
3006 			msg->im_local_addr.ia_q_key =
3007 			    gid_info->gl_redirect_qkey;
3008 			msg->im_local_addr.ia_service_level =
3009 			    gid_info->gl_redirectSL;
3010 		} else {
3011 			msg->im_local_addr.ia_remote_qno = 1;
3012 			msg->im_local_addr.ia_p_key = gid_info->gl_p_key;
3013 			msg->im_local_addr.ia_q_key = IB_GSI_QKEY;
3014 			msg->im_local_addr.ia_service_level = gid_info->gl_SL;
3015 		}
3016 
3017 		hdr			= IBDM_OUT_IBMFMSG_MADHDR(msg);
3018 		hdr->BaseVersion	= MAD_CLASS_BASE_VERS_1;
3019 		hdr->MgmtClass		= MAD_MGMT_CLASS_DEV_MGT;
3020 		hdr->ClassVersion	= IB_DM_CLASS_VERSION_1;
3021 		hdr->R_Method		= IB_DM_DEVMGT_METHOD_GET;
3022 		hdr->Status		= 0;
3023 		hdr->TransactionID	= h2b64(gid_info->gl_transactionID);
3024 		hdr->AttributeID	= h2b16(IB_DM_ATTR_SERVICE_ENTRIES);
3025 
3026 		srv_start = ii * 4;
3027 		cb_args = &ioc_info->ioc_serv[srv_start].se_cb_args;
3028 		cb_args->cb_gid_info	= gid_info;
3029 		cb_args->cb_retry_count	= ibdm_dft_retry_cnt;
3030 		cb_args->cb_req_type	= IBDM_REQ_TYPE_SRVENTS;
3031 		cb_args->cb_srvents_start = srv_start;
3032 		cb_args->cb_ioc_num	= ioc_no - 1;
3033 
3034 		if (nserv_entries >= IBDM_MAX_SERV_ENTRIES_PER_REQ) {
3035 			nserv_entries -= IBDM_MAX_SERV_ENTRIES_PER_REQ;
3036 			cb_args->cb_srvents_end = (cb_args->cb_srvents_start +
3037 			    IBDM_MAX_SERV_ENTRIES_PER_REQ - 1);
3038 		} else {
3039 			cb_args->cb_srvents_end =
3040 			    (cb_args->cb_srvents_start + nserv_entries - 1);
3041 			nserv_entries = 0;
3042 		}
3043 		_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*hdr))
3044 		ibdm_fill_srv_attr_mod(hdr, cb_args);
3045 		_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*hdr))
3046 
3047 		mutex_enter(&gid_info->gl_mutex);
3048 		ioc_info->ioc_serv[srv_start].se_timeout_id = timeout(
3049 		    ibdm_pkt_timeout_hdlr, cb_args,
3050 		    IBDM_TIMEOUT_VALUE(ibdm_dft_timeout));
3051 		mutex_exit(&gid_info->gl_mutex);
3052 
3053 		IBTF_DPRINTF_L5("ibdm", "\thandle_ioc_profile:"
3054 		    "timeout %x, ioc %d srv %d",
3055 		    ioc_info->ioc_serv[srv_start].se_timeout_id,
3056 		    ioc_no - 1, srv_start);
3057 
3058 		if (ibmf_msg_transport(ibmf_hdl, gid_info->gl_qp_hdl, msg,
3059 		    NULL, ibdm_ibmf_send_cb, cb_args, 0) != IBMF_SUCCESS) {
3060 			IBTF_DPRINTF_L2("ibdm",
3061 			    "\thandle_ioc_profile: msg send failed");
3062 			ibdm_ibmf_send_cb(ibmf_hdl, msg, cb_args);
3063 		}
3064 		(*flag) |= IBDM_IBMF_PKT_REUSED;
3065 		first = B_FALSE;
3066 		ii++;
3067 	}
3068 }
3069 
3070 
3071 /*
3072  * ibdm_handle_srventry_mad()
3073  */
3074 static void
3075 ibdm_handle_srventry_mad(ibmf_msg_t *msg,
3076     ibdm_dp_gidinfo_t *gid_info, int *flag)
3077 {
3078 	uint_t			ii, ioc_no, attrmod;
3079 	uint_t			nentries, start, end;
3080 	timeout_id_t		timeout_id;
3081 	ib_dm_srv_t		*srv_ents;
3082 	ibdm_ioc_info_t		*ioc_info;
3083 	ibdm_srvents_info_t	*gsrv_ents;
3084 
3085 	IBTF_DPRINTF_L4("ibdm", "\thandle_srventry_mad:"
3086 	    " IBMF msg %p gid info %p", msg, gid_info);
3087 
3088 	srv_ents = IBDM_IN_IBMFMSG2SRVENT(msg);
3089 	/*
3090 	 * Get the start and end index of the service entries
3091 	 * Upper 16 bits identify the IOC
3092 	 * Lower 16 bits specify the range of service entries
3093 	 * 	LSB specifies (Big endian) end of the range
3094 	 * 	MSB specifies (Big endian) start of the range
3095 	 */
3096 	attrmod = IBDM_IN_IBMFMSG_ATTRMOD(msg);
3097 	ioc_no	= ((attrmod >> 16) & IBDM_16_BIT_MASK);
3098 	end	= ((attrmod >> 8) & IBDM_8_BIT_MASK);
3099 	start	= (attrmod & IBDM_8_BIT_MASK);
3100 
3101 	/* Make sure that IOC index is with the valid range */
3102 	if ((ioc_no < 1) |
3103 	    (ioc_no > gid_info->gl_iou->iou_info.iou_num_ctrl_slots)) {
3104 		IBTF_DPRINTF_L2("ibdm", "\thandle_srventry_mad: "
3105 		    "IOC index Out of range, index %d", ioc_no);
3106 		(*flag) |= IBDM_IBMF_PKT_UNEXP_RESP;
3107 		return;
3108 	}
3109 	ioc_info = IBDM_GIDINFO2IOCINFO(gid_info, (ioc_no -1));
3110 
3111 	/*
3112 	 * Make sure that the "start" and "end" service indexes are
3113 	 * with in the valid range
3114 	 */
3115 	nentries = ioc_info->ioc_profile.ioc_service_entries;
3116 	if ((start > end) | (start >= nentries) | (end >= nentries)) {
3117 		IBTF_DPRINTF_L2("ibdm", "\thandle_srventry_mad: "
3118 		    "Attr modifier 0x%x, #Serv entries %d", attrmod, nentries);
3119 		(*flag) |= IBDM_IBMF_PKT_UNEXP_RESP;
3120 		return;
3121 	}
3122 	gsrv_ents = &ioc_info->ioc_serv[start];
3123 	mutex_enter(&gid_info->gl_mutex);
3124 	if (gsrv_ents->se_state != IBDM_SE_INVALID) {
3125 		IBTF_DPRINTF_L2("ibdm", "\thandle_srventry_mad: "
3126 		    "already known, ioc %d, srv %d, se_state %x",
3127 		    ioc_no - 1, start, gsrv_ents->se_state);
3128 		mutex_exit(&gid_info->gl_mutex);
3129 		(*flag) |= IBDM_IBMF_PKT_DUP_RESP;
3130 		return;
3131 	}
3132 	ioc_info->ioc_serv[start].se_cb_args.cb_req_type = 0;
3133 	if (ioc_info->ioc_serv[start].se_timeout_id) {
3134 		IBTF_DPRINTF_L2("ibdm",
3135 		    "\thandle_srventry_mad: ioc %d start %d", ioc_no, start);
3136 		timeout_id = ioc_info->ioc_serv[start].se_timeout_id;
3137 		ioc_info->ioc_serv[start].se_timeout_id = 0;
3138 		mutex_exit(&gid_info->gl_mutex);
3139 		IBTF_DPRINTF_L5("ibdm", "handle_srverntry_mad: "
3140 		    "se_timeout_id = 0x%x", timeout_id);
3141 		if (untimeout(timeout_id) == -1) {
3142 			IBTF_DPRINTF_L2("ibdm", "handle_srventry_mad: "
3143 			    "untimeout se_timeout_id failed");
3144 		}
3145 		mutex_enter(&gid_info->gl_mutex);
3146 	}
3147 
3148 	gsrv_ents->se_state = IBDM_SE_VALID;
3149 	mutex_exit(&gid_info->gl_mutex);
3150 	for (ii = start; ii <= end; ii++, srv_ents++, gsrv_ents++) {
3151 		gsrv_ents->se_attr.srv_id = b2h64(srv_ents->srv_id);
3152 		bcopy(srv_ents->srv_name,
3153 		    gsrv_ents->se_attr.srv_name, IB_DM_MAX_SVC_NAME_LEN);
3154 		ibdm_dump_service_entries(&gsrv_ents->se_attr);
3155 	}
3156 }
3157 
3158 
3159 /*
3160  * ibdm_get_diagcode:
3161  *	Send request to get IOU/IOC diag code
3162  *	Returns IBDM_SUCCESS/IBDM_FAILURE
3163  */
3164 static int
3165 ibdm_get_diagcode(ibdm_dp_gidinfo_t *gid_info, int attr)
3166 {
3167 	ibmf_msg_t		*msg;
3168 	ib_mad_hdr_t		*hdr;
3169 	ibdm_ioc_info_t		*ioc;
3170 	ibdm_timeout_cb_args_t	*cb_args;
3171 	timeout_id_t		*timeout_id;
3172 
3173 	IBTF_DPRINTF_L4("ibdm", "\tget_diagcode: gid info %p, attr = %d",
3174 	    gid_info, attr);
3175 
3176 	if (ibmf_alloc_msg(gid_info->gl_ibmf_hdl, IBMF_ALLOC_SLEEP,
3177 	    &msg) != IBMF_SUCCESS) {
3178 		IBTF_DPRINTF_L4("ibdm", "\tget_diagcode: pkt alloc fail");
3179 		return (IBDM_FAILURE);
3180 	}
3181 
3182 	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*msg))
3183 	ibdm_alloc_send_buffers(msg);
3184 	_NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*msg))
3185 
3186 	mutex_enter(&gid_info->gl_mutex);
3187 	ibdm_bump_transactionID(gid_info);
3188 	mutex_exit(&gid_info->gl_mutex);
3189 
3190 	msg->im_local_addr.ia_local_lid	= gid_info->gl_slid;
3191 	msg->im_local_addr.ia_remote_lid	= gid_info->gl_dlid;
3192 	if (gid_info->gl_redirected == B_TRUE) {
3193 		if (gid_info->gl_redirect_dlid != 0) {
3194 			msg->im_local_addr.ia_remote_lid =
3195 			    gid_info->gl_redirect_dlid;
3196 		}
3197 
3198 		msg->im_local_addr.ia_remote_qno = gid_info->gl_redirect_QP;
3199 		msg->im_local_addr.ia_p_key = gid_info->gl_redirect_pkey;
3200 		msg->im_local_addr.ia_q_key = gid_info->gl_redirect_qkey;
3201 		msg->im_local_addr.ia_service_level = gid_info->gl_redirectSL;
3202 	} else {
3203 		msg->im_local_addr.ia_remote_qno = 1;
3204 		msg->im_local_addr.ia_p_key = gid_info->gl_p_key;
3205 		msg->im_local_addr.ia_q_key = IB_GSI_QKEY;
3206 		msg->im_local_addr.ia_service_level = gid_info->gl_SL;
3207 	}
3208 
3209 	hdr			= IBDM_OUT_IBMFMSG_MADHDR(msg);
3210 	hdr->BaseVersion	= MAD_CLASS_BASE_VERS_1;
3211 	hdr->MgmtClass		= MAD_MGMT_CLASS_DEV_MGT;
3212 	hdr->ClassVersion	= IB_DM_CLASS_VERSION_1;
3213 	hdr->R_Method		= IB_DM_DEVMGT_METHOD_GET;
3214 	hdr->Status		= 0;
3215 	hdr->TransactionID	= h2b64(gid_info->gl_transactionID);
3216 
3217 	hdr->AttributeID	= h2b16(IB_DM_ATTR_DIAG_CODE);
3218 	hdr->AttributeModifier	= h2b32(attr);
3219 
3220 	if (attr == 0) {
3221 		cb_args = &gid_info->gl_iou_cb_args;
3222 		gid_info->gl_iou->iou_dc_valid = B_FALSE;
3223 		cb_args->cb_ioc_num	= 0;
3224 		cb_args->cb_req_type	= IBDM_REQ_TYPE_IOU_DIAGCODE;
3225 		timeout_id = &gid_info->gl_timeout_id;
3226 	} else {
3227 		ioc = IBDM_GIDINFO2IOCINFO(gid_info, (attr - 1));
3228 		ioc->ioc_dc_valid = B_FALSE;
3229 		cb_args = &ioc->ioc_dc_cb_args;
3230 		cb_args->cb_ioc_num	= attr - 1;
3231 		cb_args->cb_req_type	= IBDM_REQ_TYPE_IOC_DIAGCODE;
3232 		timeout_id = &ioc->ioc_dc_timeout_id;
3233 	}
3234 	cb_args->cb_gid_info	= gid_info;
3235 	cb_args->cb_retry_count	= ibdm_dft_retry_cnt;
3236 	cb_args->cb_srvents_start = 0;
3237 
3238 	mutex_enter(&gid_info->gl_mutex);
3239 	*timeout_id = timeout(ibdm_pkt_timeout_hdlr,
3240 	    cb_args, IBDM_TIMEOUT_VALUE(ibdm_dft_timeout));
3241 	mutex_exit(&gid_info->gl_mutex);
3242 
3243 	IBTF_DPRINTF_L5("ibdm", "\tget_diagcode:"
3244 	    "timeout %x, ioc %d", *timeout_id, cb_args->cb_ioc_num);
3245 
3246 	if (ibmf_msg_transport(gid_info->gl_ibmf_hdl, gid_info->gl_qp_hdl,
3247 	    msg, NULL, ibdm_ibmf_send_cb, cb_args, 0) != IBMF_SUCCESS) {
3248 		IBTF_DPRINTF_L2("ibdm", "\tget_diagcode: ibmf send failed");
3249 		ibdm_ibmf_send_cb(gid_info->gl_ibmf_hdl, msg, cb_args);
3250 	}
3251 	return (IBDM_SUCCESS);
3252 }
3253 
3254 /*
3255  * ibdm_handle_diagcode:
3256  *	Process the DiagCode MAD response and update local DM
3257  *	data structure.
3258  */
3259 static void
3260 ibdm_handle_diagcode(ibmf_msg_t *ibmf_msg,
3261     ibdm_dp_gidinfo_t *gid_info, int *flag)
3262 {
3263 	uint16_t	attrmod, *diagcode;
3264 	ibdm_iou_info_t	*iou;
3265 	ibdm_ioc_info_t	*ioc;
3266 	timeout_id_t	timeout_id;
3267 	ibdm_timeout_cb_args_t	*cb_args;
3268 
3269 	diagcode = (uint16_t *)ibmf_msg->im_msgbufs_recv.im_bufs_cl_data;
3270 
3271 	mutex_enter(&gid_info->gl_mutex);
3272 	attrmod = IBDM_IN_IBMFMSG_ATTRMOD(ibmf_msg);
3273 	iou = gid_info->gl_iou;
3274 	if (attrmod == 0) {
3275 		if (iou->iou_dc_valid != B_FALSE) {
3276 			(*flag) |= IBDM_IBMF_PKT_DUP_RESP;
3277 			IBTF_DPRINTF_L4("ibdm",
3278 			    "\thandle_diagcode: Duplicate IOU DiagCode");
3279 			mutex_exit(&gid_info->gl_mutex);
3280 			return;
3281 		}
3282 		cb_args = &gid_info->gl_iou_cb_args;
3283 		cb_args->cb_req_type = 0;
3284 		iou->iou_diagcode = b2h16(*diagcode);
3285 		iou->iou_dc_valid = B_TRUE;
3286 		if (gid_info->gl_timeout_id) {
3287 			timeout_id = gid_info->gl_timeout_id;
3288 			mutex_exit(&gid_info->gl_mutex);
3289 			IBTF_DPRINTF_L5("ibdm", "\thandle_diagcode: "
3290 			    "gl_timeout_id = 0x%x", timeout_id);
3291 			if (untimeout(timeout_id) == -1) {
3292 				IBTF_DPRINTF_L2("ibdm", "handle_diagcode: "
3293 				    "untimeout gl_timeout_id failed");
3294 			}
3295 			mutex_enter(&gid_info->gl_mutex);
3296 			gid_info->gl_timeout_id = 0;
3297 		}
3298 	} else {
3299 		ioc = IBDM_GIDINFO2IOCINFO(gid_info, (attrmod - 1));
3300 		if (ioc->ioc_dc_valid != B_FALSE) {
3301 			(*flag) |= IBDM_IBMF_PKT_DUP_RESP;
3302 			IBTF_DPRINTF_L4("ibdm",
3303 			    "\thandle_diagcode: Duplicate IOC DiagCode");
3304 			mutex_exit(&gid_info->gl_mutex);
3305 			return;
3306 		}
3307 		cb_args = &ioc->ioc_dc_cb_args;
3308 		cb_args->cb_req_type = 0;
3309 		ioc->ioc_diagcode = b2h16(*diagcode);
3310 		ioc->ioc_dc_valid = B_TRUE;
3311 		timeout_id = iou->iou_ioc_info[attrmod - 1].ioc_dc_timeout_id;
3312 		if (timeout_id) {
3313 			iou->iou_ioc_info[attrmod - 1].ioc_dc_timeout_id = 0;
3314 			mutex_exit(&gid_info->gl_mutex);
3315 			IBTF_DPRINTF_L5("ibdm", "handle_diagcode: "
3316 			    "timeout_id = 0x%x", timeout_id);
3317 			if (untimeout(timeout_id) == -1) {
3318 				IBTF_DPRINTF_L2("ibdm", "\thandle_diagcode: "
3319 				    "untimeout ioc_dc_timeout_id failed");
3320 			}
3321 			mutex_enter(&gid_info->gl_mutex);
3322 		}
3323 	}
3324 	mutex_exit(&gid_info->gl_mutex);
3325 
3326 	IBTF_DPRINTF_L4("ibdm", "\thandle_diagcode: DiagCode : 0x%x"
3327 	    "attrmod : 0x%x", b2h16(*diagcode), attrmod);
3328 }
3329 
3330 
3331 /*
3332  * ibdm_is_ioc_present()
3333  *	Return ibdm_ioc_info_t if IOC guid is found in the global gid list
3334  */
3335 static ibdm_ioc_info_t *
3336 ibdm_is_ioc_present(ib_guid_t ioc_guid,
3337     ibdm_dp_gidinfo_t *gid_info, int *flag)
3338 {
3339 	int				ii;
3340 	ibdm_ioc_info_t			*ioc;
3341 	ibdm_dp_gidinfo_t		*head;
3342 	ib_dm_io_unitinfo_t		*iou;
3343 
3344 	mutex_enter(&ibdm.ibdm_mutex);
3345 	head = ibdm.ibdm_dp_gidlist_head;
3346 	while (head) {
3347 		mutex_enter(&head->gl_mutex);
3348 		if (head->gl_iou == NULL) {
3349 			mutex_exit(&head->gl_mutex);
3350 			head = head->gl_next;
3351 			continue;
3352 		}
3353 		iou = &head->gl_iou->iou_info;
3354 		for (ii = 0; ii < iou->iou_num_ctrl_slots; ii++) {
3355 			ioc = IBDM_GIDINFO2IOCINFO(head, ii);
3356 			if ((ioc->ioc_state == IBDM_IOC_STATE_PROBE_SUCCESS) &&
3357 			    (ioc->ioc_profile.ioc_guid == ioc_guid)) {
3358 				if (gid_info == head) {
3359 					*flag |= IBDM_IBMF_PKT_DUP_RESP;
3360 				} else if (ibdm_check_dgid(head->gl_dgid_lo,
3361 				    head->gl_dgid_hi) != NULL) {
3362 					IBTF_DPRINTF_L4("ibdm", "\tis_ioc_"
3363 					    "present: gid not present");
3364 					ibdm_add_to_gl_gid(gid_info, head);
3365 				}
3366 				mutex_exit(&head->gl_mutex);
3367 				mutex_exit(&ibdm.ibdm_mutex);
3368 				return (ioc);
3369 			}
3370 		}
3371 		mutex_exit(&head->gl_mutex);
3372 		head = head->gl_next;
3373 	}
3374 	mutex_exit(&ibdm.ibdm_mutex);
3375 	return (NULL);
3376 }
3377 
3378 
3379 /*
3380  * ibdm_ibmf_send_cb()
3381  *	IBMF invokes this callback routine after posting the DM MAD to
3382  *	the HCA.
3383  */
3384 /*ARGSUSED*/
3385 static void
3386 ibdm_ibmf_send_cb(ibmf_handle_t ibmf_hdl, ibmf_msg_t *ibmf_msg, void *arg)
3387 {
3388 	ibdm_dump_ibmf_msg(ibmf_msg, 1);
3389 	ibdm_free_send_buffers(ibmf_msg);
3390 	if (ibmf_free_msg(ibmf_hdl, &ibmf_msg) != IBMF_SUCCESS) {
3391 		IBTF_DPRINTF_L4("ibdm",
3392 		    "\tibmf_send_cb: IBMF free msg failed");
3393 	}
3394 }
3395 
3396 
3397 /*
3398  * ibdm_ibmf_recv_cb()
3399  *	Invoked by the IBMF when a response to the one of the DM requests
3400  *	is received.
3401  */
3402 /*ARGSUSED*/
3403 static void
3404 ibdm_ibmf_recv_cb(ibmf_handle_t ibmf_hdl, ibmf_msg_t *msg, void *arg)
3405 {
3406 	ibdm_taskq_args_t	*taskq_args;
3407 
3408 	/*
3409 	 * If the taskq enable is set then dispatch a taskq to process
3410 	 * the MAD, otherwise just process it on this thread
3411 	 */
3412 	if (ibdm_taskq_enable != IBDM_ENABLE_TASKQ_HANDLING) {
3413 		ibdm_process_incoming_mad(ibmf_hdl, msg, arg);
3414 		return;
3415 	}
3416 
3417 	/*
3418 	 * create a taskq and dispatch it to process the incoming MAD
3419 	 */
3420 	taskq_args = kmem_alloc(sizeof (ibdm_taskq_args_t), KM_NOSLEEP);
3421 	if (taskq_args == NULL) {
3422 		IBTF_DPRINTF_L2("ibdm", "ibmf_recv_cb: kmem_alloc failed for"
3423 		    "taskq_args");
3424 		if (ibmf_free_msg(ibmf_hdl, &msg) != IBMF_SUCCESS) {
3425 			IBTF_DPRINTF_L4("ibmf_recv_cb",
3426 			    "\tibmf_recv_cb: IBMF free msg failed");
3427 		}
3428 		return;
3429 	}
3430 	taskq_args->tq_ibmf_handle = ibmf_hdl;
3431 	taskq_args->tq_ibmf_msg = msg;
3432 	taskq_args->tq_args = arg;
3433 
3434 	if (taskq_dispatch(system_taskq, ibdm_recv_incoming_mad, taskq_args,
3435 	    TQ_NOSLEEP) == TASKQID_INVALID) {
3436 		IBTF_DPRINTF_L2("ibdm", "ibmf_recv_cb: taskq_dispatch failed");
3437 		if (ibmf_free_msg(ibmf_hdl, &msg) != IBMF_SUCCESS) {
3438 			IBTF_DPRINTF_L4("ibmf_recv_cb",
3439 			    "\tibmf_recv_cb: IBMF free msg failed");
3440 		}
3441 		kmem_free(taskq_args, sizeof (ibdm_taskq_args_t));
3442 		return;
3443 	}
3444 
3445 	/* taskq_args are deleted in ibdm_recv_incoming_mad() */
3446 }
3447 
3448 
3449 void
3450 ibdm_recv_incoming_mad(void *args)
3451 {
3452 	ibdm_taskq_args_t	*taskq_args;
3453 
3454 	taskq_args = (ibdm_taskq_args_t *)args;
3455 
3456 	IBTF_DPRINTF_L4("ibdm", "\tibdm_recv_incoming_mad: "
3457 	    "Processing incoming MAD via taskq");
3458 
3459 	ibdm_process_incoming_mad(taskq_args->tq_ibmf_handle,
3460 	    taskq_args->tq_ibmf_msg, taskq_args->tq_args);
3461 
3462 	kmem_free(taskq_args, sizeof (ibdm_taskq_args_t));
3463 }
3464 
3465 
3466 /*
3467  * Calls ibdm_process_incoming_mad with all function arguments  extracted
3468  * from args
3469  */
3470 /*ARGSUSED*/
3471 static void
3472 ibdm_process_incoming_mad(ibmf_handle_t ibmf_hdl, ibmf_msg_t *msg, void *arg)
3473 {
3474 	int			flag = 0;
3475 	int			ret;
3476 	uint64_t		transaction_id;
3477 	ib_mad_hdr_t		*hdr;
3478 	ibdm_dp_gidinfo_t	*gid_info = NULL;
3479 
3480 	IBTF_DPRINTF_L4("ibdm",
3481 	    "\tprocess_incoming_mad: ibmf hdl %p pkt %p", ibmf_hdl, msg);
3482 	ibdm_dump_ibmf_msg(msg, 0);
3483 
3484 	/*
3485 	 * IBMF calls this routine for every DM MAD that arrives at this port.
3486 	 * But we handle only the responses for requests we sent. We drop all
3487 	 * the DM packets that does not have response bit set in the MAD
3488 	 * header(this eliminates all the requests sent to this port).
3489 	 * We handle only DM class version 1 MAD's
3490 	 */
3491 	hdr = IBDM_IN_IBMFMSG_MADHDR(msg);
3492 	if (ibdm_verify_mad_status(hdr) != IBDM_SUCCESS) {
3493 		if (ibmf_free_msg(ibmf_hdl, &msg) != IBMF_SUCCESS) {
3494 			IBTF_DPRINTF_L2("ibdm", "\tprocess_incoming_mad: "
3495 			    "IBMF free msg failed DM request drop it");
3496 		}
3497 		return;
3498 	}
3499 
3500 	transaction_id = b2h64(hdr->TransactionID);
3501 
3502 	mutex_enter(&ibdm.ibdm_mutex);
3503 	gid_info = ibdm.ibdm_dp_gidlist_head;
3504 	while (gid_info) {
3505 		if ((gid_info->gl_transactionID  &
3506 		    IBDM_GID_TRANSACTIONID_MASK) ==
3507 		    (transaction_id & IBDM_GID_TRANSACTIONID_MASK))
3508 			break;
3509 		gid_info = gid_info->gl_next;
3510 	}
3511 	mutex_exit(&ibdm.ibdm_mutex);
3512 
3513 	if (gid_info == NULL) {
3514 		/* Drop the packet */
3515 		IBTF_DPRINTF_L2("ibdm", "process_incoming_mad: transaction ID"
3516 		    " does not match: 0x%llx", transaction_id);
3517 		if (ibmf_free_msg(ibmf_hdl, &msg) != IBMF_SUCCESS) {
3518 			IBTF_DPRINTF_L2("ibdm", "process_incoming_mad: "
3519 			    "IBMF free msg failed DM request drop it");
3520 		}
3521 		return;
3522 	}
3523 
3524 	/* Handle redirection for all the MAD's, except ClassPortInfo */
3525 	if (((IBDM_IN_IBMFMSG_STATUS(msg) & MAD_STATUS_REDIRECT_REQUIRED)) &&
3526 	    (IBDM_IN_IBMFMSG_ATTR(msg) != IB_DM_ATTR_CLASSPORTINFO)) {
3527 		ret = ibdm_handle_redirection(msg, gid_info, &flag);
3528 		if (ret == IBDM_SUCCESS) {
3529 			return;
3530 		}
3531 	} else {
3532 		uint_t gl_state;
3533 
3534 		mutex_enter(&gid_info->gl_mutex);
3535 		gl_state = gid_info->gl_state;
3536 		mutex_exit(&gid_info->gl_mutex);
3537 
3538 		switch (gl_state) {
3539 
3540 		case IBDM_SET_CLASSPORTINFO:
3541 			ibdm_handle_setclassportinfo(
3542 			    ibmf_hdl, msg, gid_info, &flag);
3543 			break;
3544 
3545 		case IBDM_GET_CLASSPORTINFO:
3546 			ibdm_handle_classportinfo(
3547 			    ibmf_hdl, msg, gid_info, &flag);
3548 			break;
3549 
3550 		case IBDM_GET_IOUNITINFO:
3551 			ibdm_handle_iounitinfo(ibmf_hdl, msg, gid_info, &flag);
3552 			break;
3553 
3554 		case IBDM_GET_IOC_DETAILS:
3555 			switch (IBDM_IN_IBMFMSG_ATTR(msg)) {
3556 
3557 			case IB_DM_ATTR_SERVICE_ENTRIES:
3558 				ibdm_handle_srventry_mad(msg, gid_info, &flag);
3559 				break;
3560 
3561 			case IB_DM_ATTR_IOC_CTRL_PROFILE:
3562 				ibdm_handle_ioc_profile(
3563 				    ibmf_hdl, msg, gid_info, &flag);
3564 				break;
3565 
3566 			case IB_DM_ATTR_DIAG_CODE:
3567 				ibdm_handle_diagcode(msg, gid_info, &flag);
3568 				break;
3569 
3570 			default:
3571 				IBTF_DPRINTF_L2("ibdm", "process_incoming_mad: "
3572 				    "Error state, wrong attribute :-(");
3573 				(void) ibmf_free_msg(ibmf_hdl, &msg);
3574 				return;
3575 			}
3576 			break;
3577 		default:
3578 			IBTF_DPRINTF_L2("ibdm",
3579 			    "process_incoming_mad: Dropping the packet"
3580 			    " gl_state %x", gl_state);
3581 			if (ibmf_free_msg(ibmf_hdl, &msg) != IBMF_SUCCESS) {
3582 				IBTF_DPRINTF_L2("ibdm", "process_incoming_mad: "
3583 				    "IBMF free msg failed DM request drop it");
3584 			}
3585 			return;
3586 		}
3587 	}
3588 
3589 	if ((flag & IBDM_IBMF_PKT_DUP_RESP) ||
3590 	    (flag & IBDM_IBMF_PKT_UNEXP_RESP)) {
3591 		IBTF_DPRINTF_L2("ibdm",
3592 		    "\tprocess_incoming_mad:Dup/unexp resp : 0x%x", flag);
3593 		if (ibmf_free_msg(ibmf_hdl, &msg) != IBMF_SUCCESS) {
3594 			IBTF_DPRINTF_L2("ibdm", "process_incoming_mad: "
3595 			    "IBMF free msg failed DM request drop it");
3596 		}
3597 		return;
3598 	}
3599 
3600 	mutex_enter(&gid_info->gl_mutex);
3601 	if (gid_info->gl_pending_cmds < 1) {
3602 		IBTF_DPRINTF_L2("ibdm",
3603 		    "\tprocess_incoming_mad: pending commands negative");
3604 	}
3605 	if (--gid_info->gl_pending_cmds) {
3606 		IBTF_DPRINTF_L4("ibdm", "\tprocess_incoming_mad: "
3607 		    "gid_info %p pending cmds %d",
3608 		    gid_info, gid_info->gl_pending_cmds);
3609 		mutex_exit(&gid_info->gl_mutex);
3610 	} else {
3611 		uint_t prev_state;
3612 		IBTF_DPRINTF_L4("ibdm", "\tprocess_incoming_mad: Probing DONE");
3613 		prev_state = gid_info->gl_state;
3614 		gid_info->gl_state = IBDM_GID_PROBING_COMPLETE;
3615 		if (prev_state == IBDM_SET_CLASSPORTINFO) {
3616 			IBTF_DPRINTF_L4("ibdm",
3617 			    "\tprocess_incoming_mad: "
3618 			    "Setclassportinfo for Cisco FC GW is done.");
3619 			gid_info->gl_flag &= ~IBDM_CISCO_PROBE;
3620 			gid_info->gl_flag |= IBDM_CISCO_PROBE_DONE;
3621 			mutex_exit(&gid_info->gl_mutex);
3622 			cv_broadcast(&gid_info->gl_probe_cv);
3623 		} else {
3624 			mutex_exit(&gid_info->gl_mutex);
3625 			ibdm_notify_newgid_iocs(gid_info);
3626 			mutex_enter(&ibdm.ibdm_mutex);
3627 			if (--ibdm.ibdm_ngid_probes_in_progress == 0) {
3628 				IBTF_DPRINTF_L4("ibdm",
3629 				    "\tprocess_incoming_mad: Wakeup");
3630 				ibdm.ibdm_busy &= ~IBDM_PROBE_IN_PROGRESS;
3631 				cv_broadcast(&ibdm.ibdm_probe_cv);
3632 			}
3633 			mutex_exit(&ibdm.ibdm_mutex);
3634 		}
3635 	}
3636 
3637 	/*
3638 	 * Do not deallocate the IBMF packet if atleast one request
3639 	 * is posted. IBMF packet is reused.
3640 	 */
3641 	if (!(flag & IBDM_IBMF_PKT_REUSED)) {
3642 		if (ibmf_free_msg(ibmf_hdl, &msg) != IBMF_SUCCESS) {
3643 			IBTF_DPRINTF_L2("ibdm", "\tprocess_incoming_mad: "
3644 			    "IBMF free msg failed DM request drop it");
3645 		}
3646 	}
3647 }
3648 
3649 
3650 /*
3651  * ibdm_verify_mad_status()
3652  *	Verifies the MAD status
3653  *	Returns IBDM_SUCCESS if status is correct
3654  *	Returns IBDM_FAILURE for bogus MAD status
3655  */
3656 static int
3657 ibdm_verify_mad_status(ib_mad_hdr_t *hdr)
3658 {
3659 	int	ret = 0;
3660 
3661 	if ((hdr->R_Method != IB_DM_DEVMGT_METHOD_GET_RESP) ||
3662 	    (hdr->ClassVersion != IB_DM_CLASS_VERSION_1)) {
3663 		return (IBDM_FAILURE);
3664 	}
3665 
3666 	if (b2h16(hdr->Status) == 0)
3667 		ret = IBDM_SUCCESS;
3668 	else if ((b2h16(hdr->Status) & 0x1f) == MAD_STATUS_REDIRECT_REQUIRED)
3669 		ret = IBDM_SUCCESS;
3670 	else {
3671 		IBTF_DPRINTF_L2("ibdm",
3672 		    "\tverify_mad_status: Status : 0x%x", b2h16(hdr->Status));
3673 		ret = IBDM_FAILURE;
3674 	}
3675 	return (ret);
3676 }
3677 
3678 
3679 
3680 /*
3681  * ibdm_handle_redirection()
3682  *	Returns IBDM_SUCCESS/IBDM_FAILURE
3683  */
3684 static int
3685 ibdm_handle_redirection(ibmf_msg_t *msg,
3686     ibdm_dp_gidinfo_t *gid_info, int *flag)
3687 {
3688 	int			attrmod, ioc_no, start;
3689 	void			*data;
3690 	timeout_id_t		*timeout_id;
3691 	ib_mad_hdr_t		*hdr;
3692 	ibdm_ioc_info_t		*ioc = NULL;
3693 	ibdm_timeout_cb_args_t	*cb_args;
3694 	ib_mad_classportinfo_t	*cpi;
3695 
3696 	IBTF_DPRINTF_L4("ibdm", "\thandle_redirection: Enter");
3697 	mutex_enter(&gid_info->gl_mutex);
3698 	switch (gid_info->gl_state) {
3699 	case IBDM_GET_IOUNITINFO:
3700 		cb_args		= &gid_info->gl_iou_cb_args;
3701 		timeout_id	= &gid_info->gl_timeout_id;
3702 		break;
3703 
3704 	case IBDM_GET_IOC_DETAILS:
3705 		attrmod	= IBDM_IN_IBMFMSG_ATTRMOD(msg);
3706 		switch (IBDM_IN_IBMFMSG_ATTR(msg)) {
3707 
3708 		case IB_DM_ATTR_DIAG_CODE:
3709 			if (attrmod == 0) {
3710 				cb_args = &gid_info->gl_iou_cb_args;
3711 				timeout_id = &gid_info->gl_timeout_id;
3712 				break;
3713 			}
3714 			if (IBDM_IS_IOC_NUM_INVALID(attrmod, gid_info)) {
3715 				IBTF_DPRINTF_L2("ibdm", "\thandle_redirction:"
3716 				    "IOC# Out of range %d", attrmod);
3717 				(*flag) |= IBDM_IBMF_PKT_UNEXP_RESP;
3718 				mutex_exit(&gid_info->gl_mutex);
3719 				return (IBDM_FAILURE);
3720 			}
3721 			ioc	= IBDM_GIDINFO2IOCINFO(gid_info, (attrmod -1));
3722 			cb_args = &ioc->ioc_dc_cb_args;
3723 			timeout_id = &ioc->ioc_dc_timeout_id;
3724 			break;
3725 
3726 		case IB_DM_ATTR_IOC_CTRL_PROFILE:
3727 			if (IBDM_IS_IOC_NUM_INVALID(attrmod, gid_info)) {
3728 				IBTF_DPRINTF_L2("ibdm", "\thandle_redirction:"
3729 				    "IOC# Out of range %d", attrmod);
3730 				(*flag) |= IBDM_IBMF_PKT_UNEXP_RESP;
3731 				mutex_exit(&gid_info->gl_mutex);
3732 				return (IBDM_FAILURE);
3733 			}
3734 			ioc	= IBDM_GIDINFO2IOCINFO(gid_info, (attrmod -1));
3735 			cb_args = &ioc->ioc_cb_args;
3736 			timeout_id = &ioc->ioc_timeout_id;
3737 			break;
3738 
3739 		case IB_DM_ATTR_SERVICE_ENTRIES:
3740 			ioc_no	= ((attrmod >> 16) & IBDM_16_BIT_MASK);
3741 			if (IBDM_IS_IOC_NUM_INVALID(ioc_no, gid_info)) {
3742 				IBTF_DPRINTF_L2("ibdm", "\thandle_redirction:"
3743 				    "IOC# Out of range %d", ioc_no);
3744 				(*flag) |= IBDM_IBMF_PKT_UNEXP_RESP;
3745 				mutex_exit(&gid_info->gl_mutex);
3746 				return (IBDM_FAILURE);
3747 			}
3748 			start 	= (attrmod & IBDM_8_BIT_MASK);
3749 			ioc	= IBDM_GIDINFO2IOCINFO(gid_info, (ioc_no -1));
3750 			if (start > ioc->ioc_profile.ioc_service_entries) {
3751 				IBTF_DPRINTF_L2("ibdm", "\thandle_redirction:"
3752 				    " SE index Out of range %d", start);
3753 				(*flag) |= IBDM_IBMF_PKT_UNEXP_RESP;
3754 				mutex_exit(&gid_info->gl_mutex);
3755 				return (IBDM_FAILURE);
3756 			}
3757 			cb_args = &ioc->ioc_serv[start].se_cb_args;
3758 			timeout_id = &ioc->ioc_serv[start].se_timeout_id;
3759 			break;
3760 
3761 		default:
3762 			/* ERROR State */
3763 			IBTF_DPRINTF_L2("ibdm",
3764 			    "\thandle_redirection: wrong attribute :-(");
3765 			(*flag) |= IBDM_IBMF_PKT_UNEXP_RESP;
3766 			mutex_exit(&gid_info->gl_mutex);
3767 			return (IBDM_FAILURE);
3768 		}
3769 		break;
3770 	default:
3771 		/* ERROR State */
3772 		IBTF_DPRINTF_L2("ibdm",
3773 		    "\thandle_redirection: Error state :-(");
3774 		(*flag) |= IBDM_IBMF_PKT_UNEXP_RESP;
3775 		mutex_exit(&gid_info->gl_mutex);
3776 		return (IBDM_FAILURE);
3777 	}
3778 	if ((*timeout_id) != 0) {
3779 		mutex_exit(&gid_info->gl_mutex);
3780 		if (untimeout(*timeout_id) == -1) {
3781 			IBTF_DPRINTF_L2("ibdm", "\thandle_redirection: "
3782 			    "untimeout failed %x", *timeout_id);
3783 		} else {
3784 			IBTF_DPRINTF_L5("ibdm",
3785 			    "\thandle_redirection: timeout %x", *timeout_id);
3786 		}
3787 		mutex_enter(&gid_info->gl_mutex);
3788 		*timeout_id = 0;
3789 	}
3790 
3791 	data = msg->im_msgbufs_recv.im_bufs_cl_data;
3792 	cpi = (ib_mad_classportinfo_t *)data;
3793 
3794 	gid_info->gl_resp_timeout	=
3795 	    (b2h32(cpi->RespTimeValue) & 0x1F);
3796 
3797 	gid_info->gl_redirected		= B_TRUE;
3798 	gid_info->gl_redirect_dlid	= b2h16(cpi->RedirectLID);
3799 	gid_info->gl_redirect_QP	= (b2h32(cpi->RedirectQP) & 0xffffff);
3800 	gid_info->gl_redirect_pkey	= b2h16(cpi->RedirectP_Key);
3801 	gid_info->gl_redirect_qkey	= b2h32(cpi->RedirectQ_Key);
3802 	gid_info->gl_redirectGID_hi	= b2h64(cpi->RedirectGID_hi);
3803 	gid_info->gl_redirectGID_lo	= b2h64(cpi->RedirectGID_lo);
3804 	gid_info->gl_redirectSL		= cpi->RedirectSL;
3805 
3806 	if (gid_info->gl_redirect_dlid != 0) {
3807 		msg->im_local_addr.ia_remote_lid =
3808 		    gid_info->gl_redirect_dlid;
3809 	}
3810 	ibdm_bump_transactionID(gid_info);
3811 	mutex_exit(&gid_info->gl_mutex);
3812 
3813 	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*msg, *hdr))
3814 	ibdm_alloc_send_buffers(msg);
3815 
3816 	hdr			= IBDM_OUT_IBMFMSG_MADHDR(msg);
3817 	hdr->BaseVersion	= MAD_CLASS_BASE_VERS_1;
3818 	hdr->MgmtClass		= MAD_MGMT_CLASS_DEV_MGT;
3819 	hdr->ClassVersion	= IB_DM_CLASS_VERSION_1;
3820 	hdr->R_Method		= IB_DM_DEVMGT_METHOD_GET;
3821 	hdr->Status		= 0;
3822 	hdr->TransactionID	= h2b64(gid_info->gl_transactionID);
3823 	hdr->AttributeID	=
3824 	    msg->im_msgbufs_recv.im_bufs_mad_hdr->AttributeID;
3825 	hdr->AttributeModifier	=
3826 	    msg->im_msgbufs_recv.im_bufs_mad_hdr->AttributeModifier;
3827 	_NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*msg, *hdr))
3828 
3829 	msg->im_local_addr.ia_remote_qno = gid_info->gl_redirect_QP;
3830 	msg->im_local_addr.ia_p_key = gid_info->gl_redirect_pkey;
3831 	msg->im_local_addr.ia_q_key = gid_info->gl_redirect_qkey;
3832 	msg->im_local_addr.ia_service_level = gid_info->gl_redirectSL;
3833 
3834 	mutex_enter(&gid_info->gl_mutex);
3835 	*timeout_id = timeout(ibdm_pkt_timeout_hdlr,
3836 	    cb_args, IBDM_TIMEOUT_VALUE(ibdm_dft_timeout));
3837 	mutex_exit(&gid_info->gl_mutex);
3838 
3839 	IBTF_DPRINTF_L5("ibdm", "\thandle_redirect:"
3840 	    "timeout %x", *timeout_id);
3841 
3842 	if (ibmf_msg_transport(gid_info->gl_ibmf_hdl, gid_info->gl_qp_hdl,
3843 	    msg, NULL, ibdm_ibmf_send_cb, cb_args, 0) != IBMF_SUCCESS) {
3844 		IBTF_DPRINTF_L4("ibdm", "\thandle_redirection:"
3845 		    "message transport failed");
3846 		ibdm_ibmf_send_cb(gid_info->gl_ibmf_hdl, msg, cb_args);
3847 	}
3848 	(*flag) |= IBDM_IBMF_PKT_REUSED;
3849 	IBTF_DPRINTF_L4("ibdm", "\thandle_redirection: Exit");
3850 	return (IBDM_SUCCESS);
3851 }
3852 
3853 
3854 /*
3855  * ibdm_pkt_timeout_hdlr
3856  *	This  timeout  handler is  registed for every  IBMF  packet that is
3857  *	sent through the IBMF.  It gets called when no response is received
3858  *	within the specified time for the packet. No retries for the failed
3859  *	commands  currently.  Drops the failed  IBMF packet and  update the
3860  *	pending list commands.
3861  */
3862 static void
3863 ibdm_pkt_timeout_hdlr(void *arg)
3864 {
3865 	ibdm_iou_info_t		*iou;
3866 	ibdm_ioc_info_t		*ioc;
3867 	ibdm_timeout_cb_args_t	*cb_args = arg;
3868 	ibdm_dp_gidinfo_t	*gid_info;
3869 	int			srv_ent;
3870 	uint_t			new_gl_state;
3871 
3872 	IBTF_DPRINTF_L2("ibdm", "\tpkt_timeout_hdlr: gid_info: %p "
3873 	    "rtype 0x%x iocidx 0x%x srvidx %d", cb_args->cb_gid_info,
3874 	    cb_args->cb_req_type, cb_args->cb_ioc_num,
3875 	    cb_args->cb_srvents_start);
3876 
3877 	gid_info = cb_args->cb_gid_info;
3878 	mutex_enter(&gid_info->gl_mutex);
3879 
3880 	if ((gid_info->gl_state == IBDM_GID_PROBING_COMPLETE) ||
3881 	    (cb_args->cb_req_type == 0)) {
3882 
3883 		IBTF_DPRINTF_L2("ibdm", "\tpkt_timeout_hdlr: req completed"
3884 		    "rtype 0x%x iocidx 0x%x srvidx %d", cb_args->cb_req_type,
3885 		    cb_args->cb_ioc_num, cb_args->cb_srvents_start);
3886 
3887 		if (gid_info->gl_timeout_id)
3888 			gid_info->gl_timeout_id = 0;
3889 		mutex_exit(&gid_info->gl_mutex);
3890 		return;
3891 	}
3892 	if (cb_args->cb_retry_count) {
3893 		cb_args->cb_retry_count--;
3894 		/*
3895 		 * A new timeout_id is set inside ibdm_retry_command().
3896 		 * When the function returns an error, the timeout_id
3897 		 * is reset (to zero) in the switch statement below.
3898 		 */
3899 		if (ibdm_retry_command(cb_args) == IBDM_SUCCESS) {
3900 			mutex_exit(&gid_info->gl_mutex);
3901 			return;
3902 		}
3903 		cb_args->cb_retry_count = 0;
3904 	}
3905 
3906 	IBTF_DPRINTF_L2("ibdm", "\tpkt_timeout_hdlr: command failed: gid %p"
3907 	    " rtype 0x%x iocidx 0x%x srvidx %d", cb_args->cb_gid_info,
3908 	    cb_args->cb_req_type, cb_args->cb_ioc_num,
3909 	    cb_args->cb_srvents_start);
3910 
3911 	switch (cb_args->cb_req_type) {
3912 
3913 	case IBDM_REQ_TYPE_CLASSPORTINFO:
3914 	case IBDM_REQ_TYPE_IOUINFO:
3915 		new_gl_state = IBDM_GID_PROBING_FAILED;
3916 		if (gid_info->gl_timeout_id)
3917 			gid_info->gl_timeout_id = 0;
3918 		break;
3919 
3920 	case IBDM_REQ_TYPE_IOCINFO:
3921 		new_gl_state = IBDM_GID_PROBING_COMPLETE;
3922 		iou = gid_info->gl_iou;
3923 		ioc = &iou->iou_ioc_info[cb_args->cb_ioc_num];
3924 		ioc->ioc_state = IBDM_IOC_STATE_PROBE_FAILED;
3925 		if (ioc->ioc_timeout_id)
3926 			ioc->ioc_timeout_id = 0;
3927 		break;
3928 
3929 	case IBDM_REQ_TYPE_SRVENTS:
3930 		new_gl_state = IBDM_GID_PROBING_COMPLETE;
3931 		iou = gid_info->gl_iou;
3932 		ioc = &iou->iou_ioc_info[cb_args->cb_ioc_num];
3933 		ioc->ioc_state = IBDM_IOC_STATE_PROBE_FAILED;
3934 		srv_ent = cb_args->cb_srvents_start;
3935 		if (ioc->ioc_serv[srv_ent].se_timeout_id)
3936 			ioc->ioc_serv[srv_ent].se_timeout_id = 0;
3937 		break;
3938 
3939 	case IBDM_REQ_TYPE_IOU_DIAGCODE:
3940 		new_gl_state = IBDM_GID_PROBING_COMPLETE;
3941 		iou = gid_info->gl_iou;
3942 		iou->iou_dc_valid = B_FALSE;
3943 		if (gid_info->gl_timeout_id)
3944 			gid_info->gl_timeout_id = 0;
3945 		break;
3946 
3947 	case IBDM_REQ_TYPE_IOC_DIAGCODE:
3948 		new_gl_state = IBDM_GID_PROBING_COMPLETE;
3949 		iou = gid_info->gl_iou;
3950 		ioc = &iou->iou_ioc_info[cb_args->cb_ioc_num];
3951 		ioc->ioc_dc_valid = B_FALSE;
3952 		if (ioc->ioc_dc_timeout_id)
3953 			ioc->ioc_dc_timeout_id = 0;
3954 		break;
3955 
3956 	default: /* ERROR State */
3957 		new_gl_state = IBDM_GID_PROBING_FAILED;
3958 		if (gid_info->gl_timeout_id)
3959 			gid_info->gl_timeout_id = 0;
3960 		IBTF_DPRINTF_L2("ibdm",
3961 		    "\tpkt_timeout_hdlr: wrong request type.");
3962 		break;
3963 	}
3964 
3965 	--gid_info->gl_pending_cmds; /* decrease the counter */
3966 
3967 	if (gid_info->gl_pending_cmds == 0) {
3968 		gid_info->gl_state = new_gl_state;
3969 		mutex_exit(&gid_info->gl_mutex);
3970 		/*
3971 		 * Delete this gid_info if the gid probe fails.
3972 		 */
3973 		if (new_gl_state == IBDM_GID_PROBING_FAILED) {
3974 			ibdm_delete_glhca_list(gid_info);
3975 		}
3976 		ibdm_notify_newgid_iocs(gid_info);
3977 		mutex_enter(&ibdm.ibdm_mutex);
3978 		if (--ibdm.ibdm_ngid_probes_in_progress == 0) {
3979 			IBTF_DPRINTF_L4("ibdm", "\tpkt_timeout_hdlr: Wakeup");
3980 			ibdm.ibdm_busy &= ~IBDM_PROBE_IN_PROGRESS;
3981 			cv_broadcast(&ibdm.ibdm_probe_cv);
3982 		}
3983 		mutex_exit(&ibdm.ibdm_mutex);
3984 	} else {
3985 		/*
3986 		 * Reset gl_pending_cmd if the extra timeout happens since
3987 		 * gl_pending_cmd becomes negative as a result.
3988 		 */
3989 		if (gid_info->gl_pending_cmds < 0) {
3990 			gid_info->gl_pending_cmds = 0;
3991 			IBTF_DPRINTF_L2("ibdm",
3992 			    "\tpkt_timeout_hdlr: extra timeout request."
3993 			    " reset gl_pending_cmds");
3994 		}
3995 		mutex_exit(&gid_info->gl_mutex);
3996 		/*
3997 		 * Delete this gid_info if the gid probe fails.
3998 		 */
3999 		if (new_gl_state == IBDM_GID_PROBING_FAILED) {
4000 			ibdm_delete_glhca_list(gid_info);
4001 		}
4002 	}
4003 }
4004 
4005 
4006 /*
4007  * ibdm_retry_command()
4008  *	Retries the failed command.
4009  *	Returns IBDM_FAILURE/IBDM_SUCCESS
4010  */
4011 static int
4012 ibdm_retry_command(ibdm_timeout_cb_args_t *cb_args)
4013 {
4014 	int			ret;
4015 	ibmf_msg_t		*msg;
4016 	ib_mad_hdr_t		*hdr;
4017 	ibdm_dp_gidinfo_t	*gid_info = cb_args->cb_gid_info;
4018 	timeout_id_t		*timeout_id;
4019 	ibdm_ioc_info_t		*ioc;
4020 	int			ioc_no;
4021 	ASSERT(MUTEX_HELD(&gid_info->gl_mutex));
4022 
4023 	IBTF_DPRINTF_L2("ibdm", "\tretry_command: gid_info: %p "
4024 	    "rtype 0x%x iocidx 0x%x srvidx %d", cb_args->cb_gid_info,
4025 	    cb_args->cb_req_type, cb_args->cb_ioc_num,
4026 	    cb_args->cb_srvents_start);
4027 
4028 	ret = ibmf_alloc_msg(gid_info->gl_ibmf_hdl, IBMF_ALLOC_NOSLEEP, &msg);
4029 
4030 
4031 	/*
4032 	 * Reset the gid if alloc_msg failed with BAD_HANDLE
4033 	 * ibdm_reset_gidinfo reinits the gid_info
4034 	 */
4035 	if (ret == IBMF_BAD_HANDLE) {
4036 		IBTF_DPRINTF_L3(ibdm_string, "\tretry_command: gid %p hdl bad",
4037 		    gid_info);
4038 
4039 		mutex_exit(&gid_info->gl_mutex);
4040 		ibdm_reset_gidinfo(gid_info);
4041 		mutex_enter(&gid_info->gl_mutex);
4042 
4043 		/* Retry alloc */
4044 		ret = ibmf_alloc_msg(gid_info->gl_ibmf_hdl, IBMF_ALLOC_NOSLEEP,
4045 		    &msg);
4046 	}
4047 
4048 	if (ret != IBDM_SUCCESS) {
4049 		IBTF_DPRINTF_L2("ibdm", "\tretry_command: alloc failed: %p "
4050 		    "rtype 0x%x iocidx 0x%x srvidx %d", cb_args->cb_gid_info,
4051 		    cb_args->cb_req_type, cb_args->cb_ioc_num,
4052 		    cb_args->cb_srvents_start);
4053 		return (IBDM_FAILURE);
4054 	}
4055 
4056 	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*msg))
4057 	ibdm_alloc_send_buffers(msg);
4058 	_NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*msg))
4059 
4060 	ibdm_bump_transactionID(gid_info);
4061 
4062 	msg->im_local_addr.ia_local_lid	= gid_info->gl_slid;
4063 	msg->im_local_addr.ia_remote_lid	= gid_info->gl_dlid;
4064 	if (gid_info->gl_redirected == B_TRUE) {
4065 		if (gid_info->gl_redirect_dlid != 0) {
4066 			msg->im_local_addr.ia_remote_lid =
4067 			    gid_info->gl_redirect_dlid;
4068 		}
4069 		msg->im_local_addr.ia_remote_qno = gid_info->gl_redirect_QP;
4070 		msg->im_local_addr.ia_p_key = gid_info->gl_redirect_pkey;
4071 		msg->im_local_addr.ia_q_key = gid_info->gl_redirect_qkey;
4072 		msg->im_local_addr.ia_service_level = gid_info->gl_redirectSL;
4073 	} else {
4074 		msg->im_local_addr.ia_remote_qno = 1;
4075 		msg->im_local_addr.ia_p_key = gid_info->gl_p_key;
4076 		msg->im_local_addr.ia_q_key = IB_GSI_QKEY;
4077 		msg->im_local_addr.ia_service_level = gid_info->gl_SL;
4078 	}
4079 	hdr = IBDM_OUT_IBMFMSG_MADHDR(msg);
4080 	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*hdr))
4081 	hdr->BaseVersion	= MAD_CLASS_BASE_VERS_1;
4082 	hdr->MgmtClass		= MAD_MGMT_CLASS_DEV_MGT;
4083 	hdr->ClassVersion	= IB_DM_CLASS_VERSION_1;
4084 	hdr->R_Method		= IB_DM_DEVMGT_METHOD_GET;
4085 	hdr->Status		= 0;
4086 	hdr->TransactionID	= h2b64(gid_info->gl_transactionID);
4087 
4088 	switch (cb_args->cb_req_type) {
4089 	case IBDM_REQ_TYPE_CLASSPORTINFO:
4090 		hdr->AttributeID = h2b16(IB_DM_ATTR_CLASSPORTINFO);
4091 		hdr->AttributeModifier = 0;
4092 		timeout_id = &gid_info->gl_timeout_id;
4093 		break;
4094 	case IBDM_REQ_TYPE_IOUINFO:
4095 		hdr->AttributeID = h2b16(IB_DM_ATTR_IO_UNITINFO);
4096 		hdr->AttributeModifier = 0;
4097 		timeout_id = &gid_info->gl_timeout_id;
4098 		break;
4099 	case IBDM_REQ_TYPE_IOCINFO:
4100 		hdr->AttributeID = h2b16(IB_DM_ATTR_IOC_CTRL_PROFILE);
4101 		hdr->AttributeModifier = h2b32(cb_args->cb_ioc_num + 1);
4102 		ioc = IBDM_GIDINFO2IOCINFO(gid_info, cb_args->cb_ioc_num);
4103 		timeout_id = &ioc->ioc_timeout_id;
4104 		break;
4105 	case IBDM_REQ_TYPE_SRVENTS:
4106 		hdr->AttributeID = h2b16(IB_DM_ATTR_SERVICE_ENTRIES);
4107 		ibdm_fill_srv_attr_mod(hdr, cb_args);
4108 		ioc = IBDM_GIDINFO2IOCINFO(gid_info, cb_args->cb_ioc_num);
4109 		timeout_id =
4110 		    &ioc->ioc_serv[cb_args->cb_srvents_start].se_timeout_id;
4111 		break;
4112 	case IBDM_REQ_TYPE_IOU_DIAGCODE:
4113 		hdr->AttributeID = h2b16(IB_DM_ATTR_DIAG_CODE);
4114 		hdr->AttributeModifier = 0;
4115 		timeout_id = &gid_info->gl_timeout_id;
4116 		break;
4117 	case IBDM_REQ_TYPE_IOC_DIAGCODE:
4118 		hdr->AttributeID = h2b16(IB_DM_ATTR_DIAG_CODE);
4119 		hdr->AttributeModifier = h2b32(cb_args->cb_ioc_num + 1);
4120 		ioc_no = cb_args->cb_ioc_num;
4121 		ioc = &gid_info->gl_iou->iou_ioc_info[ioc_no];
4122 		timeout_id = &ioc->ioc_dc_timeout_id;
4123 		break;
4124 	}
4125 	_NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*hdr))
4126 
4127 	*timeout_id = timeout(ibdm_pkt_timeout_hdlr,
4128 	    cb_args, IBDM_TIMEOUT_VALUE(ibdm_dft_timeout));
4129 
4130 	mutex_exit(&gid_info->gl_mutex);
4131 
4132 	IBTF_DPRINTF_L5("ibdm", "\tretry_command: %p,%x,%d,%d:"
4133 	    "timeout %x", cb_args->cb_req_type, cb_args->cb_ioc_num,
4134 	    cb_args->cb_srvents_start, *timeout_id);
4135 
4136 	if (ibmf_msg_transport(gid_info->gl_ibmf_hdl,
4137 	    gid_info->gl_qp_hdl, msg, NULL, ibdm_ibmf_send_cb,
4138 	    cb_args, 0) != IBMF_SUCCESS) {
4139 		IBTF_DPRINTF_L2("ibdm", "\tretry_command: send failed: %p "
4140 		    "rtype 0x%x iocidx 0x%x srvidx %d", cb_args->cb_gid_info,
4141 		    cb_args->cb_req_type, cb_args->cb_ioc_num,
4142 		    cb_args->cb_srvents_start);
4143 		ibdm_ibmf_send_cb(gid_info->gl_ibmf_hdl, msg, cb_args);
4144 	}
4145 	mutex_enter(&gid_info->gl_mutex);
4146 	return (IBDM_SUCCESS);
4147 }
4148 
4149 
4150 /*
4151  * ibdm_update_ioc_port_gidlist()
4152  */
4153 static void
4154 ibdm_update_ioc_port_gidlist(ibdm_ioc_info_t *dest,
4155     ibdm_dp_gidinfo_t *gid_info)
4156 {
4157 	int		ii, ngid_ents;
4158 	ibdm_gid_t	*tmp;
4159 	ibdm_hca_list_t	*gid_hca_head, *temp;
4160 	ibdm_hca_list_t	*ioc_head = NULL;
4161 	ASSERT(MUTEX_HELD(&gid_info->gl_mutex));
4162 
4163 	IBTF_DPRINTF_L5("ibdm", "\tupdate_ioc_port_gidlist: Enter");
4164 
4165 	ngid_ents = gid_info->gl_ngids;
4166 	dest->ioc_nportgids = ngid_ents;
4167 	dest->ioc_gid_list = kmem_zalloc(sizeof (ibdm_gid_t) *
4168 	    ngid_ents, KM_SLEEP);
4169 	tmp = gid_info->gl_gid;
4170 	for (ii = 0; (ii < ngid_ents) && (tmp); ii++) {
4171 		dest->ioc_gid_list[ii].gid_dgid_hi = tmp->gid_dgid_hi;
4172 		dest->ioc_gid_list[ii].gid_dgid_lo = tmp->gid_dgid_lo;
4173 		tmp = tmp->gid_next;
4174 	}
4175 
4176 	gid_hca_head = gid_info->gl_hca_list;
4177 	while (gid_hca_head) {
4178 		temp = ibdm_dup_hca_attr(gid_hca_head);
4179 		temp->hl_next = ioc_head;
4180 		ioc_head = temp;
4181 		gid_hca_head = gid_hca_head->hl_next;
4182 	}
4183 	dest->ioc_hca_list = ioc_head;
4184 }
4185 
4186 
4187 /*
4188  * ibdm_alloc_send_buffers()
4189  *	Allocates memory for the IBMF send buffer to send and/or receive
4190  *	the Device Management MAD packet.
4191  */
4192 static void
4193 ibdm_alloc_send_buffers(ibmf_msg_t *msgp)
4194 {
4195 	msgp->im_msgbufs_send.im_bufs_mad_hdr =
4196 	    kmem_zalloc(IBDM_MAD_SIZE, KM_SLEEP);
4197 
4198 	msgp->im_msgbufs_send.im_bufs_cl_hdr = (uchar_t *)
4199 	    msgp->im_msgbufs_send.im_bufs_mad_hdr + sizeof (ib_mad_hdr_t);
4200 	msgp->im_msgbufs_send.im_bufs_cl_hdr_len = IBDM_DM_MAD_HDR_SZ;
4201 
4202 	msgp->im_msgbufs_send.im_bufs_cl_data =
4203 	    ((char *)msgp->im_msgbufs_send.im_bufs_cl_hdr + IBDM_DM_MAD_HDR_SZ);
4204 	msgp->im_msgbufs_send.im_bufs_cl_data_len =
4205 	    IBDM_MAD_SIZE - sizeof (ib_mad_hdr_t) - IBDM_DM_MAD_HDR_SZ;
4206 }
4207 
4208 
4209 /*
4210  * ibdm_alloc_send_buffers()
4211  *	De-allocates memory for the IBMF send buffer
4212  */
4213 static void
4214 ibdm_free_send_buffers(ibmf_msg_t *msgp)
4215 {
4216 	if (msgp->im_msgbufs_send.im_bufs_mad_hdr != NULL)
4217 		kmem_free(msgp->im_msgbufs_send.im_bufs_mad_hdr, IBDM_MAD_SIZE);
4218 }
4219 
4220 /*
4221  * ibdm_probe_ioc()
4222  *  	1. Gets the node records for the port GUID. This detects all the port
4223  *  		to the IOU.
4224  *	2. Selectively probes all the IOC, given it's node GUID
4225  *	3. In case of reprobe, only the IOC to be reprobed is send the IOC
4226  *		Controller Profile asynchronously
4227  */
4228 /*ARGSUSED*/
4229 static void
4230 ibdm_probe_ioc(ib_guid_t nodeguid, ib_guid_t ioc_guid, int reprobe_flag)
4231 {
4232 	int			ii, nrecords;
4233 	size_t			nr_len = 0, pi_len = 0;
4234 	ib_gid_t		sgid, dgid;
4235 	ibdm_hca_list_t		*hca_list = NULL;
4236 	sa_node_record_t	*nr, *tmp;
4237 	ibdm_port_attr_t	*port = NULL;
4238 	ibdm_dp_gidinfo_t	*reprobe_gid, *new_gid, *node_gid;
4239 	ibdm_dp_gidinfo_t	*temp_gidinfo;
4240 	ibdm_gid_t		*temp_gid;
4241 	sa_portinfo_record_t	*pi;
4242 
4243 	IBTF_DPRINTF_L4("ibdm", "\tprobe_ioc(%llx, %llx, %x): Begin",
4244 	    nodeguid, ioc_guid, reprobe_flag);
4245 
4246 	/* Rescan the GID list for any removed GIDs for reprobe */
4247 	if (reprobe_flag)
4248 		ibdm_rescan_gidlist(&ioc_guid);
4249 
4250 	mutex_enter(&ibdm.ibdm_hl_mutex);
4251 	for (ibdm_get_next_port(&hca_list, &port, 1); port;
4252 	    ibdm_get_next_port(&hca_list, &port, 1)) {
4253 		reprobe_gid = new_gid = node_gid = NULL;
4254 
4255 		nr = ibdm_get_node_records(port->pa_sa_hdl, &nr_len, nodeguid);
4256 		if (nr == NULL) {
4257 			IBTF_DPRINTF_L4("ibdm", "\tprobe_ioc: no records");
4258 			continue;
4259 		}
4260 		nrecords = (nr_len / sizeof (sa_node_record_t));
4261 		for (tmp = nr, ii = 0;  (ii < nrecords); ii++, tmp++) {
4262 			if ((pi = ibdm_get_portinfo(
4263 			    port->pa_sa_hdl, &pi_len, tmp->LID)) ==  NULL) {
4264 				IBTF_DPRINTF_L4("ibdm",
4265 				    "\tibdm_get_portinfo: no portinfo recs");
4266 				continue;
4267 			}
4268 
4269 			/*
4270 			 * If Device Management is not supported on
4271 			 * this port, skip the rest.
4272 			 */
4273 			if (!(pi->PortInfo.CapabilityMask &
4274 			    SM_CAP_MASK_IS_DM_SUPPD)) {
4275 				kmem_free(pi, pi_len);
4276 				continue;
4277 			}
4278 
4279 			/*
4280 			 * For reprobes: Check if GID, already in
4281 			 * the list. If so, set the state to SKIPPED
4282 			 */
4283 			if (((temp_gidinfo = ibdm_find_gid(nodeguid,
4284 			    tmp->NodeInfo.PortGUID)) != NULL) &&
4285 			    temp_gidinfo->gl_state ==
4286 			    IBDM_GID_PROBING_COMPLETE) {
4287 				ASSERT(reprobe_gid == NULL);
4288 				ibdm_addto_glhcalist(temp_gidinfo,
4289 				    hca_list);
4290 				reprobe_gid = temp_gidinfo;
4291 				kmem_free(pi, pi_len);
4292 				continue;
4293 			} else if (temp_gidinfo != NULL) {
4294 				kmem_free(pi, pi_len);
4295 				ibdm_addto_glhcalist(temp_gidinfo,
4296 				    hca_list);
4297 				continue;
4298 			}
4299 
4300 			IBTF_DPRINTF_L4("ibdm", "\tprobe_ioc : "
4301 			    "create_gid : prefix %llx, guid %llx\n",
4302 			    pi->PortInfo.GidPrefix,
4303 			    tmp->NodeInfo.PortGUID);
4304 
4305 			sgid.gid_prefix = port->pa_sn_prefix;
4306 			sgid.gid_guid = port->pa_port_guid;
4307 			dgid.gid_prefix = pi->PortInfo.GidPrefix;
4308 			dgid.gid_guid = tmp->NodeInfo.PortGUID;
4309 			new_gid = ibdm_create_gid_info(port, sgid,
4310 			    dgid);
4311 			if (new_gid == NULL) {
4312 				IBTF_DPRINTF_L2("ibdm", "\tprobe_ioc: "
4313 				    "create_gid_info failed\n");
4314 				kmem_free(pi, pi_len);
4315 				continue;
4316 			}
4317 			if (node_gid == NULL) {
4318 				node_gid = new_gid;
4319 				ibdm_add_to_gl_gid(node_gid, node_gid);
4320 			} else {
4321 				IBTF_DPRINTF_L4("ibdm",
4322 				    "\tprobe_ioc: new gid");
4323 				temp_gid = kmem_zalloc(
4324 				    sizeof (ibdm_gid_t), KM_SLEEP);
4325 				temp_gid->gid_dgid_hi =
4326 				    new_gid->gl_dgid_hi;
4327 				temp_gid->gid_dgid_lo =
4328 				    new_gid->gl_dgid_lo;
4329 				temp_gid->gid_next = node_gid->gl_gid;
4330 				node_gid->gl_gid = temp_gid;
4331 				node_gid->gl_ngids++;
4332 			}
4333 			new_gid->gl_is_dm_capable = B_TRUE;
4334 			new_gid->gl_nodeguid = nodeguid;
4335 			new_gid->gl_portguid = dgid.gid_guid;
4336 			ibdm_addto_glhcalist(new_gid, hca_list);
4337 
4338 			/*
4339 			 * Set the state to skipped as all these
4340 			 * gids point to the same node.
4341 			 * We (re)probe only one GID below and reset
4342 			 * state appropriately
4343 			 */
4344 			new_gid->gl_state = IBDM_GID_PROBING_SKIPPED;
4345 			new_gid->gl_devid = (*tmp).NodeInfo.DeviceID;
4346 			kmem_free(pi, pi_len);
4347 		}
4348 		kmem_free(nr, nr_len);
4349 
4350 		IBTF_DPRINTF_L4("ibdm", "\tprobe_ioc : reprobe_flag %d "
4351 		    "reprobe_gid %p new_gid %p node_gid %p",
4352 		    reprobe_flag, reprobe_gid, new_gid, node_gid);
4353 
4354 		if (reprobe_flag != 0 && reprobe_gid != NULL) {
4355 			int	niocs, jj;
4356 			ibdm_ioc_info_t *tmp_ioc;
4357 			int ioc_matched = 0;
4358 
4359 			mutex_exit(&ibdm.ibdm_hl_mutex);
4360 			mutex_enter(&reprobe_gid->gl_mutex);
4361 			reprobe_gid->gl_state = IBDM_GET_IOC_DETAILS;
4362 			niocs =
4363 			    reprobe_gid->gl_iou->iou_info.iou_num_ctrl_slots;
4364 			reprobe_gid->gl_pending_cmds++;
4365 			mutex_exit(&reprobe_gid->gl_mutex);
4366 
4367 			for (jj = 0; jj < niocs; jj++) {
4368 				tmp_ioc =
4369 				    IBDM_GIDINFO2IOCINFO(reprobe_gid, jj);
4370 				if (tmp_ioc->ioc_profile.ioc_guid != ioc_guid)
4371 					continue;
4372 
4373 				ioc_matched = 1;
4374 
4375 				/*
4376 				 * Explicitly set gl_reprobe_flag to 0 so that
4377 				 * IBnex is not notified on completion
4378 				 */
4379 				mutex_enter(&reprobe_gid->gl_mutex);
4380 				reprobe_gid->gl_reprobe_flag = 0;
4381 				mutex_exit(&reprobe_gid->gl_mutex);
4382 
4383 				mutex_enter(&ibdm.ibdm_mutex);
4384 				ibdm.ibdm_ngid_probes_in_progress++;
4385 				mutex_exit(&ibdm.ibdm_mutex);
4386 				if (ibdm_send_ioc_profile(reprobe_gid, jj) !=
4387 				    IBDM_SUCCESS) {
4388 					IBTF_DPRINTF_L4("ibdm",
4389 					    "\tprobe_ioc: "
4390 					    "send_ioc_profile failed "
4391 					    "for ioc %d", jj);
4392 					ibdm_gid_decr_pending(reprobe_gid);
4393 					break;
4394 				}
4395 				mutex_enter(&ibdm.ibdm_mutex);
4396 				ibdm_wait_probe_completion();
4397 				mutex_exit(&ibdm.ibdm_mutex);
4398 				break;
4399 			}
4400 			if (ioc_matched == 0)
4401 				ibdm_gid_decr_pending(reprobe_gid);
4402 			else {
4403 				mutex_enter(&ibdm.ibdm_hl_mutex);
4404 				break;
4405 			}
4406 		} else if (new_gid != NULL) {
4407 			mutex_exit(&ibdm.ibdm_hl_mutex);
4408 			node_gid = node_gid ? node_gid : new_gid;
4409 
4410 			/*
4411 			 * New or reinserted GID : Enable notification
4412 			 * to IBnex
4413 			 */
4414 			mutex_enter(&node_gid->gl_mutex);
4415 			node_gid->gl_reprobe_flag = 1;
4416 			mutex_exit(&node_gid->gl_mutex);
4417 
4418 			ibdm_probe_gid(node_gid);
4419 
4420 			mutex_enter(&ibdm.ibdm_hl_mutex);
4421 		}
4422 	}
4423 	mutex_exit(&ibdm.ibdm_hl_mutex);
4424 	IBTF_DPRINTF_L4("ibdm", "\tprobe_ioc : End\n");
4425 }
4426 
4427 
4428 /*
4429  * ibdm_probe_gid()
4430  *	Selectively probes the GID
4431  */
4432 static void
4433 ibdm_probe_gid(ibdm_dp_gidinfo_t *gid_info)
4434 {
4435 	IBTF_DPRINTF_L4("ibdm", "\tprobe_gid:");
4436 
4437 	/*
4438 	 * A Cisco FC GW needs the special handling to get IOUnitInfo.
4439 	 */
4440 	mutex_enter(&gid_info->gl_mutex);
4441 	if (ibdm_is_cisco_switch(gid_info)) {
4442 		gid_info->gl_pending_cmds++;
4443 		gid_info->gl_state = IBDM_SET_CLASSPORTINFO;
4444 		mutex_exit(&gid_info->gl_mutex);
4445 
4446 		if (ibdm_set_classportinfo(gid_info) != IBDM_SUCCESS) {
4447 
4448 			mutex_enter(&gid_info->gl_mutex);
4449 			gid_info->gl_state = IBDM_GID_PROBING_FAILED;
4450 			--gid_info->gl_pending_cmds;
4451 			mutex_exit(&gid_info->gl_mutex);
4452 
4453 			/* free the hca_list on this gid_info */
4454 			ibdm_delete_glhca_list(gid_info);
4455 			gid_info = gid_info->gl_next;
4456 			return;
4457 		}
4458 
4459 		mutex_enter(&gid_info->gl_mutex);
4460 		ibdm_wait_cisco_probe_completion(gid_info);
4461 
4462 		IBTF_DPRINTF_L4("ibdm",
4463 		    "\tprobe_gid: CISCO Wakeup signal received");
4464 	}
4465 
4466 	/* move on to the 'GET_CLASSPORTINFO' stage */
4467 	gid_info->gl_pending_cmds++;
4468 	gid_info->gl_state = IBDM_GET_CLASSPORTINFO;
4469 	mutex_exit(&gid_info->gl_mutex);
4470 
4471 	if (ibdm_send_classportinfo(gid_info) != IBDM_SUCCESS) {
4472 
4473 		mutex_enter(&gid_info->gl_mutex);
4474 		gid_info->gl_state = IBDM_GID_PROBING_FAILED;
4475 		--gid_info->gl_pending_cmds;
4476 		mutex_exit(&gid_info->gl_mutex);
4477 
4478 		/* free the hca_list on this gid_info */
4479 		ibdm_delete_glhca_list(gid_info);
4480 		gid_info = gid_info->gl_next;
4481 		return;
4482 	}
4483 
4484 	mutex_enter(&ibdm.ibdm_mutex);
4485 	ibdm.ibdm_ngid_probes_in_progress++;
4486 	gid_info = gid_info->gl_next;
4487 	ibdm_wait_probe_completion();
4488 	mutex_exit(&ibdm.ibdm_mutex);
4489 
4490 	IBTF_DPRINTF_L4("ibdm", "\tprobe_gid: Wakeup signal received");
4491 }
4492 
4493 
4494 /*
4495  * ibdm_create_gid_info()
4496  *	Allocates a gid_info structure and initializes
4497  *	Returns pointer to the structure on success
4498  *	and NULL on failure
4499  */
4500 static ibdm_dp_gidinfo_t *
4501 ibdm_create_gid_info(ibdm_port_attr_t *port, ib_gid_t sgid, ib_gid_t dgid)
4502 {
4503 	uint8_t			ii, npaths;
4504 	sa_path_record_t	*path;
4505 	size_t			len;
4506 	ibdm_pkey_tbl_t		*pkey_tbl;
4507 	ibdm_dp_gidinfo_t	*gid_info = NULL;
4508 	int			ret;
4509 
4510 	IBTF_DPRINTF_L4("ibdm", "\tcreate_gid_info: Begin");
4511 	npaths = 1;
4512 
4513 	/* query for reversible paths */
4514 	if (port->pa_sa_hdl)
4515 		ret = ibmf_saa_gid_to_pathrecords(port->pa_sa_hdl,
4516 		    sgid, dgid, IBMF_SAA_PKEY_WC, 0, B_TRUE, &npaths, 0,
4517 		    &len, &path);
4518 	else
4519 		return (NULL);
4520 
4521 	if (ret == IBMF_SUCCESS && path) {
4522 		ibdm_dump_path_info(path);
4523 
4524 		gid_info = kmem_zalloc(
4525 		    sizeof (ibdm_dp_gidinfo_t), KM_SLEEP);
4526 		mutex_init(&gid_info->gl_mutex, NULL, MUTEX_DEFAULT, NULL);
4527 		cv_init(&gid_info->gl_probe_cv, NULL, CV_DRIVER, NULL);
4528 		gid_info->gl_dgid_hi		= path->DGID.gid_prefix;
4529 		gid_info->gl_dgid_lo		= path->DGID.gid_guid;
4530 		gid_info->gl_sgid_hi		= path->SGID.gid_prefix;
4531 		gid_info->gl_sgid_lo		= path->SGID.gid_guid;
4532 		gid_info->gl_p_key		= path->P_Key;
4533 		gid_info->gl_sa_hdl		= port->pa_sa_hdl;
4534 		gid_info->gl_ibmf_hdl		= port->pa_ibmf_hdl;
4535 		gid_info->gl_slid		= path->SLID;
4536 		gid_info->gl_dlid		= path->DLID;
4537 		gid_info->gl_transactionID	= (++ibdm.ibdm_transactionID)
4538 		    << IBDM_GID_TRANSACTIONID_SHIFT;
4539 		gid_info->gl_min_transactionID  = gid_info->gl_transactionID;
4540 		gid_info->gl_max_transactionID  = (ibdm.ibdm_transactionID +1)
4541 		    << IBDM_GID_TRANSACTIONID_SHIFT;
4542 		gid_info->gl_SL			= path->SL;
4543 
4544 		gid_info->gl_qp_hdl = IBMF_QP_HANDLE_DEFAULT;
4545 		for (ii = 0; ii < port->pa_npkeys; ii++) {
4546 			if (port->pa_pkey_tbl == NULL)
4547 				break;
4548 
4549 			pkey_tbl = &port->pa_pkey_tbl[ii];
4550 			if ((gid_info->gl_p_key == pkey_tbl->pt_pkey) &&
4551 			    (pkey_tbl->pt_qp_hdl != NULL)) {
4552 				gid_info->gl_qp_hdl = pkey_tbl->pt_qp_hdl;
4553 				break;
4554 			}
4555 		}
4556 		kmem_free(path, len);
4557 
4558 		/*
4559 		 * QP handle for GID not initialized. No matching Pkey
4560 		 * was found!! ibdm should *not* hit this case. Flag an
4561 		 * error and drop the GID if ibdm does encounter this.
4562 		 */
4563 		if (gid_info->gl_qp_hdl == NULL) {
4564 			IBTF_DPRINTF_L2(ibdm_string,
4565 			    "\tcreate_gid_info: No matching Pkey");
4566 			ibdm_delete_gidinfo(gid_info);
4567 			return (NULL);
4568 		}
4569 
4570 		ibdm.ibdm_ngids++;
4571 		if (ibdm.ibdm_dp_gidlist_head == NULL) {
4572 			ibdm.ibdm_dp_gidlist_head = gid_info;
4573 			ibdm.ibdm_dp_gidlist_tail = gid_info;
4574 		} else {
4575 			ibdm.ibdm_dp_gidlist_tail->gl_next = gid_info;
4576 			gid_info->gl_prev = ibdm.ibdm_dp_gidlist_tail;
4577 			ibdm.ibdm_dp_gidlist_tail = gid_info;
4578 		}
4579 	}
4580 
4581 	return (gid_info);
4582 }
4583 
4584 
4585 /*
4586  * ibdm_get_node_records
4587  *	Sends a SA query to get the NODE record
4588  *	Returns pointer to the sa_node_record_t on success
4589  *	and NULL on failure
4590  */
4591 static sa_node_record_t *
4592 ibdm_get_node_records(ibmf_saa_handle_t sa_hdl, size_t *length, ib_guid_t guid)
4593 {
4594 	sa_node_record_t	req, *resp = NULL;
4595 	ibmf_saa_access_args_t	args;
4596 	int			ret;
4597 
4598 	IBTF_DPRINTF_L4("ibdm", "\tget_node_records: Begin");
4599 
4600 	bzero(&req, sizeof (sa_node_record_t));
4601 	req.NodeInfo.NodeGUID = guid;
4602 
4603 	args.sq_attr_id		= SA_NODERECORD_ATTRID;
4604 	args.sq_access_type 	= IBMF_SAA_RETRIEVE;
4605 	args.sq_component_mask 	= SA_NODEINFO_COMPMASK_NODEGUID;
4606 	args.sq_template	= &req;
4607 	args.sq_callback	= NULL;
4608 	args.sq_callback_arg 	= NULL;
4609 
4610 	ret = ibmf_sa_access(sa_hdl, &args, 0, length, (void **) &resp);
4611 	if (ret != IBMF_SUCCESS) {
4612 		IBTF_DPRINTF_L2("ibdm", "\tget_node_records:"
4613 		    " SA Retrieve Failed: %d", ret);
4614 		return (NULL);
4615 	}
4616 	if ((resp == NULL) || (*length == 0)) {
4617 		IBTF_DPRINTF_L2("ibdm", "\tget_node_records: No records");
4618 		return (NULL);
4619 	}
4620 
4621 	IBTF_DPRINTF_L4("ibdm", "\tget_node_records: NodeGuid %llx "
4622 	    "PortGUID %llx", resp->NodeInfo.NodeGUID, resp->NodeInfo.PortGUID);
4623 
4624 	return (resp);
4625 }
4626 
4627 
4628 /*
4629  * ibdm_get_portinfo()
4630  *	Sends a SA query to get the PortInfo record
4631  *	Returns pointer to the sa_portinfo_record_t on success
4632  *	and NULL on failure
4633  */
4634 static sa_portinfo_record_t *
4635 ibdm_get_portinfo(ibmf_saa_handle_t sa_hdl, size_t *length, ib_lid_t lid)
4636 {
4637 	sa_portinfo_record_t	req, *resp = NULL;
4638 	ibmf_saa_access_args_t	args;
4639 	int			ret;
4640 
4641 	IBTF_DPRINTF_L4("ibdm", "\tget_portinfo: Begin");
4642 
4643 	bzero(&req, sizeof (sa_portinfo_record_t));
4644 	req.EndportLID	= lid;
4645 
4646 	args.sq_attr_id		= SA_PORTINFORECORD_ATTRID;
4647 	args.sq_access_type	= IBMF_SAA_RETRIEVE;
4648 	args.sq_component_mask	= SA_PORTINFO_COMPMASK_PORTLID;
4649 	args.sq_template	= &req;
4650 	args.sq_callback	= NULL;
4651 	args.sq_callback_arg	= NULL;
4652 
4653 	ret = ibmf_sa_access(sa_hdl, &args, 0, length, (void **) &resp);
4654 	if (ret != IBMF_SUCCESS) {
4655 		IBTF_DPRINTF_L2("ibdm", "\tget_portinfo:"
4656 		    " SA Retrieve Failed: 0x%X", ret);
4657 		return (NULL);
4658 	}
4659 	if ((*length == 0) || (resp == NULL))
4660 		return (NULL);
4661 
4662 	IBTF_DPRINTF_L4("ibdm", "\tget_portinfo: GidPrefix %llx Cap 0x%x",
4663 	    resp->PortInfo.GidPrefix, resp->PortInfo.CapabilityMask);
4664 	return (resp);
4665 }
4666 
4667 
4668 /*
4669  * ibdm_ibnex_register_callback
4670  *	IB nexus callback routine for HCA attach and detach notification
4671  */
4672 void
4673 ibdm_ibnex_register_callback(ibdm_callback_t ibnex_dm_callback)
4674 {
4675 	IBTF_DPRINTF_L4("ibdm", "\tibnex_register_callbacks");
4676 	mutex_enter(&ibdm.ibdm_ibnex_mutex);
4677 	ibdm.ibdm_ibnex_callback = ibnex_dm_callback;
4678 	mutex_exit(&ibdm.ibdm_ibnex_mutex);
4679 }
4680 
4681 
4682 /*
4683  * ibdm_ibnex_unregister_callbacks
4684  */
4685 void
4686 ibdm_ibnex_unregister_callback()
4687 {
4688 	IBTF_DPRINTF_L4("ibdm", "\tibnex_unregister_callbacks");
4689 	mutex_enter(&ibdm.ibdm_ibnex_mutex);
4690 	ibdm.ibdm_ibnex_callback = NULL;
4691 	mutex_exit(&ibdm.ibdm_ibnex_mutex);
4692 }
4693 
4694 /*
4695  * ibdm_get_waittime()
4696  *	Calculates the wait time based on the last HCA attach time
4697  */
4698 static clock_t
4699 ibdm_get_waittime(ib_guid_t hca_guid, int dft_wait_sec)
4700 {
4701 	const hrtime_t	dft_wait = dft_wait_sec * NANOSEC;
4702 	hrtime_t	temp, wait_time = 0;
4703 	clock_t		usecs;
4704 	int		i;
4705 	ibdm_hca_list_t	*hca;
4706 
4707 	IBTF_DPRINTF_L4("ibdm", "\tget_waittime hcaguid:%llx"
4708 	    "\tport settling time %d", hca_guid, dft_wait);
4709 
4710 	ASSERT(mutex_owned(&ibdm.ibdm_hl_mutex));
4711 
4712 	hca = ibdm.ibdm_hca_list_head;
4713 
4714 	for (i = 0; i < ibdm.ibdm_hca_count; i++, hca = hca->hl_next) {
4715 		if (hca->hl_nports == hca->hl_nports_active)
4716 			continue;
4717 
4718 		if (hca_guid && (hca_guid != hca->hl_hca_guid))
4719 			continue;
4720 
4721 		temp = gethrtime() - hca->hl_attach_time;
4722 		temp = MAX(0, (dft_wait - temp));
4723 
4724 		if (hca_guid) {
4725 			wait_time = temp;
4726 			break;
4727 		}
4728 
4729 		wait_time = MAX(temp, wait_time);
4730 	}
4731 
4732 	/* convert to microseconds */
4733 	usecs = MIN(wait_time, dft_wait) / (NANOSEC / MICROSEC);
4734 
4735 	IBTF_DPRINTF_L2("ibdm", "\tget_waittime: wait_time = %ld usecs",
4736 	    (long)usecs);
4737 
4738 	return (drv_usectohz(usecs));
4739 }
4740 
4741 void
4742 ibdm_ibnex_port_settle_wait(ib_guid_t hca_guid, int dft_wait)
4743 {
4744 	clock_t wait_time;
4745 
4746 	mutex_enter(&ibdm.ibdm_hl_mutex);
4747 
4748 	while ((wait_time = ibdm_get_waittime(hca_guid, dft_wait)) > 0)
4749 		(void) cv_reltimedwait(&ibdm.ibdm_port_settle_cv,
4750 		    &ibdm.ibdm_hl_mutex, wait_time, TR_CLOCK_TICK);
4751 
4752 	mutex_exit(&ibdm.ibdm_hl_mutex);
4753 }
4754 
4755 
4756 /*
4757  * ibdm_ibnex_probe_hcaport
4758  *	Probes the presence of HCA port (with HCA dip and port number)
4759  *	Returns port attributes structure on SUCCESS
4760  */
4761 ibdm_port_attr_t *
4762 ibdm_ibnex_probe_hcaport(ib_guid_t hca_guid, uint8_t port_num)
4763 {
4764 	int			ii, jj;
4765 	ibdm_hca_list_t		*hca_list;
4766 	ibdm_port_attr_t	*port_attr;
4767 
4768 	IBTF_DPRINTF_L4("ibdm", "\tibnex_probe_hcaport:");
4769 
4770 	mutex_enter(&ibdm.ibdm_hl_mutex);
4771 	hca_list = ibdm.ibdm_hca_list_head;
4772 	for (ii = 0; ii < ibdm.ibdm_hca_count; ii++) {
4773 		if (hca_list->hl_hca_guid == hca_guid) {
4774 			for (jj = 0; jj < hca_list->hl_nports; jj++) {
4775 				if (hca_list->hl_port_attr[jj].pa_port_num ==
4776 				    port_num) {
4777 					break;
4778 				}
4779 			}
4780 			if (jj != hca_list->hl_nports)
4781 				break;
4782 		}
4783 		hca_list = hca_list->hl_next;
4784 	}
4785 	if (ii == ibdm.ibdm_hca_count) {
4786 		IBTF_DPRINTF_L2("ibdm", "\tibnex_probe_hcaport: not found");
4787 		mutex_exit(&ibdm.ibdm_hl_mutex);
4788 		return (NULL);
4789 	}
4790 	port_attr = (ibdm_port_attr_t *)kmem_zalloc(
4791 	    sizeof (ibdm_port_attr_t), KM_SLEEP);
4792 	bcopy((char *)&hca_list->hl_port_attr[jj],
4793 	    port_attr, sizeof (ibdm_port_attr_t));
4794 	ibdm_update_port_attr(port_attr);
4795 
4796 	mutex_exit(&ibdm.ibdm_hl_mutex);
4797 	return (port_attr);
4798 }
4799 
4800 
4801 /*
4802  * ibdm_ibnex_get_port_attrs
4803  *	Scan all HCAs for a matching port_guid.
4804  *	Returns "port attributes" structure on success.
4805  */
4806 ibdm_port_attr_t *
4807 ibdm_ibnex_get_port_attrs(ib_guid_t port_guid)
4808 {
4809 	int			ii, jj;
4810 	ibdm_hca_list_t		*hca_list;
4811 	ibdm_port_attr_t	*port_attr;
4812 
4813 	IBTF_DPRINTF_L4("ibdm", "\tibnex_get_port_attrs:");
4814 
4815 	mutex_enter(&ibdm.ibdm_hl_mutex);
4816 	hca_list = ibdm.ibdm_hca_list_head;
4817 
4818 	for (ii = 0; ii < ibdm.ibdm_hca_count; ii++) {
4819 		for (jj = 0; jj < hca_list->hl_nports; jj++) {
4820 			if (hca_list->hl_port_attr[jj].pa_port_guid ==
4821 			    port_guid) {
4822 				break;
4823 			}
4824 		}
4825 		if (jj != hca_list->hl_nports)
4826 			break;
4827 		hca_list = hca_list->hl_next;
4828 	}
4829 
4830 	if (ii == ibdm.ibdm_hca_count) {
4831 		IBTF_DPRINTF_L2("ibdm", "\tibnex_get_port_attrs: not found");
4832 		mutex_exit(&ibdm.ibdm_hl_mutex);
4833 		return (NULL);
4834 	}
4835 
4836 	port_attr = (ibdm_port_attr_t *)kmem_alloc(sizeof (ibdm_port_attr_t),
4837 	    KM_SLEEP);
4838 	bcopy((char *)&hca_list->hl_port_attr[jj], port_attr,
4839 	    sizeof (ibdm_port_attr_t));
4840 	ibdm_update_port_attr(port_attr);
4841 
4842 	mutex_exit(&ibdm.ibdm_hl_mutex);
4843 	return (port_attr);
4844 }
4845 
4846 
4847 /*
4848  * ibdm_ibnex_free_port_attr()
4849  */
4850 void
4851 ibdm_ibnex_free_port_attr(ibdm_port_attr_t *port_attr)
4852 {
4853 	IBTF_DPRINTF_L4("ibdm", "\tibnex_free_port_attr:");
4854 	if (port_attr) {
4855 		if (port_attr->pa_pkey_tbl != NULL) {
4856 			kmem_free(port_attr->pa_pkey_tbl,
4857 			    (port_attr->pa_npkeys * sizeof (ibdm_pkey_tbl_t)));
4858 		}
4859 		kmem_free(port_attr, sizeof (ibdm_port_attr_t));
4860 	}
4861 }
4862 
4863 
4864 /*
4865  * ibdm_ibnex_get_hca_list()
4866  *	Returns portinfo for all the port for all the HCA's
4867  */
4868 void
4869 ibdm_ibnex_get_hca_list(ibdm_hca_list_t **hca, int *count)
4870 {
4871 	ibdm_hca_list_t		*head = NULL, *temp, *temp1;
4872 	int			ii;
4873 
4874 	IBTF_DPRINTF_L4("ibdm", "\tibnex_get_hca_list:");
4875 
4876 	mutex_enter(&ibdm.ibdm_hl_mutex);
4877 	temp = ibdm.ibdm_hca_list_head;
4878 	for (ii = 0; ii < ibdm.ibdm_hca_count; ii++) {
4879 		temp1 = ibdm_dup_hca_attr(temp);
4880 		temp1->hl_next = head;
4881 		head = temp1;
4882 		temp = temp->hl_next;
4883 	}
4884 	*count = ibdm.ibdm_hca_count;
4885 	*hca = head;
4886 	mutex_exit(&ibdm.ibdm_hl_mutex);
4887 }
4888 
4889 
4890 /*
4891  * ibdm_ibnex_get_hca_info_by_guid()
4892  */
4893 ibdm_hca_list_t	*
4894 ibdm_ibnex_get_hca_info_by_guid(ib_guid_t hca_guid)
4895 {
4896 	ibdm_hca_list_t		*head = NULL, *hca = NULL;
4897 
4898 	IBTF_DPRINTF_L4("ibdm", "\tibnex_get_hca_info_by_dip");
4899 
4900 	mutex_enter(&ibdm.ibdm_hl_mutex);
4901 	head = ibdm.ibdm_hca_list_head;
4902 	while (head) {
4903 		if (head->hl_hca_guid == hca_guid) {
4904 			hca = ibdm_dup_hca_attr(head);
4905 			hca->hl_next = NULL;
4906 			break;
4907 		}
4908 		head = head->hl_next;
4909 	}
4910 	mutex_exit(&ibdm.ibdm_hl_mutex);
4911 	IBTF_DPRINTF_L4("ibdm", "\tibnex_get_hca_info_by_dip %p", hca);
4912 	return (hca);
4913 }
4914 
4915 
4916 /*
4917  * ibdm_dup_hca_attr()
4918  *	Allocate a new HCA attribute strucuture and initialize
4919  *	hca attribute structure with the incoming HCA attributes
4920  *	returned the allocated hca attributes.
4921  */
4922 static ibdm_hca_list_t *
4923 ibdm_dup_hca_attr(ibdm_hca_list_t *in_hca)
4924 {
4925 	int			len;
4926 	ibdm_hca_list_t		*out_hca;
4927 
4928 	len = sizeof (ibdm_hca_list_t) +
4929 	    (in_hca->hl_nports * sizeof (ibdm_port_attr_t));
4930 	IBTF_DPRINTF_L4("ibdm", "\tdup_hca_attr len %d", len);
4931 	out_hca = (ibdm_hca_list_t *)kmem_alloc(len, KM_SLEEP);
4932 	bcopy((char *)in_hca,
4933 	    (char *)out_hca, sizeof (ibdm_hca_list_t));
4934 	if (in_hca->hl_nports) {
4935 		out_hca->hl_port_attr = (ibdm_port_attr_t *)
4936 		    ((char *)out_hca + sizeof (ibdm_hca_list_t));
4937 		bcopy((char *)in_hca->hl_port_attr,
4938 		    (char *)out_hca->hl_port_attr,
4939 		    (in_hca->hl_nports * sizeof (ibdm_port_attr_t)));
4940 		for (len = 0; len < out_hca->hl_nports; len++)
4941 			ibdm_update_port_attr(&out_hca->hl_port_attr[len]);
4942 	}
4943 	return (out_hca);
4944 }
4945 
4946 
4947 /*
4948  * ibdm_ibnex_free_hca_list()
4949  *	Free one/more HCA lists
4950  */
4951 void
4952 ibdm_ibnex_free_hca_list(ibdm_hca_list_t *hca_list)
4953 {
4954 	int			ii;
4955 	size_t			len;
4956 	ibdm_hca_list_t 	*temp;
4957 	ibdm_port_attr_t	*port;
4958 
4959 	IBTF_DPRINTF_L4("ibdm", "\tibnex_free_hca_list:");
4960 	ASSERT(hca_list);
4961 	while (hca_list) {
4962 		temp = hca_list;
4963 		hca_list = hca_list->hl_next;
4964 		for (ii = 0; ii < temp->hl_nports; ii++) {
4965 			port = &temp->hl_port_attr[ii];
4966 			len = (port->pa_npkeys * sizeof (ibdm_pkey_tbl_t));
4967 			if (len != 0)
4968 				kmem_free(port->pa_pkey_tbl, len);
4969 		}
4970 		len = sizeof (ibdm_hca_list_t) + (temp->hl_nports *
4971 		    sizeof (ibdm_port_attr_t));
4972 		kmem_free(temp, len);
4973 	}
4974 }
4975 
4976 
4977 /*
4978  * ibdm_ibnex_probe_iocguid()
4979  *	Probes the IOC on the fabric and returns the IOC information
4980  *	if present. Otherwise, NULL is returned
4981  */
4982 /* ARGSUSED */
4983 ibdm_ioc_info_t *
4984 ibdm_ibnex_probe_ioc(ib_guid_t iou, ib_guid_t ioc_guid, int reprobe_flag)
4985 {
4986 	int			k;
4987 	ibdm_ioc_info_t		*ioc_info;
4988 	ibdm_dp_gidinfo_t	*gid_info; /* used as index and arg */
4989 	timeout_id_t		*timeout_id;
4990 
4991 	IBTF_DPRINTF_L4("ibdm", "\tibnex_probe_ioc: (%llX, %llX, %d) Begin",
4992 	    iou, ioc_guid, reprobe_flag);
4993 
4994 	if (ibdm_enumerate_iocs == 0)
4995 		return (NULL);
4996 
4997 	/* Check whether we know this already */
4998 	ioc_info = ibdm_get_ioc_info_with_gid(ioc_guid, &gid_info);
4999 	if (ioc_info == NULL) {
5000 		mutex_enter(&ibdm.ibdm_mutex);
5001 		while (ibdm.ibdm_busy & IBDM_BUSY)
5002 			cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex);
5003 		ibdm.ibdm_busy |= IBDM_BUSY;
5004 		mutex_exit(&ibdm.ibdm_mutex);
5005 		ibdm_probe_ioc(iou, ioc_guid, 0);
5006 		mutex_enter(&ibdm.ibdm_mutex);
5007 		ibdm.ibdm_busy &= ~IBDM_BUSY;
5008 		cv_broadcast(&ibdm.ibdm_busy_cv);
5009 		mutex_exit(&ibdm.ibdm_mutex);
5010 		ioc_info = ibdm_get_ioc_info_with_gid(ioc_guid, &gid_info);
5011 	} else if (reprobe_flag) {	/* Handle Reprobe for the IOC */
5012 		ASSERT(gid_info != NULL);
5013 		/* Free the ioc_list before reprobe; and cancel any timers */
5014 		mutex_enter(&ibdm.ibdm_mutex);
5015 		mutex_enter(&gid_info->gl_mutex);
5016 		if (ioc_info->ioc_timeout_id) {
5017 			timeout_id = ioc_info->ioc_timeout_id;
5018 			ioc_info->ioc_timeout_id = 0;
5019 			mutex_exit(&gid_info->gl_mutex);
5020 			IBTF_DPRINTF_L5("ibdm", "\tprobe_ioc: "
5021 			    "ioc_timeout_id = 0x%x", timeout_id);
5022 			if (untimeout(timeout_id) == -1) {
5023 				IBTF_DPRINTF_L2("ibdm", "\tprobe_ioc: "
5024 				    "untimeout ioc_timeout_id failed");
5025 			}
5026 			mutex_enter(&gid_info->gl_mutex);
5027 		}
5028 		if (ioc_info->ioc_dc_timeout_id) {
5029 			timeout_id = ioc_info->ioc_dc_timeout_id;
5030 			ioc_info->ioc_dc_timeout_id = 0;
5031 			mutex_exit(&gid_info->gl_mutex);
5032 			IBTF_DPRINTF_L5("ibdm", "\tprobe_ioc: "
5033 			    "ioc_dc_timeout_id = 0x%x", timeout_id);
5034 			if (untimeout(timeout_id) == -1) {
5035 				IBTF_DPRINTF_L2("ibdm", "\tprobe_ioc: "
5036 				    "untimeout ioc_dc_timeout_id failed");
5037 			}
5038 			mutex_enter(&gid_info->gl_mutex);
5039 		}
5040 		for (k = 0; k < ioc_info->ioc_profile.ioc_service_entries; k++)
5041 			if (ioc_info->ioc_serv[k].se_timeout_id) {
5042 				timeout_id = ioc_info->ioc_serv[k].
5043 				    se_timeout_id;
5044 				ioc_info->ioc_serv[k].se_timeout_id = 0;
5045 				mutex_exit(&gid_info->gl_mutex);
5046 				IBTF_DPRINTF_L5("ibdm", "\tprobe_ioc: "
5047 				    "ioc_info->ioc_serv[k].se_timeout_id = %x",
5048 				    k, timeout_id);
5049 				if (untimeout(timeout_id) == -1) {
5050 					IBTF_DPRINTF_L2("ibdm", "\tprobe_ioc: "
5051 					    "untimeout se_timeout_id %d "
5052 					    "failed", k);
5053 				}
5054 				mutex_enter(&gid_info->gl_mutex);
5055 			}
5056 		mutex_exit(&gid_info->gl_mutex);
5057 		mutex_exit(&ibdm.ibdm_mutex);
5058 		ibdm_ibnex_free_ioc_list(ioc_info);
5059 
5060 		mutex_enter(&ibdm.ibdm_mutex);
5061 		while (ibdm.ibdm_busy & IBDM_BUSY)
5062 			cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex);
5063 		ibdm.ibdm_busy |= IBDM_BUSY;
5064 		mutex_exit(&ibdm.ibdm_mutex);
5065 
5066 		ibdm_probe_ioc(iou, ioc_guid, 1);
5067 
5068 		/*
5069 		 * Skip if gl_reprobe_flag is set, this will be
5070 		 * a re-inserted / new GID, for which notifications
5071 		 * have already been send.
5072 		 */
5073 		for (gid_info = ibdm.ibdm_dp_gidlist_head; gid_info;
5074 		    gid_info = gid_info->gl_next) {
5075 			uint8_t			ii, niocs;
5076 			ibdm_ioc_info_t		*ioc;
5077 
5078 			if (gid_info->gl_iou == NULL)
5079 				continue;
5080 
5081 			if (gid_info->gl_reprobe_flag) {
5082 				gid_info->gl_reprobe_flag = 0;
5083 				continue;
5084 			}
5085 
5086 			niocs = gid_info->gl_iou->iou_info.iou_num_ctrl_slots;
5087 			for (ii = 0; ii < niocs; ii++) {
5088 				ioc = IBDM_GIDINFO2IOCINFO(gid_info, ii);
5089 				if (ioc->ioc_profile.ioc_guid == ioc_guid) {
5090 					mutex_enter(&ibdm.ibdm_mutex);
5091 					ibdm_reprobe_update_port_srv(ioc,
5092 					    gid_info);
5093 					mutex_exit(&ibdm.ibdm_mutex);
5094 				}
5095 			}
5096 		}
5097 		mutex_enter(&ibdm.ibdm_mutex);
5098 		ibdm.ibdm_busy &= ~IBDM_BUSY;
5099 		cv_broadcast(&ibdm.ibdm_busy_cv);
5100 		mutex_exit(&ibdm.ibdm_mutex);
5101 
5102 		ioc_info = ibdm_get_ioc_info_with_gid(ioc_guid, &gid_info);
5103 	}
5104 	return (ioc_info);
5105 }
5106 
5107 
5108 /*
5109  * ibdm_get_ioc_info_with_gid()
5110  *	Returns pointer to ibdm_ioc_info_t if it finds
5111  *	matching record for the ioc_guid. Otherwise NULL is returned.
5112  *	The pointer to gid_info is set to the second argument in case that
5113  *	the non-NULL value returns (and the second argument is not NULL).
5114  *
5115  * Note. use the same strings as "ibnex_get_ioc_info" in
5116  *       IBTF_DPRINTF() to keep compatibility.
5117  */
5118 static ibdm_ioc_info_t *
5119 ibdm_get_ioc_info_with_gid(ib_guid_t ioc_guid,
5120     ibdm_dp_gidinfo_t **gid_info)
5121 {
5122 	int			ii;
5123 	ibdm_ioc_info_t		*ioc = NULL, *tmp = NULL;
5124 	ibdm_dp_gidinfo_t	*gid_list;
5125 	ib_dm_io_unitinfo_t	*iou;
5126 
5127 	IBTF_DPRINTF_L4("ibdm", "\tibnex_get_ioc_info: GUID %llx", ioc_guid);
5128 
5129 	mutex_enter(&ibdm.ibdm_mutex);
5130 	while (ibdm.ibdm_busy & IBDM_BUSY)
5131 		cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex);
5132 	ibdm.ibdm_busy |= IBDM_BUSY;
5133 
5134 	if (gid_info)
5135 		*gid_info = NULL; /* clear the value of gid_info */
5136 
5137 	gid_list = ibdm.ibdm_dp_gidlist_head;
5138 	while (gid_list) {
5139 		mutex_enter(&gid_list->gl_mutex);
5140 		if (gid_list->gl_state != IBDM_GID_PROBING_COMPLETE) {
5141 			mutex_exit(&gid_list->gl_mutex);
5142 			gid_list = gid_list->gl_next;
5143 			continue;
5144 		}
5145 		if (gid_list->gl_iou == NULL) {
5146 			IBTF_DPRINTF_L2("ibdm",
5147 			    "\tget_ioc_info: No IOU info");
5148 			mutex_exit(&gid_list->gl_mutex);
5149 			gid_list = gid_list->gl_next;
5150 			continue;
5151 		}
5152 		iou = &gid_list->gl_iou->iou_info;
5153 		for (ii = 0; ii < iou->iou_num_ctrl_slots; ii++) {
5154 			tmp = IBDM_GIDINFO2IOCINFO(gid_list, ii);
5155 			if ((tmp->ioc_profile.ioc_guid == ioc_guid) &&
5156 			    (tmp->ioc_state == IBDM_IOC_STATE_PROBE_SUCCESS)) {
5157 				ioc = ibdm_dup_ioc_info(tmp, gid_list);
5158 				if (gid_info)
5159 					*gid_info = gid_list; /* set this ptr */
5160 				mutex_exit(&gid_list->gl_mutex);
5161 				ibdm.ibdm_busy &= ~IBDM_BUSY;
5162 				cv_broadcast(&ibdm.ibdm_busy_cv);
5163 				mutex_exit(&ibdm.ibdm_mutex);
5164 				IBTF_DPRINTF_L4("ibdm", "\tget_ioc_info: End");
5165 				return (ioc);
5166 			}
5167 		}
5168 		if (ii == iou->iou_num_ctrl_slots)
5169 			ioc = NULL;
5170 
5171 		mutex_exit(&gid_list->gl_mutex);
5172 		gid_list = gid_list->gl_next;
5173 	}
5174 
5175 	ibdm.ibdm_busy &= ~IBDM_BUSY;
5176 	cv_broadcast(&ibdm.ibdm_busy_cv);
5177 	mutex_exit(&ibdm.ibdm_mutex);
5178 	IBTF_DPRINTF_L4("ibdm", "\tget_ioc_info: failure End");
5179 	return (ioc);
5180 }
5181 
5182 /*
5183  * ibdm_ibnex_get_ioc_info()
5184  *	Returns pointer to ibdm_ioc_info_t if it finds
5185  *	matching record for the ioc_guid, otherwise NULL
5186  *	is returned
5187  *
5188  * Note. this is a wrapper function to ibdm_get_ioc_info_with_gid() now.
5189  */
5190 ibdm_ioc_info_t *
5191 ibdm_ibnex_get_ioc_info(ib_guid_t ioc_guid)
5192 {
5193 	if (ibdm_enumerate_iocs == 0)
5194 		return (NULL);
5195 
5196 	/* will not use the gid_info pointer, so the second arg is NULL */
5197 	return (ibdm_get_ioc_info_with_gid(ioc_guid, NULL));
5198 }
5199 
5200 /*
5201  * ibdm_ibnex_get_ioc_count()
5202  *	Returns number of ibdm_ioc_info_t it finds
5203  */
5204 int
5205 ibdm_ibnex_get_ioc_count(void)
5206 {
5207 	int			count = 0, k;
5208 	ibdm_ioc_info_t		*ioc;
5209 	ibdm_dp_gidinfo_t	*gid_list;
5210 
5211 	if (ibdm_enumerate_iocs == 0)
5212 		return (0);
5213 
5214 	mutex_enter(&ibdm.ibdm_mutex);
5215 	ibdm_sweep_fabric(0);
5216 
5217 	while (ibdm.ibdm_busy & IBDM_BUSY)
5218 		cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex);
5219 	ibdm.ibdm_busy |= IBDM_BUSY;
5220 
5221 	for (gid_list = ibdm.ibdm_dp_gidlist_head; gid_list;
5222 	    gid_list = gid_list->gl_next) {
5223 		mutex_enter(&gid_list->gl_mutex);
5224 		if ((gid_list->gl_state != IBDM_GID_PROBING_COMPLETE) ||
5225 		    (gid_list->gl_iou == NULL)) {
5226 			mutex_exit(&gid_list->gl_mutex);
5227 			continue;
5228 		}
5229 		for (k = 0; k < gid_list->gl_iou->iou_info.iou_num_ctrl_slots;
5230 		    k++) {
5231 			ioc = IBDM_GIDINFO2IOCINFO(gid_list, k);
5232 			if (ioc->ioc_state == IBDM_IOC_STATE_PROBE_SUCCESS)
5233 				++count;
5234 		}
5235 		mutex_exit(&gid_list->gl_mutex);
5236 	}
5237 	ibdm.ibdm_busy &= ~IBDM_BUSY;
5238 	cv_broadcast(&ibdm.ibdm_busy_cv);
5239 	mutex_exit(&ibdm.ibdm_mutex);
5240 
5241 	IBTF_DPRINTF_L4("ibdm", "\tget_ioc_count: count = %d", count);
5242 	return (count);
5243 }
5244 
5245 
5246 /*
5247  * ibdm_ibnex_get_ioc_list()
5248  *	Returns information about all the IOCs present on the fabric.
5249  *	Reprobes the IOCs and the GID list if list_flag is set to REPROBE_ALL.
5250  *	Does not sweep fabric if DONOT_PROBE is set
5251  */
5252 ibdm_ioc_info_t *
5253 ibdm_ibnex_get_ioc_list(ibdm_ibnex_get_ioclist_mtd_t list_flag)
5254 {
5255 	int			ii;
5256 	ibdm_ioc_info_t		*ioc_list = NULL, *tmp, *ioc;
5257 	ibdm_dp_gidinfo_t	*gid_list;
5258 	ib_dm_io_unitinfo_t	*iou;
5259 
5260 	IBTF_DPRINTF_L4("ibdm", "\tget_ioc_list: Enter");
5261 
5262 	if (ibdm_enumerate_iocs == 0)
5263 		return (NULL);
5264 
5265 	mutex_enter(&ibdm.ibdm_mutex);
5266 	if (list_flag != IBDM_IBNEX_DONOT_PROBE)
5267 		ibdm_sweep_fabric(list_flag == IBDM_IBNEX_REPROBE_ALL);
5268 
5269 	while (ibdm.ibdm_busy & IBDM_BUSY)
5270 		cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex);
5271 	ibdm.ibdm_busy |= IBDM_BUSY;
5272 
5273 	gid_list = ibdm.ibdm_dp_gidlist_head;
5274 	while (gid_list) {
5275 		mutex_enter(&gid_list->gl_mutex);
5276 		if (gid_list->gl_state != IBDM_GID_PROBING_COMPLETE) {
5277 			mutex_exit(&gid_list->gl_mutex);
5278 			gid_list = gid_list->gl_next;
5279 			continue;
5280 		}
5281 		if (gid_list->gl_iou == NULL) {
5282 			IBTF_DPRINTF_L2("ibdm",
5283 			    "\tget_ioc_list: No IOU info");
5284 			mutex_exit(&gid_list->gl_mutex);
5285 			gid_list = gid_list->gl_next;
5286 			continue;
5287 		}
5288 		iou = &gid_list->gl_iou->iou_info;
5289 		for (ii = 0; ii < iou->iou_num_ctrl_slots; ii++) {
5290 			ioc = IBDM_GIDINFO2IOCINFO(gid_list, ii);
5291 			if (ioc->ioc_state == IBDM_IOC_STATE_PROBE_SUCCESS) {
5292 				tmp = ibdm_dup_ioc_info(ioc, gid_list);
5293 				tmp->ioc_next = ioc_list;
5294 				ioc_list = tmp;
5295 			}
5296 		}
5297 		mutex_exit(&gid_list->gl_mutex);
5298 		gid_list = gid_list->gl_next;
5299 	}
5300 	ibdm.ibdm_busy &= ~IBDM_BUSY;
5301 	cv_broadcast(&ibdm.ibdm_busy_cv);
5302 	mutex_exit(&ibdm.ibdm_mutex);
5303 
5304 	IBTF_DPRINTF_L4("ibdm", "\tget_ioc_list: End");
5305 	return (ioc_list);
5306 }
5307 
5308 /*
5309  * ibdm_dup_ioc_info()
5310  *	Duplicate the IOC information and return the IOC
5311  *	information.
5312  */
5313 static ibdm_ioc_info_t *
5314 ibdm_dup_ioc_info(ibdm_ioc_info_t *in_ioc, ibdm_dp_gidinfo_t *gid_list)
5315 {
5316 	ibdm_ioc_info_t	*out_ioc;
5317 	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*out_ioc));
5318 	ASSERT(MUTEX_HELD(&gid_list->gl_mutex));
5319 
5320 	out_ioc = kmem_alloc(sizeof (ibdm_ioc_info_t), KM_SLEEP);
5321 	bcopy(in_ioc, out_ioc, sizeof (ibdm_ioc_info_t));
5322 	ibdm_update_ioc_port_gidlist(out_ioc, gid_list);
5323 	out_ioc->ioc_iou_dc_valid = gid_list->gl_iou->iou_dc_valid;
5324 	out_ioc->ioc_iou_diagcode = gid_list->gl_iou->iou_diagcode;
5325 
5326 	return (out_ioc);
5327 }
5328 
5329 
5330 /*
5331  * ibdm_free_ioc_list()
5332  *	Deallocate memory for IOC list structure
5333  */
5334 void
5335 ibdm_ibnex_free_ioc_list(ibdm_ioc_info_t *ioc)
5336 {
5337 	ibdm_ioc_info_t *temp;
5338 
5339 	IBTF_DPRINTF_L4("ibdm", "\tibnex_free_ioc_list:");
5340 	while (ioc) {
5341 		temp = ioc;
5342 		ioc = ioc->ioc_next;
5343 		kmem_free(temp->ioc_gid_list,
5344 		    (sizeof (ibdm_gid_t) * temp->ioc_nportgids));
5345 		if (temp->ioc_hca_list)
5346 			ibdm_ibnex_free_hca_list(temp->ioc_hca_list);
5347 		kmem_free(temp, sizeof (ibdm_ioc_info_t));
5348 	}
5349 }
5350 
5351 
5352 /*
5353  * ibdm_ibnex_update_pkey_tbls
5354  *	Updates the DM P_Key database.
5355  *	NOTE: Two cases are handled here: P_Key being added or removed.
5356  *
5357  * Arguments		: NONE
5358  * Return Values	: NONE
5359  */
5360 void
5361 ibdm_ibnex_update_pkey_tbls(void)
5362 {
5363 	int			h, pp, pidx;
5364 	uint_t			nports;
5365 	uint_t			size;
5366 	ib_pkey_t		new_pkey;
5367 	ib_pkey_t		*orig_pkey;
5368 	ibdm_hca_list_t		*hca_list;
5369 	ibdm_port_attr_t	*port;
5370 	ibt_hca_portinfo_t	*pinfop;
5371 
5372 	IBTF_DPRINTF_L4("ibdm", "\tibnex_update_pkey_tbls:");
5373 
5374 	mutex_enter(&ibdm.ibdm_hl_mutex);
5375 	hca_list = ibdm.ibdm_hca_list_head;
5376 
5377 	for (h = 0; h < ibdm.ibdm_hca_count; h++) {
5378 
5379 		/* This updates P_Key Tables for all ports of this HCA */
5380 		(void) ibt_query_hca_ports(hca_list->hl_hca_hdl, 0, &pinfop,
5381 		    &nports, &size);
5382 
5383 		/* number of ports shouldn't have changed */
5384 		ASSERT(nports == hca_list->hl_nports);
5385 
5386 		for (pp = 0; pp < hca_list->hl_nports; pp++) {
5387 			port = &hca_list->hl_port_attr[pp];
5388 
5389 			/*
5390 			 * First figure out the P_Keys from IBTL.
5391 			 * Three things could have happened:
5392 			 *	New P_Keys added
5393 			 *	Existing P_Keys removed
5394 			 *	Both of the above two
5395 			 *
5396 			 * Loop through the P_Key Indices and check if a
5397 			 * give P_Key_Ix matches that of the one seen by
5398 			 * IBDM. If they match no action is needed.
5399 			 *
5400 			 * If they don't match:
5401 			 *	1. if orig_pkey is invalid and new_pkey is valid
5402 			 *		---> add new_pkey to DM database
5403 			 *	2. if orig_pkey is valid and new_pkey is invalid
5404 			 *		---> remove orig_pkey from DM database
5405 			 *	3. if orig_pkey and new_pkey are both valid:
5406 			 *		---> remov orig_pkey from DM database
5407 			 *		---> add new_pkey to DM database
5408 			 *	4. if orig_pkey and new_pkey are both invalid:
5409 			 *		---> do nothing. Updated DM database.
5410 			 */
5411 
5412 			for (pidx = 0; pidx < port->pa_npkeys; pidx++) {
5413 				new_pkey = pinfop[pp].p_pkey_tbl[pidx];
5414 				orig_pkey = &port->pa_pkey_tbl[pidx].pt_pkey;
5415 
5416 				/* keys match - do nothing */
5417 				if (*orig_pkey == new_pkey)
5418 					continue;
5419 
5420 				if (IBDM_INVALID_PKEY(*orig_pkey) &&
5421 				    !IBDM_INVALID_PKEY(new_pkey)) {
5422 					/* P_Key was added */
5423 					IBTF_DPRINTF_L5("ibdm",
5424 					    "\tibnex_update_pkey_tbls: new "
5425 					    "P_Key added = 0x%x", new_pkey);
5426 					*orig_pkey = new_pkey;
5427 					ibdm_port_attr_ibmf_init(port,
5428 					    new_pkey, pp);
5429 				} else if (!IBDM_INVALID_PKEY(*orig_pkey) &&
5430 				    IBDM_INVALID_PKEY(new_pkey)) {
5431 					/* P_Key was removed */
5432 					IBTF_DPRINTF_L5("ibdm",
5433 					    "\tibnex_update_pkey_tbls: P_Key "
5434 					    "removed = 0x%x", *orig_pkey);
5435 					*orig_pkey = new_pkey;
5436 					(void) ibdm_port_attr_ibmf_fini(port,
5437 					    pidx);
5438 				} else if (!IBDM_INVALID_PKEY(*orig_pkey) &&
5439 				    !IBDM_INVALID_PKEY(new_pkey)) {
5440 					/* P_Key were replaced */
5441 					IBTF_DPRINTF_L5("ibdm",
5442 					    "\tibnex_update_pkey_tbls: P_Key "
5443 					    "replaced 0x%x with 0x%x",
5444 					    *orig_pkey, new_pkey);
5445 					(void) ibdm_port_attr_ibmf_fini(port,
5446 					    pidx);
5447 					*orig_pkey = new_pkey;
5448 					ibdm_port_attr_ibmf_init(port,
5449 					    new_pkey, pp);
5450 				} else {
5451 					/*
5452 					 * P_Keys are invalid
5453 					 * set anyway to reflect if
5454 					 * INVALID_FULL was changed to
5455 					 * INVALID_LIMITED or vice-versa.
5456 					 */
5457 					*orig_pkey = new_pkey;
5458 				} /* end of else */
5459 
5460 			} /* loop of p_key index */
5461 
5462 		} /* loop of #ports of HCA */
5463 
5464 		ibt_free_portinfo(pinfop, size);
5465 		hca_list = hca_list->hl_next;
5466 
5467 	} /* loop for all HCAs in the system */
5468 
5469 	mutex_exit(&ibdm.ibdm_hl_mutex);
5470 }
5471 
5472 
5473 /*
5474  * ibdm_send_ioc_profile()
5475  *	Send IOC Controller Profile request. When the request is completed
5476  *	IBMF calls ibdm_process_incoming_mad routine to inform about
5477  *	the completion.
5478  */
5479 static int
5480 ibdm_send_ioc_profile(ibdm_dp_gidinfo_t *gid_info, uint8_t ioc_no)
5481 {
5482 	ibmf_msg_t		*msg;
5483 	ib_mad_hdr_t	*hdr;
5484 	ibdm_ioc_info_t	*ioc_info = &(gid_info->gl_iou->iou_ioc_info[ioc_no]);
5485 	ibdm_timeout_cb_args_t	*cb_args;
5486 
5487 	IBTF_DPRINTF_L4("ibdm", "\tsend_ioc_profile: "
5488 	    "gid info 0x%p, ioc_no = %d", gid_info, ioc_no);
5489 
5490 	/*
5491 	 * Send command to get IOC profile.
5492 	 * Allocate a IBMF packet and initialize the packet.
5493 	 */
5494 	if (ibmf_alloc_msg(gid_info->gl_ibmf_hdl, IBMF_ALLOC_SLEEP,
5495 	    &msg) != IBMF_SUCCESS) {
5496 		IBTF_DPRINTF_L2("ibdm", "\tsend_ioc_profile: pkt alloc fail");
5497 		return (IBDM_FAILURE);
5498 	}
5499 
5500 	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*msg))
5501 	ibdm_alloc_send_buffers(msg);
5502 	_NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*msg))
5503 
5504 	mutex_enter(&gid_info->gl_mutex);
5505 	ibdm_bump_transactionID(gid_info);
5506 	mutex_exit(&gid_info->gl_mutex);
5507 
5508 	msg->im_local_addr.ia_local_lid	= gid_info->gl_slid;
5509 	msg->im_local_addr.ia_remote_lid	= gid_info->gl_dlid;
5510 	if (gid_info->gl_redirected == B_TRUE) {
5511 		if (gid_info->gl_redirect_dlid != 0) {
5512 			msg->im_local_addr.ia_remote_lid =
5513 			    gid_info->gl_redirect_dlid;
5514 		}
5515 		msg->im_local_addr.ia_remote_qno = gid_info->gl_redirect_QP;
5516 		msg->im_local_addr.ia_p_key = gid_info->gl_redirect_pkey;
5517 		msg->im_local_addr.ia_q_key = gid_info->gl_redirect_qkey;
5518 		msg->im_local_addr.ia_service_level = gid_info->gl_redirectSL;
5519 	} else {
5520 		msg->im_local_addr.ia_remote_qno = 1;
5521 		msg->im_local_addr.ia_p_key = gid_info->gl_p_key;
5522 		msg->im_local_addr.ia_q_key = IB_GSI_QKEY;
5523 		msg->im_local_addr.ia_service_level = gid_info->gl_SL;
5524 	}
5525 
5526 	hdr			= IBDM_OUT_IBMFMSG_MADHDR(msg);
5527 	hdr->BaseVersion	= MAD_CLASS_BASE_VERS_1;
5528 	hdr->MgmtClass		= MAD_MGMT_CLASS_DEV_MGT;
5529 	hdr->ClassVersion	= IB_DM_CLASS_VERSION_1;
5530 	hdr->R_Method		= IB_DM_DEVMGT_METHOD_GET;
5531 	hdr->Status		= 0;
5532 	hdr->TransactionID	= h2b64(gid_info->gl_transactionID);
5533 	hdr->AttributeID	= h2b16(IB_DM_ATTR_IOC_CTRL_PROFILE);
5534 	hdr->AttributeModifier 	= h2b32(ioc_no + 1);
5535 
5536 	ioc_info->ioc_state	= IBDM_IOC_STATE_REPROBE_PROGRESS;
5537 	cb_args			= &ioc_info->ioc_cb_args;
5538 	cb_args->cb_gid_info	= gid_info;
5539 	cb_args->cb_retry_count	= ibdm_dft_retry_cnt;
5540 	cb_args->cb_req_type	= IBDM_REQ_TYPE_IOCINFO;
5541 	cb_args->cb_ioc_num	= ioc_no;
5542 
5543 	mutex_enter(&gid_info->gl_mutex);
5544 	ioc_info->ioc_timeout_id = timeout(ibdm_pkt_timeout_hdlr,
5545 	    cb_args, IBDM_TIMEOUT_VALUE(ibdm_dft_timeout));
5546 	mutex_exit(&gid_info->gl_mutex);
5547 
5548 	IBTF_DPRINTF_L5("ibdm", "\tsend_ioc_profile:"
5549 	    "timeout %x", ioc_info->ioc_timeout_id);
5550 
5551 	if (ibmf_msg_transport(gid_info->gl_ibmf_hdl, gid_info->gl_qp_hdl, msg,
5552 	    NULL, ibdm_ibmf_send_cb, cb_args, 0) != IBMF_SUCCESS) {
5553 		IBTF_DPRINTF_L2("ibdm",
5554 		    "\tsend_ioc_profile: msg transport failed");
5555 		ibdm_ibmf_send_cb(gid_info->gl_ibmf_hdl, msg, cb_args);
5556 	}
5557 	ioc_info->ioc_state = IBDM_IOC_STATE_REPROBE_PROGRESS;
5558 	return (IBDM_SUCCESS);
5559 }
5560 
5561 
5562 /*
5563  * ibdm_port_reachable
5564  *	Returns B_TRUE if the port GID is reachable by sending
5565  *	a SA query to get the NODE record for this port GUID.
5566  */
5567 static boolean_t
5568 ibdm_port_reachable(ibmf_saa_handle_t sa_hdl, ib_guid_t guid)
5569 {
5570 	sa_node_record_t *resp;
5571 	size_t length;
5572 
5573 	/*
5574 	 * Verify if it's reachable by getting the node record.
5575 	 */
5576 	if (ibdm_get_node_record_by_port(sa_hdl, guid, &resp, &length) ==
5577 	    IBDM_SUCCESS) {
5578 		kmem_free(resp, length);
5579 		return (B_TRUE);
5580 	}
5581 	return (B_FALSE);
5582 }
5583 
5584 /*
5585  * ibdm_get_node_record_by_port
5586  *	Sends a SA query to get the NODE record for port GUID
5587  *	Returns IBDM_SUCCESS if the port GID is reachable.
5588  *
5589  *      Note: the caller must be responsible for freeing the resource
5590  *      by calling kmem_free(resp, length) later.
5591  */
5592 static int
5593 ibdm_get_node_record_by_port(ibmf_saa_handle_t sa_hdl, ib_guid_t guid,
5594     sa_node_record_t **resp, size_t *length)
5595 {
5596 	sa_node_record_t	req;
5597 	ibmf_saa_access_args_t	args;
5598 	int			ret;
5599 	ASSERT(resp != NULL && length != NULL);
5600 
5601 	IBTF_DPRINTF_L4("ibdm", "\tport_reachable: port_guid %llx",
5602 	    guid);
5603 
5604 	bzero(&req, sizeof (sa_node_record_t));
5605 	req.NodeInfo.PortGUID = guid;
5606 
5607 	args.sq_attr_id		= SA_NODERECORD_ATTRID;
5608 	args.sq_access_type 	= IBMF_SAA_RETRIEVE;
5609 	args.sq_component_mask 	= SA_NODEINFO_COMPMASK_PORTGUID;
5610 	args.sq_template	= &req;
5611 	args.sq_callback	= NULL;
5612 	args.sq_callback_arg 	= NULL;
5613 
5614 	ret = ibmf_sa_access(sa_hdl, &args, 0, length, (void **) resp);
5615 	if (ret != IBMF_SUCCESS) {
5616 		IBTF_DPRINTF_L2("ibdm", "\tport_reachable:"
5617 		    " SA Retrieve Failed: %d", ret);
5618 		return (IBDM_FAILURE);
5619 	}
5620 	if (*resp == NULL || *length == 0) {
5621 		IBTF_DPRINTF_L2("ibdm", "\tport_reachable: No records");
5622 		return (IBDM_FAILURE);
5623 	}
5624 	/*
5625 	 * There is one NodeRecord on each endport on a subnet.
5626 	 */
5627 	ASSERT(*length == sizeof (sa_node_record_t));
5628 
5629 	return (IBDM_SUCCESS);
5630 }
5631 
5632 
5633 /*
5634  * Update the gidlist for all affected IOCs when GID becomes
5635  * available/unavailable.
5636  *
5637  * Parameters :
5638  *	gidinfo - Incoming / Outgoing GID.
5639  *	add_flag - 1 for GID added, 0 for GID removed.
5640  *		- (-1) : IOC gid list updated, ioc_list required.
5641  *
5642  * This function gets the GID for the node GUID corresponding to the
5643  * port GID. Gets the IOU info
5644  */
5645 static ibdm_ioc_info_t *
5646 ibdm_update_ioc_gidlist(ibdm_dp_gidinfo_t *gid_info, int avail_flag)
5647 {
5648 	ibdm_dp_gidinfo_t	*node_gid = NULL;
5649 	uint8_t	niocs, ii;
5650 	ibdm_ioc_info_t	*ioc, *ioc_list = NULL, *tmp;
5651 
5652 	IBTF_DPRINTF_L4("ibdm", "\tupdate_ioc_gidlist");
5653 
5654 	switch (avail_flag) {
5655 		case 1 :
5656 			node_gid = ibdm_check_dest_nodeguid(gid_info);
5657 			break;
5658 		case 0 :
5659 			node_gid = ibdm_handle_gid_rm(gid_info);
5660 			break;
5661 		case -1 :
5662 			node_gid = gid_info;
5663 			break;
5664 		default :
5665 			break;
5666 	}
5667 
5668 	if (node_gid == NULL) {
5669 		IBTF_DPRINTF_L4("ibdm", "\tupdate_ioc_gidlist: "
5670 		    "No node GID found, port gid 0x%p, avail_flag %d",
5671 		    gid_info, avail_flag);
5672 		return (NULL);
5673 	}
5674 
5675 	mutex_enter(&node_gid->gl_mutex);
5676 	if ((node_gid->gl_state != IBDM_GID_PROBING_COMPLETE &&
5677 	    node_gid->gl_state != IBDM_GID_PROBING_SKIPPED) ||
5678 	    node_gid->gl_iou == NULL) {
5679 		IBTF_DPRINTF_L4("ibdm", "\tupdate_ioc_gidlist "
5680 		    "gl_state %x, gl_iou %p", node_gid->gl_state,
5681 		    node_gid->gl_iou);
5682 		mutex_exit(&node_gid->gl_mutex);
5683 		return (NULL);
5684 	}
5685 
5686 	niocs = node_gid->gl_iou->iou_info.iou_num_ctrl_slots;
5687 	IBTF_DPRINTF_L4("ibdm", "\tupdate_ioc_gidlist : niocs %x",
5688 	    niocs);
5689 	for (ii = 0; ii < niocs; ii++) {
5690 		ioc = IBDM_GIDINFO2IOCINFO(node_gid, ii);
5691 		/*
5692 		 * Skip IOCs for which probe is not complete or
5693 		 * reprobe is progress
5694 		 */
5695 		if (ioc->ioc_state == IBDM_IOC_STATE_PROBE_SUCCESS) {
5696 			tmp = ibdm_dup_ioc_info(ioc, node_gid);
5697 			tmp->ioc_info_updated.ib_gid_prop_updated = 1;
5698 			tmp->ioc_next = ioc_list;
5699 			ioc_list = tmp;
5700 		}
5701 	}
5702 	mutex_exit(&node_gid->gl_mutex);
5703 
5704 	IBTF_DPRINTF_L4("ibdm", "\tupdate_ioc_gidlist : return %p",
5705 	    ioc_list);
5706 	return (ioc_list);
5707 }
5708 
5709 /*
5710  * ibdm_saa_event_cb :
5711  *	Event handling which does *not* require ibdm_hl_mutex to be
5712  *	held are executed in the same thread. This is to prevent
5713  *	deadlocks with HCA port down notifications which hold the
5714  *	ibdm_hl_mutex.
5715  *
5716  *	GID_AVAILABLE event is handled here. A taskq is spawned to
5717  *	handle GID_UNAVAILABLE.
5718  *
5719  *	A new mutex ibdm_ibnex_mutex has been introduced to protect
5720  *	ibnex_callback. This has been done to prevent any possible
5721  *	deadlock (described above) while handling GID_AVAILABLE.
5722  *
5723  *	IBMF calls the event callback for a HCA port. The SA handle
5724  *	for this port would be valid, till the callback returns.
5725  *	IBDM calling IBDM using the above SA handle should be valid.
5726  *
5727  *	IBDM will additionally  check (SA handle != NULL), before
5728  *	calling IBMF.
5729  */
5730 /*ARGSUSED*/
5731 static void
5732 ibdm_saa_event_cb(ibmf_saa_handle_t ibmf_saa_handle,
5733     ibmf_saa_subnet_event_t ibmf_saa_event,
5734     ibmf_saa_event_details_t *event_details, void *callback_arg)
5735 {
5736 	ibdm_saa_event_arg_t *event_arg;
5737 	ib_gid_t		sgid, dgid;
5738 	ibdm_port_attr_t	*hca_port;
5739 	ibdm_dp_gidinfo_t	*gid_info, *node_gid_info = NULL;
5740 	sa_node_record_t *nrec;
5741 	size_t length;
5742 
5743 	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*event_arg));
5744 
5745 	hca_port = (ibdm_port_attr_t *)callback_arg;
5746 
5747 	IBTF_DPRINTF_L4("ibdm", "\tsaa_event_cb(%x, %x, %x, %x)\n",
5748 	    ibmf_saa_handle, ibmf_saa_event, event_details,
5749 	    callback_arg);
5750 
5751 #ifdef DEBUG
5752 	if (ibdm_ignore_saa_event)
5753 		return;
5754 #endif
5755 
5756 	if (ibmf_saa_event == IBMF_SAA_EVENT_GID_AVAILABLE) {
5757 		/*
5758 		 * Ensure no other probe / sweep fabric is in
5759 		 * progress.
5760 		 */
5761 		mutex_enter(&ibdm.ibdm_mutex);
5762 		while (ibdm.ibdm_busy & IBDM_BUSY)
5763 			cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex);
5764 		ibdm.ibdm_busy |= IBDM_BUSY;
5765 		mutex_exit(&ibdm.ibdm_mutex);
5766 
5767 		/*
5768 		 * If we already know about this GID, return.
5769 		 * GID_AVAILABLE may be reported for multiple HCA
5770 		 * ports.
5771 		 */
5772 		if ((ibdm_check_dgid(event_details->ie_gid.gid_guid,
5773 		    event_details->ie_gid.gid_prefix))  != NULL) {
5774 			mutex_enter(&ibdm.ibdm_mutex);
5775 			ibdm.ibdm_busy &= ~IBDM_BUSY;
5776 			cv_broadcast(&ibdm.ibdm_busy_cv);
5777 			mutex_exit(&ibdm.ibdm_mutex);
5778 			return;
5779 		}
5780 
5781 		IBTF_DPRINTF_L4("ibdm", "\tGID (prefix %x, guid %llx) "
5782 		    "Insertion notified",
5783 		    event_details->ie_gid.gid_prefix,
5784 		    event_details->ie_gid.gid_guid);
5785 
5786 		/* This is a new gid, insert it to GID list */
5787 		sgid.gid_prefix = hca_port->pa_sn_prefix;
5788 		sgid.gid_guid = hca_port->pa_port_guid;
5789 		dgid.gid_prefix = event_details->ie_gid.gid_prefix;
5790 		dgid.gid_guid = event_details->ie_gid.gid_guid;
5791 		gid_info = ibdm_create_gid_info(hca_port, sgid, dgid);
5792 		if (gid_info == NULL) {
5793 			IBTF_DPRINTF_L4("ibdm", "\tGID_AVAILABLE: "
5794 			    "create_gid_info returned NULL");
5795 			mutex_enter(&ibdm.ibdm_mutex);
5796 			ibdm.ibdm_busy &= ~IBDM_BUSY;
5797 			cv_broadcast(&ibdm.ibdm_busy_cv);
5798 			mutex_exit(&ibdm.ibdm_mutex);
5799 			return;
5800 		}
5801 		mutex_enter(&gid_info->gl_mutex);
5802 		gid_info->gl_state = IBDM_GID_PROBING_SKIPPED;
5803 		mutex_exit(&gid_info->gl_mutex);
5804 
5805 		/* Get the node GUID */
5806 		if (ibdm_get_node_record_by_port(ibmf_saa_handle, dgid.gid_guid,
5807 		    &nrec, &length) != IBDM_SUCCESS) {
5808 			/*
5809 			 * Set the state to PROBE_NOT_DONE for the
5810 			 * next sweep to probe it
5811 			 */
5812 			IBTF_DPRINTF_L2("ibdm", "\tsaa_event_taskq: "
5813 			    "Skipping GID : port GUID not found");
5814 			mutex_enter(&gid_info->gl_mutex);
5815 			gid_info->gl_state = IBDM_GID_PROBE_NOT_DONE;
5816 			mutex_exit(&gid_info->gl_mutex);
5817 			mutex_enter(&ibdm.ibdm_mutex);
5818 			ibdm.ibdm_busy &= ~IBDM_BUSY;
5819 			cv_broadcast(&ibdm.ibdm_busy_cv);
5820 			mutex_exit(&ibdm.ibdm_mutex);
5821 			return;
5822 		}
5823 		gid_info->gl_nodeguid = nrec->NodeInfo.NodeGUID;
5824 		gid_info->gl_devid = nrec->NodeInfo.DeviceID;
5825 		kmem_free(nrec, length);
5826 		gid_info->gl_portguid = dgid.gid_guid;
5827 
5828 		/*
5829 		 * Get the gid info with the same node GUID.
5830 		 */
5831 		mutex_enter(&ibdm.ibdm_mutex);
5832 		node_gid_info = ibdm.ibdm_dp_gidlist_head;
5833 		while (node_gid_info) {
5834 			if (node_gid_info->gl_nodeguid ==
5835 			    gid_info->gl_nodeguid &&
5836 			    node_gid_info->gl_iou != NULL) {
5837 				break;
5838 			}
5839 			node_gid_info = node_gid_info->gl_next;
5840 		}
5841 		mutex_exit(&ibdm.ibdm_mutex);
5842 
5843 		/*
5844 		 * Handling a new GID requires filling of gl_hca_list.
5845 		 * This require ibdm hca_list to be parsed and hence
5846 		 * holding the ibdm_hl_mutex. Spawning a new thread to
5847 		 * handle this.
5848 		 */
5849 		if (node_gid_info == NULL) {
5850 			if (taskq_dispatch(system_taskq,
5851 			    ibdm_saa_handle_new_gid, (void *)gid_info,
5852 			    TQ_NOSLEEP) == TASKQID_INVALID) {
5853 				IBTF_DPRINTF_L2("ibdm", "\tsaa_event_cb: "
5854 				    "new_gid taskq_dispatch failed");
5855 				return;
5856 			}
5857 		}
5858 
5859 		mutex_enter(&ibdm.ibdm_mutex);
5860 		ibdm.ibdm_busy &= ~IBDM_BUSY;
5861 		cv_broadcast(&ibdm.ibdm_busy_cv);
5862 		mutex_exit(&ibdm.ibdm_mutex);
5863 		return;
5864 	}
5865 
5866 	if (ibmf_saa_event != IBMF_SAA_EVENT_GID_UNAVAILABLE)
5867 		return;
5868 
5869 	/*
5870 	 * GID UNAVAIL EVENT: Try to locate the GID in the GID list.
5871 	 * If we don't find it we just return.
5872 	 */
5873 	mutex_enter(&ibdm.ibdm_mutex);
5874 	gid_info = ibdm.ibdm_dp_gidlist_head;
5875 	while (gid_info) {
5876 		if (gid_info->gl_portguid ==
5877 		    event_details->ie_gid.gid_guid) {
5878 			break;
5879 		}
5880 		gid_info = gid_info->gl_next;
5881 	}
5882 	mutex_exit(&ibdm.ibdm_mutex);
5883 	if (gid_info == NULL) {
5884 		IBTF_DPRINTF_L2("ibdm", "\tsaa_event_cb: "
5885 		    "GID for GUID %llX not found during GID UNAVAIL event",
5886 		    event_details->ie_gid.gid_guid);
5887 		return;
5888 	}
5889 
5890 	/*
5891 	 * If this GID is DM capable, we'll have to check whether this DGID
5892 	 * is reachable via another port.
5893 	 */
5894 	if (gid_info->gl_is_dm_capable == B_TRUE) {
5895 		event_arg = (ibdm_saa_event_arg_t *)kmem_alloc(
5896 		    sizeof (ibdm_saa_event_arg_t), KM_SLEEP);
5897 		event_arg->ibmf_saa_handle = ibmf_saa_handle;
5898 		event_arg->ibmf_saa_event = ibmf_saa_event;
5899 		bcopy(event_details, &event_arg->event_details,
5900 		    sizeof (ibmf_saa_event_details_t));
5901 		event_arg->callback_arg = callback_arg;
5902 
5903 		if (taskq_dispatch(system_taskq, ibdm_saa_event_taskq,
5904 		    (void *)event_arg, TQ_NOSLEEP) == TASKQID_INVALID) {
5905 			IBTF_DPRINTF_L2("ibdm", "\tsaa_event_cb: "
5906 			    "taskq_dispatch failed");
5907 			ibdm_free_saa_event_arg(event_arg);
5908 			return;
5909 		}
5910 	}
5911 }
5912 
5913 /*
5914  * Handle a new GID discovered by GID_AVAILABLE saa event.
5915  */
5916 void
5917 ibdm_saa_handle_new_gid(void *arg)
5918 {
5919 	ibdm_dp_gidinfo_t	*gid_info;
5920 	ibdm_hca_list_t		*hca_list = NULL;
5921 	ibdm_port_attr_t	*port = NULL;
5922 	ibdm_ioc_info_t		*ioc_list = NULL;
5923 
5924 	IBTF_DPRINTF_L4(ibdm_string, "\tsaa_handle_new_gid(%p)", arg);
5925 
5926 	gid_info = (ibdm_dp_gidinfo_t *)arg;
5927 
5928 	/*
5929 	 * Ensure that no other sweep / probe has completed
5930 	 * probing this gid.
5931 	 */
5932 	mutex_enter(&gid_info->gl_mutex);
5933 	if (gid_info->gl_state != IBDM_GID_PROBE_NOT_DONE) {
5934 		mutex_exit(&gid_info->gl_mutex);
5935 		return;
5936 	}
5937 	mutex_exit(&gid_info->gl_mutex);
5938 
5939 	/*
5940 	 * Parse HCAs to fill gl_hca_list
5941 	 */
5942 	mutex_enter(&ibdm.ibdm_hl_mutex);
5943 	for (ibdm_get_next_port(&hca_list, &port, 1); port;
5944 	    ibdm_get_next_port(&hca_list, &port, 1)) {
5945 		if (ibdm_port_reachable(port->pa_sa_hdl,
5946 		    gid_info->gl_portguid) == B_TRUE) {
5947 			ibdm_addto_glhcalist(gid_info, hca_list);
5948 		}
5949 	}
5950 	mutex_exit(&ibdm.ibdm_hl_mutex);
5951 
5952 	/*
5953 	 * Ensure no other probe / sweep fabric is in
5954 	 * progress.
5955 	 */
5956 	mutex_enter(&ibdm.ibdm_mutex);
5957 	while (ibdm.ibdm_busy & IBDM_BUSY)
5958 		cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex);
5959 	ibdm.ibdm_busy |= IBDM_BUSY;
5960 	mutex_exit(&ibdm.ibdm_mutex);
5961 
5962 	/*
5963 	 * New IOU probe it, to check if new IOCs
5964 	 */
5965 	IBTF_DPRINTF_L4(ibdm_string, "\tsaa_handle_new_gid: "
5966 	    "new GID : probing");
5967 	mutex_enter(&ibdm.ibdm_mutex);
5968 	ibdm.ibdm_ngid_probes_in_progress++;
5969 	mutex_exit(&ibdm.ibdm_mutex);
5970 	mutex_enter(&gid_info->gl_mutex);
5971 	gid_info->gl_reprobe_flag = 0;
5972 	gid_info->gl_state = IBDM_GID_PROBE_NOT_DONE;
5973 	mutex_exit(&gid_info->gl_mutex);
5974 	ibdm_probe_gid_thread((void *)gid_info);
5975 
5976 	mutex_enter(&ibdm.ibdm_mutex);
5977 	ibdm_wait_probe_completion();
5978 	mutex_exit(&ibdm.ibdm_mutex);
5979 
5980 	if (gid_info->gl_iou == NULL) {
5981 		mutex_enter(&ibdm.ibdm_mutex);
5982 		ibdm.ibdm_busy &= ~IBDM_BUSY;
5983 		cv_broadcast(&ibdm.ibdm_busy_cv);
5984 		mutex_exit(&ibdm.ibdm_mutex);
5985 		return;
5986 	}
5987 
5988 	/*
5989 	 * Update GID list in all IOCs affected by this
5990 	 */
5991 	ioc_list = ibdm_update_ioc_gidlist(gid_info, 1);
5992 
5993 	/*
5994 	 * Pass on the IOCs with updated GIDs to IBnexus
5995 	 */
5996 	if (ioc_list) {
5997 		mutex_enter(&ibdm.ibdm_ibnex_mutex);
5998 		if (ibdm.ibdm_ibnex_callback != NULL) {
5999 			(*ibdm.ibdm_ibnex_callback)((void *)ioc_list,
6000 			    IBDM_EVENT_IOC_PROP_UPDATE);
6001 		}
6002 		mutex_exit(&ibdm.ibdm_ibnex_mutex);
6003 	}
6004 
6005 	mutex_enter(&ibdm.ibdm_mutex);
6006 	ibdm.ibdm_busy &= ~IBDM_BUSY;
6007 	cv_broadcast(&ibdm.ibdm_busy_cv);
6008 	mutex_exit(&ibdm.ibdm_mutex);
6009 }
6010 
6011 /*
6012  * ibdm_saa_event_taskq :
6013  *	GID_UNAVAILABLE Event handling requires ibdm_hl_mutex to be
6014  *	held. The GID_UNAVAILABLE handling is done in a taskq to
6015  *	prevent deadlocks with HCA port down notifications which hold
6016  *	ibdm_hl_mutex.
6017  */
6018 void
6019 ibdm_saa_event_taskq(void *arg)
6020 {
6021 	ibdm_saa_event_arg_t *event_arg;
6022 	ibmf_saa_handle_t ibmf_saa_handle;
6023 	ibmf_saa_subnet_event_t ibmf_saa_event;
6024 	ibmf_saa_event_details_t *event_details;
6025 	void *callback_arg;
6026 
6027 	ibdm_dp_gidinfo_t	*gid_info;
6028 	ibdm_port_attr_t	*hca_port, *port = NULL;
6029 	ibdm_hca_list_t		*hca_list = NULL;
6030 	int	sa_handle_valid = 0;
6031 	ibdm_ioc_info_t		*ioc_list = NULL;
6032 
6033 	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*event_arg));
6034 
6035 	event_arg = (ibdm_saa_event_arg_t *)arg;
6036 	ibmf_saa_handle = event_arg->ibmf_saa_handle;
6037 	ibmf_saa_event = event_arg->ibmf_saa_event;
6038 	event_details = &event_arg->event_details;
6039 	callback_arg = event_arg->callback_arg;
6040 
6041 	ASSERT(callback_arg != NULL);
6042 	ASSERT(ibmf_saa_event == IBMF_SAA_EVENT_GID_UNAVAILABLE);
6043 	IBTF_DPRINTF_L4("ibdm", "\tsaa_event_taskq(%x, %x, %x, %x)",
6044 	    ibmf_saa_handle, ibmf_saa_event, event_details,
6045 	    callback_arg);
6046 
6047 	hca_port = (ibdm_port_attr_t *)callback_arg;
6048 
6049 	/* Check if the port_attr is still valid */
6050 	mutex_enter(&ibdm.ibdm_hl_mutex);
6051 	for (ibdm_get_next_port(&hca_list, &port, 0); port;
6052 	    ibdm_get_next_port(&hca_list, &port, 0)) {
6053 		if (port == hca_port && port->pa_port_guid ==
6054 		    hca_port->pa_port_guid) {
6055 			if (ibmf_saa_handle == hca_port->pa_sa_hdl)
6056 				sa_handle_valid = 1;
6057 			break;
6058 		}
6059 	}
6060 	mutex_exit(&ibdm.ibdm_hl_mutex);
6061 	if (sa_handle_valid == 0) {
6062 		ibdm_free_saa_event_arg(event_arg);
6063 		return;
6064 	}
6065 
6066 	if (hca_port && (hca_port->pa_sa_hdl == NULL ||
6067 	    ibmf_saa_handle != hca_port->pa_sa_hdl)) {
6068 		ibdm_free_saa_event_arg(event_arg);
6069 		return;
6070 	}
6071 	hca_list = NULL;
6072 	port = NULL;
6073 
6074 	/*
6075 	 * Check if the GID is visible to other HCA ports.
6076 	 * Return if so.
6077 	 */
6078 	mutex_enter(&ibdm.ibdm_hl_mutex);
6079 	for (ibdm_get_next_port(&hca_list, &port, 1); port;
6080 	    ibdm_get_next_port(&hca_list, &port, 1)) {
6081 		if (ibdm_port_reachable(port->pa_sa_hdl,
6082 		    event_details->ie_gid.gid_guid) == B_TRUE) {
6083 			mutex_exit(&ibdm.ibdm_hl_mutex);
6084 			ibdm_free_saa_event_arg(event_arg);
6085 			return;
6086 		}
6087 	}
6088 	mutex_exit(&ibdm.ibdm_hl_mutex);
6089 
6090 	/*
6091 	 * Ensure no other probe / sweep fabric is in
6092 	 * progress.
6093 	 */
6094 	mutex_enter(&ibdm.ibdm_mutex);
6095 	while (ibdm.ibdm_busy & IBDM_BUSY)
6096 		cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex);
6097 	ibdm.ibdm_busy |= IBDM_BUSY;
6098 	mutex_exit(&ibdm.ibdm_mutex);
6099 
6100 	/*
6101 	 * If this GID is no longer in GID list, return
6102 	 * GID_UNAVAILABLE may be reported for multiple HCA
6103 	 * ports.
6104 	 */
6105 	mutex_enter(&ibdm.ibdm_mutex);
6106 	gid_info = ibdm.ibdm_dp_gidlist_head;
6107 	while (gid_info) {
6108 		if (gid_info->gl_portguid ==
6109 		    event_details->ie_gid.gid_guid) {
6110 			break;
6111 		}
6112 		gid_info = gid_info->gl_next;
6113 	}
6114 	mutex_exit(&ibdm.ibdm_mutex);
6115 	if (gid_info == NULL) {
6116 		mutex_enter(&ibdm.ibdm_mutex);
6117 		ibdm.ibdm_busy &= ~IBDM_BUSY;
6118 		cv_broadcast(&ibdm.ibdm_busy_cv);
6119 		mutex_exit(&ibdm.ibdm_mutex);
6120 		ibdm_free_saa_event_arg(event_arg);
6121 		return;
6122 	}
6123 
6124 	IBTF_DPRINTF_L4("ibdm", "\tGID (prefix %x, guid %llx) "
6125 	    "Unavailable notification",
6126 	    event_details->ie_gid.gid_prefix,
6127 	    event_details->ie_gid.gid_guid);
6128 
6129 	/*
6130 	 * Update GID list in all IOCs affected by this
6131 	 */
6132 	if (gid_info->gl_state == IBDM_GID_PROBING_SKIPPED ||
6133 	    gid_info->gl_state == IBDM_GID_PROBING_COMPLETE)
6134 		ioc_list = ibdm_update_ioc_gidlist(gid_info, 0);
6135 
6136 	/*
6137 	 * Remove GID from the global GID list
6138 	 * Handle the case where all port GIDs for an
6139 	 * IOU have been hot-removed. Check both gid_info
6140 	 * & ioc_info for checking ngids.
6141 	 */
6142 	mutex_enter(&ibdm.ibdm_mutex);
6143 	if (gid_info->gl_iou != NULL && gid_info->gl_ngids == 0) {
6144 		mutex_enter(&gid_info->gl_mutex);
6145 		(void) ibdm_free_iou_info(gid_info, &gid_info->gl_iou);
6146 		mutex_exit(&gid_info->gl_mutex);
6147 	}
6148 	if (gid_info->gl_prev != NULL)
6149 		gid_info->gl_prev->gl_next = gid_info->gl_next;
6150 	if (gid_info->gl_next != NULL)
6151 		gid_info->gl_next->gl_prev = gid_info->gl_prev;
6152 
6153 	if (gid_info == ibdm.ibdm_dp_gidlist_head)
6154 		ibdm.ibdm_dp_gidlist_head = gid_info->gl_next;
6155 	if (gid_info == ibdm.ibdm_dp_gidlist_tail)
6156 		ibdm.ibdm_dp_gidlist_tail = gid_info->gl_prev;
6157 	ibdm.ibdm_ngids--;
6158 
6159 	ibdm.ibdm_busy &= ~IBDM_BUSY;
6160 	cv_broadcast(&ibdm.ibdm_busy_cv);
6161 	mutex_exit(&ibdm.ibdm_mutex);
6162 
6163 	/* free the hca_list on this gid_info */
6164 	ibdm_delete_glhca_list(gid_info);
6165 
6166 	mutex_destroy(&gid_info->gl_mutex);
6167 	kmem_free(gid_info, sizeof (ibdm_dp_gidinfo_t));
6168 
6169 	/*
6170 	 * Pass on the IOCs with updated GIDs to IBnexus
6171 	 */
6172 	if (ioc_list) {
6173 		IBTF_DPRINTF_L4("ibdm", "\tGID_UNAVAILABLE "
6174 		    "IOC_PROP_UPDATE for %p\n", ioc_list);
6175 		mutex_enter(&ibdm.ibdm_ibnex_mutex);
6176 		if (ibdm.ibdm_ibnex_callback != NULL) {
6177 			(*ibdm.ibdm_ibnex_callback)((void *)
6178 			    ioc_list, IBDM_EVENT_IOC_PROP_UPDATE);
6179 		}
6180 		mutex_exit(&ibdm.ibdm_ibnex_mutex);
6181 	}
6182 
6183 	ibdm_free_saa_event_arg(event_arg);
6184 }
6185 
6186 
6187 static int
6188 ibdm_cmp_gid_list(ibdm_gid_t *new, ibdm_gid_t *prev)
6189 {
6190 	ibdm_gid_t		*scan_new, *scan_prev;
6191 	int	cmp_failed = 0;
6192 
6193 	ASSERT(new != NULL);
6194 	ASSERT(prev != NULL);
6195 
6196 	/*
6197 	 * Search for each new gid anywhere in the prev GID list.
6198 	 * Note that the gid list could have been re-ordered.
6199 	 */
6200 	for (scan_new = new; scan_new; scan_new = scan_new->gid_next) {
6201 		for (scan_prev = prev, cmp_failed = 1; scan_prev;
6202 		    scan_prev = scan_prev->gid_next) {
6203 			if (scan_prev->gid_dgid_hi == scan_new->gid_dgid_hi &&
6204 			    scan_prev->gid_dgid_lo == scan_new->gid_dgid_lo) {
6205 				cmp_failed = 0;
6206 				break;
6207 			}
6208 		}
6209 
6210 		if (cmp_failed)
6211 			return (1);
6212 	}
6213 	return (0);
6214 }
6215 
6216 /*
6217  * This is always called in a single thread
6218  * This function updates the gid_list and serv_list of IOC
6219  * The current gid_list is in ioc_info_t(contains only port
6220  * guids for which probe is done) & gidinfo_t(other port gids)
6221  * The gids in both locations are used for comparision.
6222  */
6223 static void
6224 ibdm_reprobe_update_port_srv(ibdm_ioc_info_t *ioc, ibdm_dp_gidinfo_t *gidinfo)
6225 {
6226 	ibdm_gid_t		*cur_gid_list;
6227 	uint_t			cur_nportgids;
6228 
6229 	ASSERT(MUTEX_HELD(&ibdm.ibdm_mutex));
6230 
6231 	ioc->ioc_info_updated.ib_prop_updated = 0;
6232 
6233 
6234 	/* Current GID list in gid_info only */
6235 	cur_gid_list = gidinfo->gl_gid;
6236 	cur_nportgids = gidinfo->gl_ngids;
6237 
6238 	if (ioc->ioc_prev_serv_cnt !=
6239 	    ioc->ioc_profile.ioc_service_entries ||
6240 	    ibdm_serv_cmp(&ioc->ioc_serv[0], &ioc->ioc_prev_serv[0],
6241 	    ioc->ioc_prev_serv_cnt))
6242 		ioc->ioc_info_updated.ib_srv_prop_updated = 1;
6243 
6244 	if (ioc->ioc_prev_nportgids != cur_nportgids ||
6245 	    ioc->ioc_prev_gid_list == NULL || cur_gid_list == NULL) {
6246 		ioc->ioc_info_updated.ib_gid_prop_updated = 1;
6247 	} else if (ibdm_cmp_gid_list(ioc->ioc_prev_gid_list, cur_gid_list)) {
6248 		ioc->ioc_info_updated.ib_gid_prop_updated = 1;
6249 	}
6250 
6251 	/* Zero out previous entries */
6252 	ibdm_free_gid_list(ioc->ioc_prev_gid_list);
6253 	if (ioc->ioc_prev_serv)
6254 		kmem_free(ioc->ioc_prev_serv, ioc->ioc_prev_serv_cnt *
6255 		    sizeof (ibdm_srvents_info_t));
6256 	ioc->ioc_prev_serv_cnt = 0;
6257 	ioc->ioc_prev_nportgids = 0;
6258 	ioc->ioc_prev_serv = NULL;
6259 	ioc->ioc_prev_gid_list = NULL;
6260 }
6261 
6262 /*
6263  * Handle GID removal. This returns gid_info of an GID for the same
6264  * node GUID, if found.  For an GID with IOU information, the same
6265  * gid_info is returned if no gid_info with same node_guid is found.
6266  */
6267 static ibdm_dp_gidinfo_t *
6268 ibdm_handle_gid_rm(ibdm_dp_gidinfo_t *rm_gid)
6269 {
6270 	ibdm_dp_gidinfo_t	*gid_list;
6271 
6272 	IBTF_DPRINTF_L4("ibdm", "\thandle_gid_rm(0x%p)", rm_gid);
6273 
6274 	if (rm_gid->gl_iou == NULL) {
6275 		IBTF_DPRINTF_L4("ibdm", "\thandle_gid_rm NO iou");
6276 		/*
6277 		 * Search for a GID with same node_guid and
6278 		 * gl_iou != NULL
6279 		 */
6280 		for (gid_list = ibdm.ibdm_dp_gidlist_head; gid_list;
6281 		    gid_list = gid_list->gl_next) {
6282 			if (gid_list->gl_iou != NULL && (gid_list->gl_nodeguid
6283 			    == rm_gid->gl_nodeguid))
6284 				break;
6285 		}
6286 
6287 		if (gid_list)
6288 			ibdm_rmfrom_glgid_list(gid_list, rm_gid);
6289 
6290 		IBTF_DPRINTF_L4("ibdm", "\thandle_gid_rm ret %p", gid_list);
6291 		return (gid_list);
6292 	} else {
6293 		/*
6294 		 * Search for a GID with same node_guid and
6295 		 * gl_iou == NULL
6296 		 */
6297 		IBTF_DPRINTF_L4("ibdm", "\thandle_gid_rm with iou");
6298 		for (gid_list = ibdm.ibdm_dp_gidlist_head; gid_list;
6299 		    gid_list = gid_list->gl_next) {
6300 			if (gid_list->gl_iou == NULL && (gid_list->gl_nodeguid
6301 			    == rm_gid->gl_nodeguid))
6302 				break;
6303 		}
6304 
6305 		if (gid_list) {
6306 			/*
6307 			 * Copy the following fields from rm_gid :
6308 			 *	1. gl_state
6309 			 *	2. gl_iou
6310 			 *	3. gl_gid & gl_ngids
6311 			 *
6312 			 * Note :	Function is synchronized by
6313 			 *			ibdm_busy flag.
6314 			 *
6315 			 * Note :	Redirect info is initialized if
6316 			 *			any MADs for the GID fail
6317 			 */
6318 			IBTF_DPRINTF_L4("ibdm", "\thandle_gid_rm "
6319 			    "copying info to GID with gl_iou != NULl");
6320 			gid_list->gl_state = rm_gid->gl_state;
6321 			gid_list->gl_iou = rm_gid->gl_iou;
6322 			gid_list->gl_gid = rm_gid->gl_gid;
6323 			gid_list->gl_ngids = rm_gid->gl_ngids;
6324 
6325 			/* Remove the GID from gl_gid list */
6326 			ibdm_rmfrom_glgid_list(gid_list, rm_gid);
6327 		} else {
6328 			/*
6329 			 * Handle a case where all GIDs to the IOU have
6330 			 * been removed.
6331 			 */
6332 			IBTF_DPRINTF_L4("ibdm", "\thandle_gid_rm 0 GID "
6333 			    "to IOU");
6334 
6335 			ibdm_rmfrom_glgid_list(rm_gid, rm_gid);
6336 			return (rm_gid);
6337 		}
6338 		IBTF_DPRINTF_L4("ibdm", "\thandle_gid_rm ret %p", gid_list);
6339 		return (gid_list);
6340 	}
6341 }
6342 
6343 static void
6344 ibdm_rmfrom_glgid_list(ibdm_dp_gidinfo_t *gid_info,
6345     ibdm_dp_gidinfo_t *rm_gid)
6346 {
6347 	ibdm_gid_t 		*tmp, *prev;
6348 
6349 	IBTF_DPRINTF_L4("ibdm", "\trmfrom_glgid (%p, %p)",
6350 	    gid_info, rm_gid);
6351 
6352 	for (tmp = gid_info->gl_gid, prev = NULL; tmp; ) {
6353 		if (tmp->gid_dgid_hi == rm_gid->gl_dgid_hi &&
6354 		    tmp->gid_dgid_lo == rm_gid->gl_dgid_lo) {
6355 			if (prev == NULL)
6356 				gid_info->gl_gid = tmp->gid_next;
6357 			else
6358 				prev->gid_next = tmp->gid_next;
6359 
6360 			kmem_free(tmp, sizeof (ibdm_gid_t));
6361 			gid_info->gl_ngids--;
6362 			break;
6363 		} else {
6364 			prev = tmp;
6365 			tmp = tmp->gid_next;
6366 		}
6367 	}
6368 }
6369 
6370 static void
6371 ibdm_addto_gidlist(ibdm_gid_t **src_ptr, ibdm_gid_t *dest)
6372 {
6373 	ibdm_gid_t *head = NULL, *new, *tail;
6374 
6375 	/* First copy the destination */
6376 	for (; dest; dest = dest->gid_next) {
6377 		new = kmem_zalloc(sizeof (ibdm_gid_t), KM_SLEEP);
6378 		new->gid_dgid_hi = dest->gid_dgid_hi;
6379 		new->gid_dgid_lo = dest->gid_dgid_lo;
6380 		new->gid_next = head;
6381 		head = new;
6382 	}
6383 
6384 	/* Insert this to the source */
6385 	if (*src_ptr == NULL)
6386 		*src_ptr = head;
6387 	else {
6388 		for (tail = *src_ptr; tail->gid_next != NULL;
6389 		    tail = tail->gid_next)
6390 			;
6391 
6392 		tail->gid_next = head;
6393 	}
6394 }
6395 
6396 static void
6397 ibdm_free_gid_list(ibdm_gid_t	*head)
6398 {
6399 	ibdm_gid_t	*delete;
6400 
6401 	for (delete = head; delete; ) {
6402 		head = delete->gid_next;
6403 		kmem_free(delete, sizeof (ibdm_gid_t));
6404 		delete = head;
6405 	}
6406 }
6407 
6408 /*
6409  * This function rescans the DM capable GIDs (gl_state is
6410  * GID_PROBE_COMPLETE or IBDM_GID_PROBING_SKIPPED.This
6411  * basically checks if the DM capable GID is reachable. If
6412  * not this is handled the same way as GID_UNAVAILABLE,
6413  * except that notifications are not send to IBnexus.
6414  *
6415  * This function also initializes the ioc_prev_list for
6416  * a particular IOC (when called from probe_ioc, with
6417  * ioc_guidp != NULL) or all IOCs for the gid (called from
6418  * sweep_fabric, ioc_guidp == NULL).
6419  */
6420 static void
6421 ibdm_rescan_gidlist(ib_guid_t *ioc_guidp)
6422 {
6423 	ibdm_dp_gidinfo_t	*gid_info, *tmp;
6424 	int ii, niocs, found;
6425 	ibdm_hca_list_t *hca_list = NULL;
6426 	ibdm_port_attr_t *port = NULL;
6427 	ibdm_ioc_info_t *ioc_list;
6428 
6429 	for (gid_info = ibdm.ibdm_dp_gidlist_head; gid_info; ) {
6430 		found = 0;
6431 		if (gid_info->gl_state != IBDM_GID_PROBING_SKIPPED &&
6432 		    gid_info->gl_state != IBDM_GID_PROBING_COMPLETE) {
6433 			gid_info = gid_info->gl_next;
6434 			continue;
6435 		}
6436 
6437 		/*
6438 		 * Check if the GID is visible to any HCA ports.
6439 		 * Return if so.
6440 		 */
6441 		mutex_enter(&ibdm.ibdm_hl_mutex);
6442 		for (ibdm_get_next_port(&hca_list, &port, 1); port;
6443 		    ibdm_get_next_port(&hca_list, &port, 1)) {
6444 			if (ibdm_port_reachable(port->pa_sa_hdl,
6445 			    gid_info->gl_dgid_lo) == B_TRUE) {
6446 				found = 1;
6447 				break;
6448 			}
6449 		}
6450 		mutex_exit(&ibdm.ibdm_hl_mutex);
6451 
6452 		if (found) {
6453 			if (gid_info->gl_iou == NULL) {
6454 				gid_info = gid_info->gl_next;
6455 				continue;
6456 			}
6457 
6458 			/* Intialize the ioc_prev_gid_list */
6459 			niocs =
6460 			    gid_info->gl_iou->iou_info.iou_num_ctrl_slots;
6461 			for (ii = 0; ii < niocs; ii++) {
6462 				ioc_list = IBDM_GIDINFO2IOCINFO(gid_info, ii);
6463 
6464 				if (ioc_guidp == NULL || (*ioc_guidp ==
6465 				    ioc_list->ioc_profile.ioc_guid)) {
6466 					/* Add info of GIDs in gid_info also */
6467 					ibdm_addto_gidlist(
6468 					    &ioc_list->ioc_prev_gid_list,
6469 					    gid_info->gl_gid);
6470 					ioc_list->ioc_prev_nportgids =
6471 					    gid_info->gl_ngids;
6472 				}
6473 			}
6474 			gid_info = gid_info->gl_next;
6475 			continue;
6476 		}
6477 
6478 		IBTF_DPRINTF_L4("ibdm", "\trescan_gidlist "
6479 		    "deleted port GUID %llx",
6480 		    gid_info->gl_dgid_lo);
6481 
6482 		/*
6483 		 * Update GID list in all IOCs affected by this
6484 		 */
6485 		ioc_list = ibdm_update_ioc_gidlist(gid_info, 0);
6486 
6487 		/*
6488 		 * Remove GID from the global GID list
6489 		 * Handle the case where all port GIDs for an
6490 		 * IOU have been hot-removed.
6491 		 */
6492 		mutex_enter(&ibdm.ibdm_mutex);
6493 		if (gid_info->gl_iou != NULL && gid_info->gl_ngids == 0) {
6494 			mutex_enter(&gid_info->gl_mutex);
6495 			(void) ibdm_free_iou_info(gid_info, &gid_info->gl_iou);
6496 			mutex_exit(&gid_info->gl_mutex);
6497 		}
6498 
6499 		tmp = gid_info->gl_next;
6500 		if (gid_info->gl_prev != NULL)
6501 			gid_info->gl_prev->gl_next = gid_info->gl_next;
6502 		if (gid_info->gl_next != NULL)
6503 			gid_info->gl_next->gl_prev = gid_info->gl_prev;
6504 
6505 		if (gid_info == ibdm.ibdm_dp_gidlist_head)
6506 			ibdm.ibdm_dp_gidlist_head = gid_info->gl_next;
6507 		if (gid_info == ibdm.ibdm_dp_gidlist_tail)
6508 			ibdm.ibdm_dp_gidlist_tail = gid_info->gl_prev;
6509 		ibdm.ibdm_ngids--;
6510 		mutex_exit(&ibdm.ibdm_mutex);
6511 
6512 		/* free the hca_list on this gid_info */
6513 		ibdm_delete_glhca_list(gid_info);
6514 
6515 		mutex_destroy(&gid_info->gl_mutex);
6516 		kmem_free(gid_info, sizeof (ibdm_dp_gidinfo_t));
6517 
6518 		gid_info = tmp;
6519 
6520 		/*
6521 		 * Pass on the IOCs with updated GIDs to IBnexus
6522 		 */
6523 		if (ioc_list) {
6524 			IBTF_DPRINTF_L4("ibdm", "\trescan_gidlist "
6525 			    "IOC_PROP_UPDATE for %p\n", ioc_list);
6526 			mutex_enter(&ibdm.ibdm_ibnex_mutex);
6527 			if (ibdm.ibdm_ibnex_callback != NULL) {
6528 				(*ibdm.ibdm_ibnex_callback)((void *)
6529 				    ioc_list, IBDM_EVENT_IOC_PROP_UPDATE);
6530 			}
6531 			mutex_exit(&ibdm.ibdm_ibnex_mutex);
6532 		}
6533 	}
6534 }
6535 
6536 /*
6537  * This function notifies IBnex of IOCs on this GID.
6538  * Notification is for GIDs with gl_reprobe_flag set.
6539  * The flag is set when IOC probe / fabric sweep
6540  * probes a GID starting from CLASS port info.
6541  *
6542  * IBnexus will have information of a reconnected IOC
6543  * if it had probed it before. If this is a new IOC,
6544  * IBnexus ignores the notification.
6545  *
6546  * This function should be called with no locks held.
6547  */
6548 static void
6549 ibdm_notify_newgid_iocs(ibdm_dp_gidinfo_t *gid_info)
6550 {
6551 	ibdm_ioc_info_t	*ioc_list;
6552 
6553 	if (gid_info->gl_reprobe_flag == 0 ||
6554 	    gid_info->gl_iou == NULL)
6555 		return;
6556 
6557 	ioc_list = ibdm_update_ioc_gidlist(gid_info, -1);
6558 
6559 	/*
6560 	 * Pass on the IOCs with updated GIDs to IBnexus
6561 	 */
6562 	if (ioc_list) {
6563 		mutex_enter(&ibdm.ibdm_ibnex_mutex);
6564 		if (ibdm.ibdm_ibnex_callback != NULL) {
6565 			(*ibdm.ibdm_ibnex_callback)((void *)ioc_list,
6566 			    IBDM_EVENT_IOC_PROP_UPDATE);
6567 		}
6568 		mutex_exit(&ibdm.ibdm_ibnex_mutex);
6569 	}
6570 }
6571 
6572 
6573 static void
6574 ibdm_free_saa_event_arg(ibdm_saa_event_arg_t *arg)
6575 {
6576 	if (arg != NULL)
6577 		kmem_free(arg, sizeof (ibdm_saa_event_arg_t));
6578 }
6579 
6580 /*
6581  * This function parses the list of HCAs and HCA ports
6582  * to return the port_attr of the next HCA port. A port
6583  * connected to IB fabric (port_state active) is returned,
6584  * if connected_flag is set.
6585  */
6586 static void
6587 ibdm_get_next_port(ibdm_hca_list_t **inp_hcap,
6588     ibdm_port_attr_t **inp_portp, int connect_flag)
6589 {
6590 	int ii;
6591 	ibdm_port_attr_t *port, *next_port = NULL;
6592 	ibdm_port_attr_t *inp_port;
6593 	ibdm_hca_list_t	 *hca_list;
6594 	int found = 0;
6595 
6596 	ASSERT(MUTEX_HELD(&ibdm.ibdm_hl_mutex));
6597 	IBTF_DPRINTF_L4(ibdm_string, "\tget_next_port(%p, %p, %x)",
6598 	    inp_hcap, inp_portp, connect_flag);
6599 
6600 	hca_list = *inp_hcap;
6601 	inp_port = *inp_portp;
6602 
6603 	if (hca_list == NULL)
6604 		hca_list = ibdm.ibdm_hca_list_head;
6605 
6606 	for (; hca_list; hca_list = hca_list->hl_next) {
6607 		for (ii = 0; ii < hca_list->hl_nports; ii++) {
6608 			port = &hca_list->hl_port_attr[ii];
6609 
6610 			/*
6611 			 * inp_port != NULL;
6612 			 * 	Skip till we find the matching port
6613 			 */
6614 			if (inp_port && !found) {
6615 				if (inp_port == port)
6616 					found = 1;
6617 				continue;
6618 			}
6619 
6620 			if (!connect_flag) {
6621 				next_port = port;
6622 				break;
6623 			}
6624 
6625 			if (port->pa_sa_hdl == NULL)
6626 				ibdm_initialize_port(port);
6627 			if (port->pa_sa_hdl == NULL)
6628 				(void) ibdm_fini_port(port);
6629 			else if (next_port == NULL &&
6630 			    port->pa_sa_hdl != NULL &&
6631 			    port->pa_state == IBT_PORT_ACTIVE) {
6632 				next_port = port;
6633 				break;
6634 			}
6635 		}
6636 
6637 		if (next_port)
6638 			break;
6639 	}
6640 
6641 	IBTF_DPRINTF_L4(ibdm_string, "\tget_next_port : "
6642 	    "returns hca_list %p port %p", hca_list, next_port);
6643 	*inp_hcap = hca_list;
6644 	*inp_portp = next_port;
6645 }
6646 
6647 static void
6648 ibdm_add_to_gl_gid(ibdm_dp_gidinfo_t *nodegid, ibdm_dp_gidinfo_t *addgid)
6649 {
6650 	ibdm_gid_t	*tmp;
6651 
6652 	tmp = kmem_zalloc(sizeof (ibdm_gid_t), KM_SLEEP);
6653 	tmp->gid_dgid_hi = addgid->gl_dgid_hi;
6654 	tmp->gid_dgid_lo = addgid->gl_dgid_lo;
6655 
6656 	mutex_enter(&nodegid->gl_mutex);
6657 	tmp->gid_next = nodegid->gl_gid;
6658 	nodegid->gl_gid = tmp;
6659 	nodegid->gl_ngids++;
6660 	mutex_exit(&nodegid->gl_mutex);
6661 }
6662 
6663 static void
6664 ibdm_addto_glhcalist(ibdm_dp_gidinfo_t *gid_info,
6665     ibdm_hca_list_t *hca)
6666 {
6667 	ibdm_hca_list_t		*head, *prev = NULL, *temp;
6668 
6669 	IBTF_DPRINTF_L4(ibdm_string, "\taddto_glhcalist(%p, %p) "
6670 	    ": gl_hca_list %p", gid_info, hca, gid_info->gl_hca_list);
6671 	ASSERT(!MUTEX_HELD(&gid_info->gl_mutex));
6672 
6673 	mutex_enter(&gid_info->gl_mutex);
6674 	head = gid_info->gl_hca_list;
6675 	if (head == NULL) {
6676 		head = ibdm_dup_hca_attr(hca);
6677 		head->hl_next = NULL;
6678 		gid_info->gl_hca_list = head;
6679 		mutex_exit(&gid_info->gl_mutex);
6680 		IBTF_DPRINTF_L4(ibdm_string, "\tadd_to_glhcalist: "
6681 		    "gid %p, gl_hca_list %p", gid_info,
6682 		    gid_info->gl_hca_list);
6683 		return;
6684 	}
6685 
6686 	/* Check if already in the list */
6687 	while (head) {
6688 		if (head->hl_hca_guid == hca->hl_hca_guid) {
6689 			mutex_exit(&gid_info->gl_mutex);
6690 			IBTF_DPRINTF_L4(ibdm_string,
6691 			    "\taddto_glhcalist : gid %p hca %p dup",
6692 			    gid_info, hca);
6693 			return;
6694 		}
6695 		prev = head;
6696 		head = head->hl_next;
6697 	}
6698 
6699 	/* Add this HCA to gl_hca_list */
6700 	temp =  ibdm_dup_hca_attr(hca);
6701 	temp->hl_next = NULL;
6702 	prev->hl_next = temp;
6703 	mutex_exit(&gid_info->gl_mutex);
6704 
6705 	IBTF_DPRINTF_L4(ibdm_string, "\tadd_to_glhcalist: "
6706 	    "gid %p, gl_hca_list %p", gid_info, gid_info->gl_hca_list);
6707 }
6708 
6709 static void
6710 ibdm_delete_glhca_list(ibdm_dp_gidinfo_t *gid_info)
6711 {
6712 	ASSERT(!MUTEX_HELD(&gid_info->gl_mutex));
6713 	ASSERT(!MUTEX_HELD(&ibdm.ibdm_mutex));
6714 
6715 	mutex_enter(&gid_info->gl_mutex);
6716 	if (gid_info->gl_hca_list)
6717 		ibdm_ibnex_free_hca_list(gid_info->gl_hca_list);
6718 	gid_info->gl_hca_list = NULL;
6719 	mutex_exit(&gid_info->gl_mutex);
6720 }
6721 
6722 
6723 static void
6724 ibdm_reset_all_dgids(ibmf_saa_handle_t port_sa_hdl)
6725 {
6726 	IBTF_DPRINTF_L4(ibdm_string, "\treset_all_dgids(%X)",
6727 	    port_sa_hdl);
6728 
6729 	if (ibdm_enumerate_iocs == 0)
6730 		return;
6731 
6732 	ASSERT(!MUTEX_HELD(&ibdm.ibdm_mutex));
6733 	ASSERT(!MUTEX_HELD(&ibdm.ibdm_hl_mutex));
6734 
6735 	/* Check : Not busy in another probe / sweep */
6736 	mutex_enter(&ibdm.ibdm_mutex);
6737 	if ((ibdm.ibdm_busy & IBDM_BUSY) == 0) {
6738 		ibdm_dp_gidinfo_t	*gid_info;
6739 
6740 		ibdm.ibdm_busy |= IBDM_BUSY;
6741 		mutex_exit(&ibdm.ibdm_mutex);
6742 
6743 		/*
6744 		 * Check if any GID is using the SA & IBMF handle
6745 		 * of HCA port going down. Reset ibdm_dp_gidinfo_t
6746 		 * using another HCA port which can reach the GID.
6747 		 * This is for DM capable GIDs only, no need to do
6748 		 * this for others
6749 		 *
6750 		 * Delete the GID if no alternate HCA port to reach
6751 		 * it is found.
6752 		 */
6753 		for (gid_info = ibdm.ibdm_dp_gidlist_head; gid_info; ) {
6754 			ibdm_dp_gidinfo_t *tmp;
6755 
6756 			IBTF_DPRINTF_L4(ibdm_string, "\tevent_hdlr "
6757 			    "checking gidinfo %p", gid_info);
6758 
6759 			if (gid_info->gl_sa_hdl == port_sa_hdl) {
6760 				IBTF_DPRINTF_L3(ibdm_string,
6761 				    "\tevent_hdlr: down HCA port hdl "
6762 				    "matches gid %p", gid_info);
6763 
6764 				/*
6765 				 * The non-DM GIDs can come back
6766 				 * with a new subnet prefix, when
6767 				 * the HCA port commes up again. To
6768 				 * avoid issues, delete non-DM
6769 				 * capable GIDs, if the gid was
6770 				 * discovered using the HCA port
6771 				 * going down. This is ensured by
6772 				 * setting gl_disconnected to 1.
6773 				 */
6774 				if (gid_info->gl_is_dm_capable == B_FALSE)
6775 					gid_info->gl_disconnected = 1;
6776 				else
6777 					ibdm_reset_gidinfo(gid_info);
6778 
6779 				if (gid_info->gl_disconnected) {
6780 					IBTF_DPRINTF_L3(ibdm_string,
6781 					    "\tevent_hdlr: deleting"
6782 					    " gid %p", gid_info);
6783 					tmp = gid_info;
6784 					gid_info = gid_info->gl_next;
6785 					ibdm_delete_gidinfo(tmp);
6786 				} else
6787 					gid_info = gid_info->gl_next;
6788 			} else
6789 				gid_info = gid_info->gl_next;
6790 		}
6791 
6792 		mutex_enter(&ibdm.ibdm_mutex);
6793 		ibdm.ibdm_busy &= ~IBDM_BUSY;
6794 		cv_signal(&ibdm.ibdm_busy_cv);
6795 	}
6796 	mutex_exit(&ibdm.ibdm_mutex);
6797 }
6798 
6799 static void
6800 ibdm_reset_gidinfo(ibdm_dp_gidinfo_t *gidinfo)
6801 {
6802 	ibdm_hca_list_t	*hca_list = NULL;
6803 	ibdm_port_attr_t	*port = NULL;
6804 	int	gid_reinited = 0;
6805 	sa_node_record_t	*nr, *tmp;
6806 	sa_portinfo_record_t	*pi;
6807 	size_t	nr_len = 0, pi_len = 0;
6808 	size_t	path_len;
6809 	ib_gid_t	sgid, dgid;
6810 	int	ret, ii, nrecords;
6811 	sa_path_record_t	*path;
6812 	uint8_t	npaths = 1;
6813 	ibdm_pkey_tbl_t		*pkey_tbl;
6814 
6815 	IBTF_DPRINTF_L4(ibdm_string, "\treset_gidinfo(%p)", gidinfo);
6816 
6817 	/*
6818 	 * Get list of all the ports reachable from the local known HCA
6819 	 * ports which are active
6820 	 */
6821 	mutex_enter(&ibdm.ibdm_hl_mutex);
6822 	for (ibdm_get_next_port(&hca_list, &port, 1); port;
6823 	    ibdm_get_next_port(&hca_list, &port, 1)) {
6824 
6825 
6826 		/*
6827 		 * Get the path and re-populate the gidinfo.
6828 		 * Getting the path is the same probe_ioc
6829 		 * Init the gid info as in ibdm_create_gidinfo()
6830 		 */
6831 		nr = ibdm_get_node_records(port->pa_sa_hdl, &nr_len,
6832 		    gidinfo->gl_nodeguid);
6833 		if (nr == NULL) {
6834 			IBTF_DPRINTF_L4(ibdm_string,
6835 			    "\treset_gidinfo : no records");
6836 			continue;
6837 		}
6838 
6839 		nrecords = (nr_len / sizeof (sa_node_record_t));
6840 		for (tmp = nr, ii = 0;  (ii < nrecords); ii++, tmp++) {
6841 			if (tmp->NodeInfo.PortGUID == gidinfo->gl_portguid)
6842 				break;
6843 		}
6844 
6845 		if (ii == nrecords) {
6846 			IBTF_DPRINTF_L4(ibdm_string,
6847 			    "\treset_gidinfo : no record for portguid");
6848 			kmem_free(nr, nr_len);
6849 			continue;
6850 		}
6851 
6852 		pi = ibdm_get_portinfo(port->pa_sa_hdl, &pi_len, tmp->LID);
6853 		if (pi == NULL) {
6854 			IBTF_DPRINTF_L4(ibdm_string,
6855 			    "\treset_gidinfo : no portinfo");
6856 			kmem_free(nr, nr_len);
6857 			continue;
6858 		}
6859 
6860 		sgid.gid_prefix = port->pa_sn_prefix;
6861 		sgid.gid_guid = port->pa_port_guid;
6862 		dgid.gid_prefix = pi->PortInfo.GidPrefix;
6863 		dgid.gid_guid = tmp->NodeInfo.PortGUID;
6864 
6865 		ret = ibmf_saa_gid_to_pathrecords(port->pa_sa_hdl, sgid, dgid,
6866 		    IBMF_SAA_PKEY_WC, 0, B_TRUE, &npaths, 0, &path_len, &path);
6867 
6868 		if ((ret != IBMF_SUCCESS) || path == NULL) {
6869 			IBTF_DPRINTF_L4(ibdm_string,
6870 			    "\treset_gidinfo : no paths");
6871 			kmem_free(pi, pi_len);
6872 			kmem_free(nr, nr_len);
6873 			continue;
6874 		}
6875 
6876 		gidinfo->gl_dgid_hi	= path->DGID.gid_prefix;
6877 		gidinfo->gl_dgid_lo	= path->DGID.gid_guid;
6878 		gidinfo->gl_sgid_hi	= path->SGID.gid_prefix;
6879 		gidinfo->gl_sgid_lo	= path->SGID.gid_guid;
6880 		gidinfo->gl_p_key	= path->P_Key;
6881 		gidinfo->gl_sa_hdl	= port->pa_sa_hdl;
6882 		gidinfo->gl_ibmf_hdl	= port->pa_ibmf_hdl;
6883 		gidinfo->gl_slid	= path->SLID;
6884 		gidinfo->gl_dlid	= path->DLID;
6885 		/* Reset redirect info, next MAD will set if redirected */
6886 		gidinfo->gl_redirected	= 0;
6887 		gidinfo->gl_devid	= (*tmp).NodeInfo.DeviceID;
6888 		gidinfo->gl_SL		= path->SL;
6889 
6890 		gidinfo->gl_qp_hdl = IBMF_QP_HANDLE_DEFAULT;
6891 		for (ii = 0; ii < port->pa_npkeys; ii++) {
6892 			if (port->pa_pkey_tbl == NULL)
6893 				break;
6894 
6895 			pkey_tbl = &port->pa_pkey_tbl[ii];
6896 			if ((gidinfo->gl_p_key == pkey_tbl->pt_pkey) &&
6897 			    (pkey_tbl->pt_qp_hdl != NULL)) {
6898 				gidinfo->gl_qp_hdl = pkey_tbl->pt_qp_hdl;
6899 				break;
6900 			}
6901 		}
6902 
6903 		if (gidinfo->gl_qp_hdl == NULL)
6904 			IBTF_DPRINTF_L2(ibdm_string,
6905 			    "\treset_gid_info: No matching Pkey");
6906 		else
6907 			gid_reinited = 1;
6908 
6909 		kmem_free(path, path_len);
6910 		kmem_free(pi, pi_len);
6911 		kmem_free(nr, nr_len);
6912 		break;
6913 	}
6914 	mutex_exit(&ibdm.ibdm_hl_mutex);
6915 
6916 	if (!gid_reinited)
6917 		gidinfo->gl_disconnected = 1;
6918 }
6919 
6920 static void
6921 ibdm_delete_gidinfo(ibdm_dp_gidinfo_t *gidinfo)
6922 {
6923 	ibdm_ioc_info_t *ioc_list;
6924 	int	in_gidlist = 0;
6925 
6926 	/*
6927 	 * Check if gidinfo has been inserted into the
6928 	 * ibdm_dp_gidlist_head list. gl_next or gl_prev
6929 	 * != NULL, if gidinfo is the list.
6930 	 */
6931 	if (gidinfo->gl_prev != NULL ||
6932 	    gidinfo->gl_next != NULL ||
6933 	    ibdm.ibdm_dp_gidlist_head == gidinfo)
6934 		in_gidlist = 1;
6935 
6936 	ioc_list = ibdm_update_ioc_gidlist(gidinfo, 0);
6937 
6938 	/*
6939 	 * Remove GID from the global GID list
6940 	 * Handle the case where all port GIDs for an
6941 	 * IOU have been hot-removed.
6942 	 */
6943 	mutex_enter(&ibdm.ibdm_mutex);
6944 	if (gidinfo->gl_iou != NULL && gidinfo->gl_ngids == 0) {
6945 		mutex_enter(&gidinfo->gl_mutex);
6946 		(void) ibdm_free_iou_info(gidinfo, &gidinfo->gl_iou);
6947 		mutex_exit(&gidinfo->gl_mutex);
6948 	}
6949 
6950 	/* Delete gl_hca_list */
6951 	mutex_exit(&ibdm.ibdm_mutex);
6952 	ibdm_delete_glhca_list(gidinfo);
6953 	mutex_enter(&ibdm.ibdm_mutex);
6954 
6955 	if (in_gidlist) {
6956 		if (gidinfo->gl_prev != NULL)
6957 			gidinfo->gl_prev->gl_next = gidinfo->gl_next;
6958 		if (gidinfo->gl_next != NULL)
6959 			gidinfo->gl_next->gl_prev = gidinfo->gl_prev;
6960 
6961 		if (gidinfo == ibdm.ibdm_dp_gidlist_head)
6962 			ibdm.ibdm_dp_gidlist_head = gidinfo->gl_next;
6963 		if (gidinfo == ibdm.ibdm_dp_gidlist_tail)
6964 			ibdm.ibdm_dp_gidlist_tail = gidinfo->gl_prev;
6965 		ibdm.ibdm_ngids--;
6966 	}
6967 	mutex_exit(&ibdm.ibdm_mutex);
6968 
6969 	mutex_destroy(&gidinfo->gl_mutex);
6970 	cv_destroy(&gidinfo->gl_probe_cv);
6971 	kmem_free(gidinfo, sizeof (ibdm_dp_gidinfo_t));
6972 
6973 	/*
6974 	 * Pass on the IOCs with updated GIDs to IBnexus
6975 	 */
6976 	if (ioc_list) {
6977 		IBTF_DPRINTF_L4("ibdm", "\tdelete_gidinfo "
6978 		    "IOC_PROP_UPDATE for %p\n", ioc_list);
6979 		mutex_enter(&ibdm.ibdm_ibnex_mutex);
6980 		if (ibdm.ibdm_ibnex_callback != NULL) {
6981 			(*ibdm.ibdm_ibnex_callback)((void *)
6982 			    ioc_list, IBDM_EVENT_IOC_PROP_UPDATE);
6983 		}
6984 		mutex_exit(&ibdm.ibdm_ibnex_mutex);
6985 	}
6986 }
6987 
6988 
6989 static void
6990 ibdm_fill_srv_attr_mod(ib_mad_hdr_t *hdr, ibdm_timeout_cb_args_t *cb_args)
6991 {
6992 	uint32_t	attr_mod;
6993 
6994 	attr_mod = (cb_args->cb_ioc_num + 1) << 16;
6995 	attr_mod |= cb_args->cb_srvents_start;
6996 	attr_mod |= (cb_args->cb_srvents_end) << 8;
6997 	hdr->AttributeModifier = h2b32(attr_mod);
6998 }
6999 
7000 static void
7001 ibdm_bump_transactionID(ibdm_dp_gidinfo_t *gid_info)
7002 {
7003 	ASSERT(MUTEX_HELD(&gid_info->gl_mutex));
7004 	gid_info->gl_transactionID++;
7005 	if (gid_info->gl_transactionID == gid_info->gl_max_transactionID) {
7006 		IBTF_DPRINTF_L4(ibdm_string,
7007 		    "\tbump_transactionID(%p), wrapup", gid_info);
7008 		gid_info->gl_transactionID = gid_info->gl_min_transactionID;
7009 	}
7010 }
7011 
7012 /*
7013  * gl_prev_iou is set for *non-reprobe* sweeep requests, which
7014  * detected that ChangeID in IOU info has changed. The service
7015  * entry also may have changed. Check if service entry in IOC
7016  * has changed wrt the prev iou, if so notify to IB Nexus.
7017  */
7018 static ibdm_ioc_info_t *
7019 ibdm_handle_prev_iou()
7020 {
7021 	ibdm_dp_gidinfo_t *gid_info;
7022 	ibdm_ioc_info_t	*ioc_list_head = NULL, *ioc_list;
7023 	ibdm_ioc_info_t	*prev_ioc, *ioc;
7024 	int		ii, jj, niocs, prev_niocs;
7025 
7026 	ASSERT(MUTEX_HELD(&ibdm.ibdm_mutex));
7027 
7028 	IBTF_DPRINTF_L4(ibdm_string, "\thandle_prev_iou enter");
7029 	for (gid_info = ibdm.ibdm_dp_gidlist_head; gid_info;
7030 	    gid_info = gid_info->gl_next) {
7031 		if (gid_info->gl_prev_iou == NULL)
7032 			continue;
7033 
7034 		IBTF_DPRINTF_L4(ibdm_string, "\thandle_prev_iou gid %p",
7035 		    gid_info);
7036 		niocs = gid_info->gl_iou->iou_info.iou_num_ctrl_slots;
7037 		prev_niocs =
7038 		    gid_info->gl_prev_iou->iou_info.iou_num_ctrl_slots;
7039 		for (ii = 0; ii < niocs; ii++) {
7040 			ioc = IBDM_GIDINFO2IOCINFO(gid_info, ii);
7041 
7042 			/* Find matching IOC */
7043 			for (jj = 0; jj < prev_niocs; jj++) {
7044 				prev_ioc = (ibdm_ioc_info_t *)
7045 				    &gid_info->gl_prev_iou->iou_ioc_info[jj];
7046 				if (prev_ioc->ioc_profile.ioc_guid ==
7047 				    ioc->ioc_profile.ioc_guid)
7048 					break;
7049 			}
7050 			if (jj == prev_niocs)
7051 				prev_ioc = NULL;
7052 			if (ioc == NULL || prev_ioc == NULL)
7053 				continue;
7054 			if ((ioc->ioc_profile.ioc_service_entries !=
7055 			    prev_ioc->ioc_profile.ioc_service_entries) ||
7056 			    ibdm_serv_cmp(&ioc->ioc_serv[0],
7057 			    &prev_ioc->ioc_serv[0],
7058 			    ioc->ioc_profile.ioc_service_entries) != 0) {
7059 				IBTF_DPRINTF_L4(ibdm_string,
7060 				    "/thandle_prev_iou modified IOC: "
7061 				    "current ioc %p, old ioc %p",
7062 				    ioc, prev_ioc);
7063 				mutex_enter(&gid_info->gl_mutex);
7064 				ioc_list = ibdm_dup_ioc_info(ioc, gid_info);
7065 				mutex_exit(&gid_info->gl_mutex);
7066 				ioc_list->ioc_info_updated.ib_prop_updated
7067 				    = 0;
7068 				ioc_list->ioc_info_updated.ib_srv_prop_updated
7069 				    = 1;
7070 
7071 				if (ioc_list_head == NULL)
7072 					ioc_list_head = ioc_list;
7073 				else {
7074 					ioc_list_head->ioc_next = ioc_list;
7075 					ioc_list_head = ioc_list;
7076 				}
7077 			}
7078 		}
7079 
7080 		mutex_enter(&gid_info->gl_mutex);
7081 		(void) ibdm_free_iou_info(gid_info, &gid_info->gl_prev_iou);
7082 		mutex_exit(&gid_info->gl_mutex);
7083 	}
7084 	IBTF_DPRINTF_L4(ibdm_string, "\thandle_prev_iouret %p",
7085 	    ioc_list_head);
7086 	return (ioc_list_head);
7087 }
7088 
7089 /*
7090  * Compares two service entries lists, returns 0 if same, returns 1
7091  * if no match.
7092  */
7093 static int
7094 ibdm_serv_cmp(ibdm_srvents_info_t *serv1, ibdm_srvents_info_t *serv2,
7095     int nserv)
7096 {
7097 	int	ii;
7098 
7099 	IBTF_DPRINTF_L4(ibdm_string, "\tserv_cmp: enter");
7100 	for (ii = 0; ii < nserv; ii++, serv1++, serv2++) {
7101 		if (serv1->se_attr.srv_id != serv2->se_attr.srv_id ||
7102 		    bcmp(serv1->se_attr.srv_name,
7103 		    serv2->se_attr.srv_name,
7104 		    IB_DM_MAX_SVC_NAME_LEN) != 0) {
7105 			IBTF_DPRINTF_L4(ibdm_string, "\tserv_cmp: ret 1");
7106 			return (1);
7107 		}
7108 	}
7109 	IBTF_DPRINTF_L4(ibdm_string, "\tserv_cmp: ret 0");
7110 	return (0);
7111 }
7112 
7113 /* For debugging purpose only */
7114 #ifdef	DEBUG
7115 void
7116 ibdm_dump_mad_hdr(ib_mad_hdr_t	*mad_hdr)
7117 {
7118 	IBTF_DPRINTF_L4("ibdm", "\t\t MAD Header info");
7119 	IBTF_DPRINTF_L4("ibdm", "\t\t ---------------");
7120 
7121 	IBTF_DPRINTF_L4("ibdm", "\tBase version  : 0x%x"
7122 	    "\tMgmt Class : 0x%x", mad_hdr->BaseVersion, mad_hdr->MgmtClass);
7123 	IBTF_DPRINTF_L4("ibdm", "\tClass version : 0x%x"
7124 	    "\tR Method           : 0x%x",
7125 	    mad_hdr->ClassVersion, mad_hdr->R_Method);
7126 	IBTF_DPRINTF_L4("ibdm", "\tMAD  Status   : 0x%x"
7127 	    "\tTransaction ID     : 0x%llx",
7128 	    b2h16(mad_hdr->Status), b2h64(mad_hdr->TransactionID));
7129 	IBTF_DPRINTF_L4("ibdm", "\t Attribute ID  : 0x%x"
7130 	    "\tAttribute Modified : 0x%lx",
7131 	    b2h16(mad_hdr->AttributeID), b2h32(mad_hdr->AttributeModifier));
7132 }
7133 
7134 
7135 void
7136 ibdm_dump_ibmf_msg(ibmf_msg_t *ibmf_msg, int flag)
7137 {
7138 	ib_mad_hdr_t	*mad_hdr;
7139 
7140 	IBTF_DPRINTF_L4("ibdm", "\t\t(IBMF_PKT): Local address info");
7141 	IBTF_DPRINTF_L4("ibdm", "\t\t            ------------------");
7142 
7143 	IBTF_DPRINTF_L4("ibdm", "\tLocal Lid  : 0x%x\tRemote Lid : 0x%x"
7144 	    " Remote Qp  : 0x%x", ibmf_msg->im_local_addr.ia_local_lid,
7145 	    ibmf_msg->im_local_addr.ia_remote_lid,
7146 	    ibmf_msg->im_local_addr.ia_remote_qno);
7147 	IBTF_DPRINTF_L4("ibdm", "\tP_key      : 0x%x\tQ_key      : 0x%x"
7148 	    " SL  : 0x%x", ibmf_msg->im_local_addr.ia_p_key,
7149 	    ibmf_msg->im_local_addr.ia_q_key,
7150 	    ibmf_msg->im_local_addr.ia_service_level);
7151 
7152 	if (flag)
7153 		mad_hdr = (ib_mad_hdr_t *)IBDM_OUT_IBMFMSG_MADHDR(ibmf_msg);
7154 	else
7155 		mad_hdr = IBDM_IN_IBMFMSG_MADHDR(ibmf_msg);
7156 
7157 	ibdm_dump_mad_hdr(mad_hdr);
7158 }
7159 
7160 
7161 void
7162 ibdm_dump_path_info(sa_path_record_t *path)
7163 {
7164 	IBTF_DPRINTF_L4("ibdm", "\t\t Path information");
7165 	IBTF_DPRINTF_L4("ibdm", "\t\t ----------------");
7166 
7167 	IBTF_DPRINTF_L4("ibdm", "\t DGID hi  : %llx\tDGID lo  : %llx",
7168 	    path->DGID.gid_prefix, path->DGID.gid_guid);
7169 	IBTF_DPRINTF_L4("ibdm", "\t SGID hi  : %llx\tSGID lo  : %llx",
7170 	    path->SGID.gid_prefix, path->SGID.gid_guid);
7171 	IBTF_DPRINTF_L4("ibdm", "\t SLID     : %x\t\tDlID     : %x",
7172 	    path->SLID, path->DLID);
7173 	IBTF_DPRINTF_L4("ibdm", "\t P Key    : %x\t\tSL       : %x",
7174 	    path->P_Key, path->SL);
7175 }
7176 
7177 
7178 void
7179 ibdm_dump_classportinfo(ib_mad_classportinfo_t *classportinfo)
7180 {
7181 	IBTF_DPRINTF_L4("ibdm", "\t\t CLASSPORT INFO");
7182 	IBTF_DPRINTF_L4("ibdm", "\t\t --------------");
7183 
7184 	IBTF_DPRINTF_L4("ibdm", "\t Response Time Value : 0x%x",
7185 	    ((b2h32(classportinfo->RespTimeValue)) & 0x1F));
7186 
7187 	IBTF_DPRINTF_L4("ibdm", "\t Redirected GID hi   : 0x%llx",
7188 	    b2h64(classportinfo->RedirectGID_hi));
7189 	IBTF_DPRINTF_L4("ibdm", "\t Redirected GID lo   : 0x%llx",
7190 	    b2h64(classportinfo->RedirectGID_lo));
7191 	IBTF_DPRINTF_L4("ibdm", "\t Redirected TC       : 0x%x",
7192 	    classportinfo->RedirectTC);
7193 	IBTF_DPRINTF_L4("ibdm", "\t Redirected SL       : 0x%x",
7194 	    classportinfo->RedirectSL);
7195 	IBTF_DPRINTF_L4("ibdm", "\t Redirected FL       : 0x%x",
7196 	    classportinfo->RedirectFL);
7197 	IBTF_DPRINTF_L4("ibdm", "\t Redirected LID      : 0x%x",
7198 	    b2h16(classportinfo->RedirectLID));
7199 	IBTF_DPRINTF_L4("ibdm", "\t Redirected P KEY    : 0x%x",
7200 	    b2h16(classportinfo->RedirectP_Key));
7201 	IBTF_DPRINTF_L4("ibdm", "\t Redirected QP       : 0x%x",
7202 	    classportinfo->RedirectQP);
7203 	IBTF_DPRINTF_L4("ibdm", "\t Redirected Q KEY    : 0x%x",
7204 	    b2h32(classportinfo->RedirectQ_Key));
7205 	IBTF_DPRINTF_L4("ibdm", "\t Trap GID hi         : 0x%llx",
7206 	    b2h64(classportinfo->TrapGID_hi));
7207 	IBTF_DPRINTF_L4("ibdm", "\t Trap GID lo         : 0x%llx",
7208 	    b2h64(classportinfo->TrapGID_lo));
7209 	IBTF_DPRINTF_L4("ibdm", "\t Trap TC             : 0x%x",
7210 	    classportinfo->TrapTC);
7211 	IBTF_DPRINTF_L4("ibdm", "\t Trap SL             : 0x%x",
7212 	    classportinfo->TrapSL);
7213 	IBTF_DPRINTF_L4("ibdm", "\t Trap FL             : 0x%x",
7214 	    classportinfo->TrapFL);
7215 	IBTF_DPRINTF_L4("ibdm", "\t Trap LID            : 0x%x",
7216 	    b2h16(classportinfo->TrapLID));
7217 	IBTF_DPRINTF_L4("ibdm", "\t Trap P_Key          : 0x%x",
7218 	    b2h16(classportinfo->TrapP_Key));
7219 	IBTF_DPRINTF_L4("ibdm", "\t Trap HL             : 0x%x",
7220 	    classportinfo->TrapHL);
7221 	IBTF_DPRINTF_L4("ibdm", "\t Trap QP             : 0x%x",
7222 	    classportinfo->TrapQP);
7223 	IBTF_DPRINTF_L4("ibdm", "\t Trap Q_Key          : 0x%x",
7224 	    b2h32(classportinfo->TrapQ_Key));
7225 }
7226 
7227 
7228 void
7229 ibdm_dump_iounitinfo(ib_dm_io_unitinfo_t *iou_info)
7230 {
7231 	IBTF_DPRINTF_L4("ibdm", "\t\t I/O UnitInfo");
7232 	IBTF_DPRINTF_L4("ibdm", "\t\t ------------");
7233 
7234 	IBTF_DPRINTF_L4("ibdm", "\tChange ID            : 0x%x",
7235 	    b2h16(iou_info->iou_changeid));
7236 	IBTF_DPRINTF_L4("ibdm", "\t#of ctrl slots       : %d",
7237 	    iou_info->iou_num_ctrl_slots);
7238 	IBTF_DPRINTF_L4("ibdm", "\tIOU flag             : 0x%x",
7239 	    iou_info->iou_flag);
7240 	IBTF_DPRINTF_L4("ibdm", "\tContrl list byte 0   : 0x%x",
7241 	    iou_info->iou_ctrl_list[0]);
7242 	IBTF_DPRINTF_L4("ibdm", "\tContrl list byte 1   : 0x%x",
7243 	    iou_info->iou_ctrl_list[1]);
7244 	IBTF_DPRINTF_L4("ibdm", "\tContrl list byte 2   : 0x%x",
7245 	    iou_info->iou_ctrl_list[2]);
7246 }
7247 
7248 
7249 void
7250 ibdm_dump_ioc_profile(ib_dm_ioc_ctrl_profile_t *ioc)
7251 {
7252 	IBTF_DPRINTF_L4("ibdm", "\t\t IOC Controller Profile");
7253 	IBTF_DPRINTF_L4("ibdm", "\t\t ----------------------");
7254 
7255 	IBTF_DPRINTF_L4("ibdm", "\tIOC Guid    : %llx", ioc->ioc_guid);
7256 	IBTF_DPRINTF_L4("ibdm", "\tVendorID    : 0x%x", ioc->ioc_vendorid);
7257 	IBTF_DPRINTF_L4("ibdm", "\tDevice Id   : 0x%x", ioc->ioc_deviceid);
7258 	IBTF_DPRINTF_L4("ibdm", "\tDevice Ver  : 0x%x", ioc->ioc_device_ver);
7259 	IBTF_DPRINTF_L4("ibdm", "\tSubsys ID   : 0x%x", ioc->ioc_subsys_id);
7260 	IBTF_DPRINTF_L4("ibdm", "\tIO class    : 0x%x", ioc->ioc_io_class);
7261 	IBTF_DPRINTF_L4("ibdm", "\tIO subclass : 0x%x", ioc->ioc_io_subclass);
7262 	IBTF_DPRINTF_L4("ibdm", "\tProtocol    : 0x%x", ioc->ioc_protocol);
7263 	IBTF_DPRINTF_L4("ibdm", "\tProtocolV   : 0x%x", ioc->ioc_protocol_ver);
7264 	IBTF_DPRINTF_L4("ibdm", "\tmsg qdepth  : %d", ioc->ioc_send_msg_qdepth);
7265 	IBTF_DPRINTF_L4("ibdm", "\trdma qdepth : %d",
7266 	    ioc->ioc_rdma_read_qdepth);
7267 	IBTF_DPRINTF_L4("ibdm", "\tsndmsg sz   : %d", ioc->ioc_send_msg_sz);
7268 	IBTF_DPRINTF_L4("ibdm", "\trdma xfersz : %d", ioc->ioc_rdma_xfer_sz);
7269 	IBTF_DPRINTF_L4("ibdm", "\topcal mask  : 0x%x",
7270 	    ioc->ioc_ctrl_opcap_mask);
7271 	IBTF_DPRINTF_L4("ibdm", "\tsrventries  : %x", ioc->ioc_service_entries);
7272 }
7273 
7274 
7275 void
7276 ibdm_dump_service_entries(ib_dm_srv_t *srv_ents)
7277 {
7278 	IBTF_DPRINTF_L4("ibdm",
7279 	    "\thandle_srventry_mad: service id : %llx", srv_ents->srv_id);
7280 
7281 	IBTF_DPRINTF_L4("ibdm", "\thandle_srventry_mad: "
7282 	    "Service Name : %s", srv_ents->srv_name);
7283 }
7284 
7285 int ibdm_allow_sweep_fabric_timestamp = 1;
7286 
7287 void
7288 ibdm_dump_sweep_fabric_timestamp(int flag)
7289 {
7290 	static hrtime_t x;
7291 	if (flag) {
7292 		if (ibdm_allow_sweep_fabric_timestamp) {
7293 			IBTF_DPRINTF_L4("ibdm", "\tTime taken to complete "
7294 			    "sweep %lld ms", ((gethrtime() - x)/ 1000000));
7295 		}
7296 		x = 0;
7297 	} else
7298 		x = gethrtime();
7299 }
7300 #endif
7301