xref: /illumos-gate/usr/src/uts/common/io/ib/mgt/ibdm/ibdm.c (revision 66582b606a8194f7f3ba5b3a3a6dca5b0d346361)

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
 */
/*
 * Copyright 2014 Nexenta Systems, Inc.  All rights reserved.
 */

/*
 * ibdm.c
 *
 * This file contains the InifiniBand Device Manager (IBDM) support functions.
 * IB nexus driver will only be the client for the IBDM module.
 *
 * IBDM registers with IBTF for HCA arrival/removal notification.
 * IBDM registers with SA access to send DM MADs to discover the IOC's behind
 * the IOU's.
 *
 * IB nexus driver registers with IBDM to find the information about the
 * HCA's and IOC's (behind the IOU) present on the IB fabric.
 */

#include <sys/sysmacros.h>
#include <sys/systm.h>
#include <sys/taskq.h>
#include <sys/ib/mgt/ibdm/ibdm_impl.h>
#include <sys/ib/mgt/ibmf/ibmf_impl.h>
#include <sys/ib/ibtl/impl/ibtl_ibnex.h>
#include <sys/modctl.h>

/* Function Prototype declarations */
static int	ibdm_free_iou_info(ibdm_dp_gidinfo_t *, ibdm_iou_info_t **);
static int	ibdm_fini(void);
static int	ibdm_init(void);
static int	ibdm_get_reachable_ports(ibdm_port_attr_t *,
			ibdm_hca_list_t *);
static ibdm_dp_gidinfo_t *ibdm_check_dgid(ib_guid_t, ib_sn_prefix_t);
static ibdm_dp_gidinfo_t *ibdm_check_dest_nodeguid(ibdm_dp_gidinfo_t *);
static boolean_t ibdm_is_cisco(ib_guid_t);
static boolean_t ibdm_is_cisco_switch(ibdm_dp_gidinfo_t *);
static void	ibdm_wait_cisco_probe_completion(ibdm_dp_gidinfo_t *);
static int	ibdm_set_classportinfo(ibdm_dp_gidinfo_t *);
static int	ibdm_send_classportinfo(ibdm_dp_gidinfo_t *);
static int	ibdm_send_iounitinfo(ibdm_dp_gidinfo_t *);
static int	ibdm_is_dev_mgt_supported(ibdm_dp_gidinfo_t *);
static int	ibdm_get_node_port_guids(ibmf_saa_handle_t, ib_lid_t,
		    ib_guid_t *, ib_guid_t *);
static int	ibdm_retry_command(ibdm_timeout_cb_args_t *);
static int	ibdm_get_diagcode(ibdm_dp_gidinfo_t *, int);
static int	ibdm_verify_mad_status(ib_mad_hdr_t *);
static int	ibdm_handle_redirection(ibmf_msg_t *,
		    ibdm_dp_gidinfo_t *, int *);
static void	ibdm_wait_probe_completion(void);
static void	ibdm_sweep_fabric(int);
static void	ibdm_probe_gid_thread(void *);
static void	ibdm_wakeup_probe_gid_cv(void);
static void	ibdm_port_attr_ibmf_init(ibdm_port_attr_t *, ib_pkey_t, int);
static int	ibdm_port_attr_ibmf_fini(ibdm_port_attr_t *, int);
static void	ibdm_update_port_attr(ibdm_port_attr_t *);
static void	ibdm_handle_hca_attach(ib_guid_t);
static void	ibdm_handle_srventry_mad(ibmf_msg_t *,
		    ibdm_dp_gidinfo_t *, int *);
static void	ibdm_ibmf_recv_cb(ibmf_handle_t, ibmf_msg_t *, void *);
static void	ibdm_recv_incoming_mad(void *);
static void	ibdm_process_incoming_mad(ibmf_handle_t, ibmf_msg_t *, void *);
static void	ibdm_ibmf_send_cb(ibmf_handle_t, ibmf_msg_t *, void *);
static void	ibdm_pkt_timeout_hdlr(void *arg);
static void	ibdm_initialize_port(ibdm_port_attr_t *);
static void	ibdm_update_port_pkeys(ibdm_port_attr_t *port);
static void	ibdm_handle_diagcode(ibmf_msg_t *, ibdm_dp_gidinfo_t *, int *);
static void	ibdm_probe_gid(ibdm_dp_gidinfo_t *);
static void	ibdm_alloc_send_buffers(ibmf_msg_t *);
static void	ibdm_free_send_buffers(ibmf_msg_t *);
static void	ibdm_handle_hca_detach(ib_guid_t);
static void	ibdm_handle_port_change_event(ibt_async_event_t *);
static int	ibdm_fini_port(ibdm_port_attr_t *);
static int	ibdm_uninit_hca(ibdm_hca_list_t *);
static void	ibdm_handle_setclassportinfo(ibmf_handle_t, ibmf_msg_t *,
		    ibdm_dp_gidinfo_t *, int *);
static void	ibdm_handle_iounitinfo(ibmf_handle_t,
		    ibmf_msg_t *, ibdm_dp_gidinfo_t *, int *);
static void	ibdm_handle_ioc_profile(ibmf_handle_t,
		    ibmf_msg_t *, ibdm_dp_gidinfo_t *, int *);
static void	ibdm_event_hdlr(void *, ibt_hca_hdl_t,
		    ibt_async_code_t, ibt_async_event_t *);
static void	ibdm_handle_classportinfo(ibmf_handle_t,
		    ibmf_msg_t *, ibdm_dp_gidinfo_t *, int *);
static void	ibdm_update_ioc_port_gidlist(ibdm_ioc_info_t *,
		    ibdm_dp_gidinfo_t *);

static ibdm_hca_list_t		*ibdm_dup_hca_attr(ibdm_hca_list_t *);
static ibdm_ioc_info_t		*ibdm_dup_ioc_info(ibdm_ioc_info_t *,
				    ibdm_dp_gidinfo_t *gid_list);
static void			ibdm_probe_ioc(ib_guid_t, ib_guid_t, int);
static ibdm_ioc_info_t		*ibdm_is_ioc_present(ib_guid_t,
				    ibdm_dp_gidinfo_t *, int *);
static ibdm_port_attr_t		*ibdm_get_port_attr(ibt_async_event_t *,
				    ibdm_hca_list_t **);
static sa_node_record_t		*ibdm_get_node_records(ibmf_saa_handle_t,
				    size_t *, ib_guid_t);
static int			ibdm_get_node_record_by_port(ibmf_saa_handle_t,
				    ib_guid_t, sa_node_record_t **, size_t *);
static sa_portinfo_record_t	*ibdm_get_portinfo(ibmf_saa_handle_t, size_t *,
				    ib_lid_t);
static ibdm_dp_gidinfo_t	*ibdm_create_gid_info(ibdm_port_attr_t *,
				    ib_gid_t, ib_gid_t);
static ibdm_dp_gidinfo_t	*ibdm_find_gid(ib_guid_t, ib_guid_t);
static int	ibdm_send_ioc_profile(ibdm_dp_gidinfo_t *, uint8_t);
static ibdm_ioc_info_t	*ibdm_update_ioc_gidlist(ibdm_dp_gidinfo_t *, int);
static void	ibdm_saa_event_cb(ibmf_saa_handle_t, ibmf_saa_subnet_event_t,
		    ibmf_saa_event_details_t *, void *);
static void	ibdm_reprobe_update_port_srv(ibdm_ioc_info_t *,
    ibdm_dp_gidinfo_t *);
static ibdm_dp_gidinfo_t *ibdm_handle_gid_rm(ibdm_dp_gidinfo_t *);
static void ibdm_rmfrom_glgid_list(ibdm_dp_gidinfo_t *,
    ibdm_dp_gidinfo_t *);
static void ibdm_addto_gidlist(ibdm_gid_t **, ibdm_gid_t *);
static void ibdm_free_gid_list(ibdm_gid_t *);
static void ibdm_rescan_gidlist(ib_guid_t *ioc_guid);
static void ibdm_notify_newgid_iocs(ibdm_dp_gidinfo_t *);
static void ibdm_saa_event_taskq(void *);
static void ibdm_free_saa_event_arg(ibdm_saa_event_arg_t *);
static void ibdm_get_next_port(ibdm_hca_list_t **,
    ibdm_port_attr_t **, int);
static void ibdm_add_to_gl_gid(ibdm_dp_gidinfo_t *,
    ibdm_dp_gidinfo_t *);
static void ibdm_addto_glhcalist(ibdm_dp_gidinfo_t *,
    ibdm_hca_list_t *);
static void ibdm_delete_glhca_list(ibdm_dp_gidinfo_t *);
static void ibdm_saa_handle_new_gid(void *);
static void ibdm_reset_all_dgids(ibmf_saa_handle_t);
static void ibdm_reset_gidinfo(ibdm_dp_gidinfo_t *);
static void ibdm_delete_gidinfo(ibdm_dp_gidinfo_t *);
static void ibdm_fill_srv_attr_mod(ib_mad_hdr_t *, ibdm_timeout_cb_args_t *);
static void ibdm_bump_transactionID(ibdm_dp_gidinfo_t *);
static ibdm_ioc_info_t	*ibdm_handle_prev_iou();
static int ibdm_serv_cmp(ibdm_srvents_info_t *, ibdm_srvents_info_t *,
    int);
static ibdm_ioc_info_t *ibdm_get_ioc_info_with_gid(ib_guid_t,
    ibdm_dp_gidinfo_t **);

int	ibdm_dft_timeout	= IBDM_DFT_TIMEOUT;
int	ibdm_dft_retry_cnt	= IBDM_DFT_NRETRIES;
#ifdef DEBUG
int	ibdm_ignore_saa_event = 0;
#endif
int	ibdm_enumerate_iocs = 0;

/* Modload support */
static struct modlmisc ibdm_modlmisc	= {
	&mod_miscops,
	"InfiniBand Device Manager"
};

struct modlinkage ibdm_modlinkage = {
	MODREV_1,
	(void *)&ibdm_modlmisc,
	NULL
};

static ibt_clnt_modinfo_t ibdm_ibt_modinfo = {
	IBTI_V_CURR,
	IBT_DM,
	ibdm_event_hdlr,
	NULL,
	"ibdm"
};

/* Global variables */
ibdm_t	ibdm;
int	ibdm_taskq_enable = IBDM_ENABLE_TASKQ_HANDLING;
char	*ibdm_string = "ibdm";

_NOTE(SCHEME_PROTECTS_DATA("Serialized access by cv",
    ibdm.ibdm_dp_gidlist_head))

/*
 * _init
 *	Loadable module init, called before any other module.
 *	Initialize mutex
 *	Register with IBTF
 */
int
_init(void)
{
	int		err;

	IBTF_DPRINTF_L4("ibdm", "\t_init: addr of ibdm %p", &ibdm);

	if ((err = ibdm_init()) != IBDM_SUCCESS) {
		IBTF_DPRINTF_L2("ibdm", "_init: ibdm_init failed 0x%x", err);
		(void) ibdm_fini();
		return (DDI_FAILURE);
	}

	if ((err = mod_install(&ibdm_modlinkage)) != 0) {
		IBTF_DPRINTF_L2("ibdm", "_init: mod_install failed 0x%x", err);
		(void) ibdm_fini();
	}
	return (err);
}


int
_fini(void)
{
	int err;

	if ((err = ibdm_fini()) != IBDM_SUCCESS) {
		IBTF_DPRINTF_L2("ibdm", "_fini: ibdm_fini failed 0x%x", err);
		(void) ibdm_init();
		return (EBUSY);
	}

	if ((err = mod_remove(&ibdm_modlinkage)) != 0) {
		IBTF_DPRINTF_L2("ibdm", "_fini: mod_remove failed 0x%x", err);
		(void) ibdm_init();
	}
	return (err);
}


int
_info(struct modinfo *modinfop)
{
	return (mod_info(&ibdm_modlinkage, modinfop));
}


/*
 * ibdm_init():
 * 	Register with IBTF
 *	Allocate memory for the HCAs
 *	Allocate minor-nodes for the HCAs
 */
static int
ibdm_init(void)
{
	int			i, hca_count;
	ib_guid_t		*hca_guids;
	ibt_status_t		status;

	IBTF_DPRINTF_L4("ibdm", "\tibdm_init:");
	if (!(ibdm.ibdm_state & IBDM_LOCKS_ALLOCED)) {
		mutex_init(&ibdm.ibdm_mutex, NULL, MUTEX_DEFAULT, NULL);
		mutex_init(&ibdm.ibdm_hl_mutex, NULL, MUTEX_DEFAULT, NULL);
		mutex_init(&ibdm.ibdm_ibnex_mutex, NULL, MUTEX_DEFAULT, NULL);
		cv_init(&ibdm.ibdm_port_settle_cv, NULL, CV_DRIVER, NULL);
		mutex_enter(&ibdm.ibdm_mutex);
		ibdm.ibdm_state |= IBDM_LOCKS_ALLOCED;
	}

	if (!(ibdm.ibdm_state & IBDM_IBT_ATTACHED)) {
		if ((status = ibt_attach(&ibdm_ibt_modinfo, NULL, NULL,
		    (void *)&ibdm.ibdm_ibt_clnt_hdl)) != IBT_SUCCESS) {
			IBTF_DPRINTF_L2("ibdm", "ibdm_init: ibt_attach "
			    "failed %x", status);
			mutex_exit(&ibdm.ibdm_mutex);
			return (IBDM_FAILURE);
		}

		ibdm.ibdm_state |= IBDM_IBT_ATTACHED;
		mutex_exit(&ibdm.ibdm_mutex);
	}


	if (!(ibdm.ibdm_state & IBDM_HCA_ATTACHED)) {
		hca_count = ibt_get_hca_list(&hca_guids);
		IBTF_DPRINTF_L4("ibdm", "ibdm_init: num_hcas = %d", hca_count);
		for (i = 0; i < hca_count; i++)
			(void) ibdm_handle_hca_attach(hca_guids[i]);
		if (hca_count)
			ibt_free_hca_list(hca_guids, hca_count);

		mutex_enter(&ibdm.ibdm_mutex);
		ibdm.ibdm_state |= IBDM_HCA_ATTACHED;
		mutex_exit(&ibdm.ibdm_mutex);
	}

	if (!(ibdm.ibdm_state & IBDM_CVS_ALLOCED)) {
		cv_init(&ibdm.ibdm_probe_cv, NULL, CV_DRIVER, NULL);
		cv_init(&ibdm.ibdm_busy_cv, NULL, CV_DRIVER, NULL);
		mutex_enter(&ibdm.ibdm_mutex);
		ibdm.ibdm_state |= IBDM_CVS_ALLOCED;
		mutex_exit(&ibdm.ibdm_mutex);
	}
	return (IBDM_SUCCESS);
}


static int
ibdm_free_iou_info(ibdm_dp_gidinfo_t *gid_info, ibdm_iou_info_t **ioup)
{
	int			ii, k, niocs;
	size_t			size;
	ibdm_gid_t		*delete, *head;
	timeout_id_t		timeout_id;
	ibdm_ioc_info_t		*ioc;
	ibdm_iou_info_t		*gl_iou = *ioup;

	ASSERT(mutex_owned(&gid_info->gl_mutex));
	if (gl_iou == NULL) {
		IBTF_DPRINTF_L4("ibdm", "\tibdm_free_iou_info: No IOU");
		return (0);
	}

	niocs = gl_iou->iou_info.iou_num_ctrl_slots;
	IBTF_DPRINTF_L4("ibdm", "\tfree_iou_info: gid_info = %p, niocs %d",
	    gid_info, niocs);

	for (ii = 0; ii < niocs; ii++) {
		ioc = (ibdm_ioc_info_t *)&gl_iou->iou_ioc_info[ii];

		/* handle the case where an ioc_timeout_id is scheduled */
		if (ioc->ioc_timeout_id) {
			timeout_id = ioc->ioc_timeout_id;
			ioc->ioc_timeout_id = 0;
			mutex_exit(&gid_info->gl_mutex);
			IBTF_DPRINTF_L5("ibdm", "free_iou_info: "
			    "ioc_timeout_id = 0x%x", timeout_id);
			if (untimeout(timeout_id) == -1) {
				IBTF_DPRINTF_L2("ibdm", "free_iou_info: "
				    "untimeout ioc_timeout_id failed");
				mutex_enter(&gid_info->gl_mutex);
				return (-1);
			}
			mutex_enter(&gid_info->gl_mutex);
		}

		/* handle the case where an ioc_dc_timeout_id is scheduled */
		if (ioc->ioc_dc_timeout_id) {
			timeout_id = ioc->ioc_dc_timeout_id;
			ioc->ioc_dc_timeout_id = 0;
			mutex_exit(&gid_info->gl_mutex);
			IBTF_DPRINTF_L5("ibdm", "free_iou_info: "
			    "ioc_dc_timeout_id = 0x%x", timeout_id);
			if (untimeout(timeout_id) == -1) {
				IBTF_DPRINTF_L2("ibdm", "free_iou_info: "
				    "untimeout ioc_dc_timeout_id failed");
				mutex_enter(&gid_info->gl_mutex);
				return (-1);
			}
			mutex_enter(&gid_info->gl_mutex);
		}

		/* handle the case where serv[k].se_timeout_id is scheduled */
		for (k = 0; k < ioc->ioc_profile.ioc_service_entries; k++) {
			if (ioc->ioc_serv[k].se_timeout_id) {
				timeout_id = ioc->ioc_serv[k].se_timeout_id;
				ioc->ioc_serv[k].se_timeout_id = 0;
				mutex_exit(&gid_info->gl_mutex);
				IBTF_DPRINTF_L5("ibdm", "free_iou_info: "
				    "ioc->ioc_serv[%d].se_timeout_id = 0x%x",
				    k, timeout_id);
				if (untimeout(timeout_id) == -1) {
					IBTF_DPRINTF_L2("ibdm", "free_iou_info:"
					    " untimeout se_timeout_id failed");
					mutex_enter(&gid_info->gl_mutex);
					return (-1);
				}
				mutex_enter(&gid_info->gl_mutex);
			}
		}

		/* delete GID list in IOC */
		head = ioc->ioc_gid_list;
		while (head) {
			IBTF_DPRINTF_L4("ibdm", "\tibdm_free_iou_info: "
			    "Deleting gid_list struct %p", head);
			delete = head;
			head = head->gid_next;
			kmem_free(delete, sizeof (ibdm_gid_t));
		}
		ioc->ioc_gid_list = NULL;

		/* delete ioc_serv */
		size = ioc->ioc_profile.ioc_service_entries *
		    sizeof (ibdm_srvents_info_t);
		if (ioc->ioc_serv && size) {
			kmem_free(ioc->ioc_serv, size);
			ioc->ioc_serv = NULL;
		}
	}
	/*
	 * Clear the IBDM_CISCO_PROBE_DONE flag to get the IO Unit information
	 * via the switch during the probe process.
	 */
	gid_info->gl_flag &= ~IBDM_CISCO_PROBE_DONE;

	IBTF_DPRINTF_L4("ibdm", "\tibdm_free_iou_info: deleting IOU & IOC");
	size = sizeof (ibdm_iou_info_t) + niocs * sizeof (ibdm_ioc_info_t);
	kmem_free(gl_iou, size);
	*ioup = NULL;
	return (0);
}


/*
 * ibdm_fini():
 * 	Un-register with IBTF
 *	De allocate memory for the GID info
 */
static int
ibdm_fini()
{
	int			ii;
	ibdm_hca_list_t		*hca_list, *temp;
	ibdm_dp_gidinfo_t	*gid_info, *tmp;
	ibdm_gid_t		*head, *delete;

	IBTF_DPRINTF_L4("ibdm", "\tibdm_fini");

	mutex_enter(&ibdm.ibdm_hl_mutex);
	if (ibdm.ibdm_state & IBDM_IBT_ATTACHED) {
		if (ibt_detach(ibdm.ibdm_ibt_clnt_hdl) != IBT_SUCCESS) {
			IBTF_DPRINTF_L2("ibdm", "\t_fini: ibt_detach failed");
			mutex_exit(&ibdm.ibdm_hl_mutex);
			return (IBDM_FAILURE);
		}
		ibdm.ibdm_state &= ~IBDM_IBT_ATTACHED;
		ibdm.ibdm_ibt_clnt_hdl = NULL;
	}

	hca_list = ibdm.ibdm_hca_list_head;
	IBTF_DPRINTF_L4("ibdm", "\tibdm_fini: nhcas %d", ibdm.ibdm_hca_count);
	for (ii = 0; ii < ibdm.ibdm_hca_count; ii++) {
		temp = hca_list;
		hca_list = hca_list->hl_next;
		IBTF_DPRINTF_L4("ibdm", "\tibdm_fini: hca %p", temp);
		if (ibdm_uninit_hca(temp) != IBDM_SUCCESS) {
			IBTF_DPRINTF_L2("ibdm", "\tibdm_fini: "
			    "uninit_hca %p failed", temp);
			mutex_exit(&ibdm.ibdm_hl_mutex);
			return (IBDM_FAILURE);
		}
	}
	mutex_exit(&ibdm.ibdm_hl_mutex);

	mutex_enter(&ibdm.ibdm_mutex);
	if (ibdm.ibdm_state & IBDM_HCA_ATTACHED)
		ibdm.ibdm_state &= ~IBDM_HCA_ATTACHED;

	gid_info = ibdm.ibdm_dp_gidlist_head;
	while (gid_info) {
		mutex_enter(&gid_info->gl_mutex);
		(void) ibdm_free_iou_info(gid_info, &gid_info->gl_iou);
		mutex_exit(&gid_info->gl_mutex);
		ibdm_delete_glhca_list(gid_info);

		tmp = gid_info;
		gid_info = gid_info->gl_next;
		mutex_destroy(&tmp->gl_mutex);
		head = tmp->gl_gid;
		while (head) {
			IBTF_DPRINTF_L4("ibdm",
			    "\tibdm_fini: Deleting gid structs");
			delete = head;
			head = head->gid_next;
			kmem_free(delete, sizeof (ibdm_gid_t));
		}
		kmem_free(tmp, sizeof (ibdm_dp_gidinfo_t));
	}
	mutex_exit(&ibdm.ibdm_mutex);

	if (ibdm.ibdm_state & IBDM_LOCKS_ALLOCED) {
		ibdm.ibdm_state &= ~IBDM_LOCKS_ALLOCED;
		mutex_destroy(&ibdm.ibdm_mutex);
		mutex_destroy(&ibdm.ibdm_hl_mutex);
		mutex_destroy(&ibdm.ibdm_ibnex_mutex);
		cv_destroy(&ibdm.ibdm_port_settle_cv);
	}
	if (ibdm.ibdm_state & IBDM_CVS_ALLOCED) {
		ibdm.ibdm_state &= ~IBDM_CVS_ALLOCED;
		cv_destroy(&ibdm.ibdm_probe_cv);
		cv_destroy(&ibdm.ibdm_busy_cv);
	}
	return (IBDM_SUCCESS);
}


/*
 * ibdm_event_hdlr()
 *
 *	IBDM registers  this asynchronous event handler at the time of
 *	ibt_attach. IBDM support the following async events. For other
 *	event, simply returns success.
 *	IBT_HCA_ATTACH_EVENT:
 *		Retrieves the  information about all the port that are
 *		present on this HCA,  allocates  the  port  attributes
 *		structure  and calls IB  nexus  callback  routine with
 *		the port attributes structure as an input argument.
 *	IBT_HCA_DETACH_EVENT:
 *		Retrieves the information about all the ports that are
 *		present on  this HCA and  calls IB nexus callback with
 *		port guid as an argument
 *	IBT_EVENT_PORT_UP:
 *		Register with IBMF and SA access
 *		Setup IBMF receive callback routine
 *	IBT_EVENT_PORT_DOWN:
 *		Un-Register with IBMF and SA access
 *		Teardown IBMF receive callback routine
 */
/*ARGSUSED*/
static void
ibdm_event_hdlr(void *clnt_hdl,
    ibt_hca_hdl_t hca_hdl, ibt_async_code_t code, ibt_async_event_t *event)
{
	ibdm_hca_list_t		*hca_list;
	ibdm_port_attr_t	*port;
	ibmf_saa_handle_t	port_sa_hdl;

	IBTF_DPRINTF_L4("ibdm", "\tevent_hdlr: async code 0x%x", code);

	switch (code) {
	case IBT_HCA_ATTACH_EVENT:	/* New HCA registered with IBTF */
		ibdm_handle_hca_attach(event->ev_hca_guid);
		break;

	case IBT_HCA_DETACH_EVENT:	/* HCA unregistered with IBTF */
		ibdm_handle_hca_detach(event->ev_hca_guid);
		mutex_enter(&ibdm.ibdm_ibnex_mutex);
		if (ibdm.ibdm_ibnex_callback != NULL) {
			(*ibdm.ibdm_ibnex_callback)((void *)
			    &event->ev_hca_guid, IBDM_EVENT_HCA_REMOVED);
		}
		mutex_exit(&ibdm.ibdm_ibnex_mutex);
		break;

	case IBT_EVENT_PORT_UP:
		IBTF_DPRINTF_L4("ibdm", "\tevent_hdlr: PORT_UP");
		mutex_enter(&ibdm.ibdm_hl_mutex);
		port = ibdm_get_port_attr(event, &hca_list);
		if (port == NULL) {
			IBTF_DPRINTF_L2("ibdm",
			    "\tevent_hdlr: HCA not present");
			mutex_exit(&ibdm.ibdm_hl_mutex);
			break;
		}
		ibdm_initialize_port(port);
		hca_list->hl_nports_active++;
		cv_broadcast(&ibdm.ibdm_port_settle_cv);
		mutex_exit(&ibdm.ibdm_hl_mutex);

		/* Inform IB nexus driver */
		mutex_enter(&ibdm.ibdm_ibnex_mutex);
		if (ibdm.ibdm_ibnex_callback != NULL) {
			(*ibdm.ibdm_ibnex_callback)((void *)
			    &event->ev_hca_guid, IBDM_EVENT_PORT_UP);
		}
		mutex_exit(&ibdm.ibdm_ibnex_mutex);
		break;

	case IBT_ERROR_PORT_DOWN:
		IBTF_DPRINTF_L4("ibdm", "\tevent_hdlr: PORT_DOWN");
		mutex_enter(&ibdm.ibdm_hl_mutex);
		port = ibdm_get_port_attr(event, &hca_list);
		if (port == NULL) {
			IBTF_DPRINTF_L2("ibdm",
			    "\tevent_hdlr: HCA not present");
			mutex_exit(&ibdm.ibdm_hl_mutex);
			break;
		}
		hca_list->hl_nports_active--;
		port_sa_hdl = port->pa_sa_hdl;
		(void) ibdm_fini_port(port);
		port->pa_state = IBT_PORT_DOWN;
		cv_broadcast(&ibdm.ibdm_port_settle_cv);
		mutex_exit(&ibdm.ibdm_hl_mutex);
		ibdm_reset_all_dgids(port_sa_hdl);
		break;

	case IBT_PORT_CHANGE_EVENT:
		IBTF_DPRINTF_L4("ibdm", "\tevent_hdlr: PORT_CHANGE");
		if (event->ev_port_flags & IBT_PORT_CHANGE_PKEY)
			ibdm_handle_port_change_event(event);
		break;

	default:		/* Ignore all other events/errors */
		break;
	}
}

static void
ibdm_handle_port_change_event(ibt_async_event_t *event)
{
	ibdm_port_attr_t	*port;
	ibdm_hca_list_t		*hca_list;

	IBTF_DPRINTF_L2("ibdm", "\tibdm_handle_port_change_event:"
	    " HCA guid  %llx", event->ev_hca_guid);
	mutex_enter(&ibdm.ibdm_hl_mutex);
	port = ibdm_get_port_attr(event, &hca_list);
	if (port == NULL) {
		IBTF_DPRINTF_L2("ibdm", "\tevent_hdlr: HCA not present");
		mutex_exit(&ibdm.ibdm_hl_mutex);
		return;
	}
	ibdm_update_port_pkeys(port);
	cv_broadcast(&ibdm.ibdm_port_settle_cv);
	mutex_exit(&ibdm.ibdm_hl_mutex);

	/* Inform IB nexus driver */
	mutex_enter(&ibdm.ibdm_ibnex_mutex);
	if (ibdm.ibdm_ibnex_callback != NULL) {
		(*ibdm.ibdm_ibnex_callback)((void *)
		    &event->ev_hca_guid, IBDM_EVENT_PORT_PKEY_CHANGE);
	}
	mutex_exit(&ibdm.ibdm_ibnex_mutex);
}

/*
 * ibdm_update_port_pkeys()
 *	Update the pkey table
 *	Update the port attributes
 */
static void
ibdm_update_port_pkeys(ibdm_port_attr_t *port)
{
	uint_t				nports, size;
	uint_t				pkey_idx, opkey_idx;
	uint16_t			npkeys;
	ibt_hca_portinfo_t		*pinfop;
	ib_pkey_t			pkey;
	ibdm_pkey_tbl_t			*pkey_tbl;
	ibdm_port_attr_t		newport;

	IBTF_DPRINTF_L4("ibdm", "\tupdate_port_pkeys:");
	ASSERT(MUTEX_HELD(&ibdm.ibdm_hl_mutex));

	/* Check whether the port is active */
	if (ibt_get_port_state(port->pa_hca_hdl, port->pa_port_num, NULL,
	    NULL) != IBT_SUCCESS)
		return;

	if (ibt_query_hca_ports(port->pa_hca_hdl, port->pa_port_num,
	    &pinfop, &nports, &size) != IBT_SUCCESS) {
		/* This should not occur */
		port->pa_npkeys = 0;
		port->pa_pkey_tbl = NULL;
		return;
	}

	npkeys = pinfop->p_pkey_tbl_sz;
	pkey_tbl = kmem_zalloc(npkeys * sizeof (ibdm_pkey_tbl_t), KM_SLEEP);
	newport.pa_pkey_tbl = pkey_tbl;
	newport.pa_ibmf_hdl = port->pa_ibmf_hdl;

	for (pkey_idx = 0; pkey_idx < npkeys; pkey_idx++) {
		pkey = pkey_tbl[pkey_idx].pt_pkey =
		    pinfop->p_pkey_tbl[pkey_idx];
		/*
		 * Is this pkey present in the current table ?
		 */
		for (opkey_idx = 0; opkey_idx < port->pa_npkeys; opkey_idx++) {
			if (pkey == port->pa_pkey_tbl[opkey_idx].pt_pkey) {
				pkey_tbl[pkey_idx].pt_qp_hdl =
				    port->pa_pkey_tbl[opkey_idx].pt_qp_hdl;
				port->pa_pkey_tbl[opkey_idx].pt_qp_hdl = NULL;
				break;
			}
		}

		if (opkey_idx == port->pa_npkeys) {
			pkey = pkey_tbl[pkey_idx].pt_pkey;
			if (IBDM_INVALID_PKEY(pkey)) {
				pkey_tbl[pkey_idx].pt_qp_hdl = NULL;
				continue;
			}
			ibdm_port_attr_ibmf_init(&newport, pkey, pkey_idx);
		}
	}

	for (opkey_idx = 0; opkey_idx < port->pa_npkeys; opkey_idx++) {
		if (port->pa_pkey_tbl[opkey_idx].pt_qp_hdl != NULL) {
			if (ibdm_port_attr_ibmf_fini(port, opkey_idx) !=
			    IBDM_SUCCESS) {
				IBTF_DPRINTF_L2("ibdm", "\tupdate_port_pkeys: "
				    "ibdm_port_attr_ibmf_fini failed for "
				    "port pkey 0x%x",
				    port->pa_pkey_tbl[opkey_idx].pt_pkey);
			}
		}
	}

	if (port->pa_pkey_tbl != NULL) {
		kmem_free(port->pa_pkey_tbl,
		    port->pa_npkeys * sizeof (ibdm_pkey_tbl_t));
	}

	port->pa_npkeys = npkeys;
	port->pa_pkey_tbl = pkey_tbl;
	port->pa_sn_prefix = pinfop->p_sgid_tbl[0].gid_prefix;
	port->pa_state = pinfop->p_linkstate;
	ibt_free_portinfo(pinfop, size);
}

/*
 * ibdm_initialize_port()
 *	Register with IBMF
 *	Register with SA access
 *	Register a receive callback routine with IBMF. IBMF invokes
 *	this routine whenever a MAD arrives at this port.
 *	Update the port attributes
 */
static void
ibdm_initialize_port(ibdm_port_attr_t *port)
{
	int				ii;
	uint_t				nports, size;
	uint_t				pkey_idx;
	ib_pkey_t			pkey;
	ibt_hca_portinfo_t		*pinfop;
	ibmf_register_info_t		ibmf_reg;
	ibmf_saa_subnet_event_args_t	event_args;

	IBTF_DPRINTF_L4("ibdm", "\tinitialize_port:");
	ASSERT(MUTEX_HELD(&ibdm.ibdm_hl_mutex));

	/* Check whether the port is active */
	if (ibt_get_port_state(port->pa_hca_hdl, port->pa_port_num, NULL,
	    NULL) != IBT_SUCCESS)
		return;

	if (port->pa_sa_hdl != NULL || port->pa_pkey_tbl != NULL)
		return;

	if (ibt_query_hca_ports(port->pa_hca_hdl, port->pa_port_num,
	    &pinfop, &nports, &size) != IBT_SUCCESS) {
		/* This should not occur */
		port->pa_npkeys		= 0;
		port->pa_pkey_tbl	= NULL;
		return;
	}
	port->pa_sn_prefix = pinfop->p_sgid_tbl[0].gid_prefix;

	port->pa_state		= pinfop->p_linkstate;
	port->pa_npkeys		= pinfop->p_pkey_tbl_sz;
	port->pa_pkey_tbl	= (ibdm_pkey_tbl_t *)kmem_zalloc(
	    port->pa_npkeys * sizeof (ibdm_pkey_tbl_t), KM_SLEEP);

	for (pkey_idx = 0; pkey_idx < port->pa_npkeys; pkey_idx++)
		port->pa_pkey_tbl[pkey_idx].pt_pkey =
		    pinfop->p_pkey_tbl[pkey_idx];

	ibt_free_portinfo(pinfop, size);

	if (ibdm_enumerate_iocs) {
		event_args.is_event_callback = ibdm_saa_event_cb;
		event_args.is_event_callback_arg = port;
		if (ibmf_sa_session_open(port->pa_port_guid, 0, &event_args,
		    IBMF_VERSION, 0, &port->pa_sa_hdl) != IBMF_SUCCESS) {
			IBTF_DPRINTF_L2("ibdm", "\tinitialize_port: "
			    "sa access registration failed");
			(void) ibdm_fini_port(port);
			return;
		}

		ibmf_reg.ir_ci_guid		= port->pa_hca_guid;
		ibmf_reg.ir_port_num		= port->pa_port_num;
		ibmf_reg.ir_client_class	= DEV_MGT_MANAGER;

		if (ibmf_register(&ibmf_reg, IBMF_VERSION, 0, NULL, NULL,
		    &port->pa_ibmf_hdl, &port->pa_ibmf_caps) != IBMF_SUCCESS) {
			IBTF_DPRINTF_L2("ibdm", "\tinitialize_port: "
			    "IBMF registration failed");
			(void) ibdm_fini_port(port);
			return;
		}

		if (ibmf_setup_async_cb(port->pa_ibmf_hdl,
		    IBMF_QP_HANDLE_DEFAULT,
		    ibdm_ibmf_recv_cb, 0, 0) != IBMF_SUCCESS) {
			IBTF_DPRINTF_L2("ibdm", "\tinitialize_port: "
			    "IBMF setup recv cb failed");
			(void) ibdm_fini_port(port);
			return;
		}
	} else {
		port->pa_sa_hdl = NULL;
		port->pa_ibmf_hdl = NULL;
	}

	for (ii = 0; ii < port->pa_npkeys; ii++) {
		pkey = port->pa_pkey_tbl[ii].pt_pkey;
		if (IBDM_INVALID_PKEY(pkey)) {
			port->pa_pkey_tbl[ii].pt_qp_hdl = NULL;
			continue;
		}
		ibdm_port_attr_ibmf_init(port, pkey, ii);
	}
}


/*
 * ibdm_port_attr_ibmf_init:
 *	With IBMF - Alloc QP Handle and Setup Async callback
 */
static void
ibdm_port_attr_ibmf_init(ibdm_port_attr_t *port, ib_pkey_t pkey, int ii)
{
	int ret;

	if (ibdm_enumerate_iocs == 0) {
		port->pa_pkey_tbl[ii].pt_qp_hdl = NULL;
		return;
	}

	if ((ret = ibmf_alloc_qp(port->pa_ibmf_hdl, pkey, IB_GSI_QKEY,
	    IBMF_ALT_QP_MAD_NO_RMPP, &port->pa_pkey_tbl[ii].pt_qp_hdl)) !=
	    IBMF_SUCCESS) {
		IBTF_DPRINTF_L2("ibdm", "\tport_attr_ibmf_init: "
		    "IBMF failed to alloc qp %d", ret);
		port->pa_pkey_tbl[ii].pt_qp_hdl = NULL;
		return;
	}

	IBTF_DPRINTF_L4("ibdm", "\tport_attr_ibmf_init: QP handle is %p",
	    port->pa_ibmf_hdl);

	if ((ret = ibmf_setup_async_cb(port->pa_ibmf_hdl,
	    port->pa_pkey_tbl[ii].pt_qp_hdl, ibdm_ibmf_recv_cb, 0, 0)) !=
	    IBMF_SUCCESS) {
		IBTF_DPRINTF_L2("ibdm", "\tport_attr_ibmf_init: "
		    "IBMF setup recv cb failed %d", ret);
		(void) ibmf_free_qp(port->pa_ibmf_hdl,
		    &port->pa_pkey_tbl[ii].pt_qp_hdl, 0);
		port->pa_pkey_tbl[ii].pt_qp_hdl = NULL;
	}
}


/*
 * ibdm_get_port_attr()
 *	Get port attributes from HCA guid and port number
 *	Return pointer to ibdm_port_attr_t on Success
 *	and NULL on failure
 */
static ibdm_port_attr_t *
ibdm_get_port_attr(ibt_async_event_t *event, ibdm_hca_list_t **retval)
{
	ibdm_hca_list_t		*hca_list;
	ibdm_port_attr_t	*port_attr;
	int			ii;

	IBTF_DPRINTF_L4("ibdm", "\tget_port_attr: port# %d", event->ev_port);
	ASSERT(MUTEX_HELD(&ibdm.ibdm_hl_mutex));
	hca_list = ibdm.ibdm_hca_list_head;
	while (hca_list) {
		if (hca_list->hl_hca_guid == event->ev_hca_guid) {
			for (ii = 0; ii < hca_list->hl_nports; ii++) {
				port_attr = &hca_list->hl_port_attr[ii];
				if (port_attr->pa_port_num == event->ev_port) {
					*retval = hca_list;
					return (port_attr);
				}
			}
		}
		hca_list = hca_list->hl_next;
	}
	return (NULL);
}


/*
 * ibdm_update_port_attr()
 *	Update the port attributes
 */
static void
ibdm_update_port_attr(ibdm_port_attr_t *port)
{
	uint_t			nports, size;
	uint_t			pkey_idx;
	ibt_hca_portinfo_t	*portinfop;

	IBTF_DPRINTF_L4("ibdm", "\tupdate_port_attr: Begin");
	if (ibt_query_hca_ports(port->pa_hca_hdl,
	    port->pa_port_num, &portinfop, &nports, &size) != IBT_SUCCESS) {
		/* This should not occur */
		port->pa_npkeys		= 0;
		port->pa_pkey_tbl	= NULL;
		return;
	}
	port->pa_sn_prefix = portinfop->p_sgid_tbl[0].gid_prefix;

	port->pa_state		= portinfop->p_linkstate;

	/*
	 * PKey information in portinfo valid only if port is
	 * ACTIVE. Bail out if not.
	 */
	if (port->pa_state != IBT_PORT_ACTIVE) {
		port->pa_npkeys		= 0;
		port->pa_pkey_tbl	= NULL;
		ibt_free_portinfo(portinfop, size);
		return;
	}

	port->pa_npkeys		= portinfop->p_pkey_tbl_sz;
	port->pa_pkey_tbl	= (ibdm_pkey_tbl_t *)kmem_zalloc(
	    port->pa_npkeys * sizeof (ibdm_pkey_tbl_t), KM_SLEEP);

	for (pkey_idx = 0; pkey_idx < port->pa_npkeys; pkey_idx++) {
		port->pa_pkey_tbl[pkey_idx].pt_pkey =
		    portinfop->p_pkey_tbl[pkey_idx];
	}
	ibt_free_portinfo(portinfop, size);
}


/*
 * ibdm_handle_hca_attach()
 */
static void
ibdm_handle_hca_attach(ib_guid_t hca_guid)
{
	uint_t			size;
	uint_t			ii, nports;
	ibt_status_t		status;
	ibt_hca_hdl_t		hca_hdl;
	ibt_hca_attr_t		*hca_attr;
	ibdm_hca_list_t		*hca_list, *temp;
	ibdm_port_attr_t	*port_attr;
	ibt_hca_portinfo_t	*portinfop;

	IBTF_DPRINTF_L4("ibdm",
	    "\thandle_hca_attach: hca_guid = 0x%llX", hca_guid);

	/* open the HCA first */
	if ((status = ibt_open_hca(ibdm.ibdm_ibt_clnt_hdl, hca_guid,
	    &hca_hdl)) != IBT_SUCCESS) {
		IBTF_DPRINTF_L2("ibdm", "\thandle_hca_attach: "
		    "open_hca failed, status 0x%x", status);
		return;
	}

	hca_attr = (ibt_hca_attr_t *)
	    kmem_alloc(sizeof (ibt_hca_attr_t), KM_SLEEP);
	/* ibt_query_hca always returns IBT_SUCCESS */
	(void) ibt_query_hca(hca_hdl, hca_attr);

	IBTF_DPRINTF_L4("ibdm", "\tvid: 0x%x, pid: 0x%x, ver: 0x%x,"
	    " #ports: %d", hca_attr->hca_vendor_id, hca_attr->hca_device_id,
	    hca_attr->hca_version_id, hca_attr->hca_nports);

	if ((status = ibt_query_hca_ports(hca_hdl, 0, &portinfop, &nports,
	    &size)) != IBT_SUCCESS) {
		IBTF_DPRINTF_L2("ibdm", "\thandle_hca_attach: "
		    "ibt_query_hca_ports failed, status 0x%x", status);
		kmem_free(hca_attr, sizeof (ibt_hca_attr_t));
		(void) ibt_close_hca(hca_hdl);
		return;
	}
	hca_list = (ibdm_hca_list_t *)
	    kmem_zalloc((sizeof (ibdm_hca_list_t)), KM_SLEEP);
	hca_list->hl_port_attr = (ibdm_port_attr_t *)kmem_zalloc(
	    (sizeof (ibdm_port_attr_t) * hca_attr->hca_nports), KM_SLEEP);
	hca_list->hl_hca_guid = hca_attr->hca_node_guid;
	hca_list->hl_nports = hca_attr->hca_nports;
	hca_list->hl_attach_time = gethrtime();
	hca_list->hl_hca_hdl = hca_hdl;

	/*
	 * Init a dummy port attribute for the HCA node
	 * This is for Per-HCA Node. Initialize port_attr :
	 * 	hca_guid & port_guid -> hca_guid
	 *	npkeys, pkey_tbl is NULL
	 *	port_num, sn_prefix is 0
	 *	vendorid, product_id, dev_version from HCA
	 *	pa_state is IBT_PORT_ACTIVE
	 */
	hca_list->hl_hca_port_attr = (ibdm_port_attr_t *)kmem_zalloc(
	    sizeof (ibdm_port_attr_t), KM_SLEEP);
	port_attr = hca_list->hl_hca_port_attr;
	port_attr->pa_vendorid  = hca_attr->hca_vendor_id;
	port_attr->pa_productid	= hca_attr->hca_device_id;
	port_attr->pa_dev_version = hca_attr->hca_version_id;
	port_attr->pa_hca_guid	= hca_attr->hca_node_guid;
	port_attr->pa_hca_hdl	= hca_list->hl_hca_hdl;
	port_attr->pa_port_guid	= hca_attr->hca_node_guid;
	port_attr->pa_state	= IBT_PORT_ACTIVE;


	for (ii = 0; ii < nports; ii++) {
		port_attr		= &hca_list->hl_port_attr[ii];
		port_attr->pa_vendorid	= hca_attr->hca_vendor_id;
		port_attr->pa_productid	= hca_attr->hca_device_id;
		port_attr->pa_dev_version = hca_attr->hca_version_id;
		port_attr->pa_hca_guid	= hca_attr->hca_node_guid;
		port_attr->pa_hca_hdl	= hca_list->hl_hca_hdl;
		port_attr->pa_port_guid	= portinfop[ii].p_sgid_tbl->gid_guid;
		port_attr->pa_sn_prefix	= portinfop[ii].p_sgid_tbl->gid_prefix;
		port_attr->pa_port_num	= portinfop[ii].p_port_num;
		port_attr->pa_state	= portinfop[ii].p_linkstate;

		/*
		 * Register with IBMF, SA access when the port is in
		 * ACTIVE state. Also register a callback routine
		 * with IBMF to receive incoming DM MAD's.
		 * The IBDM event handler takes care of registration of
		 * port which are not active.
		 */
		IBTF_DPRINTF_L4("ibdm",
		    "\thandle_hca_attach: port guid %llx Port state 0x%x",
		    port_attr->pa_port_guid, portinfop[ii].p_linkstate);

		if (portinfop[ii].p_linkstate == IBT_PORT_ACTIVE) {
			mutex_enter(&ibdm.ibdm_hl_mutex);
			hca_list->hl_nports_active++;
			ibdm_initialize_port(port_attr);
			cv_broadcast(&ibdm.ibdm_port_settle_cv);
			mutex_exit(&ibdm.ibdm_hl_mutex);
		}
	}
	mutex_enter(&ibdm.ibdm_hl_mutex);
	for (temp = ibdm.ibdm_hca_list_head; temp; temp = temp->hl_next) {
		if (temp->hl_hca_guid == hca_guid) {
			IBTF_DPRINTF_L2("ibdm", "hca_attach: HCA %llX "
			    "already seen by IBDM", hca_guid);
			mutex_exit(&ibdm.ibdm_hl_mutex);
			(void) ibdm_uninit_hca(hca_list);
			return;
		}
	}
	ibdm.ibdm_hca_count++;
	if (ibdm.ibdm_hca_list_head == NULL) {
		ibdm.ibdm_hca_list_head = hca_list;
		ibdm.ibdm_hca_list_tail = hca_list;
	} else {
		ibdm.ibdm_hca_list_tail->hl_next = hca_list;
		ibdm.ibdm_hca_list_tail = hca_list;
	}
	mutex_exit(&ibdm.ibdm_hl_mutex);
	mutex_enter(&ibdm.ibdm_ibnex_mutex);
	if (ibdm.ibdm_ibnex_callback != NULL) {
		(*ibdm.ibdm_ibnex_callback)((void *)
		    &hca_guid, IBDM_EVENT_HCA_ADDED);
	}
	mutex_exit(&ibdm.ibdm_ibnex_mutex);

	kmem_free(hca_attr, sizeof (ibt_hca_attr_t));
	ibt_free_portinfo(portinfop, size);
}


/*
 * ibdm_handle_hca_detach()
 */
static void
ibdm_handle_hca_detach(ib_guid_t hca_guid)
{
	ibdm_hca_list_t		*head, *prev = NULL;
	size_t			len;
	ibdm_dp_gidinfo_t	*gidinfo;
	ibdm_port_attr_t	*port_attr;
	int			i;

	IBTF_DPRINTF_L4("ibdm",
	    "\thandle_hca_detach: hca_guid = 0x%llx", hca_guid);

	/* Make sure no probes are running */
	mutex_enter(&ibdm.ibdm_mutex);
	while (ibdm.ibdm_busy & IBDM_BUSY)
		cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex);
	ibdm.ibdm_busy |= IBDM_BUSY;
	mutex_exit(&ibdm.ibdm_mutex);

	mutex_enter(&ibdm.ibdm_hl_mutex);
	head = ibdm.ibdm_hca_list_head;
	while (head) {
		if (head->hl_hca_guid == hca_guid) {
			if (prev == NULL)
				ibdm.ibdm_hca_list_head = head->hl_next;
			else
				prev->hl_next = head->hl_next;
			if (ibdm.ibdm_hca_list_tail == head)
				ibdm.ibdm_hca_list_tail = prev;
			ibdm.ibdm_hca_count--;
			break;
		}
		prev = head;
		head = head->hl_next;
	}
	mutex_exit(&ibdm.ibdm_hl_mutex);
	if (ibdm_uninit_hca(head) != IBDM_SUCCESS)
		(void) ibdm_handle_hca_attach(hca_guid);

#ifdef DEBUG
	if (ibdm_enumerate_iocs == 0) {
		ASSERT(ibdm.ibdm_dp_gidlist_head == NULL);
	}
#endif

	/*
	 * Now clean up the HCA lists in the gidlist.
	 */
	for (gidinfo = ibdm.ibdm_dp_gidlist_head; gidinfo; gidinfo =
	    gidinfo->gl_next) {
		prev = NULL;
		head = gidinfo->gl_hca_list;
		while (head) {
			if (head->hl_hca_guid == hca_guid) {
				if (prev == NULL)
					gidinfo->gl_hca_list =
					    head->hl_next;
				else
					prev->hl_next = head->hl_next;
				for (i = 0; i < head->hl_nports; i++) {
					port_attr = &head->hl_port_attr[i];
					if (port_attr->pa_pkey_tbl != NULL)
						kmem_free(
						    port_attr->pa_pkey_tbl,
						    port_attr->pa_npkeys *
						    sizeof (ibdm_pkey_tbl_t));
				}
				len = sizeof (ibdm_hca_list_t) +
				    (head->hl_nports *
				    sizeof (ibdm_port_attr_t));
				kmem_free(head, len);

				break;
			}
			prev = head;
			head = head->hl_next;
		}
	}

	mutex_enter(&ibdm.ibdm_mutex);
	ibdm.ibdm_busy &= ~IBDM_BUSY;
	cv_broadcast(&ibdm.ibdm_busy_cv);
	mutex_exit(&ibdm.ibdm_mutex);
}


static int
ibdm_uninit_hca(ibdm_hca_list_t *head)
{
	int			ii;
	ibdm_port_attr_t	*port_attr;

	for (ii = 0; ii < head->hl_nports; ii++) {
		port_attr = &head->hl_port_attr[ii];
		if (ibdm_fini_port(port_attr) != IBDM_SUCCESS) {
			IBTF_DPRINTF_L2("ibdm", "uninit_hca: HCA %p port 0x%x "
			    "ibdm_fini_port() failed", head, ii);
			return (IBDM_FAILURE);
		}
	}
	if (head->hl_hca_hdl)
		if (ibt_close_hca(head->hl_hca_hdl) != IBT_SUCCESS) {
			IBTF_DPRINTF_L2("ibdm", "uninit_hca: "
			    "ibt_close_hca() failed");
			return (IBDM_FAILURE);
		}
	kmem_free(head->hl_port_attr,
	    head->hl_nports * sizeof (ibdm_port_attr_t));
	kmem_free(head->hl_hca_port_attr, sizeof (ibdm_port_attr_t));
	kmem_free(head, sizeof (ibdm_hca_list_t));
	return (IBDM_SUCCESS);
}


/*
 * For each port on the HCA,
 *	1) Teardown IBMF receive callback function
 *	2) Unregister with IBMF
 *	3) Unregister with SA access
 */
static int
ibdm_fini_port(ibdm_port_attr_t *port_attr)
{
	int	ii, ibmf_status;

	for (ii = 0; ii < port_attr->pa_npkeys; ii++) {
		if (port_attr->pa_pkey_tbl == NULL)
			break;
		if (!port_attr->pa_pkey_tbl[ii].pt_qp_hdl)
			continue;
		if (ibdm_port_attr_ibmf_fini(port_attr, ii) != IBDM_SUCCESS) {
			IBTF_DPRINTF_L4("ibdm", "\tfini_port: "
			    "ibdm_port_attr_ibmf_fini failed for "
			    "port pkey 0x%x", ii);
			return (IBDM_FAILURE);
		}
	}

	if (port_attr->pa_ibmf_hdl) {
		ibmf_status = ibmf_tear_down_async_cb(port_attr->pa_ibmf_hdl,
		    IBMF_QP_HANDLE_DEFAULT, 0);
		if (ibmf_status != IBMF_SUCCESS) {
			IBTF_DPRINTF_L4("ibdm", "\tfini_port: "
			    "ibmf_tear_down_async_cb failed %d", ibmf_status);
			return (IBDM_FAILURE);
		}

		ibmf_status = ibmf_unregister(&port_attr->pa_ibmf_hdl, 0);
		if (ibmf_status != IBMF_SUCCESS) {
			IBTF_DPRINTF_L2("ibdm", "\tfini_port: "
			    "ibmf_unregister failed %d", ibmf_status);
			return (IBDM_FAILURE);
		}

		port_attr->pa_ibmf_hdl = NULL;
	}

	if (port_attr->pa_sa_hdl) {
		ibmf_status = ibmf_sa_session_close(&port_attr->pa_sa_hdl, 0);
		if (ibmf_status != IBMF_SUCCESS) {
			IBTF_DPRINTF_L2("ibdm", "\tfini_port: "
			    "ibmf_sa_session_close failed %d", ibmf_status);
			return (IBDM_FAILURE);
		}
		port_attr->pa_sa_hdl = NULL;
	}

	if (port_attr->pa_pkey_tbl != NULL) {
		kmem_free(port_attr->pa_pkey_tbl,
		    port_attr->pa_npkeys * sizeof (ibdm_pkey_tbl_t));
		port_attr->pa_pkey_tbl = NULL;
		port_attr->pa_npkeys = 0;
	}

	return (IBDM_SUCCESS);
}


/*
 * ibdm_port_attr_ibmf_fini:
 *	With IBMF - Tear down Async callback and free QP Handle
 */
static int
ibdm_port_attr_ibmf_fini(ibdm_port_attr_t *port_attr, int ii)
{
	int ibmf_status;

	IBTF_DPRINTF_L5("ibdm", "\tport_attr_ibmf_fini:");

	if (ibdm_enumerate_iocs == 0) {
		ASSERT(port_attr->pa_pkey_tbl[ii].pt_qp_hdl == NULL);
		return (IBDM_SUCCESS);
	}

	if (port_attr->pa_pkey_tbl[ii].pt_qp_hdl) {
		ibmf_status = ibmf_tear_down_async_cb(port_attr->pa_ibmf_hdl,
		    port_attr->pa_pkey_tbl[ii].pt_qp_hdl, 0);
		if (ibmf_status != IBMF_SUCCESS) {
			IBTF_DPRINTF_L4("ibdm", "\tport_attr_ibmf_fini: "
			    "ibmf_tear_down_async_cb failed %d", ibmf_status);
			return (IBDM_FAILURE);
		}
		ibmf_status = ibmf_free_qp(port_attr->pa_ibmf_hdl,
		    &port_attr->pa_pkey_tbl[ii].pt_qp_hdl, 0);
		if (ibmf_status != IBMF_SUCCESS) {
			IBTF_DPRINTF_L4("ibdm", "\tport_attr_ibmf_fini: "
			    "ibmf_free_qp failed %d", ibmf_status);
			return (IBDM_FAILURE);
		}
		port_attr->pa_pkey_tbl[ii].pt_qp_hdl = NULL;
	}
	return (IBDM_SUCCESS);
}


/*
 * ibdm_gid_decr_pending:
 *	decrement gl_pending_cmds. If zero wakeup sleeping threads
 */
static void
ibdm_gid_decr_pending(ibdm_dp_gidinfo_t *gidinfo)
{
	mutex_enter(&ibdm.ibdm_mutex);
	mutex_enter(&gidinfo->gl_mutex);
	if (--gidinfo->gl_pending_cmds == 0) {
		/*
		 * Handle DGID getting removed.
		 */
		if (gidinfo->gl_disconnected) {
			mutex_exit(&gidinfo->gl_mutex);
			mutex_exit(&ibdm.ibdm_mutex);

			IBTF_DPRINTF_L3(ibdm_string, "\tgid_decr_pending: "
			    "gidinfo %p hot removal", gidinfo);
			ibdm_delete_gidinfo(gidinfo);

			mutex_enter(&ibdm.ibdm_mutex);
			ibdm.ibdm_ngid_probes_in_progress--;
			ibdm_wait_probe_completion();
			mutex_exit(&ibdm.ibdm_mutex);
			return;
		}
		mutex_exit(&gidinfo->gl_mutex);
		mutex_exit(&ibdm.ibdm_mutex);
		ibdm_notify_newgid_iocs(gidinfo);
		mutex_enter(&ibdm.ibdm_mutex);
		mutex_enter(&gidinfo->gl_mutex);

		ibdm.ibdm_ngid_probes_in_progress--;
		ibdm_wait_probe_completion();
	}
	mutex_exit(&gidinfo->gl_mutex);
	mutex_exit(&ibdm.ibdm_mutex);
}


/*
 * ibdm_wait_probe_completion:
 *	wait for probing to complete
 */
static void
ibdm_wait_probe_completion(void)
{
	ASSERT(MUTEX_HELD(&ibdm.ibdm_mutex));
	if (ibdm.ibdm_ngid_probes_in_progress) {
		IBTF_DPRINTF_L4("ibdm",	"\twait for probe complete");
		ibdm.ibdm_busy |= IBDM_PROBE_IN_PROGRESS;
		while (ibdm.ibdm_busy & IBDM_PROBE_IN_PROGRESS)
			cv_wait(&ibdm.ibdm_probe_cv, &ibdm.ibdm_mutex);
	}
}


/*
 * ibdm_wait_cisco_probe_completion:
 *	wait for the reply from the Cisco FC GW switch after a setclassportinfo
 *	request is sent. This wait can be achieved on each gid.
 */
static void
ibdm_wait_cisco_probe_completion(ibdm_dp_gidinfo_t *gidinfo)
{
	ASSERT(MUTEX_HELD(&gidinfo->gl_mutex));
	IBTF_DPRINTF_L4("ibdm",	"\twait for cisco probe complete");
	gidinfo->gl_flag |= IBDM_CISCO_PROBE;
	while (gidinfo->gl_flag & IBDM_CISCO_PROBE)
		cv_wait(&gidinfo->gl_probe_cv, &gidinfo->gl_mutex);
}


/*
 * ibdm_wakeup_probe_gid_cv:
 *	wakeup waiting threads (based on ibdm_ngid_probes_in_progress)
 */
static void
ibdm_wakeup_probe_gid_cv(void)
{
	ASSERT(MUTEX_HELD(&ibdm.ibdm_mutex));
	if (!ibdm.ibdm_ngid_probes_in_progress) {
		IBTF_DPRINTF_L4("ibdm", "wakeup_probe_gid_thread: Wakeup");
		ibdm.ibdm_busy &= ~IBDM_PROBE_IN_PROGRESS;
		cv_broadcast(&ibdm.ibdm_probe_cv);
	}

}


/*
 * ibdm_sweep_fabric(reprobe_flag)
 *	Find all possible Managed IOU's and their IOC's that are visible
 *	to the host. The algorithm used is as follows
 *
 *	Send a "bus walk" request for each port on the host HCA to SA access
 *	SA returns complete set of GID's that are reachable from
 *	source port. This is done in parallel.
 *
 *	Initialize GID state to IBDM_GID_PROBE_NOT_DONE
 *
 *	Sort the GID list and eliminate duplicate GID's
 *		1) Use DGID for sorting
 *		2) use PortGuid for sorting
 *			Send SA query to retrieve NodeRecord and
 *			extract PortGuid from that.
 *
 *	Set GID state to IBDM_GID_PROBE_FAILED to all the ports that dont
 *	support DM MAD's
 *		Send a "Portinfo" query to get the port capabilities and
 *		then check for DM MAD's support
 *
 *	Send "ClassPortInfo" request for all the GID's in parallel,
 *	set the GID state to IBDM_GET_CLASSPORTINFO and wait on the
 *	cv_signal to complete.
 *
 *	When DM agent on the remote GID sends back the response, IBMF
 *	invokes DM callback routine.
 *
 *	If the response is proper, send "IOUnitInfo" request and set
 *	GID state to IBDM_GET_IOUNITINFO.
 *
 *	If the response is proper, send "IocProfileInfo" request to
 *	all the IOC simultaneously and set GID state to IBDM_GET_IOC_DETAILS.
 *
 *	Send request to get Service entries simultaneously
 *
 *	Signal the waiting thread when received response for all the commands.
 *
 *	Set the GID state to IBDM_GID_PROBE_FAILED when received a error
 *	response during the probing period.
 *
 *	Note:
 *	ibdm.ibdm_ngid_probes_in_progress and ibdm_gid_list_t:gl_pending_cmds
 *	keep track of number commands in progress at any point of time.
 *	MAD transaction ID is used to identify a particular GID
 *	TBD: Consider registering the IBMF receive callback on demand
 *
 *	Note: This routine must be called with ibdm.ibdm_mutex held
 *	TBD: Re probe the failure GID (for certain failures) when requested
 *	     for fabric sweep next time
 *
 *	Parameters : If reprobe_flag is set, All IOCs will be reprobed.
 */
static void
ibdm_sweep_fabric(int reprobe_flag)
{
	int			ii;
	int			new_paths = 0;
	uint8_t			niocs;
	taskqid_t		tid;
	ibdm_ioc_info_t		*ioc;
	ibdm_hca_list_t		*hca_list = NULL;
	ibdm_port_attr_t	*port = NULL;
	ibdm_dp_gidinfo_t 	*gid_info;

	IBTF_DPRINTF_L4("ibdm", "\tsweep_fabric: Enter");
	ASSERT(MUTEX_HELD(&ibdm.ibdm_mutex));

	/*
	 * Check whether a sweep already in progress. If so, just
	 * wait for the fabric sweep to complete
	 */
	while (ibdm.ibdm_busy & IBDM_BUSY)
		cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex);
	ibdm.ibdm_busy |= IBDM_BUSY;
	mutex_exit(&ibdm.ibdm_mutex);

	ibdm_dump_sweep_fabric_timestamp(0);

	/* Rescan the GID list for any removed GIDs for reprobe */
	if (reprobe_flag)
		ibdm_rescan_gidlist(NULL);

	/*
	 * Get list of all the ports reachable from the local known HCA
	 * ports which are active
	 */
	mutex_enter(&ibdm.ibdm_hl_mutex);
	for (ibdm_get_next_port(&hca_list, &port, 1); port;
	    ibdm_get_next_port(&hca_list, &port, 1)) {
		/*
		 * Get PATHS to all the reachable ports from
		 * SGID and update the global ibdm structure.
		 */
		new_paths = ibdm_get_reachable_ports(port, hca_list);
		ibdm.ibdm_ngids += new_paths;
	}
	mutex_exit(&ibdm.ibdm_hl_mutex);

	mutex_enter(&ibdm.ibdm_mutex);
	ibdm.ibdm_ngid_probes_in_progress += ibdm.ibdm_ngids;
	mutex_exit(&ibdm.ibdm_mutex);

	/* Send a request to probe GIDs asynchronously. */
	for (gid_info = ibdm.ibdm_dp_gidlist_head; gid_info;
	    gid_info = gid_info->gl_next) {
		mutex_enter(&gid_info->gl_mutex);
		gid_info->gl_reprobe_flag = reprobe_flag;
		mutex_exit(&gid_info->gl_mutex);

		/* process newly encountered GIDs */
		tid = taskq_dispatch(system_taskq, ibdm_probe_gid_thread,
		    (void *)gid_info, TQ_NOSLEEP);
		IBTF_DPRINTF_L4("ibdm", "\tsweep_fabric: gid_info = %p"
		    " taskq_id = %x", gid_info, tid);
		/* taskq failed to dispatch call it directly */
		if (tid == TASKQID_INVALID)
			ibdm_probe_gid_thread((void *)gid_info);
	}

	mutex_enter(&ibdm.ibdm_mutex);
	ibdm_wait_probe_completion();

	/*
	 * Update the properties, if reprobe_flag is set
	 * Skip if gl_reprobe_flag is set, this will be
	 * a re-inserted / new GID, for which notifications
	 * have already been send.
	 */
	if (reprobe_flag) {
		for (gid_info = ibdm.ibdm_dp_gidlist_head; gid_info;
		    gid_info = gid_info->gl_next) {
			if (gid_info->gl_iou == NULL)
				continue;
			if (gid_info->gl_reprobe_flag) {
				gid_info->gl_reprobe_flag = 0;
				continue;
			}

			niocs = gid_info->gl_iou->iou_info.iou_num_ctrl_slots;
			for (ii = 0; ii < niocs; ii++) {
				ioc = IBDM_GIDINFO2IOCINFO(gid_info, ii);
				if (ioc)
					ibdm_reprobe_update_port_srv(ioc,
					    gid_info);
			}
		}
	} else if (ibdm.ibdm_prev_iou) {
		ibdm_ioc_info_t	*ioc_list;

		/*
		 * Get the list of IOCs which have changed.
		 * If any IOCs have changed, Notify IBNexus
		 */
		ibdm.ibdm_prev_iou = 0;
		ioc_list = ibdm_handle_prev_iou();
		if (ioc_list) {
			if (ibdm.ibdm_ibnex_callback != NULL) {
				(*ibdm.ibdm_ibnex_callback)(
				    (void *)ioc_list,
				    IBDM_EVENT_IOC_PROP_UPDATE);
			}
		}
	}

	ibdm_dump_sweep_fabric_timestamp(1);

	ibdm.ibdm_busy &= ~IBDM_BUSY;
	cv_broadcast(&ibdm.ibdm_busy_cv);
	IBTF_DPRINTF_L5("ibdm", "\tsweep_fabric: EXIT");
}


/*
 * ibdm_is_cisco:
 * 	Check if this is a Cisco device or not.
 */
static boolean_t
ibdm_is_cisco(ib_guid_t guid)
{
	if ((guid >> IBDM_OUI_GUID_SHIFT) == IBDM_CISCO_COMPANY_ID)
		return (B_TRUE);
	return (B_FALSE);
}


/*
 * ibdm_is_cisco_switch:
 * 	Check if this switch is a CISCO switch or not.
 * 	Note that if this switch is already activated, ibdm_is_cisco_switch()
 * 	returns B_FALSE not to re-activate it again.
 */
static boolean_t
ibdm_is_cisco_switch(ibdm_dp_gidinfo_t *gid_info)
{
	int company_id, device_id;
	ASSERT(gid_info != 0);
	ASSERT(MUTEX_HELD(&gid_info->gl_mutex));

	/*
	 * If this switch is already activated, don't re-activate it.
	 */
	if (gid_info->gl_flag & IBDM_CISCO_PROBE_DONE)
		return (B_FALSE);

	/*
	 * Check if this switch is a Cisco FC GW or not.
	 * Use the node guid (the OUI part) instead of the vendor id
	 * since the vendor id is zero in practice.
	 */
	company_id = gid_info->gl_nodeguid >> IBDM_OUI_GUID_SHIFT;
	device_id = gid_info->gl_devid;

	if (company_id == IBDM_CISCO_COMPANY_ID &&
	    device_id == IBDM_CISCO_DEVICE_ID)
		return (B_TRUE);
	return (B_FALSE);
}


/*
 * ibdm_probe_gid_thread:
 *	thread that does the actual work for sweeping the fabric
 *	for a given GID
 */
static void
ibdm_probe_gid_thread(void *args)
{
	int			reprobe_flag;
	ib_guid_t		node_guid;
	ib_guid_t		port_guid;
	ibdm_dp_gidinfo_t	*gid_info;

	gid_info = (ibdm_dp_gidinfo_t *)args;
	reprobe_flag = gid_info->gl_reprobe_flag;
	IBTF_DPRINTF_L4("ibdm", "\tprobe_gid_thread: gid_info = %p, flag = %d",
	    gid_info, reprobe_flag);
	ASSERT(gid_info != NULL);
	ASSERT(gid_info->gl_pending_cmds == 0);

	if (gid_info->gl_state != IBDM_GID_PROBE_NOT_DONE &&
	    reprobe_flag == 0) {
		/*
		 * This GID may have been already probed. Send
		 * in a CLP to check if IOUnitInfo changed?
		 * Explicitly set gl_reprobe_flag to 0 so that
		 * IBnex is not notified on completion
		 */
		if (gid_info->gl_state == IBDM_GID_PROBING_COMPLETE) {
			IBTF_DPRINTF_L4("ibdm", "\tprobe_gid_thread: "
			    "get new IOCs information");
			mutex_enter(&gid_info->gl_mutex);
			gid_info->gl_pending_cmds++;
			gid_info->gl_state = IBDM_GET_IOUNITINFO;
			gid_info->gl_reprobe_flag = 0;
			mutex_exit(&gid_info->gl_mutex);
			if (ibdm_send_iounitinfo(gid_info) != IBDM_SUCCESS) {
				mutex_enter(&gid_info->gl_mutex);
				--gid_info->gl_pending_cmds;
				mutex_exit(&gid_info->gl_mutex);
				mutex_enter(&ibdm.ibdm_mutex);
				--ibdm.ibdm_ngid_probes_in_progress;
				ibdm_wakeup_probe_gid_cv();
				mutex_exit(&ibdm.ibdm_mutex);
			}
		} else {
			mutex_enter(&ibdm.ibdm_mutex);
			--ibdm.ibdm_ngid_probes_in_progress;
			ibdm_wakeup_probe_gid_cv();
			mutex_exit(&ibdm.ibdm_mutex);
		}
		return;
	} else if (reprobe_flag && gid_info->gl_state ==
	    IBDM_GID_PROBING_COMPLETE) {
		/*
		 * Reprobe all IOCs for the GID which has completed
		 * probe. Skip other port GIDs to same IOU.
		 * Explicitly set gl_reprobe_flag to 0 so that
		 * IBnex is not notified on completion
		 */
		ibdm_ioc_info_t *ioc_info;
		uint8_t		niocs, ii;

		ASSERT(gid_info->gl_iou);
		mutex_enter(&gid_info->gl_mutex);
		niocs = gid_info->gl_iou->iou_info.iou_num_ctrl_slots;
		gid_info->gl_state = IBDM_GET_IOC_DETAILS;
		gid_info->gl_pending_cmds += niocs;
		gid_info->gl_reprobe_flag = 0;
		mutex_exit(&gid_info->gl_mutex);
		for (ii = 0; ii < niocs; ii++) {
			uchar_t			slot_info;
			ib_dm_io_unitinfo_t	*giou_info;

			/*
			 * Check whether IOC is present in the slot
			 * Series of nibbles (in the field
			 * iou_ctrl_list) represents a slot in the
			 * IOU.
			 * Byte format: 76543210
			 * Bits 0-3 of first byte represent Slot 2
			 * bits 4-7 of first byte represent slot 1,
			 * bits 0-3 of second byte represent slot 4
			 * and so on
			 * Each 4-bit nibble has the following meaning
			 * 0x0 : IOC not installed
			 * 0x1 : IOC is present
			 * 0xf : Slot does not exist
			 * and all other values are reserved.
			 */
			ioc_info = IBDM_GIDINFO2IOCINFO(gid_info, ii);
			giou_info = &gid_info->gl_iou->iou_info;
			slot_info = giou_info->iou_ctrl_list[(ii/2)];
			if ((ii % 2) == 0)
				slot_info = (slot_info >> 4);

			if ((slot_info & 0xf) != 1) {
				ioc_info->ioc_state =
				    IBDM_IOC_STATE_PROBE_FAILED;
				ibdm_gid_decr_pending(gid_info);
				continue;
			}

			if (ibdm_send_ioc_profile(gid_info, ii) !=
			    IBDM_SUCCESS) {
				ibdm_gid_decr_pending(gid_info);
			}
		}

		return;
	} else if (gid_info->gl_state != IBDM_GID_PROBE_NOT_DONE) {
		mutex_enter(&ibdm.ibdm_mutex);
		--ibdm.ibdm_ngid_probes_in_progress;
		ibdm_wakeup_probe_gid_cv();
		mutex_exit(&ibdm.ibdm_mutex);
		return;
	}

	/*
	 * Check whether the destination GID supports DM agents. If
	 * not, stop probing the GID and continue with the next GID
	 * in the list.
	 */
	if (ibdm_is_dev_mgt_supported(gid_info) != IBDM_SUCCESS) {
		mutex_enter(&gid_info->gl_mutex);
		gid_info->gl_state = IBDM_GID_PROBING_FAILED;
		gid_info->gl_is_dm_capable = B_FALSE;
		mutex_exit(&gid_info->gl_mutex);
		ibdm_delete_glhca_list(gid_info);
		mutex_enter(&ibdm.ibdm_mutex);
		--ibdm.ibdm_ngid_probes_in_progress;
		ibdm_wakeup_probe_gid_cv();
		mutex_exit(&ibdm.ibdm_mutex);
		return;
	}

	/*
	 * This GID is Device management capable
	 */
	mutex_enter(&gid_info->gl_mutex);
	gid_info->gl_is_dm_capable = B_TRUE;
	mutex_exit(&gid_info->gl_mutex);

	/* Get the nodeguid and portguid of the port */
	if (ibdm_get_node_port_guids(gid_info->gl_sa_hdl, gid_info->gl_dlid,
	    &node_guid, &port_guid) != IBDM_SUCCESS) {
		mutex_enter(&gid_info->gl_mutex);
		gid_info->gl_state = IBDM_GID_PROBING_FAILED;
		mutex_exit(&gid_info->gl_mutex);
		ibdm_delete_glhca_list(gid_info);
		mutex_enter(&ibdm.ibdm_mutex);
		--ibdm.ibdm_ngid_probes_in_progress;
		ibdm_wakeup_probe_gid_cv();
		mutex_exit(&ibdm.ibdm_mutex);
		return;
	}

	/*
	 * Check whether we already knew about this NodeGuid
	 * If so, do not probe the GID and continue with the
	 * next  GID  in the gid  list. Set the GID state to
	 * probing done.
	 */
	mutex_enter(&ibdm.ibdm_mutex);
	gid_info->gl_nodeguid = node_guid;
	gid_info->gl_portguid = port_guid;
	if (ibdm_check_dest_nodeguid(gid_info) != NULL) {
		mutex_exit(&ibdm.ibdm_mutex);
		mutex_enter(&gid_info->gl_mutex);
		gid_info->gl_state = IBDM_GID_PROBING_SKIPPED;
		mutex_exit(&gid_info->gl_mutex);
		ibdm_delete_glhca_list(gid_info);
		mutex_enter(&ibdm.ibdm_mutex);
		--ibdm.ibdm_ngid_probes_in_progress;
		ibdm_wakeup_probe_gid_cv();
		mutex_exit(&ibdm.ibdm_mutex);
		return;
	}
	ibdm_add_to_gl_gid(gid_info, gid_info);
	mutex_exit(&ibdm.ibdm_mutex);

	/*
	 * New or reinserted GID : Enable notification to IBnex
	 */
	mutex_enter(&gid_info->gl_mutex);
	gid_info->gl_reprobe_flag = 1;

	/*
	 * A Cisco FC GW needs the special handling to get IOUnitInfo.
	 */
	if (ibdm_is_cisco_switch(gid_info)) {
		gid_info->gl_pending_cmds++;
		gid_info->gl_state = IBDM_SET_CLASSPORTINFO;
		mutex_exit(&gid_info->gl_mutex);

		if (ibdm_set_classportinfo(gid_info) != IBDM_SUCCESS) {
			mutex_enter(&gid_info->gl_mutex);
			gid_info->gl_state = IBDM_GID_PROBING_FAILED;
			--gid_info->gl_pending_cmds;
			mutex_exit(&gid_info->gl_mutex);

			/* free the hca_list on this gid_info */
			ibdm_delete_glhca_list(gid_info);

			mutex_enter(&ibdm.ibdm_mutex);
			--ibdm.ibdm_ngid_probes_in_progress;
			ibdm_wakeup_probe_gid_cv();
			mutex_exit(&ibdm.ibdm_mutex);

			return;
		}

		mutex_enter(&gid_info->gl_mutex);
		ibdm_wait_cisco_probe_completion(gid_info);

		IBTF_DPRINTF_L4("ibdm", "\tibdm_probe_gid_thread: "
		    "CISCO Wakeup signal received");
	}

	/* move on to the 'GET_CLASSPORTINFO' stage */
	gid_info->gl_pending_cmds++;
	gid_info->gl_state = IBDM_GET_CLASSPORTINFO;
	mutex_exit(&gid_info->gl_mutex);

	IBTF_DPRINTF_L3(ibdm_string, "\tibdm_probe_gid_thread: "
	    "%d: gid_info %p gl_state %d pending_cmds %d",
	    __LINE__, gid_info, gid_info->gl_state,
	    gid_info->gl_pending_cmds);

	/*
	 * Send ClassPortInfo request to the GID asynchronously.
	 */
	if (ibdm_send_classportinfo(gid_info) != IBDM_SUCCESS) {

		mutex_enter(&gid_info->gl_mutex);
		gid_info->gl_state = IBDM_GID_PROBING_FAILED;
		--gid_info->gl_pending_cmds;
		mutex_exit(&gid_info->gl_mutex);

		/* free the hca_list on this gid_info */
		ibdm_delete_glhca_list(gid_info);

		mutex_enter(&ibdm.ibdm_mutex);
		--ibdm.ibdm_ngid_probes_in_progress;
		ibdm_wakeup_probe_gid_cv();
		mutex_exit(&ibdm.ibdm_mutex);

		return;
	}
}


/*
 * ibdm_check_dest_nodeguid
 *	Searches for the NodeGuid in the GID list
 *	Returns matching gid_info if found and otherwise NULL
 *
 *	This function is called to handle new GIDs discovered
 *	during device sweep / probe or for GID_AVAILABLE event.
 *
 *	Parameter :
 *		gid_info	GID to check
 */
static ibdm_dp_gidinfo_t *
ibdm_check_dest_nodeguid(ibdm_dp_gidinfo_t *gid_info)
{
	ibdm_dp_gidinfo_t	*gid_list;
	ibdm_gid_t		*tmp;

	IBTF_DPRINTF_L4("ibdm", "\tcheck_dest_nodeguid");

	gid_list = ibdm.ibdm_dp_gidlist_head;
	while (gid_list) {
		if ((gid_list != gid_info) &&
		    (gid_info->gl_nodeguid == gid_list->gl_nodeguid)) {
			IBTF_DPRINTF_L4("ibdm",
			    "\tcheck_dest_nodeguid: NodeGuid is present");

			/* Add to gid_list */
			tmp = kmem_zalloc(sizeof (ibdm_gid_t),
			    KM_SLEEP);
			tmp->gid_dgid_hi = gid_info->gl_dgid_hi;
			tmp->gid_dgid_lo = gid_info->gl_dgid_lo;
			tmp->gid_next = gid_list->gl_gid;
			gid_list->gl_gid = tmp;
			gid_list->gl_ngids++;
			return (gid_list);
		}

		gid_list = gid_list->gl_next;
	}

	return (NULL);
}


/*
 * ibdm_is_dev_mgt_supported
 *	Get the PortInfo attribute (SA Query)
 *	Check "CompatabilityMask" field in the Portinfo.
 *	Return IBDM_SUCCESS if DM MAD's supported (if bit 19 set)
 *	by the port, otherwise IBDM_FAILURE
 */
static int
ibdm_is_dev_mgt_supported(ibdm_dp_gidinfo_t *gid_info)
{
	int			ret;
	size_t			length = 0;
	sa_portinfo_record_t	req, *resp = NULL;
	ibmf_saa_access_args_t	qargs;

	bzero(&req, sizeof (sa_portinfo_record_t));
	req.EndportLID	= gid_info->gl_dlid;

	qargs.sq_attr_id	= SA_PORTINFORECORD_ATTRID;
	qargs.sq_access_type	= IBMF_SAA_RETRIEVE;
	qargs.sq_component_mask = SA_PORTINFO_COMPMASK_PORTLID;
	qargs.sq_template	= &req;
	qargs.sq_callback	= NULL;
	qargs.sq_callback_arg	= NULL;

	ret = ibmf_sa_access(gid_info->gl_sa_hdl,
	    &qargs, 0, &length, (void **)&resp);

	if ((ret != IBMF_SUCCESS) || (length == 0) || (resp == NULL)) {
		IBTF_DPRINTF_L2("ibdm", "\tis_dev_mgt_supported:"
		    "failed to get PORTINFO attribute %d", ret);
		return (IBDM_FAILURE);
	}

	if (resp->PortInfo.CapabilityMask & SM_CAP_MASK_IS_DM_SUPPD) {
		IBTF_DPRINTF_L4("ibdm", "\tis_dev_mgt_supported: SUPPD !!");
		ret = IBDM_SUCCESS;
	} else {
		IBTF_DPRINTF_L4("ibdm", "\tis_dev_mgt_supported: "
		    "Not SUPPD !!, cap 0x%x", resp->PortInfo.CapabilityMask);
		ret = IBDM_FAILURE;
	}
	kmem_free(resp, length);
	return (ret);
}


/*
 * ibdm_get_node_port_guids()
 *	Get the NodeInfoRecord of the port
 *	Save NodeGuid and PortGUID values in the GID list structure.
 *	Return IBDM_SUCCESS/IBDM_FAILURE
 */
static int
ibdm_get_node_port_guids(ibmf_saa_handle_t sa_hdl, ib_lid_t dlid,
    ib_guid_t *node_guid, ib_guid_t *port_guid)
{
	int			ret;
	size_t			length = 0;
	sa_node_record_t	req, *resp = NULL;
	ibmf_saa_access_args_t	qargs;

	IBTF_DPRINTF_L4("ibdm", "\tget_node_port_guids");

	bzero(&req, sizeof (sa_node_record_t));
	req.LID = dlid;

	qargs.sq_attr_id	= SA_NODERECORD_ATTRID;
	qargs.sq_access_type	= IBMF_SAA_RETRIEVE;
	qargs.sq_component_mask = SA_NODEINFO_COMPMASK_NODELID;
	qargs.sq_template	= &req;
	qargs.sq_callback	= NULL;
	qargs.sq_callback_arg	= NULL;

	ret = ibmf_sa_access(sa_hdl, &qargs, 0, &length, (void **)&resp);
	if ((ret != IBMF_SUCCESS) || (length == 0) || (resp == NULL)) {
		IBTF_DPRINTF_L2("ibdm", "\tget_node_port_guids:"
		    " SA Retrieve Failed: %d", ret);
		return (IBDM_FAILURE);
	}
	IBTF_DPRINTF_L4("ibdm", "\tget_node_port_guids: NodeGuid %llx Port"
	    "GUID %llx", resp->NodeInfo.NodeGUID, resp->NodeInfo.NodeGUID);

	*node_guid = resp->NodeInfo.NodeGUID;
	*port_guid = resp->NodeInfo.PortGUID;
	kmem_free(resp, length);
	return (IBDM_SUCCESS);
}


/*
 * ibdm_get_reachable_ports()
 *	Get list of the destination GID (and its path  records) by
 *	querying the SA access.
 *
 *	Returns Number paths
 */
static int
ibdm_get_reachable_ports(ibdm_port_attr_t *portinfo, ibdm_hca_list_t *hca)
{
	uint_t			ii, jj, nrecs;
	uint_t			npaths = 0;
	size_t			length;
	ib_gid_t		sgid;
	ibdm_pkey_tbl_t		*pkey_tbl;
	sa_path_record_t	*result;
	sa_path_record_t	*precp;
	ibdm_dp_gidinfo_t	*gid_info;

	ASSERT(MUTEX_HELD(&ibdm.ibdm_hl_mutex));
	IBTF_DPRINTF_L4("ibdm", "\tget_reachable_ports: portinfo %p", portinfo);

	sgid.gid_prefix = portinfo->pa_sn_prefix;
	sgid.gid_guid	= portinfo->pa_port_guid;

	/* get reversible paths */
	if (portinfo->pa_sa_hdl && ibmf_saa_paths_from_gid(portinfo->pa_sa_hdl,
	    sgid, IBMF_SAA_PKEY_WC, B_TRUE, 0, &nrecs, &length, &result)
	    != IBMF_SUCCESS) {
		IBTF_DPRINTF_L2("ibdm",
		    "\tget_reachable_ports: Getting path records failed");
		return (0);
	}

	for (ii = 0; ii < nrecs; ii++) {
		sa_node_record_t *nrec;
		size_t length;

		precp = &result[ii];
		if ((gid_info = ibdm_check_dgid(precp->DGID.gid_guid,
		    precp->DGID.gid_prefix)) != NULL) {
			IBTF_DPRINTF_L5("ibdm", "\tget_reachable_ports: "
			    "Already exists nrecs %d, ii %d", nrecs, ii);
			ibdm_addto_glhcalist(gid_info, hca);
			continue;
		}
		/*
		 * This is a new GID. Allocate a GID structure and
		 * initialize the structure
		 * gl_state is initialized to IBDM_GID_PROBE_NOT_DONE (0)
		 * by kmem_zalloc call
		 */
		gid_info = kmem_zalloc(sizeof (ibdm_dp_gidinfo_t), KM_SLEEP);
		mutex_init(&gid_info->gl_mutex, NULL, MUTEX_DEFAULT, NULL);
		cv_init(&gid_info->gl_probe_cv, NULL, CV_DRIVER, NULL);
		gid_info->gl_dgid_hi		= precp->DGID.gid_prefix;
		gid_info->gl_dgid_lo		= precp->DGID.gid_guid;
		gid_info->gl_sgid_hi		= precp->SGID.gid_prefix;
		gid_info->gl_sgid_lo		= precp->SGID.gid_guid;
		gid_info->gl_p_key		= precp->P_Key;
		gid_info->gl_sa_hdl		= portinfo->pa_sa_hdl;
		gid_info->gl_ibmf_hdl		= portinfo->pa_ibmf_hdl;
		gid_info->gl_slid		= precp->SLID;
		gid_info->gl_dlid		= precp->DLID;
		gid_info->gl_transactionID	= (++ibdm.ibdm_transactionID)
		    << IBDM_GID_TRANSACTIONID_SHIFT;
		gid_info->gl_min_transactionID  = gid_info->gl_transactionID;
		gid_info->gl_max_transactionID  = (ibdm.ibdm_transactionID +1)
		    << IBDM_GID_TRANSACTIONID_SHIFT;
		gid_info->gl_SL			= precp->SL;

		/*
		 * get the node record with this guid if the destination
		 * device is a Cisco one.
		 */
		if (ibdm_is_cisco(precp->DGID.gid_guid) &&
		    (gid_info->gl_nodeguid == 0 || gid_info->gl_devid == 0) &&
		    ibdm_get_node_record_by_port(portinfo->pa_sa_hdl,
		    precp->DGID.gid_guid, &nrec, &length) == IBDM_SUCCESS) {
			gid_info->gl_nodeguid = nrec->NodeInfo.NodeGUID;
			gid_info->gl_devid = nrec->NodeInfo.DeviceID;
			kmem_free(nrec, length);
		}

		ibdm_addto_glhcalist(gid_info,  hca);

		ibdm_dump_path_info(precp);

		gid_info->gl_qp_hdl = NULL;
		ASSERT(portinfo->pa_pkey_tbl != NULL &&
		    portinfo->pa_npkeys != 0);

		for (jj = 0; jj < portinfo->pa_npkeys; jj++) {
			pkey_tbl = &portinfo->pa_pkey_tbl[jj];
			if ((gid_info->gl_p_key == pkey_tbl->pt_pkey) &&
			    (pkey_tbl->pt_qp_hdl != NULL)) {
				gid_info->gl_qp_hdl = pkey_tbl->pt_qp_hdl;
				break;
			}
		}

		/*
		 * QP handle for GID not initialized. No matching Pkey
		 * was found!! ibdm should *not* hit this case. Flag an
		 * error and drop the GID if ibdm does encounter this.
		 */
		if (gid_info->gl_qp_hdl == NULL) {
			IBTF_DPRINTF_L2(ibdm_string,
			    "\tget_reachable_ports: No matching Pkey");
			ibdm_delete_gidinfo(gid_info);
			continue;
		}
		if (ibdm.ibdm_dp_gidlist_head == NULL) {
			ibdm.ibdm_dp_gidlist_head = gid_info;
			ibdm.ibdm_dp_gidlist_tail = gid_info;
		} else {
			ibdm.ibdm_dp_gidlist_tail->gl_next = gid_info;
			gid_info->gl_prev = ibdm.ibdm_dp_gidlist_tail;
			ibdm.ibdm_dp_gidlist_tail = gid_info;
		}
		npaths++;
	}
	kmem_free(result, length);
	IBTF_DPRINTF_L4("ibdm", "\tget_reachable_ports: npaths = %d", npaths);
	return (npaths);
}


/*
 * ibdm_check_dgid()
 *	Look in the global list to check whether we know this DGID already
 *	Return IBDM_GID_PRESENT/IBDM_GID_NOT_PRESENT
 */
static ibdm_dp_gidinfo_t *
ibdm_check_dgid(ib_guid_t guid, ib_sn_prefix_t prefix)
{
	ibdm_dp_gidinfo_t	*gid_list;

	for (gid_list = ibdm.ibdm_dp_gidlist_head; gid_list;
	    gid_list = gid_list->gl_next) {
		if ((guid == gid_list->gl_dgid_lo) &&
		    (prefix == gid_list->gl_dgid_hi)) {
			break;
		}
	}
	return (gid_list);
}


/*
 * ibdm_find_gid()
 *	Look in the global list to find a GID entry with matching
 *	port & node GUID.
 *	Return pointer to gidinfo if found, else return NULL
 */
static ibdm_dp_gidinfo_t *
ibdm_find_gid(ib_guid_t nodeguid, ib_guid_t portguid)
{
	ibdm_dp_gidinfo_t	*gid_list;

	IBTF_DPRINTF_L4("ibdm", "ibdm_find_gid(%llx, %llx)\n",
	    nodeguid, portguid);

	for (gid_list = ibdm.ibdm_dp_gidlist_head; gid_list;
	    gid_list = gid_list->gl_next) {
		if ((portguid == gid_list->gl_portguid) &&
		    (nodeguid == gid_list->gl_nodeguid)) {
			break;
		}
	}

	IBTF_DPRINTF_L4("ibdm", "ibdm_find_gid : returned %p\n",
	    gid_list);
	return (gid_list);
}


/*
 * ibdm_set_classportinfo()
 *	ibdm_set_classportinfo() is a function to activate a Cisco FC GW
 *	by sending the setClassPortInfo request with the trapLID, trapGID
 *	and etc. to the gateway since the gateway doesn't provide the IO
 *	Unit Information othewise. This behavior is the Cisco specific one,
 *	and this function is called to a Cisco FC GW only.
 *	Returns IBDM_SUCCESS/IBDM_FAILURE
 */
static int
ibdm_set_classportinfo(ibdm_dp_gidinfo_t *gid_info)
{
	ibmf_msg_t		*msg;
	ib_mad_hdr_t		*hdr;
	ibdm_timeout_cb_args_t	*cb_args;
	void			*data;
	ib_mad_classportinfo_t *cpi;

	IBTF_DPRINTF_L4("ibdm",
	    "\tset_classportinfo: gid info 0x%p", gid_info);

	/*
	 * Send command to set classportinfo attribute. Allocate a IBMF
	 * packet and initialize the packet.
	 */
	if (ibmf_alloc_msg(gid_info->gl_ibmf_hdl, IBMF_ALLOC_SLEEP,
	    &msg) != IBMF_SUCCESS) {
		IBTF_DPRINTF_L4("ibdm", "\tset_classportinfo: pkt alloc fail");
		return (IBDM_FAILURE);
	}

	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*msg))
	ibdm_alloc_send_buffers(msg);
	_NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*msg))

	msg->im_local_addr.ia_local_lid		= gid_info->gl_slid;
	msg->im_local_addr.ia_remote_lid	= gid_info->gl_dlid;
	msg->im_local_addr.ia_remote_qno	= 1;
	msg->im_local_addr.ia_p_key		= gid_info->gl_p_key;
	msg->im_local_addr.ia_q_key		= IB_GSI_QKEY;
	msg->im_local_addr.ia_service_level	= gid_info->gl_SL;

	hdr			= IBDM_OUT_IBMFMSG_MADHDR(msg);
	hdr->BaseVersion	= MAD_CLASS_BASE_VERS_1;
	hdr->MgmtClass		= MAD_MGMT_CLASS_DEV_MGT;
	hdr->ClassVersion	= IB_DM_CLASS_VERSION_1;
	hdr->R_Method		= IB_DM_DEVMGT_METHOD_SET;
	hdr->Status		= 0;
	hdr->TransactionID	= h2b64(gid_info->gl_transactionID);
	hdr->AttributeID	= h2b16(IB_DM_ATTR_CLASSPORTINFO);
	hdr->AttributeModifier	= 0;

	data = msg->im_msgbufs_send.im_bufs_cl_data;
	cpi = (ib_mad_classportinfo_t *)data;

	/*
	 * Set the classportinfo values to activate this Cisco FC GW.
	 */
	cpi->TrapGID_hi = h2b64(gid_info->gl_sgid_hi);
	cpi->TrapGID_lo = h2b64(gid_info->gl_sgid_lo);
	cpi->TrapLID = h2b16(gid_info->gl_slid);
	cpi->TrapSL = gid_info->gl_SL;
	cpi->TrapP_Key = h2b16(gid_info->gl_p_key);
	cpi->TrapQP = h2b32((((ibmf_alt_qp_t *)gid_info->gl_qp_hdl)->isq_qpn));
	cpi->TrapQ_Key = h2b32((((ibmf_alt_qp_t *)
	    gid_info->gl_qp_hdl)->isq_qkey));

	cb_args = &gid_info->gl_cpi_cb_args;
	cb_args->cb_gid_info = gid_info;
	cb_args->cb_retry_count	= ibdm_dft_retry_cnt;
	cb_args->cb_req_type = IBDM_REQ_TYPE_CLASSPORTINFO;

	mutex_enter(&gid_info->gl_mutex);
	gid_info->gl_timeout_id = timeout(ibdm_pkt_timeout_hdlr,
	    cb_args, IBDM_TIMEOUT_VALUE(ibdm_dft_timeout));
	mutex_exit(&gid_info->gl_mutex);

	IBTF_DPRINTF_L5("ibdm", "\tset_classportinfo: "
	    "timeout id %x", gid_info->gl_timeout_id);

	if (ibmf_msg_transport(gid_info->gl_ibmf_hdl, gid_info->gl_qp_hdl,
	    msg, NULL, ibdm_ibmf_send_cb, cb_args, 0) != IBMF_SUCCESS) {
		IBTF_DPRINTF_L2("ibdm",
		    "\tset_classportinfo: ibmf send failed");
		ibdm_ibmf_send_cb(gid_info->gl_ibmf_hdl, msg, cb_args);
	}

	return (IBDM_SUCCESS);
}


/*
 * ibdm_send_classportinfo()
 *	Send classportinfo request. When the request is completed
 *	IBMF calls ibdm_classportinfo_cb routine to inform about
 *	the completion.
 *	Returns IBDM_SUCCESS/IBDM_FAILURE
 */
static int
ibdm_send_classportinfo(ibdm_dp_gidinfo_t *gid_info)
{
	ibmf_msg_t		*msg;
	ib_mad_hdr_t		*hdr;
	ibdm_timeout_cb_args_t	*cb_args;

	IBTF_DPRINTF_L4("ibdm",
	    "\tsend_classportinfo: gid info 0x%p", gid_info);

	/*
	 * Send command to get classportinfo attribute. Allocate a IBMF
	 * packet and initialize the packet.
	 */
	if (ibmf_alloc_msg(gid_info->gl_ibmf_hdl, IBMF_ALLOC_SLEEP,
	    &msg) != IBMF_SUCCESS) {
		IBTF_DPRINTF_L4("ibdm", "\tsend_classportinfo: pkt alloc fail");
		return (IBDM_FAILURE);
	}

	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*msg))
	ibdm_alloc_send_buffers(msg);
	_NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*msg))

	msg->im_local_addr.ia_local_lid		= gid_info->gl_slid;
	msg->im_local_addr.ia_remote_lid	= gid_info->gl_dlid;
	msg->im_local_addr.ia_remote_qno	= 1;
	msg->im_local_addr.ia_p_key		= gid_info->gl_p_key;
	msg->im_local_addr.ia_q_key		= IB_GSI_QKEY;
	msg->im_local_addr.ia_service_level	= gid_info->gl_SL;

	hdr			= IBDM_OUT_IBMFMSG_MADHDR(msg);
	hdr->BaseVersion	= MAD_CLASS_BASE_VERS_1;
	hdr->MgmtClass		= MAD_MGMT_CLASS_DEV_MGT;
	hdr->ClassVersion	= IB_DM_CLASS_VERSION_1;
	hdr->R_Method		= IB_DM_DEVMGT_METHOD_GET;
	hdr->Status		= 0;
	hdr->TransactionID	= h2b64(gid_info->gl_transactionID);
	hdr->AttributeID	= h2b16(IB_DM_ATTR_CLASSPORTINFO);
	hdr->AttributeModifier	= 0;

	cb_args = &gid_info->gl_cpi_cb_args;
	cb_args->cb_gid_info = gid_info;
	cb_args->cb_retry_count	= ibdm_dft_retry_cnt;
	cb_args->cb_req_type = IBDM_REQ_TYPE_CLASSPORTINFO;

	mutex_enter(&gid_info->gl_mutex);
	gid_info->gl_timeout_id = timeout(ibdm_pkt_timeout_hdlr,
	    cb_args, IBDM_TIMEOUT_VALUE(ibdm_dft_timeout));
	mutex_exit(&gid_info->gl_mutex);

	IBTF_DPRINTF_L5("ibdm", "\tsend_classportinfo: "
	    "timeout id %x", gid_info->gl_timeout_id);

	if (ibmf_msg_transport(gid_info->gl_ibmf_hdl, gid_info->gl_qp_hdl,
	    msg, NULL, ibdm_ibmf_send_cb, cb_args, 0) != IBMF_SUCCESS) {
		IBTF_DPRINTF_L2("ibdm",
		    "\tsend_classportinfo: ibmf send failed");
		ibdm_ibmf_send_cb(gid_info->gl_ibmf_hdl, msg, cb_args);
	}

	return (IBDM_SUCCESS);
}


/*
 * ibdm_handle_setclassportinfo()
 *	Invoked by the IBMF when setClassPortInfo request is completed.
 */
static void
ibdm_handle_setclassportinfo(ibmf_handle_t ibmf_hdl,
    ibmf_msg_t *msg, ibdm_dp_gidinfo_t *gid_info, int *flag)
{
	void			*data;
	timeout_id_t		timeout_id;
	ib_mad_classportinfo_t *cpi;

	IBTF_DPRINTF_L4("ibdm", "\thandle_setclassportinfo:ibmf hdl "
	    "%p msg %p gid info %p", ibmf_hdl, msg, gid_info);

	if (IBDM_IN_IBMFMSG_ATTR(msg) != IB_DM_ATTR_CLASSPORTINFO) {
		IBTF_DPRINTF_L4("ibdm", "\thandle_setclassportinfo: "
		    "Not a ClassPortInfo resp");
		*flag |= IBDM_IBMF_PKT_UNEXP_RESP;
		return;
	}

	/*
	 * Verify whether timeout handler is created/active.
	 * If created/ active,  cancel the timeout  handler
	 */
	mutex_enter(&gid_info->gl_mutex);
	if (gid_info->gl_state != IBDM_SET_CLASSPORTINFO) {
		IBTF_DPRINTF_L2("ibdm", "\thandle_setclassportinfo:DUP resp");
		*flag |= IBDM_IBMF_PKT_DUP_RESP;
		mutex_exit(&gid_info->gl_mutex);
		return;
	}
	ibdm_bump_transactionID(gid_info);

	gid_info->gl_iou_cb_args.cb_req_type = 0;
	if (gid_info->gl_timeout_id) {
		timeout_id = gid_info->gl_timeout_id;
		mutex_exit(&gid_info->gl_mutex);
		IBTF_DPRINTF_L5("ibdm", "handle_setlassportinfo: "
		    "gl_timeout_id = 0x%x", timeout_id);
		if (untimeout(timeout_id) == -1) {
			IBTF_DPRINTF_L2("ibdm", "handle_setclassportinfo: "
			    "untimeout gl_timeout_id failed");
		}
		mutex_enter(&gid_info->gl_mutex);
		gid_info->gl_timeout_id = 0;
	}
	mutex_exit(&gid_info->gl_mutex);

	data = msg->im_msgbufs_recv.im_bufs_cl_data;
	cpi = (ib_mad_classportinfo_t *)data;

	ibdm_dump_classportinfo(cpi);
}


/*
 * ibdm_handle_classportinfo()
 *	Invoked by the IBMF when the classportinfo request is completed.
 */
static void
ibdm_handle_classportinfo(ibmf_handle_t ibmf_hdl,
    ibmf_msg_t *msg, ibdm_dp_gidinfo_t *gid_info, int *flag)
{
	void			*data;
	timeout_id_t		timeout_id;
	ib_mad_hdr_t		*hdr;
	ib_mad_classportinfo_t *cpi;

	IBTF_DPRINTF_L4("ibdm", "\thandle_classportinfo:ibmf hdl "
	    "%p msg %p gid info %p", ibmf_hdl, msg, gid_info);

	if (IBDM_IN_IBMFMSG_ATTR(msg) != IB_DM_ATTR_CLASSPORTINFO) {
		IBTF_DPRINTF_L4("ibdm", "\thandle_classportinfo: "
		    "Not a ClassPortInfo resp");
		*flag |= IBDM_IBMF_PKT_UNEXP_RESP;
		return;
	}

	/*
	 * Verify whether timeout handler is created/active.
	 * If created/ active,  cancel the timeout  handler
	 */
	mutex_enter(&gid_info->gl_mutex);
	ibdm_bump_transactionID(gid_info);
	if (gid_info->gl_state != IBDM_GET_CLASSPORTINFO) {
		IBTF_DPRINTF_L2("ibdm", "\thandle_classportinfo:DUP resp");
		*flag |= IBDM_IBMF_PKT_DUP_RESP;
		mutex_exit(&gid_info->gl_mutex);
		return;
	}
	gid_info->gl_iou_cb_args.cb_req_type = 0;
	if (gid_info->gl_timeout_id) {
		timeout_id = gid_info->gl_timeout_id;
		mutex_exit(&gid_info->gl_mutex);
		IBTF_DPRINTF_L5("ibdm", "handle_ioclassportinfo: "
		    "gl_timeout_id = 0x%x", timeout_id);
		if (untimeout(timeout_id) == -1) {
			IBTF_DPRINTF_L2("ibdm", "handle_classportinfo: "
			    "untimeout gl_timeout_id failed");
		}
		mutex_enter(&gid_info->gl_mutex);
		gid_info->gl_timeout_id = 0;
	}
	gid_info->gl_state = IBDM_GET_IOUNITINFO;
	gid_info->gl_pending_cmds++;
	mutex_exit(&gid_info->gl_mutex);

	data = msg->im_msgbufs_recv.im_bufs_cl_data;
	cpi = (ib_mad_classportinfo_t *)data;

	/*
	 * Cache the "RespTimeValue" and redirection information in the
	 * global gid list data structure. This cached information will
	 * be used to send any further requests to the GID.
	 */
	gid_info->gl_resp_timeout	=
	    (b2h32(cpi->RespTimeValue) & 0x1F);

	gid_info->gl_redirected		= ((IBDM_IN_IBMFMSG_STATUS(msg) &
	    MAD_STATUS_REDIRECT_REQUIRED) ? B_TRUE: B_FALSE);
	gid_info->gl_redirect_dlid	= b2h16(cpi->RedirectLID);
	gid_info->gl_redirect_QP	= (b2h32(cpi->RedirectQP) & 0xffffff);
	gid_info->gl_redirect_pkey	= b2h16(cpi->RedirectP_Key);
	gid_info->gl_redirect_qkey	= b2h32(cpi->RedirectQ_Key);
	gid_info->gl_redirectGID_hi	= b2h64(cpi->RedirectGID_hi);
	gid_info->gl_redirectGID_lo	= b2h64(cpi->RedirectGID_lo);
	gid_info->gl_redirectSL		= cpi->RedirectSL;

	ibdm_dump_classportinfo(cpi);

	/*
	 * Send IOUnitInfo request
	 * Reuse previously allocated IBMF packet for sending ClassPortInfo
	 * Check whether DM agent on the remote node requested redirection
	 * If so, send the request to the redirect DGID/DLID/PKEY/QP.
	 */
	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*msg))
	ibdm_alloc_send_buffers(msg);
	_NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*msg))
	msg->im_local_addr.ia_local_lid	= gid_info->gl_slid;
	msg->im_local_addr.ia_remote_lid	= gid_info->gl_dlid;

	if (gid_info->gl_redirected == B_TRUE) {
		if (gid_info->gl_redirect_dlid != 0) {
			msg->im_local_addr.ia_remote_lid =
			    gid_info->gl_redirect_dlid;
		}
		msg->im_local_addr.ia_remote_qno = gid_info->gl_redirect_QP;
		msg->im_local_addr.ia_p_key = gid_info->gl_redirect_pkey;
		msg->im_local_addr.ia_q_key = gid_info->gl_redirect_qkey;
		msg->im_local_addr.ia_service_level = gid_info->gl_redirectSL;
	} else {
		msg->im_local_addr.ia_remote_qno = 1;
		msg->im_local_addr.ia_p_key = gid_info->gl_p_key;
		msg->im_local_addr.ia_q_key = IB_GSI_QKEY;
		msg->im_local_addr.ia_service_level = gid_info->gl_SL;
	}

	hdr			= IBDM_OUT_IBMFMSG_MADHDR(msg);
	hdr->BaseVersion	= MAD_CLASS_BASE_VERS_1;
	hdr->MgmtClass		= MAD_MGMT_CLASS_DEV_MGT;
	hdr->ClassVersion	= IB_DM_CLASS_VERSION_1;
	hdr->R_Method		= IB_DM_DEVMGT_METHOD_GET;
	hdr->Status		= 0;
	hdr->TransactionID	= h2b64(gid_info->gl_transactionID);
	hdr->AttributeID	= h2b16(IB_DM_ATTR_IO_UNITINFO);
	hdr->AttributeModifier	= 0;

	gid_info->gl_iou_cb_args.cb_req_type = IBDM_REQ_TYPE_IOUINFO;
	gid_info->gl_iou_cb_args.cb_gid_info = gid_info;
	gid_info->gl_iou_cb_args.cb_retry_count = ibdm_dft_retry_cnt;

	mutex_enter(&gid_info->gl_mutex);
	gid_info->gl_timeout_id = timeout(ibdm_pkt_timeout_hdlr,
	    &gid_info->gl_iou_cb_args, IBDM_TIMEOUT_VALUE(ibdm_dft_timeout));
	mutex_exit(&gid_info->gl_mutex);

	IBTF_DPRINTF_L5("ibdm", "handle_classportinfo:"
	    "timeout %x", gid_info->gl_timeout_id);

	if (ibmf_msg_transport(ibmf_hdl, gid_info->gl_qp_hdl, msg, NULL,
	    ibdm_ibmf_send_cb, &gid_info->gl_iou_cb_args, 0) != IBMF_SUCCESS) {
		IBTF_DPRINTF_L2("ibdm",
		    "\thandle_classportinfo: msg transport failed");
		ibdm_ibmf_send_cb(ibmf_hdl, msg, &gid_info->gl_iou_cb_args);
	}
	(*flag) |= IBDM_IBMF_PKT_REUSED;
}


/*
 * ibdm_send_iounitinfo:
 *	Sends a DM request to get IOU unitinfo.
 */
static int
ibdm_send_iounitinfo(ibdm_dp_gidinfo_t *gid_info)
{
	ibmf_msg_t	*msg;
	ib_mad_hdr_t	*hdr;

	IBTF_DPRINTF_L4("ibdm", "\tsend_iounitinfo: gid info 0x%p", gid_info);

	/*
	 * Send command to get iounitinfo attribute. Allocate a IBMF
	 * packet and initialize the packet.
	 */
	if (ibmf_alloc_msg(gid_info->gl_ibmf_hdl, IBMF_ALLOC_SLEEP, &msg) !=
	    IBMF_SUCCESS) {
		IBTF_DPRINTF_L4("ibdm", "\tsend_iounitinfo: pkt alloc fail");
		return (IBDM_FAILURE);
	}

	mutex_enter(&gid_info->gl_mutex);
	ibdm_bump_transactionID(gid_info);
	mutex_exit(&gid_info->gl_mutex);


	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*msg))
	ibdm_alloc_send_buffers(msg);
	_NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*msg))
	msg->im_local_addr.ia_local_lid		= gid_info->gl_slid;
	msg->im_local_addr.ia_remote_lid	= gid_info->gl_dlid;
	msg->im_local_addr.ia_remote_qno	= 1;
	msg->im_local_addr.ia_p_key		= gid_info->gl_p_key;
	msg->im_local_addr.ia_q_key		= IB_GSI_QKEY;
	msg->im_local_addr.ia_service_level	= gid_info->gl_SL;

	hdr			= IBDM_OUT_IBMFMSG_MADHDR(msg);
	hdr->BaseVersion	= MAD_CLASS_BASE_VERS_1;
	hdr->MgmtClass		= MAD_MGMT_CLASS_DEV_MGT;
	hdr->ClassVersion	= IB_DM_CLASS_VERSION_1;
	hdr->R_Method		= IB_DM_DEVMGT_METHOD_GET;
	hdr->Status		= 0;
	hdr->TransactionID	= h2b64(gid_info->gl_transactionID);
	hdr->AttributeID	= h2b16(IB_DM_ATTR_IO_UNITINFO);
	hdr->AttributeModifier	= 0;

	gid_info->gl_iou_cb_args.cb_gid_info = gid_info;
	gid_info->gl_iou_cb_args.cb_retry_count = ibdm_dft_retry_cnt;
	gid_info->gl_iou_cb_args.cb_req_type = IBDM_REQ_TYPE_IOUINFO;

	mutex_enter(&gid_info->gl_mutex);
	gid_info->gl_timeout_id = timeout(ibdm_pkt_timeout_hdlr,
	    &gid_info->gl_iou_cb_args, IBDM_TIMEOUT_VALUE(ibdm_dft_timeout));
	mutex_exit(&gid_info->gl_mutex);

	IBTF_DPRINTF_L5("ibdm", "send_iouunitinfo:"
	    "timeout %x", gid_info->gl_timeout_id);

	if (ibmf_msg_transport(gid_info->gl_ibmf_hdl, gid_info->gl_qp_hdl, msg,
	    NULL, ibdm_ibmf_send_cb, &gid_info->gl_iou_cb_args, 0) !=
	    IBMF_SUCCESS) {
		IBTF_DPRINTF_L2("ibdm", "\tsend_iounitinfo: ibmf send failed");
		ibdm_ibmf_send_cb(gid_info->gl_ibmf_hdl,
		    msg, &gid_info->gl_iou_cb_args);
	}
	return (IBDM_SUCCESS);
}

/*
 * ibdm_handle_iounitinfo()
 *	Invoked by the IBMF when IO Unitinfo request is completed.
 */
static void
ibdm_handle_iounitinfo(ibmf_handle_t ibmf_hdl,
    ibmf_msg_t *msg, ibdm_dp_gidinfo_t *gid_info, int *flag)
{
	int			ii, first = B_TRUE;
	int			num_iocs;
	size_t			size;
	uchar_t			slot_info;
	timeout_id_t		timeout_id;
	ib_mad_hdr_t		*hdr;
	ibdm_ioc_info_t		*ioc_info;
	ib_dm_io_unitinfo_t	*iou_info;
	ib_dm_io_unitinfo_t	*giou_info;
	ibdm_timeout_cb_args_t	*cb_args;

	IBTF_DPRINTF_L4("ibdm", "\thandle_iouintinfo:"
	    " ibmf hdl %p pkt %p gid info %p", ibmf_hdl, msg, gid_info);

	if (IBDM_IN_IBMFMSG_ATTR(msg) != IB_DM_ATTR_IO_UNITINFO) {
		IBTF_DPRINTF_L4("ibdm", "\thandle_iounitinfo: "
		    "Unexpected response");
		(*flag) |= IBDM_IBMF_PKT_UNEXP_RESP;
		return;
	}

	mutex_enter(&gid_info->gl_mutex);
	if (gid_info->gl_state != IBDM_GET_IOUNITINFO) {
		IBTF_DPRINTF_L4("ibdm",
		    "\thandle_iounitinfo: DUP resp");
		mutex_exit(&gid_info->gl_mutex);
		(*flag) = IBDM_IBMF_PKT_DUP_RESP;
		return;
	}
	gid_info->gl_iou_cb_args.cb_req_type = 0;
	if (gid_info->gl_timeout_id) {
		timeout_id = gid_info->gl_timeout_id;
		mutex_exit(&gid_info->gl_mutex);
		IBTF_DPRINTF_L5("ibdm", "handle_iounitinfo: "
		    "gl_timeout_id = 0x%x", timeout_id);
		if (untimeout(timeout_id) == -1) {
			IBTF_DPRINTF_L2("ibdm", "handle_iounitinfo: "
			    "untimeout gl_timeout_id failed");
		}
		mutex_enter(&gid_info->gl_mutex);
		gid_info->gl_timeout_id = 0;
	}
	gid_info->gl_state = IBDM_GET_IOC_DETAILS;

	iou_info = IBDM_IN_IBMFMSG2IOU(msg);
	ibdm_dump_iounitinfo(iou_info);
	num_iocs = iou_info->iou_num_ctrl_slots;
	/*
	 * check if number of IOCs reported is zero? if yes, return.
	 * when num_iocs are reported zero internal IOC database needs
	 * to be updated. To ensure that save the number of IOCs in
	 * the new field "gl_num_iocs". Use a new field instead of
	 * "giou_info->iou_num_ctrl_slots" as that would prevent
	 * an unnecessary kmem_alloc/kmem_free when num_iocs is 0.
	 */
	if (num_iocs == 0 && gid_info->gl_num_iocs == 0) {
		IBTF_DPRINTF_L4("ibdm", "\thandle_iounitinfo: no IOC's");
		mutex_exit(&gid_info->gl_mutex);
		return;
	}
	IBTF_DPRINTF_L4("ibdm", "\thandle_iounitinfo: num_iocs = %d", num_iocs);

	/*
	 * if there is an existing gl_iou (IOU has been probed before)
	 * check if the "iou_changeid" is same as saved entry in
	 * "giou_info->iou_changeid".
	 * (note: this logic can prevent IOC enumeration if a given
	 * vendor doesn't support setting iou_changeid field for its IOU)
	 *
	 * if there is an existing gl_iou and iou_changeid has changed :
	 * free up existing gl_iou info and its related structures.
	 * reallocate gl_iou info all over again.
	 * if we donot free this up; then this leads to memory leaks
	 */
	if (gid_info->gl_iou) {
		giou_info = &gid_info->gl_iou->iou_info;
		if (b2h16(iou_info->iou_changeid) ==
		    giou_info->iou_changeid) {
			IBTF_DPRINTF_L3("ibdm",
			    "\thandle_iounitinfo: no IOCs changed");
			gid_info->gl_state = IBDM_GID_PROBING_COMPLETE;
			mutex_exit(&gid_info->gl_mutex);
			return;
		}

		/*
		 * Store the iou info as prev_iou to be used after
		 * sweep is done.
		 */
		ASSERT(gid_info->gl_prev_iou == NULL);
		IBTF_DPRINTF_L4(ibdm_string,
		    "\thandle_iounitinfo: setting gl_prev_iou %p",
		    gid_info->gl_prev_iou);
		gid_info->gl_prev_iou = gid_info->gl_iou;
		ibdm.ibdm_prev_iou = 1;
		gid_info->gl_iou = NULL;
	}

	size = sizeof (ibdm_iou_info_t) + num_iocs * sizeof (ibdm_ioc_info_t);
	gid_info->gl_iou = (ibdm_iou_info_t *)kmem_zalloc(size, KM_SLEEP);
	giou_info = &gid_info->gl_iou->iou_info;
	gid_info->gl_iou->iou_ioc_info = (ibdm_ioc_info_t *)
	    ((char *)gid_info->gl_iou + sizeof (ibdm_iou_info_t));

	giou_info->iou_num_ctrl_slots	= gid_info->gl_num_iocs	= num_iocs;
	giou_info->iou_flag		= iou_info->iou_flag;
	bcopy(iou_info->iou_ctrl_list, giou_info->iou_ctrl_list, 128);
	giou_info->iou_changeid	= b2h16(iou_info->iou_changeid);
	gid_info->gl_pending_cmds++; /* for diag code */
	mutex_exit(&gid_info->gl_mutex);

	if (ibdm_get_diagcode(gid_info, 0) != IBDM_SUCCESS) {
		mutex_enter(&gid_info->gl_mutex);
		gid_info->gl_pending_cmds--;
		mutex_exit(&gid_info->gl_mutex);
	}
	/*
	 * Parallelize getting IOC controller profiles from here.
	 * Allocate IBMF packets and send commands to get IOC profile for
	 * each IOC present on the IOU.
	 */
	for (ii = 0; ii < num_iocs; ii++) {
		/*
		 * Check whether IOC is present in the slot
		 * Series of nibbles (in the field iou_ctrl_list) represents
		 * a slot in the IOU.
		 * Byte format: 76543210
		 * Bits 0-3 of first byte represent Slot 2
		 * bits 4-7 of first byte represent slot 1,
		 * bits 0-3 of second byte represent slot 4 and so on
		 * Each 4-bit nibble has the following meaning
		 * 0x0 : IOC not installed
		 * 0x1 : IOC is present
		 * 0xf : Slot does not exist
		 * and all other values are reserved.
		 */
		ioc_info = IBDM_GIDINFO2IOCINFO(gid_info, ii);
		slot_info = giou_info->iou_ctrl_list[(ii/2)];
		if ((ii % 2) == 0)
			slot_info = (slot_info >> 4);

		if ((slot_info & 0xf) != 1) {
			IBTF_DPRINTF_L4("ibdm", "\thandle_iouintinfo: "
			    "No IOC is present in the slot = %d", ii);
			ioc_info->ioc_state = IBDM_IOC_STATE_PROBE_FAILED;
			continue;
		}

		mutex_enter(&gid_info->gl_mutex);
		ibdm_bump_transactionID(gid_info);
		mutex_exit(&gid_info->gl_mutex);

		/*
		 * Re use the already allocated packet (for IOUnitinfo) to
		 * send the first IOC controller attribute. Allocate new
		 * IBMF packets for the rest of the IOC's
		 */
		if (first != B_TRUE) {
			msg = NULL;
			if (ibmf_alloc_msg(ibmf_hdl, IBMF_ALLOC_SLEEP,
			    &msg) != IBMF_SUCCESS) {
				IBTF_DPRINTF_L4("ibdm", "\thandle_iouintinfo: "
				    "IBMF packet allocation failed");
				continue;
			}

		}

		/* allocate send buffers for all messages */
		_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*msg))
		ibdm_alloc_send_buffers(msg);
		_NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*msg))

		msg->im_local_addr.ia_local_lid	= gid_info->gl_slid;
		msg->im_local_addr.ia_remote_lid	= gid_info->gl_dlid;
		if (gid_info->gl_redirected == B_TRUE) {
			if (gid_info->gl_redirect_dlid != 0) {
				msg->im_local_addr.ia_remote_lid =
				    gid_info->gl_redirect_dlid;
			}
			msg->im_local_addr.ia_remote_qno =
			    gid_info->gl_redirect_QP;
			msg->im_local_addr.ia_p_key =
			    gid_info->gl_redirect_pkey;
			msg->im_local_addr.ia_q_key =
			    gid_info->gl_redirect_qkey;
			msg->im_local_addr.ia_service_level =
			    gid_info->gl_redirectSL;
		} else {
			msg->im_local_addr.ia_remote_qno = 1;
			msg->im_local_addr.ia_p_key = gid_info->gl_p_key;
			msg->im_local_addr.ia_q_key = IB_GSI_QKEY;
			msg->im_local_addr.ia_service_level = gid_info->gl_SL;
		}

		hdr			= IBDM_OUT_IBMFMSG_MADHDR(msg);
		hdr->BaseVersion	= MAD_CLASS_BASE_VERS_1;
		hdr->MgmtClass		= MAD_MGMT_CLASS_DEV_MGT;
		hdr->ClassVersion	= IB_DM_CLASS_VERSION_1;
		hdr->R_Method		= IB_DM_DEVMGT_METHOD_GET;
		hdr->Status		= 0;
		hdr->TransactionID	= h2b64(gid_info->gl_transactionID);
		hdr->AttributeID	= h2b16(IB_DM_ATTR_IOC_CTRL_PROFILE);
		hdr->AttributeModifier 	= h2b32(ii + 1);

		ioc_info->ioc_state	= IBDM_IOC_STATE_PROBE_INVALID;
		cb_args			= &ioc_info->ioc_cb_args;
		cb_args->cb_gid_info	= gid_info;
		cb_args->cb_retry_count	= ibdm_dft_retry_cnt;
		cb_args->cb_req_type	= IBDM_REQ_TYPE_IOCINFO;
		cb_args->cb_ioc_num	= ii;

		mutex_enter(&gid_info->gl_mutex);
		gid_info->gl_pending_cmds++; /* for diag code */

		ioc_info->ioc_timeout_id = timeout(ibdm_pkt_timeout_hdlr,
		    cb_args, IBDM_TIMEOUT_VALUE(ibdm_dft_timeout));
		mutex_exit(&gid_info->gl_mutex);

		IBTF_DPRINTF_L5("ibdm", "\thandle_iounitinfo:"
		    "timeout 0x%x, ioc_num %d", ioc_info->ioc_timeout_id, ii);

		if (ibmf_msg_transport(ibmf_hdl, gid_info->gl_qp_hdl, msg,
		    NULL, ibdm_ibmf_send_cb, cb_args, 0) != IBMF_SUCCESS) {
			IBTF_DPRINTF_L2("ibdm",
			    "\thandle_iounitinfo: msg transport failed");
			ibdm_ibmf_send_cb(ibmf_hdl, msg, cb_args);
		}
		(*flag) |= IBDM_IBMF_PKT_REUSED;
		first = B_FALSE;
		gid_info->gl_iou->iou_niocs_probe_in_progress++;
	}
}


/*
 * ibdm_handle_ioc_profile()
 *	Invoked by the IBMF when the IOCControllerProfile request
 *	gets completed
 */
static void
ibdm_handle_ioc_profile(ibmf_handle_t ibmf_hdl,
    ibmf_msg_t *msg, ibdm_dp_gidinfo_t *gid_info, int *flag)
{
	int				first = B_TRUE, reprobe = 0;
	uint_t				ii, ioc_no, srv_start;
	uint_t				nserv_entries;
	timeout_id_t			timeout_id;
	ib_mad_hdr_t			*hdr;
	ibdm_ioc_info_t			*ioc_info;
	ibdm_timeout_cb_args_t		*cb_args;
	ib_dm_ioc_ctrl_profile_t	*ioc, *gioc;

	IBTF_DPRINTF_L4("ibdm", "\thandle_ioc_profile:"
	    " ibmf hdl %p msg %p gid info %p", ibmf_hdl, msg, gid_info);

	ioc = IBDM_IN_IBMFMSG2IOC(msg);
	/*
	 * Check whether we know this IOC already
	 * This will return NULL if reprobe is in progress
	 * IBDM_IOC_STATE_REPROBE_PROGRESS will be set.
	 * Do not hold mutexes here.
	 */
	if (ibdm_is_ioc_present(ioc->ioc_guid, gid_info, flag) != NULL) {
		IBTF_DPRINTF_L4("ibdm", "\thandle_ioc_profile:"
		    "IOC guid %llx is present", ioc->ioc_guid);
		return;
	}
	ioc_no = IBDM_IN_IBMFMSG_ATTRMOD(msg);
	IBTF_DPRINTF_L4("ibdm", "\thandle_ioc_profile: ioc_no = %d", ioc_no-1);

	/* Make sure that IOC index is with the valid range */
	if (IBDM_IS_IOC_NUM_INVALID(ioc_no, gid_info)) {
		IBTF_DPRINTF_L2("ibdm", "\thandle_ioc_profile: "
		    "IOC index Out of range, index %d", ioc);
		(*flag) |= IBDM_IBMF_PKT_UNEXP_RESP;
		return;
	}
	ioc_info = &gid_info->gl_iou->iou_ioc_info[ioc_no - 1];
	ioc_info->ioc_iou_info = gid_info->gl_iou;

	mutex_enter(&gid_info->gl_mutex);
	if (ioc_info->ioc_state == IBDM_IOC_STATE_REPROBE_PROGRESS) {
		reprobe = 1;
		ioc_info->ioc_prev_serv = ioc_info->ioc_serv;
		ioc_info->ioc_serv = NULL;
		ioc_info->ioc_prev_serv_cnt =
		    ioc_info->ioc_profile.ioc_service_entries;
	} else if (ioc_info->ioc_state != IBDM_IOC_STATE_PROBE_INVALID) {
		IBTF_DPRINTF_L2("ibdm", "\thandle_ioc_profile: DUP response"
		    "ioc %d, ioc_state %x", ioc_no - 1, ioc_info->ioc_state);
		mutex_exit(&gid_info->gl_mutex);
		(*flag) |= IBDM_IBMF_PKT_DUP_RESP;
		return;
	}
	ioc_info->ioc_cb_args.cb_req_type = 0;
	if (ioc_info->ioc_timeout_id) {
		timeout_id = ioc_info->ioc_timeout_id;
		ioc_info->ioc_timeout_id = 0;
		mutex_exit(&gid_info->gl_mutex);
		IBTF_DPRINTF_L5("ibdm", "handle_ioc_profile: "
		    "ioc_timeout_id = 0x%x", timeout_id);
		if (untimeout(timeout_id) == -1) {
			IBTF_DPRINTF_L2("ibdm", "handle_ioc_profile: "
			    "untimeout ioc_timeout_id failed");
		}
		mutex_enter(&gid_info->gl_mutex);
	}

	ioc_info->ioc_state = IBDM_IOC_STATE_PROBE_SUCCESS;
	if (reprobe == 0) {
		ioc_info->ioc_iou_guid = gid_info->gl_nodeguid;
		ioc_info->ioc_nodeguid = gid_info->gl_nodeguid;
	}

	/*
	 * Save all the IOC information in the global structures.
	 * Note the wire format is Big Endian and the Sparc process also
	 * big endian. So, there is no need to convert the data fields
	 * The conversion routines used below are ineffective on Sparc
	 * machines where as they will be effective on little endian
	 * machines such as Intel processors.
	 */
	gioc = (ib_dm_ioc_ctrl_profile_t *)&ioc_info->ioc_profile;

	/*
	 * Restrict updates to onlyport GIDs and service entries during reprobe
	 */
	if (reprobe == 0) {
		gioc->ioc_guid			= b2h64(ioc->ioc_guid);
		gioc->ioc_vendorid		=
		    ((b2h32(ioc->ioc_vendorid) & IB_DM_VENDORID_MASK)
		    >> IB_DM_VENDORID_SHIFT);
		gioc->ioc_deviceid		= b2h32(ioc->ioc_deviceid);
		gioc->ioc_device_ver		= b2h16(ioc->ioc_device_ver);
		gioc->ioc_subsys_vendorid	=
		    ((b2h32(ioc->ioc_subsys_vendorid) & IB_DM_VENDORID_MASK)
		    >> IB_DM_VENDORID_SHIFT);
		gioc->ioc_subsys_id		= b2h32(ioc->ioc_subsys_id);
		gioc->ioc_io_class		= b2h16(ioc->ioc_io_class);
		gioc->ioc_io_subclass		= b2h16(ioc->ioc_io_subclass);
		gioc->ioc_protocol		= b2h16(ioc->ioc_protocol);
		gioc->ioc_protocol_ver		= b2h16(ioc->ioc_protocol_ver);
		gioc->ioc_send_msg_qdepth	=
		    b2h16(ioc->ioc_send_msg_qdepth);
		gioc->ioc_rdma_read_qdepth	=
		    b2h16(ioc->ioc_rdma_read_qdepth);
		gioc->ioc_send_msg_sz		= b2h32(ioc->ioc_send_msg_sz);
		gioc->ioc_rdma_xfer_sz		= b2h32(ioc->ioc_rdma_xfer_sz);
		gioc->ioc_ctrl_opcap_mask	= ioc->ioc_ctrl_opcap_mask;
		bcopy(ioc->ioc_id_string, gioc->ioc_id_string,
		    IB_DM_IOC_ID_STRING_LEN);

		ioc_info->ioc_iou_diagcode = gid_info->gl_iou->iou_diagcode;
		ioc_info->ioc_iou_dc_valid = gid_info->gl_iou->iou_dc_valid;
		ioc_info->ioc_diagdeviceid = (IB_DM_IOU_DEVICEID_MASK &
		    gid_info->gl_iou->iou_info.iou_flag) ? B_TRUE : B_FALSE;

		if (ioc_info->ioc_diagdeviceid == B_TRUE) {
			gid_info->gl_pending_cmds++;
			IBTF_DPRINTF_L3(ibdm_string,
			    "\tibdm_handle_ioc_profile: "
			    "%d: gid_info %p gl_state %d pending_cmds %d",
			    __LINE__, gid_info, gid_info->gl_state,
			    gid_info->gl_pending_cmds);
		}
	}
	gioc->ioc_service_entries	= ioc->ioc_service_entries;
	mutex_exit(&gid_info->gl_mutex);

	ibdm_dump_ioc_profile(gioc);

	if ((ioc_info->ioc_diagdeviceid == B_TRUE) && (reprobe == 0)) {
		if (ibdm_get_diagcode(gid_info, ioc_no) != IBDM_SUCCESS) {
			mutex_enter(&gid_info->gl_mutex);
			gid_info->gl_pending_cmds--;
			mutex_exit(&gid_info->gl_mutex);
		}
	}
	ioc_info->ioc_serv = (ibdm_srvents_info_t *)kmem_zalloc(
	    (gioc->ioc_service_entries * sizeof (ibdm_srvents_info_t)),
	    KM_SLEEP);

	/*
	 * In one single request, maximum number of requests that can be
	 * obtained is 4. If number of service entries are more than four,
	 * calculate number requests needed and send them parallelly.
	 */
	nserv_entries = ioc->ioc_service_entries;
	ii = 0;
	while (nserv_entries) {
		mutex_enter(&gid_info->gl_mutex);
		gid_info->gl_pending_cmds++;
		ibdm_bump_transactionID(gid_info);
		mutex_exit(&gid_info->gl_mutex);

		if (first != B_TRUE) {
			if (ibmf_alloc_msg(ibmf_hdl, IBMF_ALLOC_SLEEP,
			    &msg) != IBMF_SUCCESS) {
				continue;
			}

		}
		_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*msg))
		ibdm_alloc_send_buffers(msg);
		_NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*msg))
		msg->im_local_addr.ia_local_lid	= gid_info->gl_slid;
		msg->im_local_addr.ia_remote_lid	= gid_info->gl_dlid;
		if (gid_info->gl_redirected == B_TRUE) {
			if (gid_info->gl_redirect_dlid != 0) {
				msg->im_local_addr.ia_remote_lid =
				    gid_info->gl_redirect_dlid;
			}
			msg->im_local_addr.ia_remote_qno =
			    gid_info->gl_redirect_QP;
			msg->im_local_addr.ia_p_key =
			    gid_info->gl_redirect_pkey;
			msg->im_local_addr.ia_q_key =
			    gid_info->gl_redirect_qkey;
			msg->im_local_addr.ia_service_level =
			    gid_info->gl_redirectSL;
		} else {
			msg->im_local_addr.ia_remote_qno = 1;
			msg->im_local_addr.ia_p_key = gid_info->gl_p_key;
			msg->im_local_addr.ia_q_key = IB_GSI_QKEY;
			msg->im_local_addr.ia_service_level = gid_info->gl_SL;
		}

		hdr			= IBDM_OUT_IBMFMSG_MADHDR(msg);
		hdr->BaseVersion	= MAD_CLASS_BASE_VERS_1;
		hdr->MgmtClass		= MAD_MGMT_CLASS_DEV_MGT;
		hdr->ClassVersion	= IB_DM_CLASS_VERSION_1;
		hdr->R_Method		= IB_DM_DEVMGT_METHOD_GET;
		hdr->Status		= 0;
		hdr->TransactionID	= h2b64(gid_info->gl_transactionID);
		hdr->AttributeID	= h2b16(IB_DM_ATTR_SERVICE_ENTRIES);

		srv_start = ii * 4;
		cb_args = &ioc_info->ioc_serv[srv_start].se_cb_args;
		cb_args->cb_gid_info	= gid_info;
		cb_args->cb_retry_count	= ibdm_dft_retry_cnt;
		cb_args->cb_req_type	= IBDM_REQ_TYPE_SRVENTS;
		cb_args->cb_srvents_start = srv_start;
		cb_args->cb_ioc_num	= ioc_no - 1;

		if (nserv_entries >= IBDM_MAX_SERV_ENTRIES_PER_REQ) {
			nserv_entries -= IBDM_MAX_SERV_ENTRIES_PER_REQ;
			cb_args->cb_srvents_end = (cb_args->cb_srvents_start +
			    IBDM_MAX_SERV_ENTRIES_PER_REQ - 1);
		} else {
			cb_args->cb_srvents_end =
			    (cb_args->cb_srvents_start + nserv_entries - 1);
			nserv_entries = 0;
		}
		_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*hdr))
		ibdm_fill_srv_attr_mod(hdr, cb_args);
		_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*hdr))

		mutex_enter(&gid_info->gl_mutex);
		ioc_info->ioc_serv[srv_start].se_timeout_id = timeout(
		    ibdm_pkt_timeout_hdlr, cb_args,
		    IBDM_TIMEOUT_VALUE(ibdm_dft_timeout));
		mutex_exit(&gid_info->gl_mutex);

		IBTF_DPRINTF_L5("ibdm", "\thandle_ioc_profile:"
		    "timeout %x, ioc %d srv %d",
		    ioc_info->ioc_serv[srv_start].se_timeout_id,
		    ioc_no - 1, srv_start);

		if (ibmf_msg_transport(ibmf_hdl, gid_info->gl_qp_hdl, msg,
		    NULL, ibdm_ibmf_send_cb, cb_args, 0) != IBMF_SUCCESS) {
			IBTF_DPRINTF_L2("ibdm",
			    "\thandle_ioc_profile: msg send failed");
			ibdm_ibmf_send_cb(ibmf_hdl, msg, cb_args);
		}
		(*flag) |= IBDM_IBMF_PKT_REUSED;
		first = B_FALSE;
		ii++;
	}
}


/*
 * ibdm_handle_srventry_mad()
 */
static void
ibdm_handle_srventry_mad(ibmf_msg_t *msg,
    ibdm_dp_gidinfo_t *gid_info, int *flag)
{
	uint_t			ii, ioc_no, attrmod;
	uint_t			nentries, start, end;
	timeout_id_t		timeout_id;
	ib_dm_srv_t		*srv_ents;
	ibdm_ioc_info_t		*ioc_info;
	ibdm_srvents_info_t	*gsrv_ents;

	IBTF_DPRINTF_L4("ibdm", "\thandle_srventry_mad:"
	    " IBMF msg %p gid info %p", msg, gid_info);

	srv_ents = IBDM_IN_IBMFMSG2SRVENT(msg);
	/*
	 * Get the start and end index of the service entries
	 * Upper 16 bits identify the IOC
	 * Lower 16 bits specify the range of service entries
	 * 	LSB specifies (Big endian) end of the range
	 * 	MSB specifies (Big endian) start of the range
	 */
	attrmod = IBDM_IN_IBMFMSG_ATTRMOD(msg);
	ioc_no	= ((attrmod >> 16) & IBDM_16_BIT_MASK);
	end	= ((attrmod >> 8) & IBDM_8_BIT_MASK);
	start	= (attrmod & IBDM_8_BIT_MASK);

	/* Make sure that IOC index is with the valid range */
	if ((ioc_no < 1) |
	    (ioc_no > gid_info->gl_iou->iou_info.iou_num_ctrl_slots)) {
		IBTF_DPRINTF_L2("ibdm", "\thandle_srventry_mad: "
		    "IOC index Out of range, index %d", ioc_no);
		(*flag) |= IBDM_IBMF_PKT_UNEXP_RESP;
		return;
	}
	ioc_info = IBDM_GIDINFO2IOCINFO(gid_info, (ioc_no -1));

	/*
	 * Make sure that the "start" and "end" service indexes are
	 * with in the valid range
	 */
	nentries = ioc_info->ioc_profile.ioc_service_entries;
	if ((start > end) | (start >= nentries) | (end >= nentries)) {
		IBTF_DPRINTF_L2("ibdm", "\thandle_srventry_mad: "
		    "Attr modifier 0x%x, #Serv entries %d", attrmod, nentries);
		(*flag) |= IBDM_IBMF_PKT_UNEXP_RESP;
		return;
	}
	gsrv_ents = &ioc_info->ioc_serv[start];
	mutex_enter(&gid_info->gl_mutex);
	if (gsrv_ents->se_state != IBDM_SE_INVALID) {
		IBTF_DPRINTF_L2("ibdm", "\thandle_srventry_mad: "
		    "already known, ioc %d, srv %d, se_state %x",
		    ioc_no - 1, start, gsrv_ents->se_state);
		mutex_exit(&gid_info->gl_mutex);
		(*flag) |= IBDM_IBMF_PKT_DUP_RESP;
		return;
	}
	ioc_info->ioc_serv[start].se_cb_args.cb_req_type = 0;
	if (ioc_info->ioc_serv[start].se_timeout_id) {
		IBTF_DPRINTF_L2("ibdm",
		    "\thandle_srventry_mad: ioc %d start %d", ioc_no, start);
		timeout_id = ioc_info->ioc_serv[start].se_timeout_id;
		ioc_info->ioc_serv[start].se_timeout_id = 0;
		mutex_exit(&gid_info->gl_mutex);
		IBTF_DPRINTF_L5("ibdm", "handle_srverntry_mad: "
		    "se_timeout_id = 0x%x", timeout_id);
		if (untimeout(timeout_id) == -1) {
			IBTF_DPRINTF_L2("ibdm", "handle_srventry_mad: "
			    "untimeout se_timeout_id failed");
		}
		mutex_enter(&gid_info->gl_mutex);
	}

	gsrv_ents->se_state = IBDM_SE_VALID;
	mutex_exit(&gid_info->gl_mutex);
	for (ii = start; ii <= end; ii++, srv_ents++, gsrv_ents++) {
		gsrv_ents->se_attr.srv_id = b2h64(srv_ents->srv_id);
		bcopy(srv_ents->srv_name,
		    gsrv_ents->se_attr.srv_name, IB_DM_MAX_SVC_NAME_LEN);
		ibdm_dump_service_entries(&gsrv_ents->se_attr);
	}
}


/*
 * ibdm_get_diagcode:
 *	Send request to get IOU/IOC diag code
 *	Returns IBDM_SUCCESS/IBDM_FAILURE
 */
static int
ibdm_get_diagcode(ibdm_dp_gidinfo_t *gid_info, int attr)
{
	ibmf_msg_t		*msg;
	ib_mad_hdr_t		*hdr;
	ibdm_ioc_info_t		*ioc;
	ibdm_timeout_cb_args_t	*cb_args;
	timeout_id_t		*timeout_id;

	IBTF_DPRINTF_L4("ibdm", "\tget_diagcode: gid info %p, attr = %d",
	    gid_info, attr);

	if (ibmf_alloc_msg(gid_info->gl_ibmf_hdl, IBMF_ALLOC_SLEEP,
	    &msg) != IBMF_SUCCESS) {
		IBTF_DPRINTF_L4("ibdm", "\tget_diagcode: pkt alloc fail");
		return (IBDM_FAILURE);
	}

	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*msg))
	ibdm_alloc_send_buffers(msg);
	_NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*msg))

	mutex_enter(&gid_info->gl_mutex);
	ibdm_bump_transactionID(gid_info);
	mutex_exit(&gid_info->gl_mutex);

	msg->im_local_addr.ia_local_lid	= gid_info->gl_slid;
	msg->im_local_addr.ia_remote_lid	= gid_info->gl_dlid;
	if (gid_info->gl_redirected == B_TRUE) {
		if (gid_info->gl_redirect_dlid != 0) {
			msg->im_local_addr.ia_remote_lid =
			    gid_info->gl_redirect_dlid;
		}

		msg->im_local_addr.ia_remote_qno = gid_info->gl_redirect_QP;
		msg->im_local_addr.ia_p_key = gid_info->gl_redirect_pkey;
		msg->im_local_addr.ia_q_key = gid_info->gl_redirect_qkey;
		msg->im_local_addr.ia_service_level = gid_info->gl_redirectSL;
	} else {
		msg->im_local_addr.ia_remote_qno = 1;
		msg->im_local_addr.ia_p_key = gid_info->gl_p_key;
		msg->im_local_addr.ia_q_key = IB_GSI_QKEY;
		msg->im_local_addr.ia_service_level = gid_info->gl_SL;
	}

	hdr			= IBDM_OUT_IBMFMSG_MADHDR(msg);
	hdr->BaseVersion	= MAD_CLASS_BASE_VERS_1;
	hdr->MgmtClass		= MAD_MGMT_CLASS_DEV_MGT;
	hdr->ClassVersion	= IB_DM_CLASS_VERSION_1;
	hdr->R_Method		= IB_DM_DEVMGT_METHOD_GET;
	hdr->Status		= 0;
	hdr->TransactionID	= h2b64(gid_info->gl_transactionID);

	hdr->AttributeID	= h2b16(IB_DM_ATTR_DIAG_CODE);
	hdr->AttributeModifier	= h2b32(attr);

	if (attr == 0) {
		cb_args = &gid_info->gl_iou_cb_args;
		gid_info->gl_iou->iou_dc_valid = B_FALSE;
		cb_args->cb_ioc_num	= 0;
		cb_args->cb_req_type	= IBDM_REQ_TYPE_IOU_DIAGCODE;
		timeout_id = &gid_info->gl_timeout_id;
	} else {
		ioc = IBDM_GIDINFO2IOCINFO(gid_info, (attr - 1));
		ioc->ioc_dc_valid = B_FALSE;
		cb_args = &ioc->ioc_dc_cb_args;
		cb_args->cb_ioc_num	= attr - 1;
		cb_args->cb_req_type	= IBDM_REQ_TYPE_IOC_DIAGCODE;
		timeout_id = &ioc->ioc_dc_timeout_id;
	}
	cb_args->cb_gid_info	= gid_info;
	cb_args->cb_retry_count	= ibdm_dft_retry_cnt;
	cb_args->cb_srvents_start = 0;

	mutex_enter(&gid_info->gl_mutex);
	*timeout_id = timeout(ibdm_pkt_timeout_hdlr,
	    cb_args, IBDM_TIMEOUT_VALUE(ibdm_dft_timeout));
	mutex_exit(&gid_info->gl_mutex);

	IBTF_DPRINTF_L5("ibdm", "\tget_diagcode:"
	    "timeout %x, ioc %d", *timeout_id, cb_args->cb_ioc_num);

	if (ibmf_msg_transport(gid_info->gl_ibmf_hdl, gid_info->gl_qp_hdl,
	    msg, NULL, ibdm_ibmf_send_cb, cb_args, 0) != IBMF_SUCCESS) {
		IBTF_DPRINTF_L2("ibdm", "\tget_diagcode: ibmf send failed");
		ibdm_ibmf_send_cb(gid_info->gl_ibmf_hdl, msg, cb_args);
	}
	return (IBDM_SUCCESS);
}

/*
 * ibdm_handle_diagcode:
 *	Process the DiagCode MAD response and update local DM
 *	data structure.
 */
static void
ibdm_handle_diagcode(ibmf_msg_t *ibmf_msg,
    ibdm_dp_gidinfo_t *gid_info, int *flag)
{
	uint16_t	attrmod, *diagcode;
	ibdm_iou_info_t	*iou;
	ibdm_ioc_info_t	*ioc;
	timeout_id_t	timeout_id;
	ibdm_timeout_cb_args_t	*cb_args;

	diagcode = (uint16_t *)ibmf_msg->im_msgbufs_recv.im_bufs_cl_data;

	mutex_enter(&gid_info->gl_mutex);
	attrmod = IBDM_IN_IBMFMSG_ATTRMOD(ibmf_msg);
	iou = gid_info->gl_iou;
	if (attrmod == 0) {
		if (iou->iou_dc_valid != B_FALSE) {
			(*flag) |= IBDM_IBMF_PKT_DUP_RESP;
			IBTF_DPRINTF_L4("ibdm",
			    "\thandle_diagcode: Duplicate IOU DiagCode");
			mutex_exit(&gid_info->gl_mutex);
			return;
		}
		cb_args = &gid_info->gl_iou_cb_args;
		cb_args->cb_req_type = 0;
		iou->iou_diagcode = b2h16(*diagcode);
		iou->iou_dc_valid = B_TRUE;
		if (gid_info->gl_timeout_id) {
			timeout_id = gid_info->gl_timeout_id;
			mutex_exit(&gid_info->gl_mutex);
			IBTF_DPRINTF_L5("ibdm", "\thandle_diagcode: "
			    "gl_timeout_id = 0x%x", timeout_id);
			if (untimeout(timeout_id) == -1) {
				IBTF_DPRINTF_L2("ibdm", "handle_diagcode: "
				    "untimeout gl_timeout_id failed");
			}
			mutex_enter(&gid_info->gl_mutex);
			gid_info->gl_timeout_id = 0;
		}
	} else {
		ioc = IBDM_GIDINFO2IOCINFO(gid_info, (attrmod - 1));
		if (ioc->ioc_dc_valid != B_FALSE) {
			(*flag) |= IBDM_IBMF_PKT_DUP_RESP;
			IBTF_DPRINTF_L4("ibdm",
			    "\thandle_diagcode: Duplicate IOC DiagCode");
			mutex_exit(&gid_info->gl_mutex);
			return;
		}
		cb_args = &ioc->ioc_dc_cb_args;
		cb_args->cb_req_type = 0;
		ioc->ioc_diagcode = b2h16(*diagcode);
		ioc->ioc_dc_valid = B_TRUE;
		timeout_id = iou->iou_ioc_info[attrmod - 1].ioc_dc_timeout_id;
		if (timeout_id) {
			iou->iou_ioc_info[attrmod - 1].ioc_dc_timeout_id = 0;
			mutex_exit(&gid_info->gl_mutex);
			IBTF_DPRINTF_L5("ibdm", "handle_diagcode: "
			    "timeout_id = 0x%x", timeout_id);
			if (untimeout(timeout_id) == -1) {
				IBTF_DPRINTF_L2("ibdm", "\thandle_diagcode: "
				    "untimeout ioc_dc_timeout_id failed");
			}
			mutex_enter(&gid_info->gl_mutex);
		}
	}
	mutex_exit(&gid_info->gl_mutex);

	IBTF_DPRINTF_L4("ibdm", "\thandle_diagcode: DiagCode : 0x%x"
	    "attrmod : 0x%x", b2h16(*diagcode), attrmod);
}


/*
 * ibdm_is_ioc_present()
 *	Return ibdm_ioc_info_t if IOC guid is found in the global gid list
 */
static ibdm_ioc_info_t *
ibdm_is_ioc_present(ib_guid_t ioc_guid,
    ibdm_dp_gidinfo_t *gid_info, int *flag)
{
	int				ii;
	ibdm_ioc_info_t			*ioc;
	ibdm_dp_gidinfo_t		*head;
	ib_dm_io_unitinfo_t		*iou;

	mutex_enter(&ibdm.ibdm_mutex);
	head = ibdm.ibdm_dp_gidlist_head;
	while (head) {
		mutex_enter(&head->gl_mutex);
		if (head->gl_iou == NULL) {
			mutex_exit(&head->gl_mutex);
			head = head->gl_next;
			continue;
		}
		iou = &head->gl_iou->iou_info;
		for (ii = 0; ii < iou->iou_num_ctrl_slots; ii++) {
			ioc = IBDM_GIDINFO2IOCINFO(head, ii);
			if ((ioc->ioc_state == IBDM_IOC_STATE_PROBE_SUCCESS) &&
			    (ioc->ioc_profile.ioc_guid == ioc_guid)) {
				if (gid_info == head) {
					*flag |= IBDM_IBMF_PKT_DUP_RESP;
				} else if (ibdm_check_dgid(head->gl_dgid_lo,
				    head->gl_dgid_hi) != NULL) {
					IBTF_DPRINTF_L4("ibdm", "\tis_ioc_"
					    "present: gid not present");
					ibdm_add_to_gl_gid(gid_info, head);
				}
				mutex_exit(&head->gl_mutex);
				mutex_exit(&ibdm.ibdm_mutex);
				return (ioc);
			}
		}
		mutex_exit(&head->gl_mutex);
		head = head->gl_next;
	}
	mutex_exit(&ibdm.ibdm_mutex);
	return (NULL);
}


/*
 * ibdm_ibmf_send_cb()
 *	IBMF invokes this callback routine after posting the DM MAD to
 *	the HCA.
 */
/*ARGSUSED*/
static void
ibdm_ibmf_send_cb(ibmf_handle_t ibmf_hdl, ibmf_msg_t *ibmf_msg, void *arg)
{
	ibdm_dump_ibmf_msg(ibmf_msg, 1);
	ibdm_free_send_buffers(ibmf_msg);
	if (ibmf_free_msg(ibmf_hdl, &ibmf_msg) != IBMF_SUCCESS) {
		IBTF_DPRINTF_L4("ibdm",
		    "\tibmf_send_cb: IBMF free msg failed");
	}
}


/*
 * ibdm_ibmf_recv_cb()
 *	Invoked by the IBMF when a response to the one of the DM requests
 *	is received.
 */
/*ARGSUSED*/
static void
ibdm_ibmf_recv_cb(ibmf_handle_t ibmf_hdl, ibmf_msg_t *msg, void *arg)
{
	ibdm_taskq_args_t	*taskq_args;

	/*
	 * If the taskq enable is set then dispatch a taskq to process
	 * the MAD, otherwise just process it on this thread
	 */
	if (ibdm_taskq_enable != IBDM_ENABLE_TASKQ_HANDLING) {
		ibdm_process_incoming_mad(ibmf_hdl, msg, arg);
		return;
	}

	/*
	 * create a taskq and dispatch it to process the incoming MAD
	 */
	taskq_args = kmem_alloc(sizeof (ibdm_taskq_args_t), KM_NOSLEEP);
	if (taskq_args == NULL) {
		IBTF_DPRINTF_L2("ibdm", "ibmf_recv_cb: kmem_alloc failed for"
		    "taskq_args");
		if (ibmf_free_msg(ibmf_hdl, &msg) != IBMF_SUCCESS) {
			IBTF_DPRINTF_L4("ibmf_recv_cb",
			    "\tibmf_recv_cb: IBMF free msg failed");
		}
		return;
	}
	taskq_args->tq_ibmf_handle = ibmf_hdl;
	taskq_args->tq_ibmf_msg = msg;
	taskq_args->tq_args = arg;

	if (taskq_dispatch(system_taskq, ibdm_recv_incoming_mad, taskq_args,
	    TQ_NOSLEEP) == TASKQID_INVALID) {
		IBTF_DPRINTF_L2("ibdm", "ibmf_recv_cb: taskq_dispatch failed");
		if (ibmf_free_msg(ibmf_hdl, &msg) != IBMF_SUCCESS) {
			IBTF_DPRINTF_L4("ibmf_recv_cb",
			    "\tibmf_recv_cb: IBMF free msg failed");
		}
		kmem_free(taskq_args, sizeof (ibdm_taskq_args_t));
		return;
	}

	/* taskq_args are deleted in ibdm_recv_incoming_mad() */
}


void
ibdm_recv_incoming_mad(void *args)
{
	ibdm_taskq_args_t	*taskq_args;

	taskq_args = (ibdm_taskq_args_t *)args;

	IBTF_DPRINTF_L4("ibdm", "\tibdm_recv_incoming_mad: "
	    "Processing incoming MAD via taskq");

	ibdm_process_incoming_mad(taskq_args->tq_ibmf_handle,
	    taskq_args->tq_ibmf_msg, taskq_args->tq_args);

	kmem_free(taskq_args, sizeof (ibdm_taskq_args_t));
}


/*
 * Calls ibdm_process_incoming_mad with all function arguments  extracted
 * from args
 */
/*ARGSUSED*/
static void
ibdm_process_incoming_mad(ibmf_handle_t ibmf_hdl, ibmf_msg_t *msg, void *arg)
{
	int			flag = 0;
	int			ret;
	uint64_t		transaction_id;
	ib_mad_hdr_t		*hdr;
	ibdm_dp_gidinfo_t	*gid_info = NULL;

	IBTF_DPRINTF_L4("ibdm",
	    "\tprocess_incoming_mad: ibmf hdl %p pkt %p", ibmf_hdl, msg);
	ibdm_dump_ibmf_msg(msg, 0);

	/*
	 * IBMF calls this routine for every DM MAD that arrives at this port.
	 * But we handle only the responses for requests we sent. We drop all
	 * the DM packets that does not have response bit set in the MAD
	 * header(this eliminates all the requests sent to this port).
	 * We handle only DM class version 1 MAD's
	 */
	hdr = IBDM_IN_IBMFMSG_MADHDR(msg);
	if (ibdm_verify_mad_status(hdr) != IBDM_SUCCESS) {
		if (ibmf_free_msg(ibmf_hdl, &msg) != IBMF_SUCCESS) {
			IBTF_DPRINTF_L2("ibdm", "\tprocess_incoming_mad: "
			    "IBMF free msg failed DM request drop it");
		}
		return;
	}

	transaction_id = b2h64(hdr->TransactionID);

	mutex_enter(&ibdm.ibdm_mutex);
	gid_info = ibdm.ibdm_dp_gidlist_head;
	while (gid_info) {
		if ((gid_info->gl_transactionID  &
		    IBDM_GID_TRANSACTIONID_MASK) ==
		    (transaction_id & IBDM_GID_TRANSACTIONID_MASK))
			break;
		gid_info = gid_info->gl_next;
	}
	mutex_exit(&ibdm.ibdm_mutex);

	if (gid_info == NULL) {
		/* Drop the packet */
		IBTF_DPRINTF_L2("ibdm", "process_incoming_mad: transaction ID"
		    " does not match: 0x%llx", transaction_id);
		if (ibmf_free_msg(ibmf_hdl, &msg) != IBMF_SUCCESS) {
			IBTF_DPRINTF_L2("ibdm", "process_incoming_mad: "
			    "IBMF free msg failed DM request drop it");
		}
		return;
	}

	/* Handle redirection for all the MAD's, except ClassPortInfo */
	if (((IBDM_IN_IBMFMSG_STATUS(msg) & MAD_STATUS_REDIRECT_REQUIRED)) &&
	    (IBDM_IN_IBMFMSG_ATTR(msg) != IB_DM_ATTR_CLASSPORTINFO)) {
		ret = ibdm_handle_redirection(msg, gid_info, &flag);
		if (ret == IBDM_SUCCESS) {
			return;
		}
	} else {
		uint_t gl_state;

		mutex_enter(&gid_info->gl_mutex);
		gl_state = gid_info->gl_state;
		mutex_exit(&gid_info->gl_mutex);

		switch (gl_state) {

		case IBDM_SET_CLASSPORTINFO:
			ibdm_handle_setclassportinfo(
			    ibmf_hdl, msg, gid_info, &flag);
			break;

		case IBDM_GET_CLASSPORTINFO:
			ibdm_handle_classportinfo(
			    ibmf_hdl, msg, gid_info, &flag);
			break;

		case IBDM_GET_IOUNITINFO:
			ibdm_handle_iounitinfo(ibmf_hdl, msg, gid_info, &flag);
			break;

		case IBDM_GET_IOC_DETAILS:
			switch (IBDM_IN_IBMFMSG_ATTR(msg)) {

			case IB_DM_ATTR_SERVICE_ENTRIES:
				ibdm_handle_srventry_mad(msg, gid_info, &flag);
				break;

			case IB_DM_ATTR_IOC_CTRL_PROFILE:
				ibdm_handle_ioc_profile(
				    ibmf_hdl, msg, gid_info, &flag);
				break;

			case IB_DM_ATTR_DIAG_CODE:
				ibdm_handle_diagcode(msg, gid_info, &flag);
				break;

			default:
				IBTF_DPRINTF_L2("ibdm", "process_incoming_mad: "
				    "Error state, wrong attribute :-(");
				(void) ibmf_free_msg(ibmf_hdl, &msg);
				return;
			}
			break;
		default:
			IBTF_DPRINTF_L2("ibdm",
			    "process_incoming_mad: Dropping the packet"
			    " gl_state %x", gl_state);
			if (ibmf_free_msg(ibmf_hdl, &msg) != IBMF_SUCCESS) {
				IBTF_DPRINTF_L2("ibdm", "process_incoming_mad: "
				    "IBMF free msg failed DM request drop it");
			}
			return;
		}
	}

	if ((flag & IBDM_IBMF_PKT_DUP_RESP) ||
	    (flag & IBDM_IBMF_PKT_UNEXP_RESP)) {
		IBTF_DPRINTF_L2("ibdm",
		    "\tprocess_incoming_mad:Dup/unexp resp : 0x%x", flag);
		if (ibmf_free_msg(ibmf_hdl, &msg) != IBMF_SUCCESS) {
			IBTF_DPRINTF_L2("ibdm", "process_incoming_mad: "
			    "IBMF free msg failed DM request drop it");
		}
		return;
	}

	mutex_enter(&gid_info->gl_mutex);
	if (gid_info->gl_pending_cmds < 1) {
		IBTF_DPRINTF_L2("ibdm",
		    "\tprocess_incoming_mad: pending commands negative");
	}
	if (--gid_info->gl_pending_cmds) {
		IBTF_DPRINTF_L4("ibdm", "\tprocess_incoming_mad: "
		    "gid_info %p pending cmds %d",
		    gid_info, gid_info->gl_pending_cmds);
		mutex_exit(&gid_info->gl_mutex);
	} else {
		uint_t prev_state;
		IBTF_DPRINTF_L4("ibdm", "\tprocess_incoming_mad: Probing DONE");
		prev_state = gid_info->gl_state;
		gid_info->gl_state = IBDM_GID_PROBING_COMPLETE;
		if (prev_state == IBDM_SET_CLASSPORTINFO) {
			IBTF_DPRINTF_L4("ibdm",
			    "\tprocess_incoming_mad: "
			    "Setclassportinfo for Cisco FC GW is done.");
			gid_info->gl_flag &= ~IBDM_CISCO_PROBE;
			gid_info->gl_flag |= IBDM_CISCO_PROBE_DONE;
			mutex_exit(&gid_info->gl_mutex);
			cv_broadcast(&gid_info->gl_probe_cv);
		} else {
			mutex_exit(&gid_info->gl_mutex);
			ibdm_notify_newgid_iocs(gid_info);
			mutex_enter(&ibdm.ibdm_mutex);
			if (--ibdm.ibdm_ngid_probes_in_progress == 0) {
				IBTF_DPRINTF_L4("ibdm",
				    "\tprocess_incoming_mad: Wakeup");
				ibdm.ibdm_busy &= ~IBDM_PROBE_IN_PROGRESS;
				cv_broadcast(&ibdm.ibdm_probe_cv);
			}
			mutex_exit(&ibdm.ibdm_mutex);
		}
	}

	/*
	 * Do not deallocate the IBMF packet if atleast one request
	 * is posted. IBMF packet is reused.
	 */
	if (!(flag & IBDM_IBMF_PKT_REUSED)) {
		if (ibmf_free_msg(ibmf_hdl, &msg) != IBMF_SUCCESS) {
			IBTF_DPRINTF_L2("ibdm", "\tprocess_incoming_mad: "
			    "IBMF free msg failed DM request drop it");
		}
	}
}


/*
 * ibdm_verify_mad_status()
 *	Verifies the MAD status
 *	Returns IBDM_SUCCESS if status is correct
 *	Returns IBDM_FAILURE for bogus MAD status
 */
static int
ibdm_verify_mad_status(ib_mad_hdr_t *hdr)
{
	int	ret = 0;

	if ((hdr->R_Method != IB_DM_DEVMGT_METHOD_GET_RESP) ||
	    (hdr->ClassVersion != IB_DM_CLASS_VERSION_1)) {
		return (IBDM_FAILURE);
	}

	if (b2h16(hdr->Status) == 0)
		ret = IBDM_SUCCESS;
	else if ((b2h16(hdr->Status) & 0x1f) == MAD_STATUS_REDIRECT_REQUIRED)
		ret = IBDM_SUCCESS;
	else {
		IBTF_DPRINTF_L2("ibdm",
		    "\tverify_mad_status: Status : 0x%x", b2h16(hdr->Status));
		ret = IBDM_FAILURE;
	}
	return (ret);
}



/*
 * ibdm_handle_redirection()
 *	Returns IBDM_SUCCESS/IBDM_FAILURE
 */
static int
ibdm_handle_redirection(ibmf_msg_t *msg,
    ibdm_dp_gidinfo_t *gid_info, int *flag)
{
	int			attrmod, ioc_no, start;
	void			*data;
	timeout_id_t		*timeout_id;
	ib_mad_hdr_t		*hdr;
	ibdm_ioc_info_t		*ioc = NULL;
	ibdm_timeout_cb_args_t	*cb_args;
	ib_mad_classportinfo_t	*cpi;

	IBTF_DPRINTF_L4("ibdm", "\thandle_redirection: Enter");
	mutex_enter(&gid_info->gl_mutex);
	switch (gid_info->gl_state) {
	case IBDM_GET_IOUNITINFO:
		cb_args		= &gid_info->gl_iou_cb_args;
		timeout_id	= &gid_info->gl_timeout_id;
		break;

	case IBDM_GET_IOC_DETAILS:
		attrmod	= IBDM_IN_IBMFMSG_ATTRMOD(msg);
		switch (IBDM_IN_IBMFMSG_ATTR(msg)) {

		case IB_DM_ATTR_DIAG_CODE:
			if (attrmod == 0) {
				cb_args = &gid_info->gl_iou_cb_args;
				timeout_id = &gid_info->gl_timeout_id;
				break;
			}
			if (IBDM_IS_IOC_NUM_INVALID(attrmod, gid_info)) {
				IBTF_DPRINTF_L2("ibdm", "\thandle_redirction:"
				    "IOC# Out of range %d", attrmod);
				(*flag) |= IBDM_IBMF_PKT_UNEXP_RESP;
				mutex_exit(&gid_info->gl_mutex);
				return (IBDM_FAILURE);
			}
			ioc	= IBDM_GIDINFO2IOCINFO(gid_info, (attrmod -1));
			cb_args = &ioc->ioc_dc_cb_args;
			timeout_id = &ioc->ioc_dc_timeout_id;
			break;

		case IB_DM_ATTR_IOC_CTRL_PROFILE:
			if (IBDM_IS_IOC_NUM_INVALID(attrmod, gid_info)) {
				IBTF_DPRINTF_L2("ibdm", "\thandle_redirction:"
				    "IOC# Out of range %d", attrmod);
				(*flag) |= IBDM_IBMF_PKT_UNEXP_RESP;
				mutex_exit(&gid_info->gl_mutex);
				return (IBDM_FAILURE);
			}
			ioc	= IBDM_GIDINFO2IOCINFO(gid_info, (attrmod -1));
			cb_args = &ioc->ioc_cb_args;
			timeout_id = &ioc->ioc_timeout_id;
			break;

		case IB_DM_ATTR_SERVICE_ENTRIES:
			ioc_no	= ((attrmod >> 16) & IBDM_16_BIT_MASK);
			if (IBDM_IS_IOC_NUM_INVALID(ioc_no, gid_info)) {
				IBTF_DPRINTF_L2("ibdm", "\thandle_redirction:"
				    "IOC# Out of range %d", ioc_no);
				(*flag) |= IBDM_IBMF_PKT_UNEXP_RESP;
				mutex_exit(&gid_info->gl_mutex);
				return (IBDM_FAILURE);
			}
			start 	= (attrmod & IBDM_8_BIT_MASK);
			ioc	= IBDM_GIDINFO2IOCINFO(gid_info, (ioc_no -1));
			if (start > ioc->ioc_profile.ioc_service_entries) {
				IBTF_DPRINTF_L2("ibdm", "\thandle_redirction:"
				    " SE index Out of range %d", start);
				(*flag) |= IBDM_IBMF_PKT_UNEXP_RESP;
				mutex_exit(&gid_info->gl_mutex);
				return (IBDM_FAILURE);
			}
			cb_args = &ioc->ioc_serv[start].se_cb_args;
			timeout_id = &ioc->ioc_serv[start].se_timeout_id;
			break;

		default:
			/* ERROR State */
			IBTF_DPRINTF_L2("ibdm",
			    "\thandle_redirection: wrong attribute :-(");
			(*flag) |= IBDM_IBMF_PKT_UNEXP_RESP;
			mutex_exit(&gid_info->gl_mutex);
			return (IBDM_FAILURE);
		}
		break;
	default:
		/* ERROR State */
		IBTF_DPRINTF_L2("ibdm",
		    "\thandle_redirection: Error state :-(");
		(*flag) |= IBDM_IBMF_PKT_UNEXP_RESP;
		mutex_exit(&gid_info->gl_mutex);
		return (IBDM_FAILURE);
	}
	if ((*timeout_id) != 0) {
		mutex_exit(&gid_info->gl_mutex);
		if (untimeout(*timeout_id) == -1) {
			IBTF_DPRINTF_L2("ibdm", "\thandle_redirection: "
			    "untimeout failed %x", *timeout_id);
		} else {
			IBTF_DPRINTF_L5("ibdm",
			    "\thandle_redirection: timeout %x", *timeout_id);
		}
		mutex_enter(&gid_info->gl_mutex);
		*timeout_id = 0;
	}

	data = msg->im_msgbufs_recv.im_bufs_cl_data;
	cpi = (ib_mad_classportinfo_t *)data;

	gid_info->gl_resp_timeout	=
	    (b2h32(cpi->RespTimeValue) & 0x1F);

	gid_info->gl_redirected		= B_TRUE;
	gid_info->gl_redirect_dlid	= b2h16(cpi->RedirectLID);
	gid_info->gl_redirect_QP	= (b2h32(cpi->RedirectQP) & 0xffffff);
	gid_info->gl_redirect_pkey	= b2h16(cpi->RedirectP_Key);
	gid_info->gl_redirect_qkey	= b2h32(cpi->RedirectQ_Key);
	gid_info->gl_redirectGID_hi	= b2h64(cpi->RedirectGID_hi);
	gid_info->gl_redirectGID_lo	= b2h64(cpi->RedirectGID_lo);
	gid_info->gl_redirectSL		= cpi->RedirectSL;

	if (gid_info->gl_redirect_dlid != 0) {
		msg->im_local_addr.ia_remote_lid =
		    gid_info->gl_redirect_dlid;
	}
	ibdm_bump_transactionID(gid_info);
	mutex_exit(&gid_info->gl_mutex);

	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*msg, *hdr))
	ibdm_alloc_send_buffers(msg);

	hdr			= IBDM_OUT_IBMFMSG_MADHDR(msg);
	hdr->BaseVersion	= MAD_CLASS_BASE_VERS_1;
	hdr->MgmtClass		= MAD_MGMT_CLASS_DEV_MGT;
	hdr->ClassVersion	= IB_DM_CLASS_VERSION_1;
	hdr->R_Method		= IB_DM_DEVMGT_METHOD_GET;
	hdr->Status		= 0;
	hdr->TransactionID	= h2b64(gid_info->gl_transactionID);
	hdr->AttributeID	=
	    msg->im_msgbufs_recv.im_bufs_mad_hdr->AttributeID;
	hdr->AttributeModifier	=
	    msg->im_msgbufs_recv.im_bufs_mad_hdr->AttributeModifier;
	_NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*msg, *hdr))

	msg->im_local_addr.ia_remote_qno = gid_info->gl_redirect_QP;
	msg->im_local_addr.ia_p_key = gid_info->gl_redirect_pkey;
	msg->im_local_addr.ia_q_key = gid_info->gl_redirect_qkey;
	msg->im_local_addr.ia_service_level = gid_info->gl_redirectSL;

	mutex_enter(&gid_info->gl_mutex);
	*timeout_id = timeout(ibdm_pkt_timeout_hdlr,
	    cb_args, IBDM_TIMEOUT_VALUE(ibdm_dft_timeout));
	mutex_exit(&gid_info->gl_mutex);

	IBTF_DPRINTF_L5("ibdm", "\thandle_redirect:"
	    "timeout %x", *timeout_id);

	if (ibmf_msg_transport(gid_info->gl_ibmf_hdl, gid_info->gl_qp_hdl,
	    msg, NULL, ibdm_ibmf_send_cb, cb_args, 0) != IBMF_SUCCESS) {
		IBTF_DPRINTF_L4("ibdm", "\thandle_redirection:"
		    "message transport failed");
		ibdm_ibmf_send_cb(gid_info->gl_ibmf_hdl, msg, cb_args);
	}
	(*flag) |= IBDM_IBMF_PKT_REUSED;
	IBTF_DPRINTF_L4("ibdm", "\thandle_redirection: Exit");
	return (IBDM_SUCCESS);
}


/*
 * ibdm_pkt_timeout_hdlr
 *	This  timeout  handler is  registed for every  IBMF  packet that is
 *	sent through the IBMF.  It gets called when no response is received
 *	within the specified time for the packet. No retries for the failed
 *	commands  currently.  Drops the failed  IBMF packet and  update the
 *	pending list commands.
 */
static void
ibdm_pkt_timeout_hdlr(void *arg)
{
	ibdm_iou_info_t		*iou;
	ibdm_ioc_info_t		*ioc;
	ibdm_timeout_cb_args_t	*cb_args = arg;
	ibdm_dp_gidinfo_t	*gid_info;
	int			srv_ent;
	uint_t			new_gl_state;

	IBTF_DPRINTF_L2("ibdm", "\tpkt_timeout_hdlr: gid_info: %p "
	    "rtype 0x%x iocidx 0x%x srvidx %d", cb_args->cb_gid_info,
	    cb_args->cb_req_type, cb_args->cb_ioc_num,
	    cb_args->cb_srvents_start);

	gid_info = cb_args->cb_gid_info;
	mutex_enter(&gid_info->gl_mutex);

	if ((gid_info->gl_state == IBDM_GID_PROBING_COMPLETE) ||
	    (cb_args->cb_req_type == 0)) {

		IBTF_DPRINTF_L2("ibdm", "\tpkt_timeout_hdlr: req completed"
		    "rtype 0x%x iocidx 0x%x srvidx %d", cb_args->cb_req_type,
		    cb_args->cb_ioc_num, cb_args->cb_srvents_start);

		if (gid_info->gl_timeout_id)
			gid_info->gl_timeout_id = 0;
		mutex_exit(&gid_info->gl_mutex);
		return;
	}
	if (cb_args->cb_retry_count) {
		cb_args->cb_retry_count--;
		/*
		 * A new timeout_id is set inside ibdm_retry_command().
		 * When the function returns an error, the timeout_id
		 * is reset (to zero) in the switch statement below.
		 */
		if (ibdm_retry_command(cb_args) == IBDM_SUCCESS) {
			mutex_exit(&gid_info->gl_mutex);
			return;
		}
		cb_args->cb_retry_count = 0;
	}

	IBTF_DPRINTF_L2("ibdm", "\tpkt_timeout_hdlr: command failed: gid %p"
	    " rtype 0x%x iocidx 0x%x srvidx %d", cb_args->cb_gid_info,
	    cb_args->cb_req_type, cb_args->cb_ioc_num,
	    cb_args->cb_srvents_start);

	switch (cb_args->cb_req_type) {

	case IBDM_REQ_TYPE_CLASSPORTINFO:
	case IBDM_REQ_TYPE_IOUINFO:
		new_gl_state = IBDM_GID_PROBING_FAILED;
		if (gid_info->gl_timeout_id)
			gid_info->gl_timeout_id = 0;
		break;

	case IBDM_REQ_TYPE_IOCINFO:
		new_gl_state = IBDM_GID_PROBING_COMPLETE;
		iou = gid_info->gl_iou;
		ioc = &iou->iou_ioc_info[cb_args->cb_ioc_num];
		ioc->ioc_state = IBDM_IOC_STATE_PROBE_FAILED;
		if (ioc->ioc_timeout_id)
			ioc->ioc_timeout_id = 0;
		break;

	case IBDM_REQ_TYPE_SRVENTS:
		new_gl_state = IBDM_GID_PROBING_COMPLETE;
		iou = gid_info->gl_iou;
		ioc = &iou->iou_ioc_info[cb_args->cb_ioc_num];
		ioc->ioc_state = IBDM_IOC_STATE_PROBE_FAILED;
		srv_ent = cb_args->cb_srvents_start;
		if (ioc->ioc_serv[srv_ent].se_timeout_id)
			ioc->ioc_serv[srv_ent].se_timeout_id = 0;
		break;

	case IBDM_REQ_TYPE_IOU_DIAGCODE:
		new_gl_state = IBDM_GID_PROBING_COMPLETE;
		iou = gid_info->gl_iou;
		iou->iou_dc_valid = B_FALSE;
		if (gid_info->gl_timeout_id)
			gid_info->gl_timeout_id = 0;
		break;

	case IBDM_REQ_TYPE_IOC_DIAGCODE:
		new_gl_state = IBDM_GID_PROBING_COMPLETE;
		iou = gid_info->gl_iou;
		ioc = &iou->iou_ioc_info[cb_args->cb_ioc_num];
		ioc->ioc_dc_valid = B_FALSE;
		if (ioc->ioc_dc_timeout_id)
			ioc->ioc_dc_timeout_id = 0;
		break;

	default: /* ERROR State */
		new_gl_state = IBDM_GID_PROBING_FAILED;
		if (gid_info->gl_timeout_id)
			gid_info->gl_timeout_id = 0;
		IBTF_DPRINTF_L2("ibdm",
		    "\tpkt_timeout_hdlr: wrong request type.");
		break;
	}

	--gid_info->gl_pending_cmds; /* decrease the counter */

	if (gid_info->gl_pending_cmds == 0) {
		gid_info->gl_state = new_gl_state;
		mutex_exit(&gid_info->gl_mutex);
		/*
		 * Delete this gid_info if the gid probe fails.
		 */
		if (new_gl_state == IBDM_GID_PROBING_FAILED) {
			ibdm_delete_glhca_list(gid_info);
		}
		ibdm_notify_newgid_iocs(gid_info);
		mutex_enter(&ibdm.ibdm_mutex);
		if (--ibdm.ibdm_ngid_probes_in_progress == 0) {
			IBTF_DPRINTF_L4("ibdm", "\tpkt_timeout_hdlr: Wakeup");
			ibdm.ibdm_busy &= ~IBDM_PROBE_IN_PROGRESS;
			cv_broadcast(&ibdm.ibdm_probe_cv);
		}
		mutex_exit(&ibdm.ibdm_mutex);
	} else {
		/*
		 * Reset gl_pending_cmd if the extra timeout happens since
		 * gl_pending_cmd becomes negative as a result.
		 */
		if (gid_info->gl_pending_cmds < 0) {
			gid_info->gl_pending_cmds = 0;
			IBTF_DPRINTF_L2("ibdm",
			    "\tpkt_timeout_hdlr: extra timeout request."
			    " reset gl_pending_cmds");
		}
		mutex_exit(&gid_info->gl_mutex);
		/*
		 * Delete this gid_info if the gid probe fails.
		 */
		if (new_gl_state == IBDM_GID_PROBING_FAILED) {
			ibdm_delete_glhca_list(gid_info);
		}
	}
}


/*
 * ibdm_retry_command()
 *	Retries the failed command.
 *	Returns IBDM_FAILURE/IBDM_SUCCESS
 */
static int
ibdm_retry_command(ibdm_timeout_cb_args_t *cb_args)
{
	int			ret;
	ibmf_msg_t		*msg;
	ib_mad_hdr_t		*hdr;
	ibdm_dp_gidinfo_t	*gid_info = cb_args->cb_gid_info;
	timeout_id_t		*timeout_id;
	ibdm_ioc_info_t		*ioc;
	int			ioc_no;
	ASSERT(MUTEX_HELD(&gid_info->gl_mutex));

	IBTF_DPRINTF_L2("ibdm", "\tretry_command: gid_info: %p "
	    "rtype 0x%x iocidx 0x%x srvidx %d", cb_args->cb_gid_info,
	    cb_args->cb_req_type, cb_args->cb_ioc_num,
	    cb_args->cb_srvents_start);

	ret = ibmf_alloc_msg(gid_info->gl_ibmf_hdl, IBMF_ALLOC_NOSLEEP, &msg);


	/*
	 * Reset the gid if alloc_msg failed with BAD_HANDLE
	 * ibdm_reset_gidinfo reinits the gid_info
	 */
	if (ret == IBMF_BAD_HANDLE) {
		IBTF_DPRINTF_L3(ibdm_string, "\tretry_command: gid %p hdl bad",
		    gid_info);

		mutex_exit(&gid_info->gl_mutex);
		ibdm_reset_gidinfo(gid_info);
		mutex_enter(&gid_info->gl_mutex);

		/* Retry alloc */
		ret = ibmf_alloc_msg(gid_info->gl_ibmf_hdl, IBMF_ALLOC_NOSLEEP,
		    &msg);
	}

	if (ret != IBDM_SUCCESS) {
		IBTF_DPRINTF_L2("ibdm", "\tretry_command: alloc failed: %p "
		    "rtype 0x%x iocidx 0x%x srvidx %d", cb_args->cb_gid_info,
		    cb_args->cb_req_type, cb_args->cb_ioc_num,
		    cb_args->cb_srvents_start);
		return (IBDM_FAILURE);
	}

	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*msg))
	ibdm_alloc_send_buffers(msg);
	_NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*msg))

	ibdm_bump_transactionID(gid_info);

	msg->im_local_addr.ia_local_lid	= gid_info->gl_slid;
	msg->im_local_addr.ia_remote_lid	= gid_info->gl_dlid;
	if (gid_info->gl_redirected == B_TRUE) {
		if (gid_info->gl_redirect_dlid != 0) {
			msg->im_local_addr.ia_remote_lid =
			    gid_info->gl_redirect_dlid;
		}
		msg->im_local_addr.ia_remote_qno = gid_info->gl_redirect_QP;
		msg->im_local_addr.ia_p_key = gid_info->gl_redirect_pkey;
		msg->im_local_addr.ia_q_key = gid_info->gl_redirect_qkey;
		msg->im_local_addr.ia_service_level = gid_info->gl_redirectSL;
	} else {
		msg->im_local_addr.ia_remote_qno = 1;
		msg->im_local_addr.ia_p_key = gid_info->gl_p_key;
		msg->im_local_addr.ia_q_key = IB_GSI_QKEY;
		msg->im_local_addr.ia_service_level = gid_info->gl_SL;
	}
	hdr = IBDM_OUT_IBMFMSG_MADHDR(msg);
	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*hdr))
	hdr->BaseVersion	= MAD_CLASS_BASE_VERS_1;
	hdr->MgmtClass		= MAD_MGMT_CLASS_DEV_MGT;
	hdr->ClassVersion	= IB_DM_CLASS_VERSION_1;
	hdr->R_Method		= IB_DM_DEVMGT_METHOD_GET;
	hdr->Status		= 0;
	hdr->TransactionID	= h2b64(gid_info->gl_transactionID);

	switch (cb_args->cb_req_type) {
	case IBDM_REQ_TYPE_CLASSPORTINFO:
		hdr->AttributeID = h2b16(IB_DM_ATTR_CLASSPORTINFO);
		hdr->AttributeModifier = 0;
		timeout_id = &gid_info->gl_timeout_id;
		break;
	case IBDM_REQ_TYPE_IOUINFO:
		hdr->AttributeID = h2b16(IB_DM_ATTR_IO_UNITINFO);
		hdr->AttributeModifier = 0;
		timeout_id = &gid_info->gl_timeout_id;
		break;
	case IBDM_REQ_TYPE_IOCINFO:
		hdr->AttributeID = h2b16(IB_DM_ATTR_IOC_CTRL_PROFILE);
		hdr->AttributeModifier = h2b32(cb_args->cb_ioc_num + 1);
		ioc = IBDM_GIDINFO2IOCINFO(gid_info, cb_args->cb_ioc_num);
		timeout_id = &ioc->ioc_timeout_id;
		break;
	case IBDM_REQ_TYPE_SRVENTS:
		hdr->AttributeID = h2b16(IB_DM_ATTR_SERVICE_ENTRIES);
		ibdm_fill_srv_attr_mod(hdr, cb_args);
		ioc = IBDM_GIDINFO2IOCINFO(gid_info, cb_args->cb_ioc_num);
		timeout_id =
		    &ioc->ioc_serv[cb_args->cb_srvents_start].se_timeout_id;
		break;
	case IBDM_REQ_TYPE_IOU_DIAGCODE:
		hdr->AttributeID = h2b16(IB_DM_ATTR_DIAG_CODE);
		hdr->AttributeModifier = 0;
		timeout_id = &gid_info->gl_timeout_id;
		break;
	case IBDM_REQ_TYPE_IOC_DIAGCODE:
		hdr->AttributeID = h2b16(IB_DM_ATTR_DIAG_CODE);
		hdr->AttributeModifier = h2b32(cb_args->cb_ioc_num + 1);
		ioc_no = cb_args->cb_ioc_num;
		ioc = &gid_info->gl_iou->iou_ioc_info[ioc_no];
		timeout_id = &ioc->ioc_dc_timeout_id;
		break;
	}
	_NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*hdr))

	*timeout_id = timeout(ibdm_pkt_timeout_hdlr,
	    cb_args, IBDM_TIMEOUT_VALUE(ibdm_dft_timeout));

	mutex_exit(&gid_info->gl_mutex);

	IBTF_DPRINTF_L5("ibdm", "\tretry_command: %p,%x,%d,%d:"
	    "timeout %x", cb_args->cb_req_type, cb_args->cb_ioc_num,
	    cb_args->cb_srvents_start, *timeout_id);

	if (ibmf_msg_transport(gid_info->gl_ibmf_hdl,
	    gid_info->gl_qp_hdl, msg, NULL, ibdm_ibmf_send_cb,
	    cb_args, 0) != IBMF_SUCCESS) {
		IBTF_DPRINTF_L2("ibdm", "\tretry_command: send failed: %p "
		    "rtype 0x%x iocidx 0x%x srvidx %d", cb_args->cb_gid_info,
		    cb_args->cb_req_type, cb_args->cb_ioc_num,
		    cb_args->cb_srvents_start);
		ibdm_ibmf_send_cb(gid_info->gl_ibmf_hdl, msg, cb_args);
	}
	mutex_enter(&gid_info->gl_mutex);
	return (IBDM_SUCCESS);
}


/*
 * ibdm_update_ioc_port_gidlist()
 */
static void
ibdm_update_ioc_port_gidlist(ibdm_ioc_info_t *dest,
    ibdm_dp_gidinfo_t *gid_info)
{
	int		ii, ngid_ents;
	ibdm_gid_t	*tmp;
	ibdm_hca_list_t	*gid_hca_head, *temp;
	ibdm_hca_list_t	*ioc_head = NULL;
	ASSERT(MUTEX_HELD(&gid_info->gl_mutex));

	IBTF_DPRINTF_L5("ibdm", "\tupdate_ioc_port_gidlist: Enter");

	ngid_ents = gid_info->gl_ngids;
	dest->ioc_nportgids = ngid_ents;
	dest->ioc_gid_list = kmem_zalloc(sizeof (ibdm_gid_t) *
	    ngid_ents, KM_SLEEP);
	tmp = gid_info->gl_gid;
	for (ii = 0; (ii < ngid_ents) && (tmp); ii++) {
		dest->ioc_gid_list[ii].gid_dgid_hi = tmp->gid_dgid_hi;
		dest->ioc_gid_list[ii].gid_dgid_lo = tmp->gid_dgid_lo;
		tmp = tmp->gid_next;
	}

	gid_hca_head = gid_info->gl_hca_list;
	while (gid_hca_head) {
		temp = ibdm_dup_hca_attr(gid_hca_head);
		temp->hl_next = ioc_head;
		ioc_head = temp;
		gid_hca_head = gid_hca_head->hl_next;
	}
	dest->ioc_hca_list = ioc_head;
}


/*
 * ibdm_alloc_send_buffers()
 *	Allocates memory for the IBMF send buffer to send and/or receive
 *	the Device Management MAD packet.
 */
static void
ibdm_alloc_send_buffers(ibmf_msg_t *msgp)
{
	msgp->im_msgbufs_send.im_bufs_mad_hdr =
	    kmem_zalloc(IBDM_MAD_SIZE, KM_SLEEP);

	msgp->im_msgbufs_send.im_bufs_cl_hdr = (uchar_t *)
	    msgp->im_msgbufs_send.im_bufs_mad_hdr + sizeof (ib_mad_hdr_t);
	msgp->im_msgbufs_send.im_bufs_cl_hdr_len = IBDM_DM_MAD_HDR_SZ;

	msgp->im_msgbufs_send.im_bufs_cl_data =
	    ((char *)msgp->im_msgbufs_send.im_bufs_cl_hdr + IBDM_DM_MAD_HDR_SZ);
	msgp->im_msgbufs_send.im_bufs_cl_data_len =
	    IBDM_MAD_SIZE - sizeof (ib_mad_hdr_t) - IBDM_DM_MAD_HDR_SZ;
}


/*
 * ibdm_alloc_send_buffers()
 *	De-allocates memory for the IBMF send buffer
 */
static void
ibdm_free_send_buffers(ibmf_msg_t *msgp)
{
	if (msgp->im_msgbufs_send.im_bufs_mad_hdr != NULL)
		kmem_free(msgp->im_msgbufs_send.im_bufs_mad_hdr, IBDM_MAD_SIZE);
}

/*
 * ibdm_probe_ioc()
 *  	1. Gets the node records for the port GUID. This detects all the port
 *  		to the IOU.
 *	2. Selectively probes all the IOC, given it's node GUID
 *	3. In case of reprobe, only the IOC to be reprobed is send the IOC
 *		Controller Profile asynchronously
 */
/*ARGSUSED*/
static void
ibdm_probe_ioc(ib_guid_t nodeguid, ib_guid_t ioc_guid, int reprobe_flag)
{
	int			ii, nrecords;
	size_t			nr_len = 0, pi_len = 0;
	ib_gid_t		sgid, dgid;
	ibdm_hca_list_t		*hca_list = NULL;
	sa_node_record_t	*nr, *tmp;
	ibdm_port_attr_t	*port = NULL;
	ibdm_dp_gidinfo_t	*reprobe_gid, *new_gid, *node_gid;
	ibdm_dp_gidinfo_t	*temp_gidinfo;
	ibdm_gid_t		*temp_gid;
	sa_portinfo_record_t	*pi;

	IBTF_DPRINTF_L4("ibdm", "\tprobe_ioc(%llx, %llx, %x): Begin",
	    nodeguid, ioc_guid, reprobe_flag);

	/* Rescan the GID list for any removed GIDs for reprobe */
	if (reprobe_flag)
		ibdm_rescan_gidlist(&ioc_guid);

	mutex_enter(&ibdm.ibdm_hl_mutex);
	for (ibdm_get_next_port(&hca_list, &port, 1); port;
	    ibdm_get_next_port(&hca_list, &port, 1)) {
		reprobe_gid = new_gid = node_gid = NULL;

		nr = ibdm_get_node_records(port->pa_sa_hdl, &nr_len, nodeguid);
		if (nr == NULL) {
			IBTF_DPRINTF_L4("ibdm", "\tprobe_ioc: no records");
			continue;
		}
		nrecords = (nr_len / sizeof (sa_node_record_t));
		for (tmp = nr, ii = 0;  (ii < nrecords); ii++, tmp++) {
			if ((pi = ibdm_get_portinfo(
			    port->pa_sa_hdl, &pi_len, tmp->LID)) ==  NULL) {
				IBTF_DPRINTF_L4("ibdm",
				    "\tibdm_get_portinfo: no portinfo recs");
				continue;
			}

			/*
			 * If Device Management is not supported on
			 * this port, skip the rest.
			 */
			if (!(pi->PortInfo.CapabilityMask &
			    SM_CAP_MASK_IS_DM_SUPPD)) {
				kmem_free(pi, pi_len);
				continue;
			}

			/*
			 * For reprobes: Check if GID, already in
			 * the list. If so, set the state to SKIPPED
			 */
			if (((temp_gidinfo = ibdm_find_gid(nodeguid,
			    tmp->NodeInfo.PortGUID)) != NULL) &&
			    temp_gidinfo->gl_state ==
			    IBDM_GID_PROBING_COMPLETE) {
				ASSERT(reprobe_gid == NULL);
				ibdm_addto_glhcalist(temp_gidinfo,
				    hca_list);
				reprobe_gid = temp_gidinfo;
				kmem_free(pi, pi_len);
				continue;
			} else if (temp_gidinfo != NULL) {
				kmem_free(pi, pi_len);
				ibdm_addto_glhcalist(temp_gidinfo,
				    hca_list);
				continue;
			}

			IBTF_DPRINTF_L4("ibdm", "\tprobe_ioc : "
			    "create_gid : prefix %llx, guid %llx\n",
			    pi->PortInfo.GidPrefix,
			    tmp->NodeInfo.PortGUID);

			sgid.gid_prefix = port->pa_sn_prefix;
			sgid.gid_guid = port->pa_port_guid;
			dgid.gid_prefix = pi->PortInfo.GidPrefix;
			dgid.gid_guid = tmp->NodeInfo.PortGUID;
			new_gid = ibdm_create_gid_info(port, sgid,
			    dgid);
			if (new_gid == NULL) {
				IBTF_DPRINTF_L2("ibdm", "\tprobe_ioc: "
				    "create_gid_info failed\n");
				kmem_free(pi, pi_len);
				continue;
			}
			if (node_gid == NULL) {
				node_gid = new_gid;
				ibdm_add_to_gl_gid(node_gid, node_gid);
			} else {
				IBTF_DPRINTF_L4("ibdm",
				    "\tprobe_ioc: new gid");
				temp_gid = kmem_zalloc(
				    sizeof (ibdm_gid_t), KM_SLEEP);
				temp_gid->gid_dgid_hi =
				    new_gid->gl_dgid_hi;
				temp_gid->gid_dgid_lo =
				    new_gid->gl_dgid_lo;
				temp_gid->gid_next = node_gid->gl_gid;
				node_gid->gl_gid = temp_gid;
				node_gid->gl_ngids++;
			}
			new_gid->gl_is_dm_capable = B_TRUE;
			new_gid->gl_nodeguid = nodeguid;
			new_gid->gl_portguid = dgid.gid_guid;
			ibdm_addto_glhcalist(new_gid, hca_list);

			/*
			 * Set the state to skipped as all these
			 * gids point to the same node.
			 * We (re)probe only one GID below and reset
			 * state appropriately
			 */
			new_gid->gl_state = IBDM_GID_PROBING_SKIPPED;
			new_gid->gl_devid = (*tmp).NodeInfo.DeviceID;
			kmem_free(pi, pi_len);
		}
		kmem_free(nr, nr_len);

		IBTF_DPRINTF_L4("ibdm", "\tprobe_ioc : reprobe_flag %d "
		    "reprobe_gid %p new_gid %p node_gid %p",
		    reprobe_flag, reprobe_gid, new_gid, node_gid);

		if (reprobe_flag != 0 && reprobe_gid != NULL) {
			int	niocs, jj;
			ibdm_ioc_info_t *tmp_ioc;
			int ioc_matched = 0;

			mutex_exit(&ibdm.ibdm_hl_mutex);
			mutex_enter(&reprobe_gid->gl_mutex);
			reprobe_gid->gl_state = IBDM_GET_IOC_DETAILS;
			niocs =
			    reprobe_gid->gl_iou->iou_info.iou_num_ctrl_slots;
			reprobe_gid->gl_pending_cmds++;
			mutex_exit(&reprobe_gid->gl_mutex);

			for (jj = 0; jj < niocs; jj++) {
				tmp_ioc =
				    IBDM_GIDINFO2IOCINFO(reprobe_gid, jj);
				if (tmp_ioc->ioc_profile.ioc_guid != ioc_guid)
					continue;

				ioc_matched = 1;

				/*
				 * Explicitly set gl_reprobe_flag to 0 so that
				 * IBnex is not notified on completion
				 */
				mutex_enter(&reprobe_gid->gl_mutex);
				reprobe_gid->gl_reprobe_flag = 0;
				mutex_exit(&reprobe_gid->gl_mutex);

				mutex_enter(&ibdm.ibdm_mutex);
				ibdm.ibdm_ngid_probes_in_progress++;
				mutex_exit(&ibdm.ibdm_mutex);
				if (ibdm_send_ioc_profile(reprobe_gid, jj) !=
				    IBDM_SUCCESS) {
					IBTF_DPRINTF_L4("ibdm",
					    "\tprobe_ioc: "
					    "send_ioc_profile failed "
					    "for ioc %d", jj);
					ibdm_gid_decr_pending(reprobe_gid);
					break;
				}
				mutex_enter(&ibdm.ibdm_mutex);
				ibdm_wait_probe_completion();
				mutex_exit(&ibdm.ibdm_mutex);
				break;
			}
			if (ioc_matched == 0)
				ibdm_gid_decr_pending(reprobe_gid);
			else {
				mutex_enter(&ibdm.ibdm_hl_mutex);
				break;
			}
		} else if (new_gid != NULL) {
			mutex_exit(&ibdm.ibdm_hl_mutex);
			node_gid = node_gid ? node_gid : new_gid;

			/*
			 * New or reinserted GID : Enable notification
			 * to IBnex
			 */
			mutex_enter(&node_gid->gl_mutex);
			node_gid->gl_reprobe_flag = 1;
			mutex_exit(&node_gid->gl_mutex);

			ibdm_probe_gid(node_gid);

			mutex_enter(&ibdm.ibdm_hl_mutex);
		}
	}
	mutex_exit(&ibdm.ibdm_hl_mutex);
	IBTF_DPRINTF_L4("ibdm", "\tprobe_ioc : End\n");
}


/*
 * ibdm_probe_gid()
 *	Selectively probes the GID
 */
static void
ibdm_probe_gid(ibdm_dp_gidinfo_t *gid_info)
{
	IBTF_DPRINTF_L4("ibdm", "\tprobe_gid:");

	/*
	 * A Cisco FC GW needs the special handling to get IOUnitInfo.
	 */
	mutex_enter(&gid_info->gl_mutex);
	if (ibdm_is_cisco_switch(gid_info)) {
		gid_info->gl_pending_cmds++;
		gid_info->gl_state = IBDM_SET_CLASSPORTINFO;
		mutex_exit(&gid_info->gl_mutex);

		if (ibdm_set_classportinfo(gid_info) != IBDM_SUCCESS) {

			mutex_enter(&gid_info->gl_mutex);
			gid_info->gl_state = IBDM_GID_PROBING_FAILED;
			--gid_info->gl_pending_cmds;
			mutex_exit(&gid_info->gl_mutex);

			/* free the hca_list on this gid_info */
			ibdm_delete_glhca_list(gid_info);
			gid_info = gid_info->gl_next;
			return;
		}

		mutex_enter(&gid_info->gl_mutex);
		ibdm_wait_cisco_probe_completion(gid_info);

		IBTF_DPRINTF_L4("ibdm",
		    "\tprobe_gid: CISCO Wakeup signal received");
	}

	/* move on to the 'GET_CLASSPORTINFO' stage */
	gid_info->gl_pending_cmds++;
	gid_info->gl_state = IBDM_GET_CLASSPORTINFO;
	mutex_exit(&gid_info->gl_mutex);

	if (ibdm_send_classportinfo(gid_info) != IBDM_SUCCESS) {

		mutex_enter(&gid_info->gl_mutex);
		gid_info->gl_state = IBDM_GID_PROBING_FAILED;
		--gid_info->gl_pending_cmds;
		mutex_exit(&gid_info->gl_mutex);

		/* free the hca_list on this gid_info */
		ibdm_delete_glhca_list(gid_info);
		gid_info = gid_info->gl_next;
		return;
	}

	mutex_enter(&ibdm.ibdm_mutex);
	ibdm.ibdm_ngid_probes_in_progress++;
	gid_info = gid_info->gl_next;
	ibdm_wait_probe_completion();
	mutex_exit(&ibdm.ibdm_mutex);

	IBTF_DPRINTF_L4("ibdm", "\tprobe_gid: Wakeup signal received");
}


/*
 * ibdm_create_gid_info()
 *	Allocates a gid_info structure and initializes
 *	Returns pointer to the structure on success
 *	and NULL on failure
 */
static ibdm_dp_gidinfo_t *
ibdm_create_gid_info(ibdm_port_attr_t *port, ib_gid_t sgid, ib_gid_t dgid)
{
	uint8_t			ii, npaths;
	sa_path_record_t	*path;
	size_t			len;
	ibdm_pkey_tbl_t		*pkey_tbl;
	ibdm_dp_gidinfo_t	*gid_info = NULL;
	int			ret;

	IBTF_DPRINTF_L4("ibdm", "\tcreate_gid_info: Begin");
	npaths = 1;

	/* query for reversible paths */
	if (port->pa_sa_hdl)
		ret = ibmf_saa_gid_to_pathrecords(port->pa_sa_hdl,
		    sgid, dgid, IBMF_SAA_PKEY_WC, 0, B_TRUE, &npaths, 0,
		    &len, &path);
	else
		return (NULL);

	if (ret == IBMF_SUCCESS && path) {
		ibdm_dump_path_info(path);

		gid_info = kmem_zalloc(
		    sizeof (ibdm_dp_gidinfo_t), KM_SLEEP);
		mutex_init(&gid_info->gl_mutex, NULL, MUTEX_DEFAULT, NULL);
		cv_init(&gid_info->gl_probe_cv, NULL, CV_DRIVER, NULL);
		gid_info->gl_dgid_hi		= path->DGID.gid_prefix;
		gid_info->gl_dgid_lo		= path->DGID.gid_guid;
		gid_info->gl_sgid_hi		= path->SGID.gid_prefix;
		gid_info->gl_sgid_lo		= path->SGID.gid_guid;
		gid_info->gl_p_key		= path->P_Key;
		gid_info->gl_sa_hdl		= port->pa_sa_hdl;
		gid_info->gl_ibmf_hdl		= port->pa_ibmf_hdl;
		gid_info->gl_slid		= path->SLID;
		gid_info->gl_dlid		= path->DLID;
		gid_info->gl_transactionID	= (++ibdm.ibdm_transactionID)
		    << IBDM_GID_TRANSACTIONID_SHIFT;
		gid_info->gl_min_transactionID  = gid_info->gl_transactionID;
		gid_info->gl_max_transactionID  = (ibdm.ibdm_transactionID +1)
		    << IBDM_GID_TRANSACTIONID_SHIFT;
		gid_info->gl_SL			= path->SL;

		gid_info->gl_qp_hdl = IBMF_QP_HANDLE_DEFAULT;
		for (ii = 0; ii < port->pa_npkeys; ii++) {
			if (port->pa_pkey_tbl == NULL)
				break;

			pkey_tbl = &port->pa_pkey_tbl[ii];
			if ((gid_info->gl_p_key == pkey_tbl->pt_pkey) &&
			    (pkey_tbl->pt_qp_hdl != NULL)) {
				gid_info->gl_qp_hdl = pkey_tbl->pt_qp_hdl;
				break;
			}
		}
		kmem_free(path, len);

		/*
		 * QP handle for GID not initialized. No matching Pkey
		 * was found!! ibdm should *not* hit this case. Flag an
		 * error and drop the GID if ibdm does encounter this.
		 */
		if (gid_info->gl_qp_hdl == NULL) {
			IBTF_DPRINTF_L2(ibdm_string,
			    "\tcreate_gid_info: No matching Pkey");
			ibdm_delete_gidinfo(gid_info);
			return (NULL);
		}

		ibdm.ibdm_ngids++;
		if (ibdm.ibdm_dp_gidlist_head == NULL) {
			ibdm.ibdm_dp_gidlist_head = gid_info;
			ibdm.ibdm_dp_gidlist_tail = gid_info;
		} else {
			ibdm.ibdm_dp_gidlist_tail->gl_next = gid_info;
			gid_info->gl_prev = ibdm.ibdm_dp_gidlist_tail;
			ibdm.ibdm_dp_gidlist_tail = gid_info;
		}
	}

	return (gid_info);
}


/*
 * ibdm_get_node_records
 *	Sends a SA query to get the NODE record
 *	Returns pointer to the sa_node_record_t on success
 *	and NULL on failure
 */
static sa_node_record_t *
ibdm_get_node_records(ibmf_saa_handle_t sa_hdl, size_t *length, ib_guid_t guid)
{
	sa_node_record_t	req, *resp = NULL;
	ibmf_saa_access_args_t	args;
	int			ret;

	IBTF_DPRINTF_L4("ibdm", "\tget_node_records: Begin");

	bzero(&req, sizeof (sa_node_record_t));
	req.NodeInfo.NodeGUID = guid;

	args.sq_attr_id		= SA_NODERECORD_ATTRID;
	args.sq_access_type 	= IBMF_SAA_RETRIEVE;
	args.sq_component_mask 	= SA_NODEINFO_COMPMASK_NODEGUID;
	args.sq_template	= &req;
	args.sq_callback	= NULL;
	args.sq_callback_arg 	= NULL;

	ret = ibmf_sa_access(sa_hdl, &args, 0, length, (void **) &resp);
	if (ret != IBMF_SUCCESS) {
		IBTF_DPRINTF_L2("ibdm", "\tget_node_records:"
		    " SA Retrieve Failed: %d", ret);
		return (NULL);
	}
	if ((resp == NULL) || (*length == 0)) {
		IBTF_DPRINTF_L2("ibdm", "\tget_node_records: No records");
		return (NULL);
	}

	IBTF_DPRINTF_L4("ibdm", "\tget_node_records: NodeGuid %llx "
	    "PortGUID %llx", resp->NodeInfo.NodeGUID, resp->NodeInfo.PortGUID);

	return (resp);
}


/*
 * ibdm_get_portinfo()
 *	Sends a SA query to get the PortInfo record
 *	Returns pointer to the sa_portinfo_record_t on success
 *	and NULL on failure
 */
static sa_portinfo_record_t *
ibdm_get_portinfo(ibmf_saa_handle_t sa_hdl, size_t *length, ib_lid_t lid)
{
	sa_portinfo_record_t	req, *resp = NULL;
	ibmf_saa_access_args_t	args;
	int			ret;

	IBTF_DPRINTF_L4("ibdm", "\tget_portinfo: Begin");

	bzero(&req, sizeof (sa_portinfo_record_t));
	req.EndportLID	= lid;

	args.sq_attr_id		= SA_PORTINFORECORD_ATTRID;
	args.sq_access_type	= IBMF_SAA_RETRIEVE;
	args.sq_component_mask	= SA_PORTINFO_COMPMASK_PORTLID;
	args.sq_template	= &req;
	args.sq_callback	= NULL;
	args.sq_callback_arg	= NULL;

	ret = ibmf_sa_access(sa_hdl, &args, 0, length, (void **) &resp);
	if (ret != IBMF_SUCCESS) {
		IBTF_DPRINTF_L2("ibdm", "\tget_portinfo:"
		    " SA Retrieve Failed: 0x%X", ret);
		return (NULL);
	}
	if ((*length == 0) || (resp == NULL))
		return (NULL);

	IBTF_DPRINTF_L4("ibdm", "\tget_portinfo: GidPrefix %llx Cap 0x%x",
	    resp->PortInfo.GidPrefix, resp->PortInfo.CapabilityMask);
	return (resp);
}


/*
 * ibdm_ibnex_register_callback
 *	IB nexus callback routine for HCA attach and detach notification
 */
void
ibdm_ibnex_register_callback(ibdm_callback_t ibnex_dm_callback)
{
	IBTF_DPRINTF_L4("ibdm", "\tibnex_register_callbacks");
	mutex_enter(&ibdm.ibdm_ibnex_mutex);
	ibdm.ibdm_ibnex_callback = ibnex_dm_callback;
	mutex_exit(&ibdm.ibdm_ibnex_mutex);
}


/*
 * ibdm_ibnex_unregister_callbacks
 */
void
ibdm_ibnex_unregister_callback()
{
	IBTF_DPRINTF_L4("ibdm", "\tibnex_unregister_callbacks");
	mutex_enter(&ibdm.ibdm_ibnex_mutex);
	ibdm.ibdm_ibnex_callback = NULL;
	mutex_exit(&ibdm.ibdm_ibnex_mutex);
}

/*
 * ibdm_get_waittime()
 *	Calculates the wait time based on the last HCA attach time
 */
static clock_t
ibdm_get_waittime(ib_guid_t hca_guid, int dft_wait_sec)
{
	const hrtime_t	dft_wait = dft_wait_sec * NANOSEC;
	hrtime_t	temp, wait_time = 0;
	clock_t		usecs;
	int		i;
	ibdm_hca_list_t	*hca;

	IBTF_DPRINTF_L4("ibdm", "\tget_waittime hcaguid:%llx"
	    "\tport settling time %d", hca_guid, dft_wait);

	ASSERT(mutex_owned(&ibdm.ibdm_hl_mutex));

	hca = ibdm.ibdm_hca_list_head;

	for (i = 0; i < ibdm.ibdm_hca_count; i++, hca = hca->hl_next) {
		if (hca->hl_nports == hca->hl_nports_active)
			continue;

		if (hca_guid && (hca_guid != hca->hl_hca_guid))
			continue;

		temp = gethrtime() - hca->hl_attach_time;
		temp = MAX(0, (dft_wait - temp));

		if (hca_guid) {
			wait_time = temp;
			break;
		}

		wait_time = MAX(temp, wait_time);
	}

	/* convert to microseconds */
	usecs = MIN(wait_time, dft_wait) / (NANOSEC / MICROSEC);

	IBTF_DPRINTF_L2("ibdm", "\tget_waittime: wait_time = %ld usecs",
	    (long)usecs);

	return (drv_usectohz(usecs));
}

void
ibdm_ibnex_port_settle_wait(ib_guid_t hca_guid, int dft_wait)
{
	clock_t wait_time;

	mutex_enter(&ibdm.ibdm_hl_mutex);

	while ((wait_time = ibdm_get_waittime(hca_guid, dft_wait)) > 0)
		(void) cv_reltimedwait(&ibdm.ibdm_port_settle_cv,
		    &ibdm.ibdm_hl_mutex, wait_time, TR_CLOCK_TICK);

	mutex_exit(&ibdm.ibdm_hl_mutex);
}


/*
 * ibdm_ibnex_probe_hcaport
 *	Probes the presence of HCA port (with HCA dip and port number)
 *	Returns port attributes structure on SUCCESS
 */
ibdm_port_attr_t *
ibdm_ibnex_probe_hcaport(ib_guid_t hca_guid, uint8_t port_num)
{
	int			ii, jj;
	ibdm_hca_list_t		*hca_list;
	ibdm_port_attr_t	*port_attr;

	IBTF_DPRINTF_L4("ibdm", "\tibnex_probe_hcaport:");

	mutex_enter(&ibdm.ibdm_hl_mutex);
	hca_list = ibdm.ibdm_hca_list_head;
	for (ii = 0; ii < ibdm.ibdm_hca_count; ii++) {
		if (hca_list->hl_hca_guid == hca_guid) {
			for (jj = 0; jj < hca_list->hl_nports; jj++) {
				if (hca_list->hl_port_attr[jj].pa_port_num ==
				    port_num) {
					break;
				}
			}
			if (jj != hca_list->hl_nports)
				break;
		}
		hca_list = hca_list->hl_next;
	}
	if (ii == ibdm.ibdm_hca_count) {
		IBTF_DPRINTF_L2("ibdm", "\tibnex_probe_hcaport: not found");
		mutex_exit(&ibdm.ibdm_hl_mutex);
		return (NULL);
	}
	port_attr = (ibdm_port_attr_t *)kmem_zalloc(
	    sizeof (ibdm_port_attr_t), KM_SLEEP);
	bcopy((char *)&hca_list->hl_port_attr[jj],
	    port_attr, sizeof (ibdm_port_attr_t));
	ibdm_update_port_attr(port_attr);

	mutex_exit(&ibdm.ibdm_hl_mutex);
	return (port_attr);
}


/*
 * ibdm_ibnex_get_port_attrs
 *	Scan all HCAs for a matching port_guid.
 *	Returns "port attributes" structure on success.
 */
ibdm_port_attr_t *
ibdm_ibnex_get_port_attrs(ib_guid_t port_guid)
{
	int			ii, jj;
	ibdm_hca_list_t		*hca_list;
	ibdm_port_attr_t	*port_attr;

	IBTF_DPRINTF_L4("ibdm", "\tibnex_get_port_attrs:");

	mutex_enter(&ibdm.ibdm_hl_mutex);
	hca_list = ibdm.ibdm_hca_list_head;

	for (ii = 0; ii < ibdm.ibdm_hca_count; ii++) {
		for (jj = 0; jj < hca_list->hl_nports; jj++) {
			if (hca_list->hl_port_attr[jj].pa_port_guid ==
			    port_guid) {
				break;
			}
		}
		if (jj != hca_list->hl_nports)
			break;
		hca_list = hca_list->hl_next;
	}

	if (ii == ibdm.ibdm_hca_count) {
		IBTF_DPRINTF_L2("ibdm", "\tibnex_get_port_attrs: not found");
		mutex_exit(&ibdm.ibdm_hl_mutex);
		return (NULL);
	}

	port_attr = (ibdm_port_attr_t *)kmem_alloc(sizeof (ibdm_port_attr_t),
	    KM_SLEEP);
	bcopy((char *)&hca_list->hl_port_attr[jj], port_attr,
	    sizeof (ibdm_port_attr_t));
	ibdm_update_port_attr(port_attr);

	mutex_exit(&ibdm.ibdm_hl_mutex);
	return (port_attr);
}


/*
 * ibdm_ibnex_free_port_attr()
 */
void
ibdm_ibnex_free_port_attr(ibdm_port_attr_t *port_attr)
{
	IBTF_DPRINTF_L4("ibdm", "\tibnex_free_port_attr:");
	if (port_attr) {
		if (port_attr->pa_pkey_tbl != NULL) {
			kmem_free(port_attr->pa_pkey_tbl,
			    (port_attr->pa_npkeys * sizeof (ibdm_pkey_tbl_t)));
		}
		kmem_free(port_attr, sizeof (ibdm_port_attr_t));
	}
}


/*
 * ibdm_ibnex_get_hca_list()
 *	Returns portinfo for all the port for all the HCA's
 */
void
ibdm_ibnex_get_hca_list(ibdm_hca_list_t **hca, int *count)
{
	ibdm_hca_list_t		*head = NULL, *temp, *temp1;
	int			ii;

	IBTF_DPRINTF_L4("ibdm", "\tibnex_get_hca_list:");

	mutex_enter(&ibdm.ibdm_hl_mutex);
	temp = ibdm.ibdm_hca_list_head;
	for (ii = 0; ii < ibdm.ibdm_hca_count; ii++) {
		temp1 = ibdm_dup_hca_attr(temp);
		temp1->hl_next = head;
		head = temp1;
		temp = temp->hl_next;
	}
	*count = ibdm.ibdm_hca_count;
	*hca = head;
	mutex_exit(&ibdm.ibdm_hl_mutex);
}


/*
 * ibdm_ibnex_get_hca_info_by_guid()
 */
ibdm_hca_list_t	*
ibdm_ibnex_get_hca_info_by_guid(ib_guid_t hca_guid)
{
	ibdm_hca_list_t		*head = NULL, *hca = NULL;

	IBTF_DPRINTF_L4("ibdm", "\tibnex_get_hca_info_by_dip");

	mutex_enter(&ibdm.ibdm_hl_mutex);
	head = ibdm.ibdm_hca_list_head;
	while (head) {
		if (head->hl_hca_guid == hca_guid) {
			hca = ibdm_dup_hca_attr(head);
			hca->hl_next = NULL;
			break;
		}
		head = head->hl_next;
	}
	mutex_exit(&ibdm.ibdm_hl_mutex);
	IBTF_DPRINTF_L4("ibdm", "\tibnex_get_hca_info_by_dip %p", hca);
	return (hca);
}


/*
 * ibdm_dup_hca_attr()
 *	Allocate a new HCA attribute strucuture and initialize
 *	hca attribute structure with the incoming HCA attributes
 *	returned the allocated hca attributes.
 */
static ibdm_hca_list_t *
ibdm_dup_hca_attr(ibdm_hca_list_t *in_hca)
{
	int			len;
	ibdm_hca_list_t		*out_hca;

	len = sizeof (ibdm_hca_list_t) +
	    (in_hca->hl_nports * sizeof (ibdm_port_attr_t));
	IBTF_DPRINTF_L4("ibdm", "\tdup_hca_attr len %d", len);
	out_hca = (ibdm_hca_list_t *)kmem_alloc(len, KM_SLEEP);
	bcopy((char *)in_hca,
	    (char *)out_hca, sizeof (ibdm_hca_list_t));
	if (in_hca->hl_nports) {
		out_hca->hl_port_attr = (ibdm_port_attr_t *)
		    ((char *)out_hca + sizeof (ibdm_hca_list_t));
		bcopy((char *)in_hca->hl_port_attr,
		    (char *)out_hca->hl_port_attr,
		    (in_hca->hl_nports * sizeof (ibdm_port_attr_t)));
		for (len = 0; len < out_hca->hl_nports; len++)
			ibdm_update_port_attr(&out_hca->hl_port_attr[len]);
	}
	return (out_hca);
}


/*
 * ibdm_ibnex_free_hca_list()
 *	Free one/more HCA lists
 */
void
ibdm_ibnex_free_hca_list(ibdm_hca_list_t *hca_list)
{
	int			ii;
	size_t			len;
	ibdm_hca_list_t 	*temp;
	ibdm_port_attr_t	*port;

	IBTF_DPRINTF_L4("ibdm", "\tibnex_free_hca_list:");
	ASSERT(hca_list);
	while (hca_list) {
		temp = hca_list;
		hca_list = hca_list->hl_next;
		for (ii = 0; ii < temp->hl_nports; ii++) {
			port = &temp->hl_port_attr[ii];
			len = (port->pa_npkeys * sizeof (ibdm_pkey_tbl_t));
			if (len != 0)
				kmem_free(port->pa_pkey_tbl, len);
		}
		len = sizeof (ibdm_hca_list_t) + (temp->hl_nports *
		    sizeof (ibdm_port_attr_t));
		kmem_free(temp, len);
	}
}


/*
 * ibdm_ibnex_probe_iocguid()
 *	Probes the IOC on the fabric and returns the IOC information
 *	if present. Otherwise, NULL is returned
 */
/* ARGSUSED */
ibdm_ioc_info_t *
ibdm_ibnex_probe_ioc(ib_guid_t iou, ib_guid_t ioc_guid, int reprobe_flag)
{
	int			k;
	ibdm_ioc_info_t		*ioc_info;
	ibdm_dp_gidinfo_t	*gid_info; /* used as index and arg */
	timeout_id_t		*timeout_id;

	IBTF_DPRINTF_L4("ibdm", "\tibnex_probe_ioc: (%llX, %llX, %d) Begin",
	    iou, ioc_guid, reprobe_flag);

	if (ibdm_enumerate_iocs == 0)
		return (NULL);

	/* Check whether we know this already */
	ioc_info = ibdm_get_ioc_info_with_gid(ioc_guid, &gid_info);
	if (ioc_info == NULL) {
		mutex_enter(&ibdm.ibdm_mutex);
		while (ibdm.ibdm_busy & IBDM_BUSY)
			cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex);
		ibdm.ibdm_busy |= IBDM_BUSY;
		mutex_exit(&ibdm.ibdm_mutex);
		ibdm_probe_ioc(iou, ioc_guid, 0);
		mutex_enter(&ibdm.ibdm_mutex);
		ibdm.ibdm_busy &= ~IBDM_BUSY;
		cv_broadcast(&ibdm.ibdm_busy_cv);
		mutex_exit(&ibdm.ibdm_mutex);
		ioc_info = ibdm_get_ioc_info_with_gid(ioc_guid, &gid_info);
	} else if (reprobe_flag) {	/* Handle Reprobe for the IOC */
		ASSERT(gid_info != NULL);
		/* Free the ioc_list before reprobe; and cancel any timers */
		mutex_enter(&ibdm.ibdm_mutex);
		mutex_enter(&gid_info->gl_mutex);
		if (ioc_info->ioc_timeout_id) {
			timeout_id = ioc_info->ioc_timeout_id;
			ioc_info->ioc_timeout_id = 0;
			mutex_exit(&gid_info->gl_mutex);
			IBTF_DPRINTF_L5("ibdm", "\tprobe_ioc: "
			    "ioc_timeout_id = 0x%x", timeout_id);
			if (untimeout(timeout_id) == -1) {
				IBTF_DPRINTF_L2("ibdm", "\tprobe_ioc: "
				    "untimeout ioc_timeout_id failed");
			}
			mutex_enter(&gid_info->gl_mutex);
		}
		if (ioc_info->ioc_dc_timeout_id) {
			timeout_id = ioc_info->ioc_dc_timeout_id;
			ioc_info->ioc_dc_timeout_id = 0;
			mutex_exit(&gid_info->gl_mutex);
			IBTF_DPRINTF_L5("ibdm", "\tprobe_ioc: "
			    "ioc_dc_timeout_id = 0x%x", timeout_id);
			if (untimeout(timeout_id) == -1) {
				IBTF_DPRINTF_L2("ibdm", "\tprobe_ioc: "
				    "untimeout ioc_dc_timeout_id failed");
			}
			mutex_enter(&gid_info->gl_mutex);
		}
		for (k = 0; k < ioc_info->ioc_profile.ioc_service_entries; k++)
			if (ioc_info->ioc_serv[k].se_timeout_id) {
				timeout_id = ioc_info->ioc_serv[k].
				    se_timeout_id;
				ioc_info->ioc_serv[k].se_timeout_id = 0;
				mutex_exit(&gid_info->gl_mutex);
				IBTF_DPRINTF_L5("ibdm", "\tprobe_ioc: "
				    "ioc_info->ioc_serv[k].se_timeout_id = %x",
				    k, timeout_id);
				if (untimeout(timeout_id) == -1) {
					IBTF_DPRINTF_L2("ibdm", "\tprobe_ioc: "
					    "untimeout se_timeout_id %d "
					    "failed", k);
				}
				mutex_enter(&gid_info->gl_mutex);
			}
		mutex_exit(&gid_info->gl_mutex);
		mutex_exit(&ibdm.ibdm_mutex);
		ibdm_ibnex_free_ioc_list(ioc_info);

		mutex_enter(&ibdm.ibdm_mutex);
		while (ibdm.ibdm_busy & IBDM_BUSY)
			cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex);
		ibdm.ibdm_busy |= IBDM_BUSY;
		mutex_exit(&ibdm.ibdm_mutex);

		ibdm_probe_ioc(iou, ioc_guid, 1);

		/*
		 * Skip if gl_reprobe_flag is set, this will be
		 * a re-inserted / new GID, for which notifications
		 * have already been send.
		 */
		for (gid_info = ibdm.ibdm_dp_gidlist_head; gid_info;
		    gid_info = gid_info->gl_next) {
			uint8_t			ii, niocs;
			ibdm_ioc_info_t		*ioc;

			if (gid_info->gl_iou == NULL)
				continue;

			if (gid_info->gl_reprobe_flag) {
				gid_info->gl_reprobe_flag = 0;
				continue;
			}

			niocs = gid_info->gl_iou->iou_info.iou_num_ctrl_slots;
			for (ii = 0; ii < niocs; ii++) {
				ioc = IBDM_GIDINFO2IOCINFO(gid_info, ii);
				if (ioc->ioc_profile.ioc_guid == ioc_guid) {
					mutex_enter(&ibdm.ibdm_mutex);
					ibdm_reprobe_update_port_srv(ioc,
					    gid_info);
					mutex_exit(&ibdm.ibdm_mutex);
				}
			}
		}
		mutex_enter(&ibdm.ibdm_mutex);
		ibdm.ibdm_busy &= ~IBDM_BUSY;
		cv_broadcast(&ibdm.ibdm_busy_cv);
		mutex_exit(&ibdm.ibdm_mutex);

		ioc_info = ibdm_get_ioc_info_with_gid(ioc_guid, &gid_info);
	}
	return (ioc_info);
}


/*
 * ibdm_get_ioc_info_with_gid()
 *	Returns pointer to ibdm_ioc_info_t if it finds
 *	matching record for the ioc_guid. Otherwise NULL is returned.
 *	The pointer to gid_info is set to the second argument in case that
 *	the non-NULL value returns (and the second argument is not NULL).
 *
 * Note. use the same strings as "ibnex_get_ioc_info" in
 *       IBTF_DPRINTF() to keep compatibility.
 */
static ibdm_ioc_info_t *
ibdm_get_ioc_info_with_gid(ib_guid_t ioc_guid,
    ibdm_dp_gidinfo_t **gid_info)
{
	int			ii;
	ibdm_ioc_info_t		*ioc = NULL, *tmp = NULL;
	ibdm_dp_gidinfo_t	*gid_list;
	ib_dm_io_unitinfo_t	*iou;

	IBTF_DPRINTF_L4("ibdm", "\tibnex_get_ioc_info: GUID %llx", ioc_guid);

	mutex_enter(&ibdm.ibdm_mutex);
	while (ibdm.ibdm_busy & IBDM_BUSY)
		cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex);
	ibdm.ibdm_busy |= IBDM_BUSY;

	if (gid_info)
		*gid_info = NULL; /* clear the value of gid_info */

	gid_list = ibdm.ibdm_dp_gidlist_head;
	while (gid_list) {
		mutex_enter(&gid_list->gl_mutex);
		if (gid_list->gl_state != IBDM_GID_PROBING_COMPLETE) {
			mutex_exit(&gid_list->gl_mutex);
			gid_list = gid_list->gl_next;
			continue;
		}
		if (gid_list->gl_iou == NULL) {
			IBTF_DPRINTF_L2("ibdm",
			    "\tget_ioc_info: No IOU info");
			mutex_exit(&gid_list->gl_mutex);
			gid_list = gid_list->gl_next;
			continue;
		}
		iou = &gid_list->gl_iou->iou_info;
		for (ii = 0; ii < iou->iou_num_ctrl_slots; ii++) {
			tmp = IBDM_GIDINFO2IOCINFO(gid_list, ii);
			if ((tmp->ioc_profile.ioc_guid == ioc_guid) &&
			    (tmp->ioc_state == IBDM_IOC_STATE_PROBE_SUCCESS)) {
				ioc = ibdm_dup_ioc_info(tmp, gid_list);
				if (gid_info)
					*gid_info = gid_list; /* set this ptr */
				mutex_exit(&gid_list->gl_mutex);
				ibdm.ibdm_busy &= ~IBDM_BUSY;
				cv_broadcast(&ibdm.ibdm_busy_cv);
				mutex_exit(&ibdm.ibdm_mutex);
				IBTF_DPRINTF_L4("ibdm", "\tget_ioc_info: End");
				return (ioc);
			}
		}
		if (ii == iou->iou_num_ctrl_slots)
			ioc = NULL;

		mutex_exit(&gid_list->gl_mutex);
		gid_list = gid_list->gl_next;
	}

	ibdm.ibdm_busy &= ~IBDM_BUSY;
	cv_broadcast(&ibdm.ibdm_busy_cv);
	mutex_exit(&ibdm.ibdm_mutex);
	IBTF_DPRINTF_L4("ibdm", "\tget_ioc_info: failure End");
	return (ioc);
}

/*
 * ibdm_ibnex_get_ioc_info()
 *	Returns pointer to ibdm_ioc_info_t if it finds
 *	matching record for the ioc_guid, otherwise NULL
 *	is returned
 *
 * Note. this is a wrapper function to ibdm_get_ioc_info_with_gid() now.
 */
ibdm_ioc_info_t *
ibdm_ibnex_get_ioc_info(ib_guid_t ioc_guid)
{
	if (ibdm_enumerate_iocs == 0)
		return (NULL);

	/* will not use the gid_info pointer, so the second arg is NULL */
	return (ibdm_get_ioc_info_with_gid(ioc_guid, NULL));
}

/*
 * ibdm_ibnex_get_ioc_count()
 *	Returns number of ibdm_ioc_info_t it finds
 */
int
ibdm_ibnex_get_ioc_count(void)
{
	int			count = 0, k;
	ibdm_ioc_info_t		*ioc;
	ibdm_dp_gidinfo_t	*gid_list;

	if (ibdm_enumerate_iocs == 0)
		return (0);

	mutex_enter(&ibdm.ibdm_mutex);
	ibdm_sweep_fabric(0);

	while (ibdm.ibdm_busy & IBDM_BUSY)
		cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex);
	ibdm.ibdm_busy |= IBDM_BUSY;

	for (gid_list = ibdm.ibdm_dp_gidlist_head; gid_list;
	    gid_list = gid_list->gl_next) {
		mutex_enter(&gid_list->gl_mutex);
		if ((gid_list->gl_state != IBDM_GID_PROBING_COMPLETE) ||
		    (gid_list->gl_iou == NULL)) {
			mutex_exit(&gid_list->gl_mutex);
			continue;
		}
		for (k = 0; k < gid_list->gl_iou->iou_info.iou_num_ctrl_slots;
		    k++) {
			ioc = IBDM_GIDINFO2IOCINFO(gid_list, k);
			if (ioc->ioc_state == IBDM_IOC_STATE_PROBE_SUCCESS)
				++count;
		}
		mutex_exit(&gid_list->gl_mutex);
	}
	ibdm.ibdm_busy &= ~IBDM_BUSY;
	cv_broadcast(&ibdm.ibdm_busy_cv);
	mutex_exit(&ibdm.ibdm_mutex);

	IBTF_DPRINTF_L4("ibdm", "\tget_ioc_count: count = %d", count);
	return (count);
}


/*
 * ibdm_ibnex_get_ioc_list()
 *	Returns information about all the IOCs present on the fabric.
 *	Reprobes the IOCs and the GID list if list_flag is set to REPROBE_ALL.
 *	Does not sweep fabric if DONOT_PROBE is set
 */
ibdm_ioc_info_t *
ibdm_ibnex_get_ioc_list(ibdm_ibnex_get_ioclist_mtd_t list_flag)
{
	int			ii;
	ibdm_ioc_info_t		*ioc_list = NULL, *tmp, *ioc;
	ibdm_dp_gidinfo_t	*gid_list;
	ib_dm_io_unitinfo_t	*iou;

	IBTF_DPRINTF_L4("ibdm", "\tget_ioc_list: Enter");

	if (ibdm_enumerate_iocs == 0)
		return (NULL);

	mutex_enter(&ibdm.ibdm_mutex);
	if (list_flag != IBDM_IBNEX_DONOT_PROBE)
		ibdm_sweep_fabric(list_flag == IBDM_IBNEX_REPROBE_ALL);

	while (ibdm.ibdm_busy & IBDM_BUSY)
		cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex);
	ibdm.ibdm_busy |= IBDM_BUSY;

	gid_list = ibdm.ibdm_dp_gidlist_head;
	while (gid_list) {
		mutex_enter(&gid_list->gl_mutex);
		if (gid_list->gl_state != IBDM_GID_PROBING_COMPLETE) {
			mutex_exit(&gid_list->gl_mutex);
			gid_list = gid_list->gl_next;
			continue;
		}
		if (gid_list->gl_iou == NULL) {
			IBTF_DPRINTF_L2("ibdm",
			    "\tget_ioc_list: No IOU info");
			mutex_exit(&gid_list->gl_mutex);
			gid_list = gid_list->gl_next;
			continue;
		}
		iou = &gid_list->gl_iou->iou_info;
		for (ii = 0; ii < iou->iou_num_ctrl_slots; ii++) {
			ioc = IBDM_GIDINFO2IOCINFO(gid_list, ii);
			if (ioc->ioc_state == IBDM_IOC_STATE_PROBE_SUCCESS) {
				tmp = ibdm_dup_ioc_info(ioc, gid_list);
				tmp->ioc_next = ioc_list;
				ioc_list = tmp;
			}
		}
		mutex_exit(&gid_list->gl_mutex);
		gid_list = gid_list->gl_next;
	}
	ibdm.ibdm_busy &= ~IBDM_BUSY;
	cv_broadcast(&ibdm.ibdm_busy_cv);
	mutex_exit(&ibdm.ibdm_mutex);

	IBTF_DPRINTF_L4("ibdm", "\tget_ioc_list: End");
	return (ioc_list);
}

/*
 * ibdm_dup_ioc_info()
 *	Duplicate the IOC information and return the IOC
 *	information.
 */
static ibdm_ioc_info_t *
ibdm_dup_ioc_info(ibdm_ioc_info_t *in_ioc, ibdm_dp_gidinfo_t *gid_list)
{
	ibdm_ioc_info_t	*out_ioc;
	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*out_ioc));
	ASSERT(MUTEX_HELD(&gid_list->gl_mutex));

	out_ioc = kmem_alloc(sizeof (ibdm_ioc_info_t), KM_SLEEP);
	bcopy(in_ioc, out_ioc, sizeof (ibdm_ioc_info_t));
	ibdm_update_ioc_port_gidlist(out_ioc, gid_list);
	out_ioc->ioc_iou_dc_valid = gid_list->gl_iou->iou_dc_valid;
	out_ioc->ioc_iou_diagcode = gid_list->gl_iou->iou_diagcode;

	return (out_ioc);
}


/*
 * ibdm_free_ioc_list()
 *	Deallocate memory for IOC list structure
 */
void
ibdm_ibnex_free_ioc_list(ibdm_ioc_info_t *ioc)
{
	ibdm_ioc_info_t *temp;

	IBTF_DPRINTF_L4("ibdm", "\tibnex_free_ioc_list:");
	while (ioc) {
		temp = ioc;
		ioc = ioc->ioc_next;
		kmem_free(temp->ioc_gid_list,
		    (sizeof (ibdm_gid_t) * temp->ioc_nportgids));
		if (temp->ioc_hca_list)
			ibdm_ibnex_free_hca_list(temp->ioc_hca_list);
		kmem_free(temp, sizeof (ibdm_ioc_info_t));
	}
}


/*
 * ibdm_ibnex_update_pkey_tbls
 *	Updates the DM P_Key database.
 *	NOTE: Two cases are handled here: P_Key being added or removed.
 *
 * Arguments		: NONE
 * Return Values	: NONE
 */
void
ibdm_ibnex_update_pkey_tbls(void)
{
	int			h, pp, pidx;
	uint_t			nports;
	uint_t			size;
	ib_pkey_t		new_pkey;
	ib_pkey_t		*orig_pkey;
	ibdm_hca_list_t		*hca_list;
	ibdm_port_attr_t	*port;
	ibt_hca_portinfo_t	*pinfop;

	IBTF_DPRINTF_L4("ibdm", "\tibnex_update_pkey_tbls:");

	mutex_enter(&ibdm.ibdm_hl_mutex);
	hca_list = ibdm.ibdm_hca_list_head;

	for (h = 0; h < ibdm.ibdm_hca_count; h++) {

		/* This updates P_Key Tables for all ports of this HCA */
		(void) ibt_query_hca_ports(hca_list->hl_hca_hdl, 0, &pinfop,
		    &nports, &size);

		/* number of ports shouldn't have changed */
		ASSERT(nports == hca_list->hl_nports);

		for (pp = 0; pp < hca_list->hl_nports; pp++) {
			port = &hca_list->hl_port_attr[pp];

			/*
			 * First figure out the P_Keys from IBTL.
			 * Three things could have happened:
			 *	New P_Keys added
			 *	Existing P_Keys removed
			 *	Both of the above two
			 *
			 * Loop through the P_Key Indices and check if a
			 * give P_Key_Ix matches that of the one seen by
			 * IBDM. If they match no action is needed.
			 *
			 * If they don't match:
			 *	1. if orig_pkey is invalid and new_pkey is valid
			 *		---> add new_pkey to DM database
			 *	2. if orig_pkey is valid and new_pkey is invalid
			 *		---> remove orig_pkey from DM database
			 *	3. if orig_pkey and new_pkey are both valid:
			 *		---> remov orig_pkey from DM database
			 *		---> add new_pkey to DM database
			 *	4. if orig_pkey and new_pkey are both invalid:
			 *		---> do nothing. Updated DM database.
			 */

			for (pidx = 0; pidx < port->pa_npkeys; pidx++) {
				new_pkey = pinfop[pp].p_pkey_tbl[pidx];
				orig_pkey = &port->pa_pkey_tbl[pidx].pt_pkey;

				/* keys match - do nothing */
				if (*orig_pkey == new_pkey)
					continue;

				if (IBDM_INVALID_PKEY(*orig_pkey) &&
				    !IBDM_INVALID_PKEY(new_pkey)) {
					/* P_Key was added */
					IBTF_DPRINTF_L5("ibdm",
					    "\tibnex_update_pkey_tbls: new "
					    "P_Key added = 0x%x", new_pkey);
					*orig_pkey = new_pkey;
					ibdm_port_attr_ibmf_init(port,
					    new_pkey, pp);
				} else if (!IBDM_INVALID_PKEY(*orig_pkey) &&
				    IBDM_INVALID_PKEY(new_pkey)) {
					/* P_Key was removed */
					IBTF_DPRINTF_L5("ibdm",
					    "\tibnex_update_pkey_tbls: P_Key "
					    "removed = 0x%x", *orig_pkey);
					*orig_pkey = new_pkey;
					(void) ibdm_port_attr_ibmf_fini(port,
					    pidx);
				} else if (!IBDM_INVALID_PKEY(*orig_pkey) &&
				    !IBDM_INVALID_PKEY(new_pkey)) {
					/* P_Key were replaced */
					IBTF_DPRINTF_L5("ibdm",
					    "\tibnex_update_pkey_tbls: P_Key "
					    "replaced 0x%x with 0x%x",
					    *orig_pkey, new_pkey);
					(void) ibdm_port_attr_ibmf_fini(port,
					    pidx);
					*orig_pkey = new_pkey;
					ibdm_port_attr_ibmf_init(port,
					    new_pkey, pp);
				} else {
					/*
					 * P_Keys are invalid
					 * set anyway to reflect if
					 * INVALID_FULL was changed to
					 * INVALID_LIMITED or vice-versa.
					 */
					*orig_pkey = new_pkey;
				} /* end of else */

			} /* loop of p_key index */

		} /* loop of #ports of HCA */

		ibt_free_portinfo(pinfop, size);
		hca_list = hca_list->hl_next;

	} /* loop for all HCAs in the system */

	mutex_exit(&ibdm.ibdm_hl_mutex);
}


/*
 * ibdm_send_ioc_profile()
 *	Send IOC Controller Profile request. When the request is completed
 *	IBMF calls ibdm_process_incoming_mad routine to inform about
 *	the completion.
 */
static int
ibdm_send_ioc_profile(ibdm_dp_gidinfo_t *gid_info, uint8_t ioc_no)
{
	ibmf_msg_t		*msg;
	ib_mad_hdr_t	*hdr;
	ibdm_ioc_info_t	*ioc_info = &(gid_info->gl_iou->iou_ioc_info[ioc_no]);
	ibdm_timeout_cb_args_t	*cb_args;

	IBTF_DPRINTF_L4("ibdm", "\tsend_ioc_profile: "
	    "gid info 0x%p, ioc_no = %d", gid_info, ioc_no);

	/*
	 * Send command to get IOC profile.
	 * Allocate a IBMF packet and initialize the packet.
	 */
	if (ibmf_alloc_msg(gid_info->gl_ibmf_hdl, IBMF_ALLOC_SLEEP,
	    &msg) != IBMF_SUCCESS) {
		IBTF_DPRINTF_L2("ibdm", "\tsend_ioc_profile: pkt alloc fail");
		return (IBDM_FAILURE);
	}

	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*msg))
	ibdm_alloc_send_buffers(msg);
	_NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*msg))

	mutex_enter(&gid_info->gl_mutex);
	ibdm_bump_transactionID(gid_info);
	mutex_exit(&gid_info->gl_mutex);

	msg->im_local_addr.ia_local_lid	= gid_info->gl_slid;
	msg->im_local_addr.ia_remote_lid	= gid_info->gl_dlid;
	if (gid_info->gl_redirected == B_TRUE) {
		if (gid_info->gl_redirect_dlid != 0) {
			msg->im_local_addr.ia_remote_lid =
			    gid_info->gl_redirect_dlid;
		}
		msg->im_local_addr.ia_remote_qno = gid_info->gl_redirect_QP;
		msg->im_local_addr.ia_p_key = gid_info->gl_redirect_pkey;
		msg->im_local_addr.ia_q_key = gid_info->gl_redirect_qkey;
		msg->im_local_addr.ia_service_level = gid_info->gl_redirectSL;
	} else {
		msg->im_local_addr.ia_remote_qno = 1;
		msg->im_local_addr.ia_p_key = gid_info->gl_p_key;
		msg->im_local_addr.ia_q_key = IB_GSI_QKEY;
		msg->im_local_addr.ia_service_level = gid_info->gl_SL;
	}

	hdr			= IBDM_OUT_IBMFMSG_MADHDR(msg);
	hdr->BaseVersion	= MAD_CLASS_BASE_VERS_1;
	hdr->MgmtClass		= MAD_MGMT_CLASS_DEV_MGT;
	hdr->ClassVersion	= IB_DM_CLASS_VERSION_1;
	hdr->R_Method		= IB_DM_DEVMGT_METHOD_GET;
	hdr->Status		= 0;
	hdr->TransactionID	= h2b64(gid_info->gl_transactionID);
	hdr->AttributeID	= h2b16(IB_DM_ATTR_IOC_CTRL_PROFILE);
	hdr->AttributeModifier 	= h2b32(ioc_no + 1);

	ioc_info->ioc_state	= IBDM_IOC_STATE_REPROBE_PROGRESS;
	cb_args			= &ioc_info->ioc_cb_args;
	cb_args->cb_gid_info	= gid_info;
	cb_args->cb_retry_count	= ibdm_dft_retry_cnt;
	cb_args->cb_req_type	= IBDM_REQ_TYPE_IOCINFO;
	cb_args->cb_ioc_num	= ioc_no;

	mutex_enter(&gid_info->gl_mutex);
	ioc_info->ioc_timeout_id = timeout(ibdm_pkt_timeout_hdlr,
	    cb_args, IBDM_TIMEOUT_VALUE(ibdm_dft_timeout));
	mutex_exit(&gid_info->gl_mutex);

	IBTF_DPRINTF_L5("ibdm", "\tsend_ioc_profile:"
	    "timeout %x", ioc_info->ioc_timeout_id);

	if (ibmf_msg_transport(gid_info->gl_ibmf_hdl, gid_info->gl_qp_hdl, msg,
	    NULL, ibdm_ibmf_send_cb, cb_args, 0) != IBMF_SUCCESS) {
		IBTF_DPRINTF_L2("ibdm",
		    "\tsend_ioc_profile: msg transport failed");
		ibdm_ibmf_send_cb(gid_info->gl_ibmf_hdl, msg, cb_args);
	}
	ioc_info->ioc_state = IBDM_IOC_STATE_REPROBE_PROGRESS;
	return (IBDM_SUCCESS);
}


/*
 * ibdm_port_reachable
 *	Returns B_TRUE if the port GID is reachable by sending
 *	a SA query to get the NODE record for this port GUID.
 */
static boolean_t
ibdm_port_reachable(ibmf_saa_handle_t sa_hdl, ib_guid_t guid)
{
	sa_node_record_t *resp;
	size_t length;

	/*
	 * Verify if it's reachable by getting the node record.
	 */
	if (ibdm_get_node_record_by_port(sa_hdl, guid, &resp, &length) ==
	    IBDM_SUCCESS) {
		kmem_free(resp, length);
		return (B_TRUE);
	}
	return (B_FALSE);
}

/*
 * ibdm_get_node_record_by_port
 *	Sends a SA query to get the NODE record for port GUID
 *	Returns IBDM_SUCCESS if the port GID is reachable.
 *
 *      Note: the caller must be responsible for freeing the resource
 *      by calling kmem_free(resp, length) later.
 */
static int
ibdm_get_node_record_by_port(ibmf_saa_handle_t sa_hdl, ib_guid_t guid,
    sa_node_record_t **resp, size_t *length)
{
	sa_node_record_t	req;
	ibmf_saa_access_args_t	args;
	int			ret;
	ASSERT(resp != NULL && length != NULL);

	IBTF_DPRINTF_L4("ibdm", "\tport_reachable: port_guid %llx",
	    guid);

	bzero(&req, sizeof (sa_node_record_t));
	req.NodeInfo.PortGUID = guid;

	args.sq_attr_id		= SA_NODERECORD_ATTRID;
	args.sq_access_type 	= IBMF_SAA_RETRIEVE;
	args.sq_component_mask 	= SA_NODEINFO_COMPMASK_PORTGUID;
	args.sq_template	= &req;
	args.sq_callback	= NULL;
	args.sq_callback_arg 	= NULL;

	ret = ibmf_sa_access(sa_hdl, &args, 0, length, (void **) resp);
	if (ret != IBMF_SUCCESS) {
		IBTF_DPRINTF_L2("ibdm", "\tport_reachable:"
		    " SA Retrieve Failed: %d", ret);
		return (IBDM_FAILURE);
	}
	if (*resp == NULL || *length == 0) {
		IBTF_DPRINTF_L2("ibdm", "\tport_reachable: No records");
		return (IBDM_FAILURE);
	}
	/*
	 * There is one NodeRecord on each endport on a subnet.
	 */
	ASSERT(*length == sizeof (sa_node_record_t));

	return (IBDM_SUCCESS);
}


/*
 * Update the gidlist for all affected IOCs when GID becomes
 * available/unavailable.
 *
 * Parameters :
 *	gidinfo - Incoming / Outgoing GID.
 *	add_flag - 1 for GID added, 0 for GID removed.
 *		- (-1) : IOC gid list updated, ioc_list required.
 *
 * This function gets the GID for the node GUID corresponding to the
 * port GID. Gets the IOU info
 */
static ibdm_ioc_info_t *
ibdm_update_ioc_gidlist(ibdm_dp_gidinfo_t *gid_info, int avail_flag)
{
	ibdm_dp_gidinfo_t	*node_gid = NULL;
	uint8_t	niocs, ii;
	ibdm_ioc_info_t	*ioc, *ioc_list = NULL, *tmp;

	IBTF_DPRINTF_L4("ibdm", "\tupdate_ioc_gidlist");

	switch (avail_flag) {
		case 1 :
			node_gid = ibdm_check_dest_nodeguid(gid_info);
			break;
		case 0 :
			node_gid = ibdm_handle_gid_rm(gid_info);
			break;
		case -1 :
			node_gid = gid_info;
			break;
		default :
			break;
	}

	if (node_gid == NULL) {
		IBTF_DPRINTF_L4("ibdm", "\tupdate_ioc_gidlist: "
		    "No node GID found, port gid 0x%p, avail_flag %d",
		    gid_info, avail_flag);
		return (NULL);
	}

	mutex_enter(&node_gid->gl_mutex);
	if ((node_gid->gl_state != IBDM_GID_PROBING_COMPLETE &&
	    node_gid->gl_state != IBDM_GID_PROBING_SKIPPED) ||
	    node_gid->gl_iou == NULL) {
		IBTF_DPRINTF_L4("ibdm", "\tupdate_ioc_gidlist "
		    "gl_state %x, gl_iou %p", node_gid->gl_state,
		    node_gid->gl_iou);
		mutex_exit(&node_gid->gl_mutex);
		return (NULL);
	}

	niocs = node_gid->gl_iou->iou_info.iou_num_ctrl_slots;
	IBTF_DPRINTF_L4("ibdm", "\tupdate_ioc_gidlist : niocs %x",
	    niocs);
	for (ii = 0; ii < niocs; ii++) {
		ioc = IBDM_GIDINFO2IOCINFO(node_gid, ii);
		/*
		 * Skip IOCs for which probe is not complete or
		 * reprobe is progress
		 */
		if (ioc->ioc_state == IBDM_IOC_STATE_PROBE_SUCCESS) {
			tmp = ibdm_dup_ioc_info(ioc, node_gid);
			tmp->ioc_info_updated.ib_gid_prop_updated = 1;
			tmp->ioc_next = ioc_list;
			ioc_list = tmp;
		}
	}
	mutex_exit(&node_gid->gl_mutex);

	IBTF_DPRINTF_L4("ibdm", "\tupdate_ioc_gidlist : return %p",
	    ioc_list);
	return (ioc_list);
}

/*
 * ibdm_saa_event_cb :
 *	Event handling which does *not* require ibdm_hl_mutex to be
 *	held are executed in the same thread. This is to prevent
 *	deadlocks with HCA port down notifications which hold the
 *	ibdm_hl_mutex.
 *
 *	GID_AVAILABLE event is handled here. A taskq is spawned to
 *	handle GID_UNAVAILABLE.
 *
 *	A new mutex ibdm_ibnex_mutex has been introduced to protect
 *	ibnex_callback. This has been done to prevent any possible
 *	deadlock (described above) while handling GID_AVAILABLE.
 *
 *	IBMF calls the event callback for a HCA port. The SA handle
 *	for this port would be valid, till the callback returns.
 *	IBDM calling IBDM using the above SA handle should be valid.
 *
 *	IBDM will additionally  check (SA handle != NULL), before
 *	calling IBMF.
 */
/*ARGSUSED*/
static void
ibdm_saa_event_cb(ibmf_saa_handle_t ibmf_saa_handle,
    ibmf_saa_subnet_event_t ibmf_saa_event,
    ibmf_saa_event_details_t *event_details, void *callback_arg)
{
	ibdm_saa_event_arg_t *event_arg;
	ib_gid_t		sgid, dgid;
	ibdm_port_attr_t	*hca_port;
	ibdm_dp_gidinfo_t	*gid_info, *node_gid_info = NULL;
	sa_node_record_t *nrec;
	size_t length;

	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*event_arg));

	hca_port = (ibdm_port_attr_t *)callback_arg;

	IBTF_DPRINTF_L4("ibdm", "\tsaa_event_cb(%x, %x, %x, %x)\n",
	    ibmf_saa_handle, ibmf_saa_event, event_details,
	    callback_arg);

#ifdef DEBUG
	if (ibdm_ignore_saa_event)
		return;
#endif

	if (ibmf_saa_event == IBMF_SAA_EVENT_GID_AVAILABLE) {
		/*
		 * Ensure no other probe / sweep fabric is in
		 * progress.
		 */
		mutex_enter(&ibdm.ibdm_mutex);
		while (ibdm.ibdm_busy & IBDM_BUSY)
			cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex);
		ibdm.ibdm_busy |= IBDM_BUSY;
		mutex_exit(&ibdm.ibdm_mutex);

		/*
		 * If we already know about this GID, return.
		 * GID_AVAILABLE may be reported for multiple HCA
		 * ports.
		 */
		if ((ibdm_check_dgid(event_details->ie_gid.gid_guid,
		    event_details->ie_gid.gid_prefix))  != NULL) {
			mutex_enter(&ibdm.ibdm_mutex);
			ibdm.ibdm_busy &= ~IBDM_BUSY;
			cv_broadcast(&ibdm.ibdm_busy_cv);
			mutex_exit(&ibdm.ibdm_mutex);
			return;
		}

		IBTF_DPRINTF_L4("ibdm", "\tGID (prefix %x, guid %llx) "
		    "Insertion notified",
		    event_details->ie_gid.gid_prefix,
		    event_details->ie_gid.gid_guid);

		/* This is a new gid, insert it to GID list */
		sgid.gid_prefix = hca_port->pa_sn_prefix;
		sgid.gid_guid = hca_port->pa_port_guid;
		dgid.gid_prefix = event_details->ie_gid.gid_prefix;
		dgid.gid_guid = event_details->ie_gid.gid_guid;
		gid_info = ibdm_create_gid_info(hca_port, sgid, dgid);
		if (gid_info == NULL) {
			IBTF_DPRINTF_L4("ibdm", "\tGID_AVAILABLE: "
			    "create_gid_info returned NULL");
			mutex_enter(&ibdm.ibdm_mutex);
			ibdm.ibdm_busy &= ~IBDM_BUSY;
			cv_broadcast(&ibdm.ibdm_busy_cv);
			mutex_exit(&ibdm.ibdm_mutex);
			return;
		}
		mutex_enter(&gid_info->gl_mutex);
		gid_info->gl_state = IBDM_GID_PROBING_SKIPPED;
		mutex_exit(&gid_info->gl_mutex);

		/* Get the node GUID */
		if (ibdm_get_node_record_by_port(ibmf_saa_handle, dgid.gid_guid,
		    &nrec, &length) != IBDM_SUCCESS) {
			/*
			 * Set the state to PROBE_NOT_DONE for the
			 * next sweep to probe it
			 */
			IBTF_DPRINTF_L2("ibdm", "\tsaa_event_taskq: "
			    "Skipping GID : port GUID not found");
			mutex_enter(&gid_info->gl_mutex);
			gid_info->gl_state = IBDM_GID_PROBE_NOT_DONE;
			mutex_exit(&gid_info->gl_mutex);
			mutex_enter(&ibdm.ibdm_mutex);
			ibdm.ibdm_busy &= ~IBDM_BUSY;
			cv_broadcast(&ibdm.ibdm_busy_cv);
			mutex_exit(&ibdm.ibdm_mutex);
			return;
		}
		gid_info->gl_nodeguid = nrec->NodeInfo.NodeGUID;
		gid_info->gl_devid = nrec->NodeInfo.DeviceID;
		kmem_free(nrec, length);
		gid_info->gl_portguid = dgid.gid_guid;

		/*
		 * Get the gid info with the same node GUID.
		 */
		mutex_enter(&ibdm.ibdm_mutex);
		node_gid_info = ibdm.ibdm_dp_gidlist_head;
		while (node_gid_info) {
			if (node_gid_info->gl_nodeguid ==
			    gid_info->gl_nodeguid &&
			    node_gid_info->gl_iou != NULL) {
				break;
			}
			node_gid_info = node_gid_info->gl_next;
		}
		mutex_exit(&ibdm.ibdm_mutex);

		/*
		 * Handling a new GID requires filling of gl_hca_list.
		 * This require ibdm hca_list to be parsed and hence
		 * holding the ibdm_hl_mutex. Spawning a new thread to
		 * handle this.
		 */
		if (node_gid_info == NULL) {
			if (taskq_dispatch(system_taskq,
			    ibdm_saa_handle_new_gid, (void *)gid_info,
			    TQ_NOSLEEP) == TASKQID_INVALID) {
				IBTF_DPRINTF_L2("ibdm", "\tsaa_event_cb: "
				    "new_gid taskq_dispatch failed");
				return;
			}
		}

		mutex_enter(&ibdm.ibdm_mutex);
		ibdm.ibdm_busy &= ~IBDM_BUSY;
		cv_broadcast(&ibdm.ibdm_busy_cv);
		mutex_exit(&ibdm.ibdm_mutex);
		return;
	}

	if (ibmf_saa_event != IBMF_SAA_EVENT_GID_UNAVAILABLE)
		return;

	/*
	 * GID UNAVAIL EVENT: Try to locate the GID in the GID list.
	 * If we don't find it we just return.
	 */
	mutex_enter(&ibdm.ibdm_mutex);
	gid_info = ibdm.ibdm_dp_gidlist_head;
	while (gid_info) {
		if (gid_info->gl_portguid ==
		    event_details->ie_gid.gid_guid) {
			break;
		}
		gid_info = gid_info->gl_next;
	}
	mutex_exit(&ibdm.ibdm_mutex);
	if (gid_info == NULL) {
		IBTF_DPRINTF_L2("ibdm", "\tsaa_event_cb: "
		    "GID for GUID %llX not found during GID UNAVAIL event",
		    event_details->ie_gid.gid_guid);
		return;
	}

	/*
	 * If this GID is DM capable, we'll have to check whether this DGID
	 * is reachable via another port.
	 */
	if (gid_info->gl_is_dm_capable == B_TRUE) {
		event_arg = (ibdm_saa_event_arg_t *)kmem_alloc(
		    sizeof (ibdm_saa_event_arg_t), KM_SLEEP);
		event_arg->ibmf_saa_handle = ibmf_saa_handle;
		event_arg->ibmf_saa_event = ibmf_saa_event;
		bcopy(event_details, &event_arg->event_details,
		    sizeof (ibmf_saa_event_details_t));
		event_arg->callback_arg = callback_arg;

		if (taskq_dispatch(system_taskq, ibdm_saa_event_taskq,
		    (void *)event_arg, TQ_NOSLEEP) == TASKQID_INVALID) {
			IBTF_DPRINTF_L2("ibdm", "\tsaa_event_cb: "
			    "taskq_dispatch failed");
			ibdm_free_saa_event_arg(event_arg);
			return;
		}
	}
}

/*
 * Handle a new GID discovered by GID_AVAILABLE saa event.
 */
void
ibdm_saa_handle_new_gid(void *arg)
{
	ibdm_dp_gidinfo_t	*gid_info;
	ibdm_hca_list_t		*hca_list = NULL;
	ibdm_port_attr_t	*port = NULL;
	ibdm_ioc_info_t		*ioc_list = NULL;

	IBTF_DPRINTF_L4(ibdm_string, "\tsaa_handle_new_gid(%p)", arg);

	gid_info = (ibdm_dp_gidinfo_t *)arg;

	/*
	 * Ensure that no other sweep / probe has completed
	 * probing this gid.
	 */
	mutex_enter(&gid_info->gl_mutex);
	if (gid_info->gl_state != IBDM_GID_PROBE_NOT_DONE) {
		mutex_exit(&gid_info->gl_mutex);
		return;
	}
	mutex_exit(&gid_info->gl_mutex);

	/*
	 * Parse HCAs to fill gl_hca_list
	 */
	mutex_enter(&ibdm.ibdm_hl_mutex);
	for (ibdm_get_next_port(&hca_list, &port, 1); port;
	    ibdm_get_next_port(&hca_list, &port, 1)) {
		if (ibdm_port_reachable(port->pa_sa_hdl,
		    gid_info->gl_portguid) == B_TRUE) {
			ibdm_addto_glhcalist(gid_info, hca_list);
		}
	}
	mutex_exit(&ibdm.ibdm_hl_mutex);

	/*
	 * Ensure no other probe / sweep fabric is in
	 * progress.
	 */
	mutex_enter(&ibdm.ibdm_mutex);
	while (ibdm.ibdm_busy & IBDM_BUSY)
		cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex);
	ibdm.ibdm_busy |= IBDM_BUSY;
	mutex_exit(&ibdm.ibdm_mutex);

	/*
	 * New IOU probe it, to check if new IOCs
	 */
	IBTF_DPRINTF_L4(ibdm_string, "\tsaa_handle_new_gid: "
	    "new GID : probing");
	mutex_enter(&ibdm.ibdm_mutex);
	ibdm.ibdm_ngid_probes_in_progress++;
	mutex_exit(&ibdm.ibdm_mutex);
	mutex_enter(&gid_info->gl_mutex);
	gid_info->gl_reprobe_flag = 0;
	gid_info->gl_state = IBDM_GID_PROBE_NOT_DONE;
	mutex_exit(&gid_info->gl_mutex);
	ibdm_probe_gid_thread((void *)gid_info);

	mutex_enter(&ibdm.ibdm_mutex);
	ibdm_wait_probe_completion();
	mutex_exit(&ibdm.ibdm_mutex);

	if (gid_info->gl_iou == NULL) {
		mutex_enter(&ibdm.ibdm_mutex);
		ibdm.ibdm_busy &= ~IBDM_BUSY;
		cv_broadcast(&ibdm.ibdm_busy_cv);
		mutex_exit(&ibdm.ibdm_mutex);
		return;
	}

	/*
	 * Update GID list in all IOCs affected by this
	 */
	ioc_list = ibdm_update_ioc_gidlist(gid_info, 1);

	/*
	 * Pass on the IOCs with updated GIDs to IBnexus
	 */
	if (ioc_list) {
		mutex_enter(&ibdm.ibdm_ibnex_mutex);
		if (ibdm.ibdm_ibnex_callback != NULL) {
			(*ibdm.ibdm_ibnex_callback)((void *)ioc_list,
			    IBDM_EVENT_IOC_PROP_UPDATE);
		}
		mutex_exit(&ibdm.ibdm_ibnex_mutex);
	}

	mutex_enter(&ibdm.ibdm_mutex);
	ibdm.ibdm_busy &= ~IBDM_BUSY;
	cv_broadcast(&ibdm.ibdm_busy_cv);
	mutex_exit(&ibdm.ibdm_mutex);
}

/*
 * ibdm_saa_event_taskq :
 *	GID_UNAVAILABLE Event handling requires ibdm_hl_mutex to be
 *	held. The GID_UNAVAILABLE handling is done in a taskq to
 *	prevent deadlocks with HCA port down notifications which hold
 *	ibdm_hl_mutex.
 */
void
ibdm_saa_event_taskq(void *arg)
{
	ibdm_saa_event_arg_t *event_arg;
	ibmf_saa_handle_t ibmf_saa_handle;
	ibmf_saa_subnet_event_t ibmf_saa_event;
	ibmf_saa_event_details_t *event_details;
	void *callback_arg;

	ibdm_dp_gidinfo_t	*gid_info;
	ibdm_port_attr_t	*hca_port, *port = NULL;
	ibdm_hca_list_t		*hca_list = NULL;
	int	sa_handle_valid = 0;
	ibdm_ioc_info_t		*ioc_list = NULL;

	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*event_arg));

	event_arg = (ibdm_saa_event_arg_t *)arg;
	ibmf_saa_handle = event_arg->ibmf_saa_handle;
	ibmf_saa_event = event_arg->ibmf_saa_event;
	event_details = &event_arg->event_details;
	callback_arg = event_arg->callback_arg;

	ASSERT(callback_arg != NULL);
	ASSERT(ibmf_saa_event == IBMF_SAA_EVENT_GID_UNAVAILABLE);
	IBTF_DPRINTF_L4("ibdm", "\tsaa_event_taskq(%x, %x, %x, %x)",
	    ibmf_saa_handle, ibmf_saa_event, event_details,
	    callback_arg);

	hca_port = (ibdm_port_attr_t *)callback_arg;

	/* Check if the port_attr is still valid */
	mutex_enter(&ibdm.ibdm_hl_mutex);
	for (ibdm_get_next_port(&hca_list, &port, 0); port;
	    ibdm_get_next_port(&hca_list, &port, 0)) {
		if (port == hca_port && port->pa_port_guid ==
		    hca_port->pa_port_guid) {
			if (ibmf_saa_handle == hca_port->pa_sa_hdl)
				sa_handle_valid = 1;
			break;
		}
	}
	mutex_exit(&ibdm.ibdm_hl_mutex);
	if (sa_handle_valid == 0) {
		ibdm_free_saa_event_arg(event_arg);
		return;
	}

	if (hca_port && (hca_port->pa_sa_hdl == NULL ||
	    ibmf_saa_handle != hca_port->pa_sa_hdl)) {
		ibdm_free_saa_event_arg(event_arg);
		return;
	}
	hca_list = NULL;
	port = NULL;

	/*
	 * Check if the GID is visible to other HCA ports.
	 * Return if so.
	 */
	mutex_enter(&ibdm.ibdm_hl_mutex);
	for (ibdm_get_next_port(&hca_list, &port, 1); port;
	    ibdm_get_next_port(&hca_list, &port, 1)) {
		if (ibdm_port_reachable(port->pa_sa_hdl,
		    event_details->ie_gid.gid_guid) == B_TRUE) {
			mutex_exit(&ibdm.ibdm_hl_mutex);
			ibdm_free_saa_event_arg(event_arg);
			return;
		}
	}
	mutex_exit(&ibdm.ibdm_hl_mutex);

	/*
	 * Ensure no other probe / sweep fabric is in
	 * progress.
	 */
	mutex_enter(&ibdm.ibdm_mutex);
	while (ibdm.ibdm_busy & IBDM_BUSY)
		cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex);
	ibdm.ibdm_busy |= IBDM_BUSY;
	mutex_exit(&ibdm.ibdm_mutex);

	/*
	 * If this GID is no longer in GID list, return
	 * GID_UNAVAILABLE may be reported for multiple HCA
	 * ports.
	 */
	mutex_enter(&ibdm.ibdm_mutex);
	gid_info = ibdm.ibdm_dp_gidlist_head;
	while (gid_info) {
		if (gid_info->gl_portguid ==
		    event_details->ie_gid.gid_guid) {
			break;
		}
		gid_info = gid_info->gl_next;
	}
	mutex_exit(&ibdm.ibdm_mutex);
	if (gid_info == NULL) {
		mutex_enter(&ibdm.ibdm_mutex);
		ibdm.ibdm_busy &= ~IBDM_BUSY;
		cv_broadcast(&ibdm.ibdm_busy_cv);
		mutex_exit(&ibdm.ibdm_mutex);
		ibdm_free_saa_event_arg(event_arg);
		return;
	}

	IBTF_DPRINTF_L4("ibdm", "\tGID (prefix %x, guid %llx) "
	    "Unavailable notification",
	    event_details->ie_gid.gid_prefix,
	    event_details->ie_gid.gid_guid);

	/*
	 * Update GID list in all IOCs affected by this
	 */
	if (gid_info->gl_state == IBDM_GID_PROBING_SKIPPED ||
	    gid_info->gl_state == IBDM_GID_PROBING_COMPLETE)
		ioc_list = ibdm_update_ioc_gidlist(gid_info, 0);

	/*
	 * Remove GID from the global GID list
	 * Handle the case where all port GIDs for an
	 * IOU have been hot-removed. Check both gid_info
	 * & ioc_info for checking ngids.
	 */
	mutex_enter(&ibdm.ibdm_mutex);
	if (gid_info->gl_iou != NULL && gid_info->gl_ngids == 0) {
		mutex_enter(&gid_info->gl_mutex);
		(void) ibdm_free_iou_info(gid_info, &gid_info->gl_iou);
		mutex_exit(&gid_info->gl_mutex);
	}
	if (gid_info->gl_prev != NULL)
		gid_info->gl_prev->gl_next = gid_info->gl_next;
	if (gid_info->gl_next != NULL)
		gid_info->gl_next->gl_prev = gid_info->gl_prev;

	if (gid_info == ibdm.ibdm_dp_gidlist_head)
		ibdm.ibdm_dp_gidlist_head = gid_info->gl_next;
	if (gid_info == ibdm.ibdm_dp_gidlist_tail)
		ibdm.ibdm_dp_gidlist_tail = gid_info->gl_prev;
	ibdm.ibdm_ngids--;

	ibdm.ibdm_busy &= ~IBDM_BUSY;
	cv_broadcast(&ibdm.ibdm_busy_cv);
	mutex_exit(&ibdm.ibdm_mutex);

	/* free the hca_list on this gid_info */
	ibdm_delete_glhca_list(gid_info);

	mutex_destroy(&gid_info->gl_mutex);
	kmem_free(gid_info, sizeof (ibdm_dp_gidinfo_t));

	/*
	 * Pass on the IOCs with updated GIDs to IBnexus
	 */
	if (ioc_list) {
		IBTF_DPRINTF_L4("ibdm", "\tGID_UNAVAILABLE "
		    "IOC_PROP_UPDATE for %p\n", ioc_list);
		mutex_enter(&ibdm.ibdm_ibnex_mutex);
		if (ibdm.ibdm_ibnex_callback != NULL) {
			(*ibdm.ibdm_ibnex_callback)((void *)
			    ioc_list, IBDM_EVENT_IOC_PROP_UPDATE);
		}
		mutex_exit(&ibdm.ibdm_ibnex_mutex);
	}

	ibdm_free_saa_event_arg(event_arg);
}


static int
ibdm_cmp_gid_list(ibdm_gid_t *new, ibdm_gid_t *prev)
{
	ibdm_gid_t		*scan_new, *scan_prev;
	int	cmp_failed = 0;

	ASSERT(new != NULL);
	ASSERT(prev != NULL);

	/*
	 * Search for each new gid anywhere in the prev GID list.
	 * Note that the gid list could have been re-ordered.
	 */
	for (scan_new = new; scan_new; scan_new = scan_new->gid_next) {
		for (scan_prev = prev, cmp_failed = 1; scan_prev;
		    scan_prev = scan_prev->gid_next) {
			if (scan_prev->gid_dgid_hi == scan_new->gid_dgid_hi &&
			    scan_prev->gid_dgid_lo == scan_new->gid_dgid_lo) {
				cmp_failed = 0;
				break;
			}
		}

		if (cmp_failed)
			return (1);
	}
	return (0);
}

/*
 * This is always called in a single thread
 * This function updates the gid_list and serv_list of IOC
 * The current gid_list is in ioc_info_t(contains only port
 * guids for which probe is done) & gidinfo_t(other port gids)
 * The gids in both locations are used for comparision.
 */
static void
ibdm_reprobe_update_port_srv(ibdm_ioc_info_t *ioc, ibdm_dp_gidinfo_t *gidinfo)
{
	ibdm_gid_t		*cur_gid_list;
	uint_t			cur_nportgids;

	ASSERT(MUTEX_HELD(&ibdm.ibdm_mutex));

	ioc->ioc_info_updated.ib_prop_updated = 0;


	/* Current GID list in gid_info only */
	cur_gid_list = gidinfo->gl_gid;
	cur_nportgids = gidinfo->gl_ngids;

	if (ioc->ioc_prev_serv_cnt !=
	    ioc->ioc_profile.ioc_service_entries ||
	    ibdm_serv_cmp(&ioc->ioc_serv[0], &ioc->ioc_prev_serv[0],
	    ioc->ioc_prev_serv_cnt))
		ioc->ioc_info_updated.ib_srv_prop_updated = 1;

	if (ioc->ioc_prev_nportgids != cur_nportgids ||
	    ioc->ioc_prev_gid_list == NULL || cur_gid_list == NULL) {
		ioc->ioc_info_updated.ib_gid_prop_updated = 1;
	} else if (ibdm_cmp_gid_list(ioc->ioc_prev_gid_list, cur_gid_list)) {
		ioc->ioc_info_updated.ib_gid_prop_updated = 1;
	}

	/* Zero out previous entries */
	ibdm_free_gid_list(ioc->ioc_prev_gid_list);
	if (ioc->ioc_prev_serv)
		kmem_free(ioc->ioc_prev_serv, ioc->ioc_prev_serv_cnt *
		    sizeof (ibdm_srvents_info_t));
	ioc->ioc_prev_serv_cnt = 0;
	ioc->ioc_prev_nportgids = 0;
	ioc->ioc_prev_serv = NULL;
	ioc->ioc_prev_gid_list = NULL;
}

/*
 * Handle GID removal. This returns gid_info of an GID for the same
 * node GUID, if found.  For an GID with IOU information, the same
 * gid_info is returned if no gid_info with same node_guid is found.
 */
static ibdm_dp_gidinfo_t *
ibdm_handle_gid_rm(ibdm_dp_gidinfo_t *rm_gid)
{
	ibdm_dp_gidinfo_t	*gid_list;

	IBTF_DPRINTF_L4("ibdm", "\thandle_gid_rm(0x%p)", rm_gid);

	if (rm_gid->gl_iou == NULL) {
		IBTF_DPRINTF_L4("ibdm", "\thandle_gid_rm NO iou");
		/*
		 * Search for a GID with same node_guid and
		 * gl_iou != NULL
		 */
		for (gid_list = ibdm.ibdm_dp_gidlist_head; gid_list;
		    gid_list = gid_list->gl_next) {
			if (gid_list->gl_iou != NULL && (gid_list->gl_nodeguid
			    == rm_gid->gl_nodeguid))
				break;
		}

		if (gid_list)
			ibdm_rmfrom_glgid_list(gid_list, rm_gid);

		IBTF_DPRINTF_L4("ibdm", "\thandle_gid_rm ret %p", gid_list);
		return (gid_list);
	} else {
		/*
		 * Search for a GID with same node_guid and
		 * gl_iou == NULL
		 */
		IBTF_DPRINTF_L4("ibdm", "\thandle_gid_rm with iou");
		for (gid_list = ibdm.ibdm_dp_gidlist_head; gid_list;
		    gid_list = gid_list->gl_next) {
			if (gid_list->gl_iou == NULL && (gid_list->gl_nodeguid
			    == rm_gid->gl_nodeguid))
				break;
		}

		if (gid_list) {
			/*
			 * Copy the following fields from rm_gid :
			 *	1. gl_state
			 *	2. gl_iou
			 *	3. gl_gid & gl_ngids
			 *
			 * Note :	Function is synchronized by
			 *			ibdm_busy flag.
			 *
			 * Note :	Redirect info is initialized if
			 *			any MADs for the GID fail
			 */
			IBTF_DPRINTF_L4("ibdm", "\thandle_gid_rm "
			    "copying info to GID with gl_iou != NULl");
			gid_list->gl_state = rm_gid->gl_state;
			gid_list->gl_iou = rm_gid->gl_iou;
			gid_list->gl_gid = rm_gid->gl_gid;
			gid_list->gl_ngids = rm_gid->gl_ngids;

			/* Remove the GID from gl_gid list */
			ibdm_rmfrom_glgid_list(gid_list, rm_gid);
		} else {
			/*
			 * Handle a case where all GIDs to the IOU have
			 * been removed.
			 */
			IBTF_DPRINTF_L4("ibdm", "\thandle_gid_rm 0 GID "
			    "to IOU");

			ibdm_rmfrom_glgid_list(rm_gid, rm_gid);
			return (rm_gid);
		}
		IBTF_DPRINTF_L4("ibdm", "\thandle_gid_rm ret %p", gid_list);
		return (gid_list);
	}
}

static void
ibdm_rmfrom_glgid_list(ibdm_dp_gidinfo_t *gid_info,
    ibdm_dp_gidinfo_t *rm_gid)
{
	ibdm_gid_t 		*tmp, *prev;

	IBTF_DPRINTF_L4("ibdm", "\trmfrom_glgid (%p, %p)",
	    gid_info, rm_gid);

	for (tmp = gid_info->gl_gid, prev = NULL; tmp; ) {
		if (tmp->gid_dgid_hi == rm_gid->gl_dgid_hi &&
		    tmp->gid_dgid_lo == rm_gid->gl_dgid_lo) {
			if (prev == NULL)
				gid_info->gl_gid = tmp->gid_next;
			else
				prev->gid_next = tmp->gid_next;

			kmem_free(tmp, sizeof (ibdm_gid_t));
			gid_info->gl_ngids--;
			break;
		} else {
			prev = tmp;
			tmp = tmp->gid_next;
		}
	}
}

static void
ibdm_addto_gidlist(ibdm_gid_t **src_ptr, ibdm_gid_t *dest)
{
	ibdm_gid_t *head = NULL, *new, *tail;

	/* First copy the destination */
	for (; dest; dest = dest->gid_next) {
		new = kmem_zalloc(sizeof (ibdm_gid_t), KM_SLEEP);
		new->gid_dgid_hi = dest->gid_dgid_hi;
		new->gid_dgid_lo = dest->gid_dgid_lo;
		new->gid_next = head;
		head = new;
	}

	/* Insert this to the source */
	if (*src_ptr == NULL)
		*src_ptr = head;
	else {
		for (tail = *src_ptr; tail->gid_next != NULL;
		    tail = tail->gid_next)
			;

		tail->gid_next = head;
	}
}

static void
ibdm_free_gid_list(ibdm_gid_t	*head)
{
	ibdm_gid_t	*delete;

	for (delete = head; delete; ) {
		head = delete->gid_next;
		kmem_free(delete, sizeof (ibdm_gid_t));
		delete = head;
	}
}

/*
 * This function rescans the DM capable GIDs (gl_state is
 * GID_PROBE_COMPLETE or IBDM_GID_PROBING_SKIPPED.This
 * basically checks if the DM capable GID is reachable. If
 * not this is handled the same way as GID_UNAVAILABLE,
 * except that notifications are not send to IBnexus.
 *
 * This function also initializes the ioc_prev_list for
 * a particular IOC (when called from probe_ioc, with
 * ioc_guidp != NULL) or all IOCs for the gid (called from
 * sweep_fabric, ioc_guidp == NULL).
 */
static void
ibdm_rescan_gidlist(ib_guid_t *ioc_guidp)
{
	ibdm_dp_gidinfo_t	*gid_info, *tmp;
	int ii, niocs, found;
	ibdm_hca_list_t *hca_list = NULL;
	ibdm_port_attr_t *port = NULL;
	ibdm_ioc_info_t *ioc_list;

	for (gid_info = ibdm.ibdm_dp_gidlist_head; gid_info; ) {
		found = 0;
		if (gid_info->gl_state != IBDM_GID_PROBING_SKIPPED &&
		    gid_info->gl_state != IBDM_GID_PROBING_COMPLETE) {
			gid_info = gid_info->gl_next;
			continue;
		}

		/*
		 * Check if the GID is visible to any HCA ports.
		 * Return if so.
		 */
		mutex_enter(&ibdm.ibdm_hl_mutex);
		for (ibdm_get_next_port(&hca_list, &port, 1); port;
		    ibdm_get_next_port(&hca_list, &port, 1)) {
			if (ibdm_port_reachable(port->pa_sa_hdl,
			    gid_info->gl_dgid_lo) == B_TRUE) {
				found = 1;
				break;
			}
		}
		mutex_exit(&ibdm.ibdm_hl_mutex);

		if (found) {
			if (gid_info->gl_iou == NULL) {
				gid_info = gid_info->gl_next;
				continue;
			}

			/* Intialize the ioc_prev_gid_list */
			niocs =
			    gid_info->gl_iou->iou_info.iou_num_ctrl_slots;
			for (ii = 0; ii < niocs; ii++) {
				ioc_list = IBDM_GIDINFO2IOCINFO(gid_info, ii);

				if (ioc_guidp == NULL || (*ioc_guidp ==
				    ioc_list->ioc_profile.ioc_guid)) {
					/* Add info of GIDs in gid_info also */
					ibdm_addto_gidlist(
					    &ioc_list->ioc_prev_gid_list,
					    gid_info->gl_gid);
					ioc_list->ioc_prev_nportgids =
					    gid_info->gl_ngids;
				}
			}
			gid_info = gid_info->gl_next;
			continue;
		}

		IBTF_DPRINTF_L4("ibdm", "\trescan_gidlist "
		    "deleted port GUID %llx",
		    gid_info->gl_dgid_lo);

		/*
		 * Update GID list in all IOCs affected by this
		 */
		ioc_list = ibdm_update_ioc_gidlist(gid_info, 0);

		/*
		 * Remove GID from the global GID list
		 * Handle the case where all port GIDs for an
		 * IOU have been hot-removed.
		 */
		mutex_enter(&ibdm.ibdm_mutex);
		if (gid_info->gl_iou != NULL && gid_info->gl_ngids == 0) {
			mutex_enter(&gid_info->gl_mutex);
			(void) ibdm_free_iou_info(gid_info, &gid_info->gl_iou);
			mutex_exit(&gid_info->gl_mutex);
		}

		tmp = gid_info->gl_next;
		if (gid_info->gl_prev != NULL)
			gid_info->gl_prev->gl_next = gid_info->gl_next;
		if (gid_info->gl_next != NULL)
			gid_info->gl_next->gl_prev = gid_info->gl_prev;

		if (gid_info == ibdm.ibdm_dp_gidlist_head)
			ibdm.ibdm_dp_gidlist_head = gid_info->gl_next;
		if (gid_info == ibdm.ibdm_dp_gidlist_tail)
			ibdm.ibdm_dp_gidlist_tail = gid_info->gl_prev;
		ibdm.ibdm_ngids--;
		mutex_exit(&ibdm.ibdm_mutex);

		/* free the hca_list on this gid_info */
		ibdm_delete_glhca_list(gid_info);

		mutex_destroy(&gid_info->gl_mutex);
		kmem_free(gid_info, sizeof (ibdm_dp_gidinfo_t));

		gid_info = tmp;

		/*
		 * Pass on the IOCs with updated GIDs to IBnexus
		 */
		if (ioc_list) {
			IBTF_DPRINTF_L4("ibdm", "\trescan_gidlist "
			    "IOC_PROP_UPDATE for %p\n", ioc_list);
			mutex_enter(&ibdm.ibdm_ibnex_mutex);
			if (ibdm.ibdm_ibnex_callback != NULL) {
				(*ibdm.ibdm_ibnex_callback)((void *)
				    ioc_list, IBDM_EVENT_IOC_PROP_UPDATE);
			}
			mutex_exit(&ibdm.ibdm_ibnex_mutex);
		}
	}
}

/*
 * This function notifies IBnex of IOCs on this GID.
 * Notification is for GIDs with gl_reprobe_flag set.
 * The flag is set when IOC probe / fabric sweep
 * probes a GID starting from CLASS port info.
 *
 * IBnexus will have information of a reconnected IOC
 * if it had probed it before. If this is a new IOC,
 * IBnexus ignores the notification.
 *
 * This function should be called with no locks held.
 */
static void
ibdm_notify_newgid_iocs(ibdm_dp_gidinfo_t *gid_info)
{
	ibdm_ioc_info_t	*ioc_list;

	if (gid_info->gl_reprobe_flag == 0 ||
	    gid_info->gl_iou == NULL)
		return;

	ioc_list = ibdm_update_ioc_gidlist(gid_info, -1);

	/*
	 * Pass on the IOCs with updated GIDs to IBnexus
	 */
	if (ioc_list) {
		mutex_enter(&ibdm.ibdm_ibnex_mutex);
		if (ibdm.ibdm_ibnex_callback != NULL) {
			(*ibdm.ibdm_ibnex_callback)((void *)ioc_list,
			    IBDM_EVENT_IOC_PROP_UPDATE);
		}
		mutex_exit(&ibdm.ibdm_ibnex_mutex);
	}
}


static void
ibdm_free_saa_event_arg(ibdm_saa_event_arg_t *arg)
{
	if (arg != NULL)
		kmem_free(arg, sizeof (ibdm_saa_event_arg_t));
}

/*
 * This function parses the list of HCAs and HCA ports
 * to return the port_attr of the next HCA port. A port
 * connected to IB fabric (port_state active) is returned,
 * if connected_flag is set.
 */
static void
ibdm_get_next_port(ibdm_hca_list_t **inp_hcap,
    ibdm_port_attr_t **inp_portp, int connect_flag)
{
	int ii;
	ibdm_port_attr_t *port, *next_port = NULL;
	ibdm_port_attr_t *inp_port;
	ibdm_hca_list_t	 *hca_list;
	int found = 0;

	ASSERT(MUTEX_HELD(&ibdm.ibdm_hl_mutex));
	IBTF_DPRINTF_L4(ibdm_string, "\tget_next_port(%p, %p, %x)",
	    inp_hcap, inp_portp, connect_flag);

	hca_list = *inp_hcap;
	inp_port = *inp_portp;

	if (hca_list == NULL)
		hca_list = ibdm.ibdm_hca_list_head;

	for (; hca_list; hca_list = hca_list->hl_next) {
		for (ii = 0; ii < hca_list->hl_nports; ii++) {
			port = &hca_list->hl_port_attr[ii];

			/*
			 * inp_port != NULL;
			 * 	Skip till we find the matching port
			 */
			if (inp_port && !found) {
				if (inp_port == port)
					found = 1;
				continue;
			}

			if (!connect_flag) {
				next_port = port;
				break;
			}

			if (port->pa_sa_hdl == NULL)
				ibdm_initialize_port(port);
			if (port->pa_sa_hdl == NULL)
				(void) ibdm_fini_port(port);
			else if (next_port == NULL &&
			    port->pa_sa_hdl != NULL &&
			    port->pa_state == IBT_PORT_ACTIVE) {
				next_port = port;
				break;
			}
		}

		if (next_port)
			break;
	}

	IBTF_DPRINTF_L4(ibdm_string, "\tget_next_port : "
	    "returns hca_list %p port %p", hca_list, next_port);
	*inp_hcap = hca_list;
	*inp_portp = next_port;
}

static void
ibdm_add_to_gl_gid(ibdm_dp_gidinfo_t *nodegid, ibdm_dp_gidinfo_t *addgid)
{
	ibdm_gid_t	*tmp;

	tmp = kmem_zalloc(sizeof (ibdm_gid_t), KM_SLEEP);
	tmp->gid_dgid_hi = addgid->gl_dgid_hi;
	tmp->gid_dgid_lo = addgid->gl_dgid_lo;

	mutex_enter(&nodegid->gl_mutex);
	tmp->gid_next = nodegid->gl_gid;
	nodegid->gl_gid = tmp;
	nodegid->gl_ngids++;
	mutex_exit(&nodegid->gl_mutex);
}

static void
ibdm_addto_glhcalist(ibdm_dp_gidinfo_t *gid_info,
    ibdm_hca_list_t *hca)
{
	ibdm_hca_list_t		*head, *prev = NULL, *temp;

	IBTF_DPRINTF_L4(ibdm_string, "\taddto_glhcalist(%p, %p) "
	    ": gl_hca_list %p", gid_info, hca, gid_info->gl_hca_list);
	ASSERT(!MUTEX_HELD(&gid_info->gl_mutex));

	mutex_enter(&gid_info->gl_mutex);
	head = gid_info->gl_hca_list;
	if (head == NULL) {
		head = ibdm_dup_hca_attr(hca);
		head->hl_next = NULL;
		gid_info->gl_hca_list = head;
		mutex_exit(&gid_info->gl_mutex);
		IBTF_DPRINTF_L4(ibdm_string, "\tadd_to_glhcalist: "
		    "gid %p, gl_hca_list %p", gid_info,
		    gid_info->gl_hca_list);
		return;
	}

	/* Check if already in the list */
	while (head) {
		if (head->hl_hca_guid == hca->hl_hca_guid) {
			mutex_exit(&gid_info->gl_mutex);
			IBTF_DPRINTF_L4(ibdm_string,
			    "\taddto_glhcalist : gid %p hca %p dup",
			    gid_info, hca);
			return;
		}
		prev = head;
		head = head->hl_next;
	}

	/* Add this HCA to gl_hca_list */
	temp =  ibdm_dup_hca_attr(hca);
	temp->hl_next = NULL;
	prev->hl_next = temp;
	mutex_exit(&gid_info->gl_mutex);

	IBTF_DPRINTF_L4(ibdm_string, "\tadd_to_glhcalist: "
	    "gid %p, gl_hca_list %p", gid_info, gid_info->gl_hca_list);
}

static void
ibdm_delete_glhca_list(ibdm_dp_gidinfo_t *gid_info)
{
	ASSERT(!MUTEX_HELD(&gid_info->gl_mutex));
	ASSERT(!MUTEX_HELD(&ibdm.ibdm_mutex));

	mutex_enter(&gid_info->gl_mutex);
	if (gid_info->gl_hca_list)
		ibdm_ibnex_free_hca_list(gid_info->gl_hca_list);
	gid_info->gl_hca_list = NULL;
	mutex_exit(&gid_info->gl_mutex);
}


static void
ibdm_reset_all_dgids(ibmf_saa_handle_t port_sa_hdl)
{
	IBTF_DPRINTF_L4(ibdm_string, "\treset_all_dgids(%X)",
	    port_sa_hdl);

	if (ibdm_enumerate_iocs == 0)
		return;

	ASSERT(!MUTEX_HELD(&ibdm.ibdm_mutex));
	ASSERT(!MUTEX_HELD(&ibdm.ibdm_hl_mutex));

	/* Check : Not busy in another probe / sweep */
	mutex_enter(&ibdm.ibdm_mutex);
	if ((ibdm.ibdm_busy & IBDM_BUSY) == 0) {
		ibdm_dp_gidinfo_t	*gid_info;

		ibdm.ibdm_busy |= IBDM_BUSY;
		mutex_exit(&ibdm.ibdm_mutex);

		/*
		 * Check if any GID is using the SA & IBMF handle
		 * of HCA port going down. Reset ibdm_dp_gidinfo_t
		 * using another HCA port which can reach the GID.
		 * This is for DM capable GIDs only, no need to do
		 * this for others
		 *
		 * Delete the GID if no alternate HCA port to reach
		 * it is found.
		 */
		for (gid_info = ibdm.ibdm_dp_gidlist_head; gid_info; ) {
			ibdm_dp_gidinfo_t *tmp;

			IBTF_DPRINTF_L4(ibdm_string, "\tevent_hdlr "
			    "checking gidinfo %p", gid_info);

			if (gid_info->gl_sa_hdl == port_sa_hdl) {
				IBTF_DPRINTF_L3(ibdm_string,
				    "\tevent_hdlr: down HCA port hdl "
				    "matches gid %p", gid_info);

				/*
				 * The non-DM GIDs can come back
				 * with a new subnet prefix, when
				 * the HCA port commes up again. To
				 * avoid issues, delete non-DM
				 * capable GIDs, if the gid was
				 * discovered using the HCA port
				 * going down. This is ensured by
				 * setting gl_disconnected to 1.
				 */
				if (gid_info->gl_is_dm_capable == B_FALSE)
					gid_info->gl_disconnected = 1;
				else
					ibdm_reset_gidinfo(gid_info);

				if (gid_info->gl_disconnected) {
					IBTF_DPRINTF_L3(ibdm_string,
					    "\tevent_hdlr: deleting"
					    " gid %p", gid_info);
					tmp = gid_info;
					gid_info = gid_info->gl_next;
					ibdm_delete_gidinfo(tmp);
				} else
					gid_info = gid_info->gl_next;
			} else
				gid_info = gid_info->gl_next;
		}

		mutex_enter(&ibdm.ibdm_mutex);
		ibdm.ibdm_busy &= ~IBDM_BUSY;
		cv_signal(&ibdm.ibdm_busy_cv);
	}
	mutex_exit(&ibdm.ibdm_mutex);
}

static void
ibdm_reset_gidinfo(ibdm_dp_gidinfo_t *gidinfo)
{
	ibdm_hca_list_t	*hca_list = NULL;
	ibdm_port_attr_t	*port = NULL;
	int	gid_reinited = 0;
	sa_node_record_t	*nr, *tmp;
	sa_portinfo_record_t	*pi;
	size_t	nr_len = 0, pi_len = 0;
	size_t	path_len;
	ib_gid_t	sgid, dgid;
	int	ret, ii, nrecords;
	sa_path_record_t	*path;
	uint8_t	npaths = 1;
	ibdm_pkey_tbl_t		*pkey_tbl;

	IBTF_DPRINTF_L4(ibdm_string, "\treset_gidinfo(%p)", gidinfo);

	/*
	 * Get list of all the ports reachable from the local known HCA
	 * ports which are active
	 */
	mutex_enter(&ibdm.ibdm_hl_mutex);
	for (ibdm_get_next_port(&hca_list, &port, 1); port;
	    ibdm_get_next_port(&hca_list, &port, 1)) {


		/*
		 * Get the path and re-populate the gidinfo.
		 * Getting the path is the same probe_ioc
		 * Init the gid info as in ibdm_create_gidinfo()
		 */
		nr = ibdm_get_node_records(port->pa_sa_hdl, &nr_len,
		    gidinfo->gl_nodeguid);
		if (nr == NULL) {
			IBTF_DPRINTF_L4(ibdm_string,
			    "\treset_gidinfo : no records");
			continue;
		}

		nrecords = (nr_len / sizeof (sa_node_record_t));
		for (tmp = nr, ii = 0;  (ii < nrecords); ii++, tmp++) {
			if (tmp->NodeInfo.PortGUID == gidinfo->gl_portguid)
				break;
		}

		if (ii == nrecords) {
			IBTF_DPRINTF_L4(ibdm_string,
			    "\treset_gidinfo : no record for portguid");
			kmem_free(nr, nr_len);
			continue;
		}

		pi = ibdm_get_portinfo(port->pa_sa_hdl, &pi_len, tmp->LID);
		if (pi == NULL) {
			IBTF_DPRINTF_L4(ibdm_string,
			    "\treset_gidinfo : no portinfo");
			kmem_free(nr, nr_len);
			continue;
		}

		sgid.gid_prefix = port->pa_sn_prefix;
		sgid.gid_guid = port->pa_port_guid;
		dgid.gid_prefix = pi->PortInfo.GidPrefix;
		dgid.gid_guid = tmp->NodeInfo.PortGUID;

		ret = ibmf_saa_gid_to_pathrecords(port->pa_sa_hdl, sgid, dgid,
		    IBMF_SAA_PKEY_WC, 0, B_TRUE, &npaths, 0, &path_len, &path);

		if ((ret != IBMF_SUCCESS) || path == NULL) {
			IBTF_DPRINTF_L4(ibdm_string,
			    "\treset_gidinfo : no paths");
			kmem_free(pi, pi_len);
			kmem_free(nr, nr_len);
			continue;
		}

		gidinfo->gl_dgid_hi	= path->DGID.gid_prefix;
		gidinfo->gl_dgid_lo	= path->DGID.gid_guid;
		gidinfo->gl_sgid_hi	= path->SGID.gid_prefix;
		gidinfo->gl_sgid_lo	= path->SGID.gid_guid;
		gidinfo->gl_p_key	= path->P_Key;
		gidinfo->gl_sa_hdl	= port->pa_sa_hdl;
		gidinfo->gl_ibmf_hdl	= port->pa_ibmf_hdl;
		gidinfo->gl_slid	= path->SLID;
		gidinfo->gl_dlid	= path->DLID;
		/* Reset redirect info, next MAD will set if redirected */
		gidinfo->gl_redirected	= 0;
		gidinfo->gl_devid	= (*tmp).NodeInfo.DeviceID;
		gidinfo->gl_SL		= path->SL;

		gidinfo->gl_qp_hdl = IBMF_QP_HANDLE_DEFAULT;
		for (ii = 0; ii < port->pa_npkeys; ii++) {
			if (port->pa_pkey_tbl == NULL)
				break;

			pkey_tbl = &port->pa_pkey_tbl[ii];
			if ((gidinfo->gl_p_key == pkey_tbl->pt_pkey) &&
			    (pkey_tbl->pt_qp_hdl != NULL)) {
				gidinfo->gl_qp_hdl = pkey_tbl->pt_qp_hdl;
				break;
			}
		}

		if (gidinfo->gl_qp_hdl == NULL)
			IBTF_DPRINTF_L2(ibdm_string,
			    "\treset_gid_info: No matching Pkey");
		else
			gid_reinited = 1;

		kmem_free(path, path_len);
		kmem_free(pi, pi_len);
		kmem_free(nr, nr_len);
		break;
	}
	mutex_exit(&ibdm.ibdm_hl_mutex);

	if (!gid_reinited)
		gidinfo->gl_disconnected = 1;
}

static void
ibdm_delete_gidinfo(ibdm_dp_gidinfo_t *gidinfo)
{
	ibdm_ioc_info_t *ioc_list;
	int	in_gidlist = 0;

	/*
	 * Check if gidinfo has been inserted into the
	 * ibdm_dp_gidlist_head list. gl_next or gl_prev
	 * != NULL, if gidinfo is the list.
	 */
	if (gidinfo->gl_prev != NULL ||
	    gidinfo->gl_next != NULL ||
	    ibdm.ibdm_dp_gidlist_head == gidinfo)
		in_gidlist = 1;

	ioc_list = ibdm_update_ioc_gidlist(gidinfo, 0);

	/*
	 * Remove GID from the global GID list
	 * Handle the case where all port GIDs for an
	 * IOU have been hot-removed.
	 */
	mutex_enter(&ibdm.ibdm_mutex);
	if (gidinfo->gl_iou != NULL && gidinfo->gl_ngids == 0) {
		mutex_enter(&gidinfo->gl_mutex);
		(void) ibdm_free_iou_info(gidinfo, &gidinfo->gl_iou);
		mutex_exit(&gidinfo->gl_mutex);
	}

	/* Delete gl_hca_list */
	mutex_exit(&ibdm.ibdm_mutex);
	ibdm_delete_glhca_list(gidinfo);
	mutex_enter(&ibdm.ibdm_mutex);

	if (in_gidlist) {
		if (gidinfo->gl_prev != NULL)
			gidinfo->gl_prev->gl_next = gidinfo->gl_next;
		if (gidinfo->gl_next != NULL)
			gidinfo->gl_next->gl_prev = gidinfo->gl_prev;

		if (gidinfo == ibdm.ibdm_dp_gidlist_head)
			ibdm.ibdm_dp_gidlist_head = gidinfo->gl_next;
		if (gidinfo == ibdm.ibdm_dp_gidlist_tail)
			ibdm.ibdm_dp_gidlist_tail = gidinfo->gl_prev;
		ibdm.ibdm_ngids--;
	}
	mutex_exit(&ibdm.ibdm_mutex);

	mutex_destroy(&gidinfo->gl_mutex);
	cv_destroy(&gidinfo->gl_probe_cv);
	kmem_free(gidinfo, sizeof (ibdm_dp_gidinfo_t));

	/*
	 * Pass on the IOCs with updated GIDs to IBnexus
	 */
	if (ioc_list) {
		IBTF_DPRINTF_L4("ibdm", "\tdelete_gidinfo "
		    "IOC_PROP_UPDATE for %p\n", ioc_list);
		mutex_enter(&ibdm.ibdm_ibnex_mutex);
		if (ibdm.ibdm_ibnex_callback != NULL) {
			(*ibdm.ibdm_ibnex_callback)((void *)
			    ioc_list, IBDM_EVENT_IOC_PROP_UPDATE);
		}
		mutex_exit(&ibdm.ibdm_ibnex_mutex);
	}
}


static void
ibdm_fill_srv_attr_mod(ib_mad_hdr_t *hdr, ibdm_timeout_cb_args_t *cb_args)
{
	uint32_t	attr_mod;

	attr_mod = (cb_args->cb_ioc_num + 1) << 16;
	attr_mod |= cb_args->cb_srvents_start;
	attr_mod |= (cb_args->cb_srvents_end) << 8;
	hdr->AttributeModifier = h2b32(attr_mod);
}

static void
ibdm_bump_transactionID(ibdm_dp_gidinfo_t *gid_info)
{
	ASSERT(MUTEX_HELD(&gid_info->gl_mutex));
	gid_info->gl_transactionID++;
	if (gid_info->gl_transactionID == gid_info->gl_max_transactionID) {
		IBTF_DPRINTF_L4(ibdm_string,
		    "\tbump_transactionID(%p), wrapup", gid_info);
		gid_info->gl_transactionID = gid_info->gl_min_transactionID;
	}
}

/*
 * gl_prev_iou is set for *non-reprobe* sweeep requests, which
 * detected that ChangeID in IOU info has changed. The service
 * entry also may have changed. Check if service entry in IOC
 * has changed wrt the prev iou, if so notify to IB Nexus.
 */
static ibdm_ioc_info_t *
ibdm_handle_prev_iou()
{
	ibdm_dp_gidinfo_t *gid_info;
	ibdm_ioc_info_t	*ioc_list_head = NULL, *ioc_list;
	ibdm_ioc_info_t	*prev_ioc, *ioc;
	int		ii, jj, niocs, prev_niocs;

	ASSERT(MUTEX_HELD(&ibdm.ibdm_mutex));

	IBTF_DPRINTF_L4(ibdm_string, "\thandle_prev_iou enter");
	for (gid_info = ibdm.ibdm_dp_gidlist_head; gid_info;
	    gid_info = gid_info->gl_next) {
		if (gid_info->gl_prev_iou == NULL)
			continue;

		IBTF_DPRINTF_L4(ibdm_string, "\thandle_prev_iou gid %p",
		    gid_info);
		niocs = gid_info->gl_iou->iou_info.iou_num_ctrl_slots;
		prev_niocs =
		    gid_info->gl_prev_iou->iou_info.iou_num_ctrl_slots;
		for (ii = 0; ii < niocs; ii++) {
			ioc = IBDM_GIDINFO2IOCINFO(gid_info, ii);

			/* Find matching IOC */
			for (jj = 0; jj < prev_niocs; jj++) {
				prev_ioc = (ibdm_ioc_info_t *)
				    &gid_info->gl_prev_iou->iou_ioc_info[jj];
				if (prev_ioc->ioc_profile.ioc_guid ==
				    ioc->ioc_profile.ioc_guid)
					break;
			}
			if (jj == prev_niocs)
				prev_ioc = NULL;
			if (ioc == NULL || prev_ioc == NULL)
				continue;
			if ((ioc->ioc_profile.ioc_service_entries !=
			    prev_ioc->ioc_profile.ioc_service_entries) ||
			    ibdm_serv_cmp(&ioc->ioc_serv[0],
			    &prev_ioc->ioc_serv[0],
			    ioc->ioc_profile.ioc_service_entries) != 0) {
				IBTF_DPRINTF_L4(ibdm_string,
				    "/thandle_prev_iou modified IOC: "
				    "current ioc %p, old ioc %p",
				    ioc, prev_ioc);
				mutex_enter(&gid_info->gl_mutex);
				ioc_list = ibdm_dup_ioc_info(ioc, gid_info);
				mutex_exit(&gid_info->gl_mutex);
				ioc_list->ioc_info_updated.ib_prop_updated
				    = 0;
				ioc_list->ioc_info_updated.ib_srv_prop_updated
				    = 1;

				if (ioc_list_head == NULL)
					ioc_list_head = ioc_list;
				else {
					ioc_list_head->ioc_next = ioc_list;
					ioc_list_head = ioc_list;
				}
			}
		}

		mutex_enter(&gid_info->gl_mutex);
		(void) ibdm_free_iou_info(gid_info, &gid_info->gl_prev_iou);
		mutex_exit(&gid_info->gl_mutex);
	}
	IBTF_DPRINTF_L4(ibdm_string, "\thandle_prev_iouret %p",
	    ioc_list_head);
	return (ioc_list_head);
}

/*
 * Compares two service entries lists, returns 0 if same, returns 1
 * if no match.
 */
static int
ibdm_serv_cmp(ibdm_srvents_info_t *serv1, ibdm_srvents_info_t *serv2,
    int nserv)
{
	int	ii;

	IBTF_DPRINTF_L4(ibdm_string, "\tserv_cmp: enter");
	for (ii = 0; ii < nserv; ii++, serv1++, serv2++) {
		if (serv1->se_attr.srv_id != serv2->se_attr.srv_id ||
		    bcmp(serv1->se_attr.srv_name,
		    serv2->se_attr.srv_name,
		    IB_DM_MAX_SVC_NAME_LEN) != 0) {
			IBTF_DPRINTF_L4(ibdm_string, "\tserv_cmp: ret 1");
			return (1);
		}
	}
	IBTF_DPRINTF_L4(ibdm_string, "\tserv_cmp: ret 0");
	return (0);
}

/* For debugging purpose only */
#ifdef	DEBUG
void
ibdm_dump_mad_hdr(ib_mad_hdr_t	*mad_hdr)
{
	IBTF_DPRINTF_L4("ibdm", "\t\t MAD Header info");
	IBTF_DPRINTF_L4("ibdm", "\t\t ---------------");

	IBTF_DPRINTF_L4("ibdm", "\tBase version  : 0x%x"
	    "\tMgmt Class : 0x%x", mad_hdr->BaseVersion, mad_hdr->MgmtClass);
	IBTF_DPRINTF_L4("ibdm", "\tClass version : 0x%x"
	    "\tR Method           : 0x%x",
	    mad_hdr->ClassVersion, mad_hdr->R_Method);
	IBTF_DPRINTF_L4("ibdm", "\tMAD  Status   : 0x%x"
	    "\tTransaction ID     : 0x%llx",
	    b2h16(mad_hdr->Status), b2h64(mad_hdr->TransactionID));
	IBTF_DPRINTF_L4("ibdm", "\t Attribute ID  : 0x%x"
	    "\tAttribute Modified : 0x%lx",
	    b2h16(mad_hdr->AttributeID), b2h32(mad_hdr->AttributeModifier));
}


void
ibdm_dump_ibmf_msg(ibmf_msg_t *ibmf_msg, int flag)
{
	ib_mad_hdr_t	*mad_hdr;

	IBTF_DPRINTF_L4("ibdm", "\t\t(IBMF_PKT): Local address info");
	IBTF_DPRINTF_L4("ibdm", "\t\t            ------------------");

	IBTF_DPRINTF_L4("ibdm", "\tLocal Lid  : 0x%x\tRemote Lid : 0x%x"
	    " Remote Qp  : 0x%x", ibmf_msg->im_local_addr.ia_local_lid,
	    ibmf_msg->im_local_addr.ia_remote_lid,
	    ibmf_msg->im_local_addr.ia_remote_qno);
	IBTF_DPRINTF_L4("ibdm", "\tP_key      : 0x%x\tQ_key      : 0x%x"
	    " SL  : 0x%x", ibmf_msg->im_local_addr.ia_p_key,
	    ibmf_msg->im_local_addr.ia_q_key,
	    ibmf_msg->im_local_addr.ia_service_level);

	if (flag)
		mad_hdr = (ib_mad_hdr_t *)IBDM_OUT_IBMFMSG_MADHDR(ibmf_msg);
	else
		mad_hdr = IBDM_IN_IBMFMSG_MADHDR(ibmf_msg);

	ibdm_dump_mad_hdr(mad_hdr);
}


void
ibdm_dump_path_info(sa_path_record_t *path)
{
	IBTF_DPRINTF_L4("ibdm", "\t\t Path information");
	IBTF_DPRINTF_L4("ibdm", "\t\t ----------------");

	IBTF_DPRINTF_L4("ibdm", "\t DGID hi  : %llx\tDGID lo  : %llx",
	    path->DGID.gid_prefix, path->DGID.gid_guid);
	IBTF_DPRINTF_L4("ibdm", "\t SGID hi  : %llx\tSGID lo  : %llx",
	    path->SGID.gid_prefix, path->SGID.gid_guid);
	IBTF_DPRINTF_L4("ibdm", "\t SLID     : %x\t\tDlID     : %x",
	    path->SLID, path->DLID);
	IBTF_DPRINTF_L4("ibdm", "\t P Key    : %x\t\tSL       : %x",
	    path->P_Key, path->SL);
}


void
ibdm_dump_classportinfo(ib_mad_classportinfo_t *classportinfo)
{
	IBTF_DPRINTF_L4("ibdm", "\t\t CLASSPORT INFO");
	IBTF_DPRINTF_L4("ibdm", "\t\t --------------");

	IBTF_DPRINTF_L4("ibdm", "\t Response Time Value : 0x%x",
	    ((b2h32(classportinfo->RespTimeValue)) & 0x1F));

	IBTF_DPRINTF_L4("ibdm", "\t Redirected GID hi   : 0x%llx",
	    b2h64(classportinfo->RedirectGID_hi));
	IBTF_DPRINTF_L4("ibdm", "\t Redirected GID lo   : 0x%llx",
	    b2h64(classportinfo->RedirectGID_lo));
	IBTF_DPRINTF_L4("ibdm", "\t Redirected TC       : 0x%x",
	    classportinfo->RedirectTC);
	IBTF_DPRINTF_L4("ibdm", "\t Redirected SL       : 0x%x",
	    classportinfo->RedirectSL);
	IBTF_DPRINTF_L4("ibdm", "\t Redirected FL       : 0x%x",
	    classportinfo->RedirectFL);
	IBTF_DPRINTF_L4("ibdm", "\t Redirected LID      : 0x%x",
	    b2h16(classportinfo->RedirectLID));
	IBTF_DPRINTF_L4("ibdm", "\t Redirected P KEY    : 0x%x",
	    b2h16(classportinfo->RedirectP_Key));
	IBTF_DPRINTF_L4("ibdm", "\t Redirected QP       : 0x%x",
	    classportinfo->RedirectQP);
	IBTF_DPRINTF_L4("ibdm", "\t Redirected Q KEY    : 0x%x",
	    b2h32(classportinfo->RedirectQ_Key));
	IBTF_DPRINTF_L4("ibdm", "\t Trap GID hi         : 0x%llx",
	    b2h64(classportinfo->TrapGID_hi));
	IBTF_DPRINTF_L4("ibdm", "\t Trap GID lo         : 0x%llx",
	    b2h64(classportinfo->TrapGID_lo));
	IBTF_DPRINTF_L4("ibdm", "\t Trap TC             : 0x%x",
	    classportinfo->TrapTC);
	IBTF_DPRINTF_L4("ibdm", "\t Trap SL             : 0x%x",
	    classportinfo->TrapSL);
	IBTF_DPRINTF_L4("ibdm", "\t Trap FL             : 0x%x",
	    classportinfo->TrapFL);
	IBTF_DPRINTF_L4("ibdm", "\t Trap LID            : 0x%x",
	    b2h16(classportinfo->TrapLID));
	IBTF_DPRINTF_L4("ibdm", "\t Trap P_Key          : 0x%x",
	    b2h16(classportinfo->TrapP_Key));
	IBTF_DPRINTF_L4("ibdm", "\t Trap HL             : 0x%x",
	    classportinfo->TrapHL);
	IBTF_DPRINTF_L4("ibdm", "\t Trap QP             : 0x%x",
	    classportinfo->TrapQP);
	IBTF_DPRINTF_L4("ibdm", "\t Trap Q_Key          : 0x%x",
	    b2h32(classportinfo->TrapQ_Key));
}


void
ibdm_dump_iounitinfo(ib_dm_io_unitinfo_t *iou_info)
{
	IBTF_DPRINTF_L4("ibdm", "\t\t I/O UnitInfo");
	IBTF_DPRINTF_L4("ibdm", "\t\t ------------");

	IBTF_DPRINTF_L4("ibdm", "\tChange ID            : 0x%x",
	    b2h16(iou_info->iou_changeid));
	IBTF_DPRINTF_L4("ibdm", "\t#of ctrl slots       : %d",
	    iou_info->iou_num_ctrl_slots);
	IBTF_DPRINTF_L4("ibdm", "\tIOU flag             : 0x%x",
	    iou_info->iou_flag);
	IBTF_DPRINTF_L4("ibdm", "\tContrl list byte 0   : 0x%x",
	    iou_info->iou_ctrl_list[0]);
	IBTF_DPRINTF_L4("ibdm", "\tContrl list byte 1   : 0x%x",
	    iou_info->iou_ctrl_list[1]);
	IBTF_DPRINTF_L4("ibdm", "\tContrl list byte 2   : 0x%x",
	    iou_info->iou_ctrl_list[2]);
}


void
ibdm_dump_ioc_profile(ib_dm_ioc_ctrl_profile_t *ioc)
{
	IBTF_DPRINTF_L4("ibdm", "\t\t IOC Controller Profile");
	IBTF_DPRINTF_L4("ibdm", "\t\t ----------------------");

	IBTF_DPRINTF_L4("ibdm", "\tIOC Guid    : %llx", ioc->ioc_guid);
	IBTF_DPRINTF_L4("ibdm", "\tVendorID    : 0x%x", ioc->ioc_vendorid);
	IBTF_DPRINTF_L4("ibdm", "\tDevice Id   : 0x%x", ioc->ioc_deviceid);
	IBTF_DPRINTF_L4("ibdm", "\tDevice Ver  : 0x%x", ioc->ioc_device_ver);
	IBTF_DPRINTF_L4("ibdm", "\tSubsys ID   : 0x%x", ioc->ioc_subsys_id);
	IBTF_DPRINTF_L4("ibdm", "\tIO class    : 0x%x", ioc->ioc_io_class);
	IBTF_DPRINTF_L4("ibdm", "\tIO subclass : 0x%x", ioc->ioc_io_subclass);
	IBTF_DPRINTF_L4("ibdm", "\tProtocol    : 0x%x", ioc->ioc_protocol);
	IBTF_DPRINTF_L4("ibdm", "\tProtocolV   : 0x%x", ioc->ioc_protocol_ver);
	IBTF_DPRINTF_L4("ibdm", "\tmsg qdepth  : %d", ioc->ioc_send_msg_qdepth);
	IBTF_DPRINTF_L4("ibdm", "\trdma qdepth : %d",
	    ioc->ioc_rdma_read_qdepth);
	IBTF_DPRINTF_L4("ibdm", "\tsndmsg sz   : %d", ioc->ioc_send_msg_sz);
	IBTF_DPRINTF_L4("ibdm", "\trdma xfersz : %d", ioc->ioc_rdma_xfer_sz);
	IBTF_DPRINTF_L4("ibdm", "\topcal mask  : 0x%x",
	    ioc->ioc_ctrl_opcap_mask);
	IBTF_DPRINTF_L4("ibdm", "\tsrventries  : %x", ioc->ioc_service_entries);
}


void
ibdm_dump_service_entries(ib_dm_srv_t *srv_ents)
{
	IBTF_DPRINTF_L4("ibdm",
	    "\thandle_srventry_mad: service id : %llx", srv_ents->srv_id);

	IBTF_DPRINTF_L4("ibdm", "\thandle_srventry_mad: "
	    "Service Name : %s", srv_ents->srv_name);
}

int ibdm_allow_sweep_fabric_timestamp = 1;

void
ibdm_dump_sweep_fabric_timestamp(int flag)
{
	static hrtime_t x;
	if (flag) {
		if (ibdm_allow_sweep_fabric_timestamp) {
			IBTF_DPRINTF_L4("ibdm", "\tTime taken to complete "
			    "sweep %lld ms", ((gethrtime() - x)/ 1000000));
		}
		x = 0;
	} else
		x = gethrtime();
}
#endif