xref: /titanic_44/usr/src/uts/common/io/rsm/rsm.c (revision de6732348404877079e00a1631babd13d8a5ad14)

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */


/*
 * Overview of the RSM Kernel Agent:
 * ---------------------------------
 *
 * rsm.c constitutes the implementation of the RSM kernel agent. The RSM
 * kernel agent is a pseudo device driver which makes use of the RSMPI
 * interface on behalf of the RSMAPI user library.
 *
 * The kernel agent functionality can be categorized into the following
 * components:
 * 1. Driver Infrastructure
 * 2. Export/Import Segment Management
 * 3. Internal resource allocation/deallocation
 *
 * The driver infrastructure includes the basic module loading entry points
 * like _init, _info, _fini to load, unload and report information about
 * the driver module. The driver infrastructure also includes the
 * autoconfiguration entry points namely, attach, detach and getinfo for
 * the device autoconfiguration.
 *
 * The kernel agent is a pseudo character device driver and exports
 * a cb_ops structure which defines the driver entry points for character
 * device access. This includes the open and close entry points. The
 * other entry points provided include ioctl, devmap and segmap and chpoll.
 * read and write entry points are not used since the device is memory
 * mapped. Also ddi_prop_op is used for the prop_op entry point.
 *
 * The ioctl entry point supports a number of commands, which are used by
 * the RSMAPI library in order to export and import segments. These
 * commands include commands for binding and rebinding the physical pages
 * allocated to the virtual address range, publishing the export segment,
 * unpublishing and republishing an export segment, creating an
 * import segment and a virtual connection from this import segment to
 * an export segment, performing scatter-gather data transfer, barrier
 * operations.
 *
 *
 * Export and Import segments:
 * ---------------------------
 *
 * In order to create an RSM export segment a process allocates a range in its
 * virtual address space for the segment using standard Solaris interfaces.
 * The process then calls RSMAPI, which in turn makes an ioctl call to the
 * RSM kernel agent for an allocation of physical memory pages and for
 * creation of the export segment by binding these pages to the virtual
 * address range. These pages are locked in memory so that remote accesses
 * are always applied to the correct page. Then the RSM segment is published,
 * again via RSMAPI making an ioctl to the RSM kernel agent, and a segment id
 * is assigned to it.
 *
 * In order to import a published RSM segment, RSMAPI creates an import
 * segment and forms a virtual connection across the interconnect to the
 * export segment, via an ioctl into the kernel agent with the connect
 * command. The import segment setup is completed by mapping the
 * local device memory into the importers virtual address space. The
 * mapping of the import segment is handled by the segmap/devmap
 * infrastructure described as follows.
 *
 * Segmap and Devmap interfaces:
 *
 * The RSM kernel agent allows device memory to be directly accessed by user
 * threads via memory mapping. In order to do so, the RSM kernel agent
 * supports the devmap and segmap entry points.
 *
 * The segmap entry point(rsm_segmap) is responsible for setting up a memory
 * mapping as requested by mmap. The devmap entry point(rsm_devmap) is
 * responsible for exporting the device memory to the user applications.
 * rsm_segmap calls RSMPI rsm_map to allocate device memory. Then the
 * control is transfered to the devmap_setup call which calls rsm_devmap.
 *
 * rsm_devmap validates the user mapping to the device or kernel memory
 * and passes the information to the system for setting up the mapping. The
 * actual setting up of the mapping is done by devmap_devmem_setup(for
 * device memory) or devmap_umem_setup(for kernel memory). Callbacks are
 * registered for device context management via the devmap_devmem_setup
 * or devmap_umem_setup calls. The callbacks are rsmmap_map, rsmmap_unmap,
 * rsmmap_access, rsmmap_dup. The callbacks are called when a new mapping
 * is created, a mapping is freed, a mapping is accessed or an existing
 * mapping is duplicated respectively. These callbacks allow the RSM kernel
 * agent to maintain state information associated with the mappings.
 * The state information is mainly in the form of a cookie list for the import
 * segment for which mapping has been done.
 *
 * Forced disconnect of import segments:
 *
 * When an exported segment is unpublished, the exporter sends a forced
 * disconnect message to all its importers. The importer segments are
 * unloaded and disconnected. This involves unloading the original
 * mappings and remapping to a preallocated kernel trash page. This is
 * done by devmap_umem_remap. The trash/dummy page is a kernel page,
 * preallocated by the kernel agent during attach using ddi_umem_alloc with
 * the DDI_UMEM_TRASH flag set. This avoids a core dump in the application
 * due to unloading of the original mappings.
 *
 * Additionally every segment has a mapping generation number associated
 * with it. This is an entry in the barrier generation page, created
 * during attach time. This mapping generation number for the import
 * segments is incremented on a force disconnect to notify the application
 * of the force disconnect. On this notification, the application needs
 * to reconnect the segment to establish a new legitimate mapping.
 *
 *
 * Locks used in the kernel agent:
 * -------------------------------
 *
 * The kernel agent uses a variety of mutexes and condition variables for
 * mutual exclusion of the shared data structures and for synchronization
 * between the various threads. Some of the locks are described as follows.
 *
 * Each resource structure, which represents either an export/import segment
 * has a lock associated with it. The lock is the resource mutex, rsmrc_lock.
 * This is used directly by RSMRC_LOCK and RSMRC_UNLOCK macros and in the
 * rsmseglock_acquire and rsmseglock_release macros. An additional
 * lock called the rsmsi_lock is used for the shared import data structure
 * that is relevant for resources representing import segments. There is
 * also a condition variable associated with the resource called s_cv. This
 * is used to wait for events like the segment state change etc.
 *
 * The resource structures are allocated from a pool of resource structures,
 * called rsm_resource. This pool is protected via a reader-writer lock,
 * called rsmrc_lock.
 *
 * There are two separate hash tables, one for the export segments and
 * one for the import segments. The export segments are inserted into the
 * export segment hash table only after they have been published and the
 * import segments are inserted in the import segments list only after they
 * have successfully connected to an exported segment. These tables are
 * protected via reader-writer locks.
 *
 * Debug Support in the kernel agent:
 * ----------------------------------
 *
 * Debugging support in the kernel agent is provided by the following
 * macros.
 *
 * DBG_PRINTF((category, level, message)) is a macro which logs a debug
 * message to the kernel agents debug buffer, rsmka_dbg. This debug buffer
 * can be viewed in kmdb as *rsmka_dbg/s. The message is logged based
 * on the definition of the category and level. All messages that belong to
 * the specified category(rsmdbg_category) and are of an equal or greater
 * severity than the specified level(rsmdbg_level) are logged. The message
 * is a string which uses the same formatting rules as the strings used in
 * printf.
 *
 * The category defines which component of the kernel agent has logged this
 * message. There are a number of categories that have been defined such as
 * RSM_KERNEL_AGENT, RSM_OPS, RSM_IMPORT, RSM_EXPORT etc. A macro,
 * DBG_ADDCATEGORY is used to add in another category to the currently
 * specified category value so that the component using this new category
 * can also effectively log debug messages. Thus, the category of a specific
 * message is some combination of the available categories and we can define
 * sub-categories if we want a finer level of granularity.
 *
 * The level defines the severity of the message. Different level values are
 * defined, with RSM_ERR being the most severe and RSM_DEBUG_VERBOSE being
 * the least severe(debug level is 0).
 *
 * DBG_DEFINE and DBG_DEFINE_STR are macros provided to declare a debug
 * variable or a string respectively.
 *
 *
 * NOTES:
 *
 * Special Fork and Exec Handling:
 * -------------------------------
 *
 * The backing physical pages of an exported segment are always locked down.
 * Thus, there are two cases in which a process having exported segments
 * will cause a cpu to hang: (1) the process invokes exec; (2) a process
 * forks and invokes exit before the duped file descriptors for the export
 * segments are closed in the child process. The hang is caused because the
 * address space release algorithm in Solaris VM subsystem is based on a
 * non-blocking loop which does not terminate while segments are locked
 * down. In addition to this, Solaris VM subsystem lacks a callback
 * mechanism to the rsm kernel agent to allow unlocking these export
 * segment pages.
 *
 * In order to circumvent this problem, the kernel agent does the following.
 * The Solaris VM subsystem keeps memory segments in increasing order of
 * virtual addressses. Thus a special page(special_exit_offset) is allocated
 * by the kernel agent and is mmapped into the heap area of the process address
 * space(the mmap is done by the RSMAPI library). During the mmap processing
 * of this special page by the devmap infrastructure, a callback(the same
 * devmap context management callbacks discussed above) is registered for an
 * unmap.
 *
 * As discussed above, this page is processed by the Solaris address space
 * release code before any of the exported segments pages(which are allocated
 * from high memory). It is during this processing that the unmap callback gets
 * called and this callback is responsible for force destroying the exported
 * segments and thus eliminating the problem of locked pages.
 *
 * Flow-control:
 * ------------
 *
 * A credit based flow control algorithm is used for messages whose
 * processing cannot be done in the interrupt context because it might
 * involve invoking rsmpi calls, or might take a long time to complete
 * or might need to allocate resources. The algorithm operates on a per
 * path basis. To send a message the pathend needs to have a credit and
 * it consumes one for every message that is flow controlled. On the
 * receiving pathend the message is put on a msgbuf_queue and a task is
 * dispatched on the worker thread - recv_taskq where it is processed.
 * After processing the message, the receiving pathend dequeues the message,
 * and if it has processed > RSMIPC_LOTSFREE_MSGBUFS messages sends
 * credits to the sender pathend.
 *
 * RSM_DRTEST:
 * -----------
 *
 * This is used to enable the DR testing using a test driver on test
 * platforms which do not supported DR.
 *
 */

#include <sys/types.h>
#include <sys/param.h>
#include <sys/user.h>
#include <sys/buf.h>
#include <sys/systm.h>
#include <sys/cred.h>
#include <sys/vm.h>
#include <sys/uio.h>
#include <vm/seg.h>
#include <vm/page.h>
#include <sys/stat.h>

#include <sys/time.h>
#include <sys/errno.h>

#include <sys/file.h>
#include <sys/uio.h>
#include <sys/proc.h>
#include <sys/mman.h>
#include <sys/open.h>
#include <sys/atomic.h>
#include <sys/mem_config.h>


#include <sys/ddi.h>
#include <sys/devops.h>
#include <sys/ddidevmap.h>
#include <sys/sunddi.h>
#include <sys/esunddi.h>
#include <sys/ddi_impldefs.h>

#include <sys/kmem.h>
#include <sys/conf.h>
#include <sys/devops.h>
#include <sys/ddi_impldefs.h>

#include <sys/modctl.h>

#include <sys/policy.h>
#include <sys/types.h>
#include <sys/conf.h>
#include <sys/param.h>

#include <sys/taskq.h>

#include <sys/rsm/rsm_common.h>
#include <sys/rsm/rsmapi_common.h>
#include <sys/rsm/rsm.h>
#include <rsm_in.h>
#include <sys/rsm/rsmka_path_int.h>
#include <sys/rsm/rsmpi.h>

#include <sys/modctl.h>
#include <sys/debug.h>

#include <sys/tuneable.h>

#ifdef	RSM_DRTEST
extern int rsm_kphysm_setup_func_register(kphysm_setup_vector_t *vec,
		void *arg);
extern void rsm_kphysm_setup_func_unregister(kphysm_setup_vector_t *vec,
		void *arg);
#endif

extern void dbg_printf(int category, int level, char *fmt, ...);
extern void rsmka_pathmanager_init();
extern void rsmka_pathmanager_cleanup();
extern void rele_sendq_token();
extern rsm_addr_t get_remote_hwaddr(adapter_t *, rsm_node_id_t);
extern rsm_node_id_t get_remote_nodeid(adapter_t *, rsm_addr_t);
extern int rsmka_topology_ioctl(caddr_t, int, int);

extern pri_t maxclsyspri;
extern work_queue_t work_queue;
extern kmutex_t ipc_info_lock;
extern kmutex_t ipc_info_cvlock;
extern kcondvar_t ipc_info_cv;
extern kmutex_t path_hold_cvlock;
extern kcondvar_t path_hold_cv;

extern kmutex_t rsmka_buf_lock;

extern path_t *rsm_find_path(char *, int, rsm_addr_t);
extern adapter_t *rsmka_lookup_adapter(char *, int);
extern sendq_token_t *rsmka_get_sendq_token(rsm_node_id_t, sendq_token_t *);
extern boolean_t rsmka_do_path_active(path_t *, int);
extern boolean_t rsmka_check_node_alive(rsm_node_id_t);
extern void rsmka_release_adapter(adapter_t *);
extern void rsmka_enqueue_msgbuf(path_t *path, void *data);
extern void rsmka_dequeue_msgbuf(path_t *path);
extern msgbuf_elem_t *rsmka_gethead_msgbuf(path_t *path);
/* lint -w2 */

static int rsm_open(dev_t *, int, int, cred_t *);
static int rsm_close(dev_t, int, int, cred_t *);
static int rsm_ioctl(dev_t dev, int cmd, intptr_t arg, int mode,
    cred_t *credp, int *rvalp);
static int rsm_devmap(dev_t, devmap_cookie_t, offset_t, size_t, size_t *,
    uint_t);
static int rsm_segmap(dev_t, off_t, struct as *, caddr_t *, off_t, uint_t,
    uint_t, uint_t, cred_t *);
static int rsm_chpoll(dev_t dev, short events, int anyyet, short *reventsp,
    struct pollhead **phpp);

static int rsm_info(dev_info_t *, ddi_info_cmd_t, void *, void **);
static int rsm_attach(dev_info_t *, ddi_attach_cmd_t);
static int rsm_detach(dev_info_t *, ddi_detach_cmd_t);

static int rsmipc_send(rsm_node_id_t, rsmipc_request_t *, rsmipc_reply_t *);
static void rsm_force_unload(rsm_node_id_t, rsm_memseg_id_t, boolean_t);
static void rsm_send_importer_disconnects(rsm_memseg_id_t, rsm_node_id_t);
static void rsm_send_republish(rsm_memseg_id_t, rsmapi_access_entry_t *, int,
				rsm_permission_t);
static void rsm_export_force_destroy(ddi_umem_cookie_t *);
static void rsmacl_free(rsmapi_access_entry_t *, int);
static void rsmpiacl_free(rsm_access_entry_t *, int);

static int rsm_inc_pgcnt(pgcnt_t);
static void rsm_dec_pgcnt(pgcnt_t);
static void rsm_free_mapinfo(rsm_mapinfo_t *mapinfop);
static rsm_mapinfo_t *rsm_get_mapinfo(rsmseg_t *, off_t, size_t, off_t *,
					size_t *);
static void exporter_quiesce();
static void rsmseg_suspend(rsmseg_t *, int *);
static void rsmsegshare_suspend(rsmseg_t *);
static int rsmseg_resume(rsmseg_t *, void **);
static int rsmsegshare_resume(rsmseg_t *);

static struct cb_ops rsm_cb_ops = {
	rsm_open,		/* open */
	rsm_close,		/* close */
	nodev,			/* strategy */
	nodev,			/* print */
	nodev,			/* dump */
	nodev,			/* read */
	nodev,			/* write */
	rsm_ioctl,		/* ioctl */
	rsm_devmap,		/* devmap */
	NULL,			/* mmap */
	rsm_segmap,		/* segmap */
	rsm_chpoll,		/* poll */
	ddi_prop_op,		/* cb_prop_op */
	0,			/* streamtab  */
	D_NEW|D_MP|D_DEVMAP,	/* Driver compatibility flag */
	0,
	0,
	0
};

static struct dev_ops rsm_ops = {
	DEVO_REV,		/* devo_rev, */
	0,			/* refcnt  */
	rsm_info,		/* get_dev_info */
	nulldev,		/* identify */
	nulldev,		/* probe */
	rsm_attach,		/* attach */
	rsm_detach,		/* detach */
	nodev,			/* reset */
	&rsm_cb_ops,		/* driver operations */
	(struct bus_ops *)0,	/* bus operations */
	0,
	ddi_quiesce_not_needed,		/* quiesce */
};

/*
 * Module linkage information for the kernel.
 */

static struct modldrv modldrv = {
	&mod_driverops, /* Type of module.  This one is a pseudo driver */
	"Remote Shared Memory Driver",
	&rsm_ops,	/* driver ops */
};

static struct modlinkage modlinkage = {
	MODREV_1,
	(void *)&modldrv,
	0,
	0,
	0
};

static void rsm_dr_callback_post_add(void *arg, pgcnt_t delta);
static int rsm_dr_callback_pre_del(void *arg, pgcnt_t delta);
static void rsm_dr_callback_post_del(void *arg, pgcnt_t delta, int cancelled);

static kphysm_setup_vector_t rsm_dr_callback_vec = {
	KPHYSM_SETUP_VECTOR_VERSION,
	rsm_dr_callback_post_add,
	rsm_dr_callback_pre_del,
	rsm_dr_callback_post_del
};

/* This flag can be changed to 0 to help with PIT testing */
int rsmka_modunloadok = 1;
int no_reply_cnt = 0;

uint64_t rsm_ctrlmsg_errcnt = 0;
uint64_t rsm_ipcsend_errcnt = 0;

#define	MAX_NODES 64

static struct rsm_driver_data rsm_drv_data;
static struct rsmresource_table rsm_resource;

static void rsmresource_insert(minor_t, rsmresource_t *, rsm_resource_type_t);
static void rsmresource_destroy(void);
static int rsmresource_alloc(minor_t *);
static rsmresource_t *rsmresource_free(minor_t rnum);
static int rsm_closeconnection(rsmseg_t *seg, void **cookie);
static int rsm_unpublish(rsmseg_t *seg, int mode);
static int rsm_unbind(rsmseg_t *seg);
static uint_t rsmhash(rsm_memseg_id_t key);
static void rsmhash_alloc(rsmhash_table_t *rhash, int size);
static void rsmhash_free(rsmhash_table_t *rhash, int size);
static void *rsmhash_getbkt(rsmhash_table_t *rhash, uint_t hashval);
static void **rsmhash_bktaddr(rsmhash_table_t *rhash, uint_t hashval);
static int rsm_send_notimporting(rsm_node_id_t dest, rsm_memseg_id_t segid,
					void *cookie);
int rsm_disconnect(rsmseg_t *seg);
void rsmseg_unload(rsmseg_t *);
void rsm_suspend_complete(rsm_node_id_t src_node, int flag);

rsm_intr_hand_ret_t rsm_srv_func(rsm_controller_object_t *chd,
    rsm_intr_q_op_t opcode, rsm_addr_t src,
    void *data, size_t size, rsm_intr_hand_arg_t arg);

static void rsm_intr_callback(void *, rsm_addr_t, rsm_intr_hand_arg_t);

rsm_node_id_t my_nodeid;

/* cookie, va, offsets and length for the barrier */
static rsm_gnum_t		*bar_va;
static ddi_umem_cookie_t	bar_cookie;
static off_t			barrier_offset;
static size_t			barrier_size;
static int			max_segs;

/* cookie for the trash memory */
static ddi_umem_cookie_t	remap_cookie;

static rsm_memseg_id_t	rsm_nextavail_segmentid;

extern taskq_t *work_taskq;
extern char *taskq_name;

static dev_info_t *rsm_dip;	/* private copy of devinfo pointer */

static rsmhash_table_t rsm_export_segs;		/* list of exported segs */
rsmhash_table_t rsm_import_segs;		/* list of imported segs */
static rsmhash_table_t rsm_event_queues;	/* list of event queues */

static	rsm_ipc_t	rsm_ipc;		/* ipc info */

/* list of nodes to which RSMIPC_MSG_SUSPEND has been sent */
static list_head_t	rsm_suspend_list;

/* list of descriptors for remote importers */
static importers_table_t importer_list;

kmutex_t rsm_suspend_cvlock;
kcondvar_t rsm_suspend_cv;

static kmutex_t rsm_lock;

adapter_t loopback_adapter;
rsm_controller_attr_t loopback_attr;

int rsmipc_send_controlmsg(path_t *path, int msgtype);

void rsmka_init_loopback();

int rsmka_null_seg_create(
    rsm_controller_handle_t,
    rsm_memseg_export_handle_t *,
    size_t,
    uint_t,
    rsm_memory_local_t *,
    rsm_resource_callback_t,
    rsm_resource_callback_arg_t);

int rsmka_null_seg_destroy(
    rsm_memseg_export_handle_t);

int rsmka_null_bind(
    rsm_memseg_export_handle_t,
    off_t,
    rsm_memory_local_t *,
    rsm_resource_callback_t,
    rsm_resource_callback_arg_t);

int rsmka_null_unbind(
    rsm_memseg_export_handle_t,
    off_t,
    size_t);

int rsmka_null_rebind(
    rsm_memseg_export_handle_t,
    off_t,
    rsm_memory_local_t *,
    rsm_resource_callback_t,
    rsm_resource_callback_arg_t);

int rsmka_null_publish(
    rsm_memseg_export_handle_t,
    rsm_access_entry_t [],
    uint_t,
    rsm_memseg_id_t,
    rsm_resource_callback_t,
    rsm_resource_callback_arg_t);


int rsmka_null_republish(
    rsm_memseg_export_handle_t,
    rsm_access_entry_t [],
    uint_t,
    rsm_resource_callback_t,
    rsm_resource_callback_arg_t);

int rsmka_null_unpublish(
    rsm_memseg_export_handle_t);

rsm_ops_t null_rsmpi_ops;

/*
 * data and locks to keep track of total amount of exported memory
 */
static	pgcnt_t		rsm_pgcnt;
static	pgcnt_t		rsm_pgcnt_max;	/* max allowed */
static	kmutex_t	rsm_pgcnt_lock;

static	int		rsm_enable_dr;

static	char		loopback_str[] = "loopback";

int		rsm_hash_size;

/*
 * The locking model is as follows:
 *
 * Local operations:
 *		find resource - grab reader lock on resouce list
 *		insert rc     - grab writer lock
 *		delete rc     - grab writer lock and resource mutex
 *		read/write    - no lock
 *
 * Remote invocations:
 *		find resource - grab read lock and resource mutex
 *
 * State:
 *		resource state - grab resource mutex
 */

int
_init(void)
{
	int e;

	e = mod_install(&modlinkage);
	if (e != 0) {
		return (e);
	}

	mutex_init(&rsm_lock, NULL, MUTEX_DRIVER, NULL);

	mutex_init(&rsmka_buf_lock, NULL, MUTEX_DEFAULT, NULL);


	rw_init(&rsm_resource.rsmrc_lock, NULL, RW_DRIVER, NULL);

	rsm_hash_size = RSM_HASHSZ;

	rw_init(&rsm_export_segs.rsmhash_rw, NULL, RW_DRIVER, NULL);

	rw_init(&rsm_import_segs.rsmhash_rw, NULL, RW_DRIVER, NULL);

	mutex_init(&importer_list.lock, NULL, MUTEX_DRIVER, NULL);

	mutex_init(&rsm_ipc.lock, NULL, MUTEX_DRIVER, NULL);
	cv_init(&rsm_ipc.cv, NULL, CV_DRIVER, 0);

	mutex_init(&rsm_suspend_cvlock, NULL, MUTEX_DRIVER, NULL);
	cv_init(&rsm_suspend_cv, NULL, CV_DRIVER, 0);

	mutex_init(&rsm_drv_data.drv_lock, NULL, MUTEX_DRIVER, NULL);
	cv_init(&rsm_drv_data.drv_cv, NULL, CV_DRIVER, 0);

	rsm_ipc.count = RSMIPC_SZ;
	rsm_ipc.wanted = 0;
	rsm_ipc.sequence = 0;

	(void) mutex_init(&rsm_pgcnt_lock, NULL, MUTEX_DRIVER, NULL);

	for (e = 0; e < RSMIPC_SZ; e++) {
		rsmipc_slot_t *slot = &rsm_ipc.slots[e];

		RSMIPC_SET(slot, RSMIPC_FREE);
		mutex_init(&slot->rsmipc_lock, NULL, MUTEX_DRIVER, NULL);
		cv_init(&slot->rsmipc_cv, NULL, CV_DRIVER, 0);
	}

	/*
	 * Initialize the suspend message list
	 */
	rsm_suspend_list.list_head = NULL;
	mutex_init(&rsm_suspend_list.list_lock, NULL, MUTEX_DRIVER, NULL);

	/*
	 * It is assumed here that configuration data is available
	 * during system boot since _init may be called at that time.
	 */

	rsmka_pathmanager_init();

	DBG_PRINTF((RSM_KERNEL_AGENT, RSM_DEBUG_VERBOSE,
	    "rsm: _init done\n"));

	return (DDI_SUCCESS);

}

int
_info(struct modinfo *modinfop)
{

	return (mod_info(&modlinkage, modinfop));
}

int
_fini(void)
{
	int e;

	DBG_PRINTF((RSM_KERNEL_AGENT, RSM_DEBUG_VERBOSE,
	    "rsm: _fini enter\n"));

	/*
	 * The rsmka_modunloadok flag is simply used to help with
	 * the PIT testing. Make this flag 0 to disallow modunload.
	 */
	if (rsmka_modunloadok == 0)
		return (EBUSY);

	/* rsm_detach will be called as a result of mod_remove */
	e = mod_remove(&modlinkage);
	if (e) {
		DBG_PRINTF((RSM_KERNEL_AGENT, RSM_ERR,
		    "Unable to fini RSM %x\n", e));
		return (e);
	}

	rsmka_pathmanager_cleanup();

	rw_destroy(&rsm_resource.rsmrc_lock);

	rw_destroy(&rsm_export_segs.rsmhash_rw);
	rw_destroy(&rsm_import_segs.rsmhash_rw);
	rw_destroy(&rsm_event_queues.rsmhash_rw);

	mutex_destroy(&importer_list.lock);

	mutex_destroy(&rsm_ipc.lock);
	cv_destroy(&rsm_ipc.cv);

	(void) mutex_destroy(&rsm_suspend_list.list_lock);

	(void) mutex_destroy(&rsm_pgcnt_lock);

	DBG_PRINTF((RSM_KERNEL_AGENT, RSM_DEBUG_VERBOSE, "_fini done\n"));

	return (DDI_SUCCESS);

}

/*ARGSUSED1*/
static int
rsm_attach(dev_info_t *devi, ddi_attach_cmd_t cmd)
{
	minor_t	rnum;
	int	percent;
	int	ret;
	DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_DDI);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_attach enter\n"));

	switch (cmd) {
	case DDI_ATTACH:
		break;
	case DDI_RESUME:
	default:
		DBG_PRINTF((category, RSM_ERR,
		    "rsm:rsm_attach - cmd not supported\n"));
		return (DDI_FAILURE);
	}

	if (rsm_dip != NULL) {
		DBG_PRINTF((category, RSM_ERR,
		    "rsm:rsm_attach - supports only "
		    "one instance\n"));
		return (DDI_FAILURE);
	}

	rsm_enable_dr = ddi_prop_get_int(DDI_DEV_T_ANY, devi,
	    DDI_PROP_DONTPASS | DDI_PROP_NOTPROM,
	    "enable-dynamic-reconfiguration", 1);

	mutex_enter(&rsm_drv_data.drv_lock);
	rsm_drv_data.drv_state = RSM_DRV_REG_PROCESSING;
	mutex_exit(&rsm_drv_data.drv_lock);

	if (rsm_enable_dr) {
#ifdef	RSM_DRTEST
		ret = rsm_kphysm_setup_func_register(&rsm_dr_callback_vec,
		    (void *)NULL);
#else
		ret = kphysm_setup_func_register(&rsm_dr_callback_vec,
		    (void *)NULL);
#endif
		if (ret != 0) {
			mutex_exit(&rsm_drv_data.drv_lock);
			cmn_err(CE_CONT, "rsm:rsm_attach - Dynamic "
			    "reconfiguration setup failed\n");
			return (DDI_FAILURE);
		}
	}

	mutex_enter(&rsm_drv_data.drv_lock);
	ASSERT(rsm_drv_data.drv_state == RSM_DRV_REG_PROCESSING);
	rsm_drv_data.drv_state = RSM_DRV_OK;
	cv_broadcast(&rsm_drv_data.drv_cv);
	mutex_exit(&rsm_drv_data.drv_lock);

	/*
	 * page_list_read_lock();
	 * xx_setup();
	 * page_list_read_unlock();
	 */

	rsm_hash_size = ddi_prop_get_int(DDI_DEV_T_ANY, devi,
	    DDI_PROP_DONTPASS | DDI_PROP_NOTPROM,
	    "segment-hashtable-size", RSM_HASHSZ);
	if (rsm_hash_size == 0) {
		DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
		    "rsm: segment-hashtable-size in rsm.conf "
		    "must be greater than 0, defaulting to 128\n"));
		rsm_hash_size = RSM_HASHSZ;
	}

	DBG_PRINTF((category, RSM_DEBUG, "rsm_attach rsm_hash_size: %d\n",
	    rsm_hash_size));

	rsm_pgcnt = 0;

	percent = ddi_prop_get_int(DDI_DEV_T_ANY, devi,
	    DDI_PROP_DONTPASS | DDI_PROP_NOTPROM,
	    "max-exported-memory", 0);
	if (percent < 0) {
		DBG_PRINTF((category, RSM_ERR,
		    "rsm:rsm_attach not enough memory available to "
		    "export, or max-exported-memory set incorrectly.\n"));
		return (DDI_FAILURE);
	}
	/* 0 indicates no fixed upper limit. maxmem is the max	*/
	/* available pageable physical mem			*/
	rsm_pgcnt_max = (percent*maxmem)/100;

	if (rsm_pgcnt_max > 0) {
		DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
		    "rsm: Available physical memory = %lu pages, "
		    "Max exportable memory = %lu pages",
		    maxmem, rsm_pgcnt_max));
	}

	/*
	 * Create minor number
	 */
	if (rsmresource_alloc(&rnum) != RSM_SUCCESS) {
		DBG_PRINTF((category, RSM_ERR,
		    "rsm: rsm_attach - Unable to get "
		    "minor number\n"));
		return (DDI_FAILURE);
	}

	ASSERT(rnum == RSM_DRIVER_MINOR);

	if (ddi_create_minor_node(devi, DRIVER_NAME, S_IFCHR,
	    rnum, DDI_PSEUDO, NULL) == DDI_FAILURE) {
		DBG_PRINTF((category, RSM_ERR,
		    "rsm: rsm_attach - unable to allocate "
		    "minor #\n"));
		return (DDI_FAILURE);
	}

	rsm_dip = devi;
	/*
	 * Allocate the hashtables
	 */
	rsmhash_alloc(&rsm_export_segs, rsm_hash_size);
	rsmhash_alloc(&rsm_import_segs, rsm_hash_size);

	importer_list.bucket = (importing_token_t **)
	    kmem_zalloc(rsm_hash_size * sizeof (importing_token_t *), KM_SLEEP);

	/*
	 * Allocate a resource struct
	 */
	{
		rsmresource_t *p;

		p = (rsmresource_t *)kmem_zalloc(sizeof (*p), KM_SLEEP);

		mutex_init(&p->rsmrc_lock, NULL, MUTEX_DRIVER, (void *) NULL);

		rsmresource_insert(rnum, p, RSM_RESOURCE_BAR);
	}

	/*
	 * Based on the rsm.conf property max-segments, determine the maximum
	 * number of segments that can be exported/imported. This is then used
	 * to determine the size for barrier failure pages.
	 */

	/* First get the max number of segments from the rsm.conf file */
	max_segs = ddi_prop_get_int(DDI_DEV_T_ANY, devi,
	    DDI_PROP_DONTPASS | DDI_PROP_NOTPROM,
	    "max-segments", 0);
	if (max_segs == 0) {
		/* Use default number of segments */
		max_segs = RSM_MAX_NUM_SEG;
	}

	/*
	 * Based on the max number of segments allowed, determine the barrier
	 * page size. add 1 to max_segs since the barrier page itself uses
	 * a slot
	 */
	barrier_size = roundup((max_segs + 1) * sizeof (rsm_gnum_t),
	    PAGESIZE);

	/*
	 * allocation of the barrier failure page
	 */
	bar_va = (rsm_gnum_t *)ddi_umem_alloc(barrier_size,
	    DDI_UMEM_SLEEP, &bar_cookie);

	/*
	 * Set the barrier_offset
	 */
	barrier_offset = 0;

	/*
	 * Allocate a trash memory and get a cookie for it. This will be used
	 * when remapping segments during force disconnects. Allocate the
	 * trash memory with a large size which is page aligned.
	 */
	(void) ddi_umem_alloc((size_t)TRASHSIZE,
	    DDI_UMEM_TRASH, &remap_cookie);

	/* initialize user segment id allocation variable */
	rsm_nextavail_segmentid = (rsm_memseg_id_t)RSM_USER_APP_ID_BASE;

	/*
	 * initialize the null_rsmpi_ops vector and the loopback adapter
	 */
	rsmka_init_loopback();


	ddi_report_dev(devi);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_attach done\n"));

	return (DDI_SUCCESS);
}

/*
 * The call to mod_remove in the _fine routine will cause the system
 * to call rsm_detach
 */
/*ARGSUSED*/
static int
rsm_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
{
	DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_DDI);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_detach enter\n"));

	switch (cmd) {
	case DDI_DETACH:
		break;
	default:
		DBG_PRINTF((category, RSM_ERR,
		    "rsm:rsm_detach - cmd %x not supported\n",
		    cmd));
		return (DDI_FAILURE);
	}

	mutex_enter(&rsm_drv_data.drv_lock);
	while (rsm_drv_data.drv_state != RSM_DRV_OK)
		cv_wait(&rsm_drv_data.drv_cv, &rsm_drv_data.drv_lock);
	rsm_drv_data.drv_state = RSM_DRV_UNREG_PROCESSING;
	mutex_exit(&rsm_drv_data.drv_lock);

	/*
	 * Unregister the DR callback functions
	 */
	if (rsm_enable_dr) {
#ifdef	RSM_DRTEST
		rsm_kphysm_setup_func_unregister(&rsm_dr_callback_vec,
		    (void *)NULL);
#else
		kphysm_setup_func_unregister(&rsm_dr_callback_vec,
		    (void *)NULL);
#endif
	}

	mutex_enter(&rsm_drv_data.drv_lock);
	ASSERT(rsm_drv_data.drv_state == RSM_DRV_UNREG_PROCESSING);
	rsm_drv_data.drv_state = RSM_DRV_NEW;
	mutex_exit(&rsm_drv_data.drv_lock);

	ASSERT(rsm_suspend_list.list_head == NULL);

	/*
	 * Release all resources, seglist, controller, ...
	 */

	/* remove intersend queues */
	/* remove registered services */


	ddi_remove_minor_node(dip, DRIVER_NAME);
	rsm_dip = NULL;

	/*
	 * Free minor zero resource
	 */
	{
		rsmresource_t *p;

		p = rsmresource_free(RSM_DRIVER_MINOR);
		if (p) {
			mutex_destroy(&p->rsmrc_lock);
			kmem_free((void *)p, sizeof (*p));
		}
	}

	/*
	 * Free resource table
	 */

	rsmresource_destroy();

	/*
	 * Free the hash tables
	 */
	rsmhash_free(&rsm_export_segs, rsm_hash_size);
	rsmhash_free(&rsm_import_segs, rsm_hash_size);

	kmem_free((void *)importer_list.bucket,
	    rsm_hash_size * sizeof (importing_token_t *));
	importer_list.bucket = NULL;


	/* free barrier page */
	if (bar_cookie != NULL) {
		ddi_umem_free(bar_cookie);
	}
	bar_va = NULL;
	bar_cookie = NULL;

	/*
	 * Free the memory allocated for the trash
	 */
	if (remap_cookie != NULL) {
		ddi_umem_free(remap_cookie);
	}
	remap_cookie = NULL;

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_detach done\n"));

	return (DDI_SUCCESS);
}

/*ARGSUSED*/
static int
rsm_info(dev_info_t *dip, ddi_info_cmd_t infocmd, void *arg, void **result)
{
	register int error;
	DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_DDI);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_info enter\n"));

	switch (infocmd) {
	case DDI_INFO_DEVT2DEVINFO:
		if (rsm_dip == NULL)
			error = DDI_FAILURE;
		else {
			*result = (void *)rsm_dip;
			error = DDI_SUCCESS;
		}
		break;
	case DDI_INFO_DEVT2INSTANCE:
		*result = (void *)0;
		error = DDI_SUCCESS;
		break;
	default:
		error = DDI_FAILURE;
	}

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_info done\n"));
	return (error);
}

adapter_t *
rsm_getadapter(rsm_ioctlmsg_t *msg, int mode)
{
	adapter_t *adapter;
	char adapter_devname[MAXNAMELEN];
	int instance;
	DBG_DEFINE(category,
	    RSM_KERNEL_AGENT | RSM_IMPORT | RSM_EXPORT | RSM_IOCTL);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_getadapter enter\n"));

	instance = msg->cnum;

	if ((msg->cname_len <= 0) || (msg->cname_len > MAXNAMELEN)) {
		return (NULL);
	}

	if (ddi_copyin(msg->cname, adapter_devname, msg->cname_len, mode))
		return (NULL);

	if (strcmp(adapter_devname, "loopback") == 0)
		return (&loopback_adapter);

	adapter = rsmka_lookup_adapter(adapter_devname, instance);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_getadapter done\n"));

	return (adapter);
}


/*
 * *********************** Resource Number Management ********************
 * All resources are stored in a simple hash table. The table is an array
 * of pointers to resource blks. Each blk contains:
 *	base	- base number of this blk
 *	used	- number of used slots in this blk.
 *	blks    - array of pointers to resource items.
 * An entry in a resource blk is empty if it's NULL.
 *
 * We start with no resource array. Each time we run out of slots, we
 * reallocate a new larger array and copy the pointer to the new array and
 * a new resource blk is allocated and added to the hash table.
 *
 * The resource control block contains:
 *      root    - array of pointer of resource blks
 *      sz      - current size of array.
 *      len     - last valid entry in array.
 *
 * A search operation based on a resource number is as follows:
 *      index = rnum / RESOURCE_BLKSZ;
 *      ASSERT(index < resource_block.len);
 *      ASSERT(index < resource_block.sz);
 *	offset = rnum % RESOURCE_BLKSZ;
 *      ASSERT(offset >= resource_block.root[index]->base);
 *	ASSERT(offset < resource_block.root[index]->base + RESOURCE_BLKSZ);
 *	return resource_block.root[index]->blks[offset];
 *
 * A resource blk is freed with its used count reachs zero.
 */
static int
rsmresource_alloc(minor_t *rnum)
{

	/* search for available resource slot */
	int i, j, empty = -1;
	rsmresource_blk_t *blk;

	DBG_PRINTF((RSM_KERNEL_ALL, RSM_DEBUG_VERBOSE,
	    "rsmresource_alloc enter\n"));

	rw_enter(&rsm_resource.rsmrc_lock, RW_WRITER);

	/* Try to find an empty slot */
	for (i = 0; i < rsm_resource.rsmrc_len; i++) {
		blk = rsm_resource.rsmrc_root[i];
		if (blk != NULL && blk->rsmrcblk_avail > 0) {
			/* found an empty slot in this blk */
			for (j = 0; j < RSMRC_BLKSZ; j++) {
				if (blk->rsmrcblk_blks[j] == NULL) {
					*rnum = (minor_t)
					    (j + (i * RSMRC_BLKSZ));
					/*
					 * obey gen page limits
					 */
					if (*rnum >= max_segs + 1) {
						if (empty < 0) {
							rw_exit(&rsm_resource.
							    rsmrc_lock);
							DBG_PRINTF((
							    RSM_KERNEL_ALL,
							    RSM_ERR,
							    "rsmresource"
							    "_alloc failed:"
							    "not enough res"
							    "%d\n", *rnum));
					return (RSMERR_INSUFFICIENT_RESOURCES);
						} else {
							/* use empty slot */
							break;
						}

					}

					blk->rsmrcblk_blks[j] = RSMRC_RESERVED;
					blk->rsmrcblk_avail--;
					rw_exit(&rsm_resource.rsmrc_lock);
					DBG_PRINTF((RSM_KERNEL_ALL,
					    RSM_DEBUG_VERBOSE,
					    "rsmresource_alloc done\n"));
					return (RSM_SUCCESS);
				}
			}
		} else if (blk == NULL && empty < 0) {
			/* remember first empty slot */
			empty = i;
		}
	}

	/* Couldn't find anything, allocate a new blk */
	/*
	 * Do we need to reallocate the root array
	 */
	if (empty < 0) {
		if (rsm_resource.rsmrc_len == rsm_resource.rsmrc_sz) {
			/*
			 * Allocate new array and copy current stuff into it
			 */
			rsmresource_blk_t	**p;
			uint_t newsz = (uint_t)rsm_resource.rsmrc_sz +
			    RSMRC_BLKSZ;
			/*
			 * Don't allocate more that max valid rnum
			 */
			if (rsm_resource.rsmrc_len*RSMRC_BLKSZ >=
			    max_segs + 1) {
				rw_exit(&rsm_resource.rsmrc_lock);
				return (RSMERR_INSUFFICIENT_RESOURCES);
			}

			p = (rsmresource_blk_t **)kmem_zalloc(
			    newsz * sizeof (*p),
			    KM_SLEEP);

			if (rsm_resource.rsmrc_root) {
				uint_t oldsz;

				oldsz = (uint_t)(rsm_resource.rsmrc_sz *
				    (int)sizeof (*p));

				/*
				 * Copy old data into new space and
				 * free old stuff
				 */
				bcopy(rsm_resource.rsmrc_root, p, oldsz);
				kmem_free(rsm_resource.rsmrc_root, oldsz);
			}

			rsm_resource.rsmrc_root = p;
			rsm_resource.rsmrc_sz = (int)newsz;
		}

		empty = rsm_resource.rsmrc_len;
		rsm_resource.rsmrc_len++;
	}

	/*
	 * Allocate a new blk
	 */
	blk = (rsmresource_blk_t *)kmem_zalloc(sizeof (*blk), KM_SLEEP);
	ASSERT(rsm_resource.rsmrc_root[empty] == NULL);
	rsm_resource.rsmrc_root[empty] = blk;
	blk->rsmrcblk_avail = RSMRC_BLKSZ - 1;

	/*
	 * Allocate slot
	 */

	*rnum = (minor_t)(empty * RSMRC_BLKSZ);

	/*
	 * watch out not to exceed bounds of barrier page
	 */
	if (*rnum >= max_segs + 1) {
		rw_exit(&rsm_resource.rsmrc_lock);
		DBG_PRINTF((RSM_KERNEL_ALL, RSM_ERR,
		    "rsmresource_alloc failed %d\n", *rnum));

		return (RSMERR_INSUFFICIENT_RESOURCES);
	}
	blk->rsmrcblk_blks[0] = RSMRC_RESERVED;


	rw_exit(&rsm_resource.rsmrc_lock);

	DBG_PRINTF((RSM_KERNEL_ALL, RSM_DEBUG_VERBOSE,
	    "rsmresource_alloc done\n"));

	return (RSM_SUCCESS);
}

static rsmresource_t *
rsmresource_free(minor_t rnum)
{

	/* search for available resource slot */
	int i, j;
	rsmresource_blk_t *blk;
	rsmresource_t *p;

	DBG_PRINTF((RSM_KERNEL_ALL, RSM_DEBUG_VERBOSE,
	    "rsmresource_free enter\n"));

	i = (int)(rnum / RSMRC_BLKSZ);
	j = (int)(rnum % RSMRC_BLKSZ);

	if (i >= rsm_resource.rsmrc_len) {
		DBG_PRINTF((RSM_KERNEL_ALL, RSM_DEBUG_VERBOSE,
		    "rsmresource_free done\n"));
		return (NULL);
	}

	rw_enter(&rsm_resource.rsmrc_lock, RW_WRITER);

	ASSERT(rsm_resource.rsmrc_root);
	ASSERT(i < rsm_resource.rsmrc_len);
	ASSERT(i < rsm_resource.rsmrc_sz);
	blk = rsm_resource.rsmrc_root[i];
	if (blk == NULL) {
		rw_exit(&rsm_resource.rsmrc_lock);
		DBG_PRINTF((RSM_KERNEL_ALL, RSM_DEBUG_VERBOSE,
		    "rsmresource_free done\n"));
		return (NULL);
	}

	ASSERT(blk->rsmrcblk_blks[j]); /* reserved or full */

	p = blk->rsmrcblk_blks[j];
	if (p == RSMRC_RESERVED) {
		p = NULL;
	}

	blk->rsmrcblk_blks[j] = NULL;
	blk->rsmrcblk_avail++;
	if (blk->rsmrcblk_avail == RSMRC_BLKSZ) {
		/* free this blk */
		kmem_free(blk, sizeof (*blk));
		rsm_resource.rsmrc_root[i] = NULL;
	}

	rw_exit(&rsm_resource.rsmrc_lock);

	DBG_PRINTF((RSM_KERNEL_ALL, RSM_DEBUG_VERBOSE,
	    "rsmresource_free done\n"));

	return (p);
}

static rsmresource_t *
rsmresource_lookup(minor_t rnum, int lock)
{
	int i, j;
	rsmresource_blk_t *blk;
	rsmresource_t *p;

	DBG_PRINTF((RSM_KERNEL_ALL, RSM_DEBUG_VERBOSE,
	    "rsmresource_lookup enter\n"));

	/* Find resource and lock it in READER mode */
	/* search for available resource slot */

	i = (int)(rnum / RSMRC_BLKSZ);
	j = (int)(rnum % RSMRC_BLKSZ);

	if (i >= rsm_resource.rsmrc_len) {
		DBG_PRINTF((RSM_KERNEL_ALL, RSM_DEBUG_VERBOSE,
		    "rsmresource_lookup done\n"));
		return (NULL);
	}

	rw_enter(&rsm_resource.rsmrc_lock, RW_READER);

	blk = rsm_resource.rsmrc_root[i];
	if (blk != NULL) {
		ASSERT(i < rsm_resource.rsmrc_len);
		ASSERT(i < rsm_resource.rsmrc_sz);

		p = blk->rsmrcblk_blks[j];
		if (lock == RSM_LOCK) {
			if (p != RSMRC_RESERVED) {
				mutex_enter(&p->rsmrc_lock);
			} else {
				p = NULL;
			}
		}
	} else {
		p = NULL;
	}
	rw_exit(&rsm_resource.rsmrc_lock);

	DBG_PRINTF((RSM_KERNEL_ALL, RSM_DEBUG_VERBOSE,
	    "rsmresource_lookup done\n"));

	return (p);
}

static void
rsmresource_insert(minor_t rnum, rsmresource_t *p, rsm_resource_type_t type)
{
	/* Find resource and lock it in READER mode */
	/* Caller can upgrade if need be */
	/* search for available resource slot */
	int i, j;
	rsmresource_blk_t *blk;

	DBG_PRINTF((RSM_KERNEL_ALL, RSM_DEBUG_VERBOSE,
	    "rsmresource_insert enter\n"));

	i = (int)(rnum / RSMRC_BLKSZ);
	j = (int)(rnum % RSMRC_BLKSZ);

	p->rsmrc_type = type;
	p->rsmrc_num = rnum;

	rw_enter(&rsm_resource.rsmrc_lock, RW_READER);

	ASSERT(rsm_resource.rsmrc_root);
	ASSERT(i < rsm_resource.rsmrc_len);
	ASSERT(i < rsm_resource.rsmrc_sz);

	blk = rsm_resource.rsmrc_root[i];
	ASSERT(blk);

	ASSERT(blk->rsmrcblk_blks[j] == RSMRC_RESERVED);

	blk->rsmrcblk_blks[j] = p;

	DBG_PRINTF((RSM_KERNEL_ALL, RSM_DEBUG_VERBOSE,
	    "rsmresource_insert done\n"));

	rw_exit(&rsm_resource.rsmrc_lock);
}

static void
rsmresource_destroy()
{
	int i, j;

	DBG_PRINTF((RSM_KERNEL_ALL, RSM_DEBUG_VERBOSE,
	    "rsmresource_destroy enter\n"));

	rw_enter(&rsm_resource.rsmrc_lock, RW_WRITER);

	for (i = 0; i < rsm_resource.rsmrc_len; i++) {
		rsmresource_blk_t	*blk;

		blk = rsm_resource.rsmrc_root[i];
		if (blk == NULL) {
			continue;
		}
		for (j = 0; j < RSMRC_BLKSZ; j++) {
			if (blk->rsmrcblk_blks[j] != NULL) {
				DBG_PRINTF((RSM_KERNEL_ALL, RSM_DEBUG_VERBOSE,
				    "Not null slot %d, %lx\n", j,
				    (size_t)blk->rsmrcblk_blks[j]));
			}
		}
		kmem_free(blk, sizeof (*blk));
		rsm_resource.rsmrc_root[i] = NULL;
	}
	if (rsm_resource.rsmrc_root) {
		i = rsm_resource.rsmrc_sz * (int)sizeof (rsmresource_blk_t *);
		kmem_free(rsm_resource.rsmrc_root, (uint_t)i);
		rsm_resource.rsmrc_root = NULL;
		rsm_resource.rsmrc_len = 0;
		rsm_resource.rsmrc_sz = 0;
	}

	DBG_PRINTF((RSM_KERNEL_ALL, RSM_DEBUG_VERBOSE,
	    "rsmresource_destroy done\n"));

	rw_exit(&rsm_resource.rsmrc_lock);
}


/* ******************** Generic Key Hash Table Management ********* */
static rsmresource_t *
rsmhash_lookup(rsmhash_table_t *rhash, rsm_memseg_id_t key,
    rsm_resource_state_t state)
{
	rsmresource_t	*p;
	uint_t		hashval;
	DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_FUNC_ALL);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmhash_lookup enter\n"));

	hashval = rsmhash(key);

	DBG_PRINTF((category, RSM_DEBUG_LVL2, "rsmhash_lookup %u=%d\n",
	    key, hashval));

	rw_enter(&rhash->rsmhash_rw, RW_READER);

	p = (rsmresource_t *)rsmhash_getbkt(rhash, hashval);

	for (; p; p = p->rsmrc_next) {
		if (p->rsmrc_key == key) {
			/* acquire resource lock */
			RSMRC_LOCK(p);
			break;
		}
	}

	rw_exit(&rhash->rsmhash_rw);

	if (p != NULL && p->rsmrc_state != state) {
		/* state changed, release lock and return null */
		RSMRC_UNLOCK(p);
		DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
		    "rsmhash_lookup done: state changed\n"));
		return (NULL);
	}

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmhash_lookup done\n"));

	return (p);
}

static void
rsmhash_rm(rsmhash_table_t *rhash, rsmresource_t *rcelm)
{
	rsmresource_t		*p, **back;
	uint_t			hashval;
	DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_FUNC_ALL);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmhash_rm enter\n"));

	hashval = rsmhash(rcelm->rsmrc_key);

	DBG_PRINTF((category, RSM_DEBUG_LVL2, "rsmhash_rm %u=%d\n",
	    rcelm->rsmrc_key, hashval));

	/*
	 * It's ok not to find the segment.
	 */
	rw_enter(&rhash->rsmhash_rw, RW_WRITER);

	back = (rsmresource_t **)rsmhash_bktaddr(rhash, hashval);

	for (; (p = *back) != NULL;  back = &p->rsmrc_next) {
		if (p == rcelm) {
			*back = rcelm->rsmrc_next;
			break;
		}
	}

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmhash_rm done\n"));

	rw_exit(&rhash->rsmhash_rw);
}

static int
rsmhash_add(rsmhash_table_t *rhash, rsmresource_t *new, rsm_memseg_id_t key,
    int dup_check, rsm_resource_state_t state)
{
	rsmresource_t	*p = NULL, **bktp;
	uint_t		hashval;
	DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_FUNC_ALL);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmhash_add enter\n"));

	/* lock table */
	rw_enter(&rhash->rsmhash_rw, RW_WRITER);

	/*
	 * If the current resource state is other than the state passed in
	 * then the resource is (probably) already on the list. eg. for an
	 * import segment if the state is not RSM_STATE_NEW then it's on the
	 * list already.
	 */
	RSMRC_LOCK(new);
	if (new->rsmrc_state != state) {
		RSMRC_UNLOCK(new);
		rw_exit(&rhash->rsmhash_rw);
		return (RSMERR_BAD_SEG_HNDL);
	}

	hashval = rsmhash(key);
	DBG_PRINTF((category, RSM_DEBUG_LVL2, "rsmhash_add %d\n", hashval));

	if (dup_check) {
		/*
		 * Used for checking export segments; don't want to have
		 * the same key used for multiple segments.
		 */

		p = (rsmresource_t *)rsmhash_getbkt(rhash, hashval);

		for (; p; p = p->rsmrc_next) {
			if (p->rsmrc_key == key) {
				RSMRC_UNLOCK(new);
				break;
			}
		}
	}

	if (p == NULL) {
		/* Key doesn't exist, add it */

		bktp = (rsmresource_t **)rsmhash_bktaddr(rhash, hashval);

		new->rsmrc_key = key;
		new->rsmrc_next = *bktp;
		*bktp = new;
	}

	rw_exit(&rhash->rsmhash_rw);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmhash_add done\n"));

	return (p == NULL ? RSM_SUCCESS : RSMERR_SEGID_IN_USE);
}

/*
 * XOR each byte of the key.
 */
static uint_t
rsmhash(rsm_memseg_id_t key)
{
	uint_t	hash = key;

	hash ^=  (key >> 8);
	hash ^=  (key >> 16);
	hash ^=  (key >> 24);

	return (hash % rsm_hash_size);

}

/*
 * generic function to get a specific bucket
 */
static void *
rsmhash_getbkt(rsmhash_table_t *rhash, uint_t hashval)
{

	if (rhash->bucket == NULL)
		return (NULL);
	else
		return ((void *)rhash->bucket[hashval]);
}

/*
 * generic function to get a specific bucket's address
 */
static void **
rsmhash_bktaddr(rsmhash_table_t *rhash, uint_t hashval)
{
	if (rhash->bucket == NULL)
		return (NULL);
	else
		return ((void **)&(rhash->bucket[hashval]));
}

/*
 * generic function to alloc a hash table
 */
static void
rsmhash_alloc(rsmhash_table_t *rhash, int size)
{
	rhash->bucket = (rsmresource_t **)
	    kmem_zalloc(size * sizeof (rsmresource_t *), KM_SLEEP);
}

/*
 * generic function to free a hash table
 */
static void
rsmhash_free(rsmhash_table_t *rhash, int size)
{

	kmem_free((void *)rhash->bucket, size * sizeof (caddr_t));
	rhash->bucket = NULL;

}
/* *********************** Exported Segment Key Management ************ */

#define	rsmexport_add(new, key)		\
	rsmhash_add(&rsm_export_segs, (rsmresource_t *)new, key, 1, \
	    RSM_STATE_BIND)

#define	rsmexport_rm(arg)	\
	rsmhash_rm(&rsm_export_segs, (rsmresource_t *)(arg))

#define	rsmexport_lookup(key)	\
	(rsmseg_t *)rsmhash_lookup(&rsm_export_segs, key, RSM_STATE_EXPORT)

/* ************************** Import Segment List Management ********** */

/*
 *  Add segment to import list. This will be useful for paging and loopback
 * segment unloading.
 */
#define	rsmimport_add(arg, key)	\
	rsmhash_add(&rsm_import_segs, (rsmresource_t *)(arg), (key), 0, \
	    RSM_STATE_NEW)

#define	rsmimport_rm(arg)	\
	rsmhash_rm(&rsm_import_segs, (rsmresource_t *)(arg))

/*
 *	#define	rsmimport_lookup(key)	\
 *	(rsmseg_t *)rsmhash_lookup(&rsm_import_segs, (key), RSM_STATE_CONNECT)
 */

/*
 * increase the ref count and make the import segment point to the
 * shared data structure. Return a pointer to the share data struct
 * and the shared data struct is locked upon return
 */
static rsm_import_share_t *
rsmshare_get(rsm_memseg_id_t key, rsm_node_id_t node, adapter_t *adapter,
    rsmseg_t *segp)
{
	uint_t		hash;
	rsmresource_t		*p;
	rsm_import_share_t	*shdatap;
	DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_FUNC_ALL);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmshare_get enter\n"));

	hash = rsmhash(key);
	/* lock table */
	rw_enter(&rsm_import_segs.rsmhash_rw, RW_WRITER);
	DBG_PRINTF((category, RSM_DEBUG_LVL2, "rsmshare_get:key=%u, hash=%d\n",
	    key, hash));

	p = (rsmresource_t *)rsmhash_getbkt(&rsm_import_segs, hash);

	for (; p; p = p->rsmrc_next) {
		/*
		 * Look for an entry that is importing the same exporter
		 * with the share data structure allocated.
		 */
		if ((p->rsmrc_key == key) &&
		    (p->rsmrc_node == node) &&
		    (p->rsmrc_adapter == adapter) &&
		    (((rsmseg_t *)p)->s_share != NULL)) {
			shdatap = ((rsmseg_t *)p)->s_share;
			break;
		}
	}

	if (p == NULL) {
		/* we are the first importer, create the shared data struct */
		shdatap = kmem_zalloc(sizeof (rsm_import_share_t), KM_SLEEP);
		shdatap->rsmsi_state = RSMSI_STATE_NEW;
		shdatap->rsmsi_segid = key;
		shdatap->rsmsi_node = node;
		mutex_init(&shdatap->rsmsi_lock, NULL, MUTEX_DRIVER, NULL);
		cv_init(&shdatap->rsmsi_cv, NULL, CV_DRIVER, 0);
	}

	rsmseglock_acquire(segp);

	/* we grab the shared lock before returning from this function */
	mutex_enter(&shdatap->rsmsi_lock);

	shdatap->rsmsi_refcnt++;
	segp->s_share = shdatap;

	rsmseglock_release(segp);

	rw_exit(&rsm_import_segs.rsmhash_rw);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmshare_get done\n"));

	return (shdatap);
}

/*
 * the shared data structure should be locked before calling
 * rsmsharecv_signal().
 * Change the state and signal any waiting segments.
 */
void
rsmsharecv_signal(rsmseg_t *seg, int oldstate, int newstate)
{
	ASSERT(rsmsharelock_held(seg));

	if (seg->s_share->rsmsi_state == oldstate) {
		seg->s_share->rsmsi_state = newstate;
		cv_broadcast(&seg->s_share->rsmsi_cv);
	}
}

/*
 * Add to the hash table
 */
static void
importer_list_add(rsm_node_id_t node, rsm_memseg_id_t key, rsm_addr_t hwaddr,
    void *cookie)
{

	importing_token_t	*head;
	importing_token_t	*new_token;
	int			index;

	DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_FUNC_ALL);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "importer_list_add enter\n"));

	new_token = kmem_zalloc(sizeof (importing_token_t), KM_SLEEP);
	new_token->importing_node = node;
	new_token->key = key;
	new_token->import_segment_cookie = cookie;
	new_token->importing_adapter_hwaddr = hwaddr;

	index = rsmhash(key);

	mutex_enter(&importer_list.lock);

	head = importer_list.bucket[index];
	importer_list.bucket[index] = new_token;
	new_token->next = head;
	mutex_exit(&importer_list.lock);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "importer_list_add done\n"));
}

static void
importer_list_rm(rsm_node_id_t node,  rsm_memseg_id_t key, void *cookie)
{

	importing_token_t	*prev, *token = NULL;
	int			index;
	DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_FUNC_ALL);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "importer_list_rm enter\n"));

	index = rsmhash(key);

	mutex_enter(&importer_list.lock);

	token = importer_list.bucket[index];

	prev = token;
	while (token != NULL) {
		if (token->importing_node == node &&
		    token->import_segment_cookie == cookie) {
			if (prev == token)
				importer_list.bucket[index] = token->next;
			else
				prev->next = token->next;
			kmem_free((void *)token, sizeof (*token));
			break;
		} else {
			prev = token;
			token = token->next;
		}
	}

	mutex_exit(&importer_list.lock);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "importer_list_rm done\n"));


}

/* **************************Segment Structure Management ************* */

/*
 * Free segment structure
 */
static void
rsmseg_free(rsmseg_t *seg)
{

	DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_FUNC_ALL);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmseg_free enter\n"));

	/* need to take seglock here to avoid race with rsmmap_unmap() */
	rsmseglock_acquire(seg);
	if (seg->s_ckl != NULL) {
		/* Segment is still busy */
		seg->s_state = RSM_STATE_END;
		rsmseglock_release(seg);
		DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
		    "rsmseg_free done\n"));
		return;
	}

	rsmseglock_release(seg);

	ASSERT(seg->s_state == RSM_STATE_END || seg->s_state == RSM_STATE_NEW);

	/*
	 * If it's an importer decrement the refcount
	 * and if its down to zero free the shared data structure.
	 * This is where failures during rsm_connect() are unrefcounted
	 */
	if (seg->s_share != NULL) {

		ASSERT(seg->s_type == RSM_RESOURCE_IMPORT_SEGMENT);

		rsmsharelock_acquire(seg);

		ASSERT(seg->s_share->rsmsi_refcnt > 0);

		seg->s_share->rsmsi_refcnt--;

		if (seg->s_share->rsmsi_refcnt == 0) {
			rsmsharelock_release(seg);
			mutex_destroy(&seg->s_share->rsmsi_lock);
			cv_destroy(&seg->s_share->rsmsi_cv);
			kmem_free((void *)(seg->s_share),
			    sizeof (rsm_import_share_t));
		} else {
			rsmsharelock_release(seg);
		}
		/*
		 * The following needs to be done after any
		 * rsmsharelock calls which use seg->s_share.
		 */
		seg->s_share = NULL;
	}

	cv_destroy(&seg->s_cv);
	mutex_destroy(&seg->s_lock);
	rsmacl_free(seg->s_acl, seg->s_acl_len);
	rsmpiacl_free(seg->s_acl_in, seg->s_acl_len);
	if (seg->s_adapter)
		rsmka_release_adapter(seg->s_adapter);

	kmem_free((void *)seg, sizeof (*seg));

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmseg_free done\n"));

}


static rsmseg_t *
rsmseg_alloc(minor_t num, struct cred *cred)
{
	rsmseg_t	*new;
	DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_FUNC_ALL);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmseg_alloc enter\n"));
	/*
	 * allocate memory for new segment. This should be a segkmem cache.
	 */
	new = (rsmseg_t *)kmem_zalloc(sizeof (*new), KM_SLEEP);

	new->s_state = RSM_STATE_NEW;
	new->s_minor	= num;
	new->s_acl_len	= 0;
	new->s_cookie = NULL;
	new->s_adapter = NULL;

	new->s_mode = 0777 & ~PTOU((ttoproc(curthread)))->u_cmask;
	/* we don't have a key yet, will set at export/connect */
	new->s_uid  = crgetuid(cred);
	new->s_gid  = crgetgid(cred);

	mutex_init(&new->s_lock, NULL, MUTEX_DRIVER, (void *)NULL);
	cv_init(&new->s_cv, NULL, CV_DRIVER, 0);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmseg_alloc done\n"));

	return (new);
}

/* ******************************** Driver Open/Close/Poll *************** */

/*ARGSUSED1*/
static int
rsm_open(dev_t *devp, int flag, int otyp, struct cred *cred)
{
	minor_t rnum;
	DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_FUNC_ALL| RSM_DDI);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_open enter\n"));
	/*
	 * Char only
	 */
	if (otyp != OTYP_CHR) {
		DBG_PRINTF((category, RSM_ERR, "rsm_open: bad otyp\n"));
		return (EINVAL);
	}

	/*
	 * Only zero can be opened, clones are used for resources.
	 */
	if (getminor(*devp) != RSM_DRIVER_MINOR) {
		DBG_PRINTF((category, RSM_ERR,
		    "rsm_open: bad minor %d\n", getminor(*devp)));
		return (ENODEV);
	}

	if ((flag & FEXCL) != 0 && secpolicy_excl_open(cred) != 0) {
		DBG_PRINTF((category, RSM_ERR, "rsm_open: bad perm\n"));
		return (EPERM);
	}

	if (!(flag & FWRITE)) {
		/*
		 * The library function _rsm_librsm_init calls open for
		 * /dev/rsm with flag set to O_RDONLY.  We want a valid
		 * file descriptor to be returned for minor device zero.
		 */

		DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
		    "rsm_open RDONLY done\n"));
		return (DDI_SUCCESS);
	}

	/*
	 * - allocate new minor number and segment.
	 * - add segment to list of all segments.
	 * - set minordev data to segment
	 * - update devp argument to new device
	 * - update s_cred to cred; make sure you do crhold(cred);
	 */

	/* allocate a new resource number */
	if (rsmresource_alloc(&rnum) == RSM_SUCCESS) {
		/*
		 * We will bind this minor to a specific resource in first
		 * ioctl
		 */
		*devp = makedevice(getmajor(*devp), rnum);
	} else {
		return (EAGAIN);
	}

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_open done\n"));
	return (DDI_SUCCESS);
}

static void
rsmseg_close(rsmseg_t *seg, int force_flag)
{
	int e = RSM_SUCCESS;

	DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_FUNC_ALL| RSM_DDI);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmseg_close enter\n"));

	rsmseglock_acquire(seg);
	if (!force_flag && (seg->s_hdr.rsmrc_type ==
	    RSM_RESOURCE_EXPORT_SEGMENT)) {
		/*
		 * If we are processing rsm_close wait for force_destroy
		 * processing to complete since force_destroy processing
		 * needs to finish first before we can free the segment.
		 * force_destroy is only for export segments
		 */
		while (seg->s_flags & RSM_FORCE_DESTROY_WAIT) {
			cv_wait(&seg->s_cv, &seg->s_lock);
		}
	}
	rsmseglock_release(seg);

	/* It's ok to read the state without a lock */
	switch (seg->s_state) {
	case RSM_STATE_EXPORT:
	case RSM_STATE_EXPORT_QUIESCING:
	case RSM_STATE_EXPORT_QUIESCED:
		e = rsm_unpublish(seg, 1);
		/* FALLTHRU */
	case RSM_STATE_BIND_QUIESCED:
		/* FALLTHRU */
	case RSM_STATE_BIND:
		e = rsm_unbind(seg);
		if (e != RSM_SUCCESS && force_flag == 1)
			return;
		ASSERT(seg->s_hdr.rsmrc_type == RSM_RESOURCE_EXPORT_SEGMENT);
		/* FALLTHRU */
	case RSM_STATE_NEW_QUIESCED:
		rsmseglock_acquire(seg);
		seg->s_state = RSM_STATE_NEW;
		cv_broadcast(&seg->s_cv);
		rsmseglock_release(seg);
		break;
	case RSM_STATE_NEW:
		break;
	case RSM_STATE_ZOMBIE:
		/*
		 * Segments in this state have been removed off the
		 * exported segments list and have been unpublished
		 * and unbind. These segments have been removed during
		 * a callback to the rsm_export_force_destroy, which
		 * is called for the purpose of unlocking these
		 * exported memory segments when a process exits but
		 * leaves the segments locked down since rsm_close is
		 * is not called for the segments. This can happen
		 * when a process calls fork or exec and then exits.
		 * Once the segments are in the ZOMBIE state, all that
		 * remains is to destroy them when rsm_close is called.
		 * This is done here. Thus, for such segments the
		 * the state is changed to new so that later in this
		 * function rsmseg_free is called.
		 */
		rsmseglock_acquire(seg);
		seg->s_state = RSM_STATE_NEW;
		rsmseglock_release(seg);
		break;
	case RSM_STATE_MAP_QUIESCE:
	case RSM_STATE_ACTIVE:
		/* Disconnect will handle the unmap */
	case RSM_STATE_CONN_QUIESCE:
	case RSM_STATE_CONNECT:
	case RSM_STATE_DISCONNECT:
		ASSERT(seg->s_hdr.rsmrc_type == RSM_RESOURCE_IMPORT_SEGMENT);
		(void) rsm_disconnect(seg);
		break;
	case RSM_STATE_MAPPING:
		/*FALLTHRU*/
	case RSM_STATE_END:
		DBG_PRINTF((category, RSM_ERR,
		    "Invalid segment state %d in rsm_close\n", seg->s_state));
		break;
	default:
		DBG_PRINTF((category, RSM_ERR,
		    "Invalid segment state %d in rsm_close\n", seg->s_state));
		break;
	}

	/*
	 * check state.
	 * - make sure you do crfree(s_cred);
	 * release segment and minor number
	 */
	ASSERT(seg->s_state == RSM_STATE_NEW);

	/*
	 * The export_force_destroy callback is created to unlock
	 * the exported segments of a process
	 * when the process does a fork or exec and then exits calls this
	 * function with the force flag set to 1 which indicates that the
	 * segment state must be converted to ZOMBIE. This state means that the
	 * segments still exist and have been unlocked and most importantly the
	 * only operation allowed is to destroy them on an rsm_close.
	 */
	if (force_flag) {
		rsmseglock_acquire(seg);
		seg->s_state = RSM_STATE_ZOMBIE;
		rsmseglock_release(seg);
	} else {
		rsmseg_free(seg);
	}

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmseg_close done\n"));
}

static int
rsm_close(dev_t dev, int flag, int otyp, cred_t *cred)
{
	minor_t	rnum = getminor(dev);
	rsmresource_t *res;
	DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_FUNC_ALL| RSM_DDI);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_close enter\n"));

	flag = flag; cred = cred;

	if (otyp != OTYP_CHR)
		return (EINVAL);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rnum = %d\n", rnum));

	/*
	 * At this point we are the last reference to the resource.
	 * Free resource number from resource table.
	 * It's ok to remove number before we free the segment.
	 * We need to lock the resource to protect against remote calls.
	 */
	if (rnum == RSM_DRIVER_MINOR ||
	    (res = rsmresource_free(rnum)) == NULL) {
		DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_close done\n"));
		return (DDI_SUCCESS);
	}

	switch (res->rsmrc_type) {
	case RSM_RESOURCE_EXPORT_SEGMENT:
	case RSM_RESOURCE_IMPORT_SEGMENT:
		rsmseg_close((rsmseg_t *)res, 0);
		break;
	case RSM_RESOURCE_BAR:
		DBG_PRINTF((category, RSM_ERR, "bad resource in rsm_close\n"));
		break;
	default:
		break;
	}

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_close done\n"));

	return (DDI_SUCCESS);
}

/*
 * rsm_inc_pgcnt
 *
 * Description: increment rsm page counter.
 *
 * Parameters:	pgcnt_t	pnum;	number of pages to be used
 *
 * Returns:	RSM_SUCCESS	if memory limit not exceeded
 *		ENOSPC		if memory limit exceeded. In this case, the
 *				page counter remains unchanged.
 *
 */
static int
rsm_inc_pgcnt(pgcnt_t pnum)
{
	DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_EXPORT);
	if (rsm_pgcnt_max == 0) { /* no upper limit has been set */
		return (RSM_SUCCESS);
	}

	mutex_enter(&rsm_pgcnt_lock);

	if (rsm_pgcnt + pnum > rsm_pgcnt_max) {
		/* ensure that limits have not been exceeded */
		mutex_exit(&rsm_pgcnt_lock);
		return (RSMERR_INSUFFICIENT_MEM);
	}

	rsm_pgcnt += pnum;
	DBG_PRINTF((category, RSM_DEBUG, "rsm_pgcnt incr to %d.\n",
	    rsm_pgcnt));
	mutex_exit(&rsm_pgcnt_lock);

	return (RSM_SUCCESS);
}

/*
 * rsm_dec_pgcnt
 *
 * Description:	decrement rsm page counter.
 *
 * Parameters:	pgcnt_t	pnum;	number of pages freed
 *
 */
static void
rsm_dec_pgcnt(pgcnt_t pnum)
{
	DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_EXPORT);

	if (rsm_pgcnt_max == 0) { /* no upper limit has been set */
		return;
	}

	mutex_enter(&rsm_pgcnt_lock);
	ASSERT(rsm_pgcnt >= pnum);
	rsm_pgcnt -= pnum;
	DBG_PRINTF((category, RSM_DEBUG, "rsm_pgcnt decr to %d.\n",
	    rsm_pgcnt));
	mutex_exit(&rsm_pgcnt_lock);
}

static struct umem_callback_ops rsm_as_ops = {
	UMEM_CALLBACK_VERSION, /* version number */
	rsm_export_force_destroy,
};

static int
rsm_bind_pages(ddi_umem_cookie_t *cookie, caddr_t vaddr, size_t len,
    proc_t *procp)
{
	int error = RSM_SUCCESS;
	ulong_t pnum;
	struct umem_callback_ops *callbackops = &rsm_as_ops;

	DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_EXPORT);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_bind_pages enter\n"));

	/*
	 * Make sure vaddr and len are aligned on a page boundary
	 */
	if ((uintptr_t)vaddr & (PAGESIZE - 1)) {
		return (RSMERR_BAD_ADDR);
	}

	if (len & (PAGESIZE - 1)) {
		return (RSMERR_BAD_LENGTH);
	}

	/*
	 * Find number of pages
	 */
	pnum = btopr(len);
	error = rsm_inc_pgcnt(pnum);
	if (error != RSM_SUCCESS) {
		DBG_PRINTF((category, RSM_ERR,
		    "rsm_bind_pages:mem limit exceeded\n"));
		return (RSMERR_INSUFFICIENT_MEM);
	}

	error = umem_lockmemory(vaddr, len,
	    DDI_UMEMLOCK_WRITE|DDI_UMEMLOCK_READ|DDI_UMEMLOCK_LONGTERM,
	    cookie,
	    callbackops, procp);

	if (error) {
		rsm_dec_pgcnt(pnum);
		DBG_PRINTF((category, RSM_ERR,
		    "rsm_bind_pages:ddi_umem_lock failed\n"));
		/*
		 * ddi_umem_lock, in the case of failure, returns one of
		 * the following three errors. These are translated into
		 * the RSMERR namespace and returned.
		 */
		if (error == EFAULT)
			return (RSMERR_BAD_ADDR);
		else if (error == EACCES)
			return (RSMERR_PERM_DENIED);
		else
			return (RSMERR_INSUFFICIENT_MEM);
	}

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_bind_pages done\n"));

	return (error);

}

static int
rsm_unbind_pages(rsmseg_t *seg)
{
	DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_EXPORT);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_unbind_pages enter\n"));

	ASSERT(rsmseglock_held(seg));

	if (seg->s_cookie != NULL) {
		/* unlock address range */
		ddi_umem_unlock(seg->s_cookie);
		rsm_dec_pgcnt(btopr(seg->s_len));
		seg->s_cookie = NULL;
	}

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_unbind_pages done\n"));

	return (RSM_SUCCESS);
}


static int
rsm_bind(rsmseg_t *seg, rsm_ioctlmsg_t *msg, intptr_t dataptr, int mode)
{
	int e;
	adapter_t *adapter;
	DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_EXPORT);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_bind enter\n"));

	adapter = rsm_getadapter(msg, mode);
	if (adapter == NULL) {
		DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
		    "rsm_bind done:no adapter\n"));
		return (RSMERR_CTLR_NOT_PRESENT);
	}

	/* lock address range */
	if (msg->vaddr == NULL) {
		rsmka_release_adapter(adapter);
		DBG_PRINTF((category, RSM_ERR,
		    "rsm: rsm_bind done: invalid vaddr\n"));
		return (RSMERR_BAD_ADDR);
	}
	if (msg->len <= 0) {
		rsmka_release_adapter(adapter);
		DBG_PRINTF((category, RSM_ERR,
		    "rsm_bind: invalid length\n"));
		return (RSMERR_BAD_LENGTH);
	}

	/* Lock segment */
	rsmseglock_acquire(seg);

	while (seg->s_state == RSM_STATE_NEW_QUIESCED) {
		if (cv_wait_sig(&seg->s_cv, &seg->s_lock) == 0) {
			DBG_PRINTF((category, RSM_DEBUG,
			    "rsm_bind done: cv_wait INTERRUPTED"));
			rsmka_release_adapter(adapter);
			rsmseglock_release(seg);
			return (RSMERR_INTERRUPTED);
		}
	}

	ASSERT(seg->s_state == RSM_STATE_NEW);

	ASSERT(seg->s_cookie == NULL);

	e = rsm_bind_pages(&seg->s_cookie, msg->vaddr, msg->len, curproc);
	if (e == RSM_SUCCESS) {
		seg->s_flags |= RSM_USER_MEMORY;
		if (msg->perm & RSM_ALLOW_REBIND) {
			seg->s_flags |= RSMKA_ALLOW_UNBIND_REBIND;
		}
		if (msg->perm & RSM_CREATE_SEG_DONTWAIT) {
			seg->s_flags |= RSMKA_SET_RESOURCE_DONTWAIT;
		}
		seg->s_region.r_vaddr = msg->vaddr;
		/*
		 * Set the s_pid value in the segment structure. This is used
		 * to identify exported segments belonging to a particular
		 * process so that when the process exits, these segments can
		 * be unlocked forcefully even if rsm_close is not called on
		 * process exit since there maybe other processes referencing
		 * them (for example on a fork or exec).
		 * The s_pid value is also used to authenticate the process
		 * doing a publish or unpublish on the export segment. Only
		 * the creator of the export segment has a right to do a
		 * publish or unpublish and unbind on the segment.
		 */
		seg->s_pid = ddi_get_pid();
		seg->s_len = msg->len;
		seg->s_state = RSM_STATE_BIND;
		seg->s_adapter = adapter;
		seg->s_proc = curproc;
	} else {
		rsmka_release_adapter(adapter);
		DBG_PRINTF((category, RSM_WARNING,
		    "unable to lock down pages\n"));
	}

	msg->rnum = seg->s_minor;
	/* Unlock segment */
	rsmseglock_release(seg);

	if (e == RSM_SUCCESS) {
		/* copyout the resource number */
#ifdef _MULTI_DATAMODEL
		if ((mode & DATAMODEL_MASK) == DATAMODEL_ILP32) {
			rsm_ioctlmsg32_t msg32;

			msg32.rnum = msg->rnum;
			if (ddi_copyout((caddr_t)&msg32.rnum,
			    (caddr_t)&((rsm_ioctlmsg32_t *)dataptr)->rnum,
			    sizeof (minor_t), mode)) {
				rsmka_release_adapter(adapter);
				e = RSMERR_BAD_ADDR;
			}
		}
#endif
		if (ddi_copyout((caddr_t)&msg->rnum,
		    (caddr_t)&((rsm_ioctlmsg_t *)dataptr)->rnum,
		    sizeof (minor_t), mode)) {
			rsmka_release_adapter(adapter);
			e = RSMERR_BAD_ADDR;
		}
	}

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_bind done\n"));

	return (e);
}

static void
rsm_remap_local_importers(rsm_node_id_t src_nodeid,
    rsm_memseg_id_t ex_segid,
    ddi_umem_cookie_t cookie)

{
	rsmresource_t	*p = NULL;
	rsmhash_table_t *rhash = &rsm_import_segs;
	uint_t		index;

	DBG_PRINTF((RSM_KERNEL_AGENT | RSM_FUNC_ALL, RSM_DEBUG_VERBOSE,
	    "rsm_remap_local_importers enter\n"));

	index = rsmhash(ex_segid);

	rw_enter(&rhash->rsmhash_rw, RW_READER);

	p = rsmhash_getbkt(rhash, index);

	for (; p; p = p->rsmrc_next) {
		rsmseg_t *seg = (rsmseg_t *)p;
		rsmseglock_acquire(seg);
		/*
		 * Change the s_cookie value of only the local importers
		 * which have been mapped (in state RSM_STATE_ACTIVE).
		 * Note that there is no need to change the s_cookie value
		 * if the imported segment is in RSM_STATE_MAPPING since
		 * eventually the s_cookie will be updated via the mapping
		 * functionality.
		 */
		if ((seg->s_segid == ex_segid) && (seg->s_node == src_nodeid) &&
		    (seg->s_state == RSM_STATE_ACTIVE)) {
			seg->s_cookie = cookie;
		}
		rsmseglock_release(seg);
	}
	rw_exit(&rhash->rsmhash_rw);

	DBG_PRINTF((RSM_KERNEL_AGENT | RSM_FUNC_ALL, RSM_DEBUG_VERBOSE,
	    "rsm_remap_local_importers done\n"));
}

static int
rsm_rebind(rsmseg_t *seg, rsm_ioctlmsg_t *msg)
{
	int e;
	adapter_t *adapter;
	ddi_umem_cookie_t cookie;
	DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_EXPORT);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_rebind enter\n"));

	/* Check for permissions to rebind */
	if (!(seg->s_flags & RSMKA_ALLOW_UNBIND_REBIND)) {
		return (RSMERR_REBIND_NOT_ALLOWED);
	}

	if (seg->s_pid != ddi_get_pid() &&
	    ddi_get_pid() != 0) {
		DBG_PRINTF((category, RSM_ERR, "rsm_rebind: Not owner\n"));
		return (RSMERR_NOT_CREATOR);
	}

	/*
	 * We will not be allowing partial rebind and hence length passed
	 * in must be same as segment length
	 */
	if (msg->vaddr == NULL) {
		DBG_PRINTF((category, RSM_ERR,
		    "rsm_rebind done: null msg->vaddr\n"));
		return (RSMERR_BAD_ADDR);
	}
	if (msg->len != seg->s_len) {
		DBG_PRINTF((category, RSM_ERR,
		    "rsm_rebind: invalid length\n"));
		return (RSMERR_BAD_LENGTH);
	}

	/* Lock segment */
	rsmseglock_acquire(seg);

	while ((seg->s_state == RSM_STATE_BIND_QUIESCED) ||
	    (seg->s_state == RSM_STATE_EXPORT_QUIESCING) ||
	    (seg->s_state == RSM_STATE_EXPORT_QUIESCED)) {
		if (cv_wait_sig(&seg->s_cv, &seg->s_lock) == 0) {
			rsmseglock_release(seg);
			DBG_PRINTF((category, RSM_DEBUG,
			    "rsm_rebind done: cv_wait INTERRUPTED"));
			return (RSMERR_INTERRUPTED);
		}
	}

	/* verify segment state */
	if ((seg->s_state != RSM_STATE_BIND) &&
	    (seg->s_state != RSM_STATE_EXPORT)) {
		/* Unlock segment */
		rsmseglock_release(seg);
		DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
		    "rsm_rebind done: invalid state\n"));
		return (RSMERR_BAD_SEG_HNDL);
	}

	ASSERT(seg->s_cookie != NULL);

	if (msg->vaddr == seg->s_region.r_vaddr) {
		rsmseglock_release(seg);
		DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_rebind done\n"));
		return (RSM_SUCCESS);
	}

	e = rsm_bind_pages(&cookie, msg->vaddr, msg->len, curproc);
	if (e == RSM_SUCCESS) {
		struct buf *xbuf;
		dev_t sdev = 0;
		rsm_memory_local_t mem;

		xbuf = ddi_umem_iosetup(cookie, 0, msg->len, B_WRITE,
		    sdev, 0, NULL, DDI_UMEM_SLEEP);
		ASSERT(xbuf != NULL);

		mem.ms_type = RSM_MEM_BUF;
		mem.ms_bp = xbuf;

		adapter = seg->s_adapter;
		e = adapter->rsmpi_ops->rsm_rebind(
		    seg->s_handle.out, 0, &mem,
		    RSM_RESOURCE_DONTWAIT, NULL);

		if (e == RSM_SUCCESS) {
			/*
			 * unbind the older pages, and unload local importers;
			 * but don't disconnect importers
			 */
			(void) rsm_unbind_pages(seg);
			seg->s_cookie = cookie;
			seg->s_region.r_vaddr = msg->vaddr;
			rsm_remap_local_importers(my_nodeid, seg->s_segid,
			    cookie);
		} else {
			/*
			 * Unbind the pages associated with "cookie" by the
			 * rsm_bind_pages calls prior to this. This is
			 * similar to what is done in the rsm_unbind_pages
			 * routine for the seg->s_cookie.
			 */
			ddi_umem_unlock(cookie);
			rsm_dec_pgcnt(btopr(msg->len));
			DBG_PRINTF((category, RSM_ERR,
			    "rsm_rebind failed with %d\n", e));
		}
		/*
		 * At present there is no dependency on the existence of xbuf.
		 * So we can free it here. If in the future this changes, it can
		 * be freed sometime during the segment destroy.
		 */
		freerbuf(xbuf);
	}

	/* Unlock segment */
	rsmseglock_release(seg);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_rebind done\n"));

	return (e);
}

static int
rsm_unbind(rsmseg_t *seg)
{
	DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_EXPORT);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_unbind enter\n"));

	rsmseglock_acquire(seg);

	/* verify segment state */
	if ((seg->s_state != RSM_STATE_BIND) &&
	    (seg->s_state != RSM_STATE_BIND_QUIESCED)) {
		rsmseglock_release(seg);
		DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
		    "rsm_unbind: invalid state\n"));
		return (RSMERR_BAD_SEG_HNDL);
	}

	/* unlock current range */
	(void) rsm_unbind_pages(seg);

	if (seg->s_state == RSM_STATE_BIND) {
		seg->s_state = RSM_STATE_NEW;
	} else if (seg->s_state == RSM_STATE_BIND_QUIESCED) {
		seg->s_state = RSM_STATE_NEW_QUIESCED;
	}

	rsmseglock_release(seg);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_unbind done\n"));

	return (RSM_SUCCESS);
}

/* **************************** Exporter Access List Management ******* */
static void
rsmacl_free(rsmapi_access_entry_t *acl, int acl_len)
{
	int	acl_sz;
	DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_EXPORT);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmacl_free enter\n"));

	/* acl could be NULL */

	if (acl != NULL && acl_len > 0) {
		acl_sz = acl_len * sizeof (rsmapi_access_entry_t);
		kmem_free((void *)acl, acl_sz);
	}

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmacl_free done\n"));
}

static void
rsmpiacl_free(rsm_access_entry_t *acl, int acl_len)
{
	int	acl_sz;
	DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_EXPORT);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmpiacl_free enter\n"));

	if (acl != NULL && acl_len > 0) {
		acl_sz = acl_len * sizeof (rsm_access_entry_t);
		kmem_free((void *)acl, acl_sz);
	}

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmpiacl_free done\n"));

}

static int
rsmacl_build(rsm_ioctlmsg_t *msg, int mode,
    rsmapi_access_entry_t **list, int *len, int loopback)
{
	rsmapi_access_entry_t *acl;
	int	acl_len;
	int i;
	DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_EXPORT);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmacl_build enter\n"));

	*len = 0;
	*list = NULL;

	acl_len = msg->acl_len;
	if ((loopback && acl_len > 1) || (acl_len < 0) ||
	    (acl_len > MAX_NODES)) {
		DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
		    "rsmacl_build done: acl invalid\n"));
		return (RSMERR_BAD_ACL);
	}

	if (acl_len > 0 && acl_len <= MAX_NODES) {
		size_t acl_size = acl_len * sizeof (rsmapi_access_entry_t);

		acl = kmem_alloc(acl_size, KM_SLEEP);

		if (ddi_copyin((caddr_t)msg->acl, (caddr_t)acl,
		    acl_size, mode)) {
			kmem_free((void *) acl, acl_size);
			DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
			    "rsmacl_build done: BAD_ADDR\n"));
			return (RSMERR_BAD_ADDR);
		}

		/*
		 * Verify access list
		 */
		for (i = 0; i < acl_len; i++) {
			if (acl[i].ae_node > MAX_NODES ||
			    (loopback && (acl[i].ae_node != my_nodeid)) ||
			    acl[i].ae_permission > RSM_ACCESS_TRUSTED) {
				/* invalid entry */
				kmem_free((void *) acl, acl_size);
				DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
				    "rsmacl_build done: EINVAL\n"));
				return (RSMERR_BAD_ACL);
			}
		}

		*len = acl_len;
		*list = acl;
	}

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmacl_build done\n"));

	return (DDI_SUCCESS);
}

static int
rsmpiacl_create(rsmapi_access_entry_t *src, rsm_access_entry_t **dest,
    int acl_len, adapter_t *adapter)
{
	rsm_access_entry_t *acl;
	rsm_addr_t hwaddr;
	int i;
	DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_EXPORT);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmpiacl_create enter\n"));

	if (src != NULL) {
		size_t acl_size = acl_len * sizeof (rsm_access_entry_t);
		acl = kmem_alloc(acl_size, KM_SLEEP);

		/*
		 * translate access list
		 */
		for (i = 0; i < acl_len; i++) {
			if (src[i].ae_node == my_nodeid) {
				acl[i].ae_addr = adapter->hwaddr;
			} else {
				hwaddr = get_remote_hwaddr(adapter,
				    src[i].ae_node);
				if ((int64_t)hwaddr < 0) {
					/* invalid hwaddr */
					kmem_free((void *) acl, acl_size);
					DBG_PRINTF((category,
					    RSM_DEBUG_VERBOSE,
					    "rsmpiacl_create done:"
					    "EINVAL hwaddr\n"));
					return (RSMERR_INTERNAL_ERROR);
				}
				acl[i].ae_addr = hwaddr;
			}
			/* rsmpi understands only RSM_PERM_XXXX */
			acl[i].ae_permission =
			    src[i].ae_permission & RSM_PERM_RDWR;
		}
		*dest = acl;
	} else {
		*dest = NULL;
	}

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmpiacl_create done\n"));

	return (RSM_SUCCESS);
}

static int
rsmsegacl_validate(rsmipc_request_t *req, rsm_node_id_t rnode,
    rsmipc_reply_t *reply)
{

	int		i;
	rsmseg_t	*seg;
	rsm_memseg_id_t key = req->rsmipc_key;
	rsm_permission_t perm = req->rsmipc_perm;
	DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_EXPORT);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
	    "rsmsegacl_validate enter\n"));

	/*
	 * Find segment and grab its lock. The reason why we grab the segment
	 * lock in side the search is to avoid the race when the segment is
	 * being deleted and we already have a pointer to it.
	 */
	seg = rsmexport_lookup(key);
	if (!seg) {
		DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
		    "rsmsegacl_validate done: %u ENXIO\n", key));
		return (RSMERR_SEG_NOT_PUBLISHED);
	}

	ASSERT(rsmseglock_held(seg));
	ASSERT(seg->s_state == RSM_STATE_EXPORT);

	/*
	 * We implement a 2-level protection scheme.
	 * First, we check if local/remote host has access rights.
	 * Second, we check if the user has access rights.
	 *
	 * This routine only validates the rnode access_list
	 */
	if (seg->s_acl_len > 0) {
		/*
		 * Check host access list
		 */
		ASSERT(seg->s_acl != NULL);
		for (i = 0; i < seg->s_acl_len; i++) {
			if (seg->s_acl[i].ae_node == rnode) {
				perm &= seg->s_acl[i].ae_permission;
				goto found;
			}
		}
		/* rnode is not found in the list */
		rsmseglock_release(seg);
		DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
		    "rsmsegacl_validate done: EPERM\n"));
		return (RSMERR_SEG_NOT_PUBLISHED_TO_NODE);
	} else {
		/* use default owner creation umask */
		perm &= seg->s_mode;
	}

found:
	/* update perm for this node */
	reply->rsmipc_mode = perm;
	reply->rsmipc_uid = seg->s_uid;
	reply->rsmipc_gid = seg->s_gid;
	reply->rsmipc_segid = seg->s_segid;
	reply->rsmipc_seglen = seg->s_len;

	/*
	 * Perm of requesting node is valid; source will validate user
	 */
	rsmseglock_release(seg);

	/*
	 * Add the importer to the list right away, if connect fails
	 * the importer will ask the exporter to remove it.
	 */
	importer_list_add(rnode, key, req->rsmipc_adapter_hwaddr,
	    req->rsmipc_segment_cookie);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmsegacl_validate done\n"));

	return (RSM_SUCCESS);
}


/* ************************** Exporter Calls ************************* */

static int
rsm_publish(rsmseg_t *seg, rsm_ioctlmsg_t *msg, intptr_t dataptr, int mode)
{
	int			e;
	int			acl_len;
	rsmapi_access_entry_t	*acl;
	rsm_access_entry_t	*rsmpi_acl;
	rsm_memory_local_t	mem;
	struct buf		*xbuf;
	dev_t 			sdev = 0;
	adapter_t		*adapter;
	rsm_memseg_id_t		segment_id = 0;
	int			loopback_flag = 0;
	int			create_flags = 0;
	rsm_resource_callback_t	callback_flag;
	DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_EXPORT);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_publish enter\n"));

	if (seg->s_adapter == &loopback_adapter)
		loopback_flag = 1;

	if (seg->s_pid != ddi_get_pid() &&
	    ddi_get_pid() != 0) {
		DBG_PRINTF((category, RSM_ERR,
		    "rsm_publish: Not creator\n"));
		return (RSMERR_NOT_CREATOR);
	}

	/*
	 * Get per node access list
	 */
	e = rsmacl_build(msg, mode, &acl, &acl_len, loopback_flag);
	if (e != DDI_SUCCESS) {
		DBG_PRINTF((category, RSM_ERR,
		    "rsm_publish done: rsmacl_build failed\n"));
		return (e);
	}

	/*
	 * The application provided msg->key is used for resolving a
	 * segment id according to the following:
	 *    key = 0   		Kernel Agent selects the segment id
	 *    key <= RSM_DLPI_ID_END	Reserved for system usage except
	 *				RSMLIB range
	 *    key < RSM_USER_APP_ID_BASE segment id = key
	 *    key >= RSM_USER_APP_ID_BASE Reserved for KA selections
	 *
	 * rsm_nextavail_segmentid is initialized to 0x80000000 and
	 * overflows to zero after 0x80000000 allocations.
	 * An algorithm is needed which allows reinitialization and provides
	 * for reallocation after overflow.  For now, ENOMEM is returned
	 * once the overflow condition has occurred.
	 */
	if (msg->key == 0) {
		mutex_enter(&rsm_lock);
		segment_id = rsm_nextavail_segmentid;
		if (segment_id != 0) {
			rsm_nextavail_segmentid++;
			mutex_exit(&rsm_lock);
		} else {
			mutex_exit(&rsm_lock);
			DBG_PRINTF((category, RSM_ERR,
			    "rsm_publish done: no more keys avlbl\n"));
			return (RSMERR_INSUFFICIENT_RESOURCES);
		}
	} else	if BETWEEN(msg->key, RSM_RSMLIB_ID_BASE, RSM_RSMLIB_ID_END)
		/* range reserved for internal use by base/ndi libraries */
		segment_id = msg->key;
	else	if (msg->key <= RSM_DLPI_ID_END)
		return (RSMERR_RESERVED_SEGID);
	else if (msg->key <= (uint_t)RSM_USER_APP_ID_BASE -1)
		segment_id = msg->key;
	else {
		DBG_PRINTF((category, RSM_ERR,
		    "rsm_publish done: invalid key %u\n", msg->key));
		return (RSMERR_RESERVED_SEGID);
	}

	/* Add key to exportlist; The segment lock is held on success */
	e = rsmexport_add(seg, segment_id);
	if (e) {
		rsmacl_free(acl, acl_len);
		DBG_PRINTF((category, RSM_ERR,
		    "rsm_publish done: export_add failed: %d\n", e));
		return (e);
	}

	seg->s_segid = segment_id;

	if ((seg->s_state != RSM_STATE_BIND) &&
	    (seg->s_state != RSM_STATE_BIND_QUIESCED)) {
		/* state changed since then, free acl and return */
		rsmseglock_release(seg);
		rsmexport_rm(seg);
		rsmacl_free(acl, acl_len);
		DBG_PRINTF((category, RSM_ERR,
		    "rsm_publish done: segment in wrong state: %d\n",
		    seg->s_state));
		return (RSMERR_BAD_SEG_HNDL);
	}

	/*
	 * If this is for a local memory handle and permissions are zero,
	 * then the surrogate segment is very large and we want to skip
	 * allocation of DVMA space.
	 *
	 * Careful!  If the user didn't use an ACL list, acl will be a NULL
	 * pointer.  Check that before dereferencing it.
	 */
	if (acl != (rsmapi_access_entry_t *)NULL) {
		if (acl[0].ae_node == my_nodeid && acl[0].ae_permission == 0)
			goto skipdriver;
	}

	/* create segment  */
	xbuf = ddi_umem_iosetup(seg->s_cookie, 0, seg->s_len, B_WRITE,
	    sdev, 0, NULL, DDI_UMEM_SLEEP);
	ASSERT(xbuf != NULL);

	mem.ms_type = RSM_MEM_BUF;
	mem.ms_bp = xbuf;

	/* This call includes a bind operations */

	adapter = seg->s_adapter;
	/*
	 * create a acl list with hwaddr for RSMPI publish
	 */
	e = rsmpiacl_create(acl, &rsmpi_acl, acl_len, adapter);

	if (e != RSM_SUCCESS) {
		rsmseglock_release(seg);
		rsmexport_rm(seg);
		rsmacl_free(acl, acl_len);
		freerbuf(xbuf);
		DBG_PRINTF((category, RSM_ERR,
		    "rsm_publish done: rsmpiacl_create failed: %d\n", e));
		return (e);
	}

	if (seg->s_state == RSM_STATE_BIND) {
		/* create segment  */

		/* This call includes a bind operations */

		if (seg->s_flags & RSMKA_ALLOW_UNBIND_REBIND) {
			create_flags = RSM_ALLOW_UNBIND_REBIND;
		}

		if (seg->s_flags & RSMKA_SET_RESOURCE_DONTWAIT) {
			callback_flag  = RSM_RESOURCE_DONTWAIT;
		} else {
			callback_flag  = RSM_RESOURCE_SLEEP;
		}

		e = adapter->rsmpi_ops->rsm_seg_create(
		    adapter->rsmpi_handle,
		    &seg->s_handle.out, seg->s_len,
		    create_flags, &mem,
		    callback_flag, NULL);
		/*
		 * At present there is no dependency on the existence of xbuf.
		 * So we can free it here. If in the future this changes, it can
		 * be freed sometime during the segment destroy.
		 */
		freerbuf(xbuf);

		if (e != RSM_SUCCESS) {
			rsmseglock_release(seg);
			rsmexport_rm(seg);
			rsmacl_free(acl, acl_len);
			rsmpiacl_free(rsmpi_acl, acl_len);
			DBG_PRINTF((category, RSM_ERR,
			    "rsm_publish done: export_create failed: %d\n", e));
			/*
			 * The following assertion ensures that the two errors
			 * related to the length and its alignment do not occur
			 * since they have been checked during export_create
			 */
			ASSERT(e != RSMERR_BAD_MEM_ALIGNMENT &&
			    e != RSMERR_BAD_LENGTH);
			if (e == RSMERR_NOT_MEM)
				e = RSMERR_INSUFFICIENT_MEM;

			return (e);
		}
		/* export segment, this should create an IMMU mapping */
		e = adapter->rsmpi_ops->rsm_publish(
		    seg->s_handle.out,
		    rsmpi_acl, acl_len,
		    seg->s_segid,
		    RSM_RESOURCE_DONTWAIT, NULL);

		if (e != RSM_SUCCESS) {
			adapter->rsmpi_ops->rsm_seg_destroy(seg->s_handle.out);
			rsmseglock_release(seg);
			rsmexport_rm(seg);
			rsmacl_free(acl, acl_len);
			rsmpiacl_free(rsmpi_acl, acl_len);
			DBG_PRINTF((category, RSM_ERR,
			    "rsm_publish done: export_publish failed: %d\n",
			    e));
			return (e);
		}
	}

	seg->s_acl_in = rsmpi_acl;

skipdriver:
	/* defer s_acl/s_acl_len -> avoid crash in rsmseg_free */
	seg->s_acl_len	= acl_len;
	seg->s_acl	= acl;

	if (seg->s_state == RSM_STATE_BIND) {
		seg->s_state = RSM_STATE_EXPORT;
	} else if (seg->s_state == RSM_STATE_BIND_QUIESCED) {
		seg->s_state = RSM_STATE_EXPORT_QUIESCED;
		cv_broadcast(&seg->s_cv);
	}

	rsmseglock_release(seg);

	/*
	 * If the segment id was solicited, then return it in
	 * the original incoming message.
	 */
	if (msg->key == 0) {
		msg->key = segment_id;
#ifdef _MULTI_DATAMODEL
		if ((mode & DATAMODEL_MASK) == DATAMODEL_ILP32) {
			rsm_ioctlmsg32_t msg32;

			msg32.key = msg->key;
			DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
			    "rsm_publish done\n"));
			return (ddi_copyout((caddr_t)&msg32,
			    (caddr_t)dataptr, sizeof (msg32), mode));
		}
#endif
		DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
		    "rsm_publish done\n"));
		return (ddi_copyout((caddr_t)msg,
		    (caddr_t)dataptr, sizeof (*msg), mode));
	}

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_publish done\n"));
	return (DDI_SUCCESS);
}

/*
 * This function modifies the access control list of an already published
 * segment.  There is no effect on import segments which are already
 * connected.
 */
static int
rsm_republish(rsmseg_t *seg, rsm_ioctlmsg_t *msg, int mode)
{
	rsmapi_access_entry_t	*new_acl, *old_acl, *tmp_acl;
	rsm_access_entry_t	*rsmpi_new_acl, *rsmpi_old_acl;
	int			new_acl_len, old_acl_len, tmp_acl_len;
	int			e, i;
	adapter_t		*adapter;
	int			loopback_flag = 0;
	rsm_memseg_id_t		key;
	rsm_permission_t	permission;
	DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_EXPORT);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_republish enter\n"));

	if ((seg->s_state != RSM_STATE_EXPORT) &&
	    (seg->s_state != RSM_STATE_EXPORT_QUIESCED) &&
	    (seg->s_state != RSM_STATE_EXPORT_QUIESCING))
		return (RSMERR_SEG_NOT_PUBLISHED);

	if (seg->s_pid != ddi_get_pid() &&
	    ddi_get_pid() != 0) {
		DBG_PRINTF((category, RSM_ERR,
		    "rsm_republish: Not owner\n"));
		return (RSMERR_NOT_CREATOR);
	}

	if (seg->s_adapter == &loopback_adapter)
		loopback_flag = 1;

	/*
	 * Build new list first
	 */
	e = rsmacl_build(msg, mode, &new_acl, &new_acl_len, loopback_flag);
	if (e) {
		DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
		    "rsm_republish done: rsmacl_build failed %d", e));
		return (e);
	}

	/* Lock segment */
	rsmseglock_acquire(seg);
	/*
	 * a republish is in progress - REPUBLISH message is being
	 * sent to the importers so wait for it to complete OR
	 * wait till DR completes
	 */
	while (((seg->s_state == RSM_STATE_EXPORT) &&
	    (seg->s_flags & RSM_REPUBLISH_WAIT)) ||
	    (seg->s_state == RSM_STATE_EXPORT_QUIESCED) ||
	    (seg->s_state == RSM_STATE_EXPORT_QUIESCING)) {
		if (cv_wait_sig(&seg->s_cv, &seg->s_lock) == 0) {
			DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
			    "rsm_republish done: cv_wait  INTERRUPTED"));
			rsmseglock_release(seg);
			rsmacl_free(new_acl, new_acl_len);
			return (RSMERR_INTERRUPTED);
		}
	}

	/* recheck if state is valid */
	if (seg->s_state != RSM_STATE_EXPORT) {
		rsmseglock_release(seg);
		rsmacl_free(new_acl, new_acl_len);
		return (RSMERR_SEG_NOT_PUBLISHED);
	}

	key = seg->s_key;
	old_acl = seg->s_acl;
	old_acl_len = seg->s_acl_len;

	seg->s_acl = new_acl;
	seg->s_acl_len = new_acl_len;

	/*
	 * This call will only be meaningful if and when the interconnect
	 * layer makes use of the access list
	 */
	adapter = seg->s_adapter;
	/*
	 * create a acl list with hwaddr for RSMPI publish
	 */
	e = rsmpiacl_create(new_acl, &rsmpi_new_acl, new_acl_len, adapter);

	if (e != RSM_SUCCESS) {
		seg->s_acl = old_acl;
		seg->s_acl_len = old_acl_len;
		rsmseglock_release(seg);
		rsmacl_free(new_acl, new_acl_len);
		DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
		    "rsm_republish done: rsmpiacl_create failed %d", e));
		return (e);
	}
	rsmpi_old_acl = seg->s_acl_in;
	seg->s_acl_in = rsmpi_new_acl;

	e = adapter->rsmpi_ops->rsm_republish(seg->s_handle.out,
	    seg->s_acl_in, seg->s_acl_len,
	    RSM_RESOURCE_DONTWAIT, NULL);

	if (e != RSM_SUCCESS) {
		seg->s_acl = old_acl;
		seg->s_acl_in = rsmpi_old_acl;
		seg->s_acl_len = old_acl_len;
		rsmseglock_release(seg);
		rsmacl_free(new_acl, new_acl_len);
		rsmpiacl_free(rsmpi_new_acl, new_acl_len);

		DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
		    "rsm_republish done: rsmpi republish failed %d\n", e));
		return (e);
	}

	/* create a tmp copy of the new acl */
	tmp_acl_len = new_acl_len;
	if (tmp_acl_len > 0) {
		tmp_acl = kmem_zalloc(new_acl_len*sizeof (*tmp_acl), KM_SLEEP);
		for (i = 0; i < tmp_acl_len; i++) {
			tmp_acl[i].ae_node = new_acl[i].ae_node;
			tmp_acl[i].ae_permission = new_acl[i].ae_permission;
		}
		/*
		 * The default permission of a node which was in the old
		 * ACL but not in the new ACL is 0 ie no access.
		 */
		permission = 0;
	} else {
		/*
		 * NULL acl means all importers can connect and
		 * default permission will be owner creation umask
		 */
		tmp_acl = NULL;
		permission = seg->s_mode;
	}

	/* make other republishers to wait for republish to complete */
	seg->s_flags |= RSM_REPUBLISH_WAIT;

	rsmseglock_release(seg);

	/* send the new perms to the importing nodes */
	rsm_send_republish(key, tmp_acl, tmp_acl_len, permission);

	rsmseglock_acquire(seg);
	seg->s_flags &= ~RSM_REPUBLISH_WAIT;
	/* wake up any one waiting for republish to complete */
	cv_broadcast(&seg->s_cv);
	rsmseglock_release(seg);

	rsmacl_free(tmp_acl, tmp_acl_len);
	rsmacl_free(old_acl, old_acl_len);
	rsmpiacl_free(rsmpi_old_acl, old_acl_len);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_republish done\n"));
	return (DDI_SUCCESS);
}

static int
rsm_unpublish(rsmseg_t *seg, int mode)
{
	rsmapi_access_entry_t	*acl;
	rsm_access_entry_t	*rsmpi_acl;
	int			acl_len;
	int			e;
	adapter_t *adapter;
	DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_EXPORT);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_unpublish enter\n"));

	if (seg->s_pid != ddi_get_pid() &&
	    ddi_get_pid() != 0) {
		DBG_PRINTF((category, RSM_ERR,
		    "rsm_unpublish: Not creator\n"));
		return (RSMERR_NOT_CREATOR);
	}

	rsmseglock_acquire(seg);
	/*
	 * wait for QUIESCING to complete here before rsmexport_rm
	 * is called because the SUSPEND_COMPLETE mesg which changes
	 * the seg state from EXPORT_QUIESCING to EXPORT_QUIESCED and
	 * signals the cv_wait needs to find it in the hashtable.
	 */
	while ((seg->s_state == RSM_STATE_EXPORT_QUIESCING) ||
	    ((seg->s_state == RSM_STATE_EXPORT) && (seg->s_rdmacnt > 0))) {
		if (cv_wait_sig(&seg->s_cv, &seg->s_lock) == 0) {
			rsmseglock_release(seg);
			DBG_PRINTF((category, RSM_ERR,
			    "rsm_unpublish done: cv_wait INTR qscing"
			    "getv/putv in progress"));
			return (RSMERR_INTERRUPTED);
		}
	}

	/* verify segment state */
	if ((seg->s_state != RSM_STATE_EXPORT) &&
	    (seg->s_state != RSM_STATE_EXPORT_QUIESCED)) {
		rsmseglock_release(seg);
		DBG_PRINTF((category, RSM_ERR,
		    "rsm_unpublish done: bad state %x\n", seg->s_state));
		return (RSMERR_SEG_NOT_PUBLISHED);
	}

	rsmseglock_release(seg);

	rsmexport_rm(seg);

	rsm_send_importer_disconnects(seg->s_segid, my_nodeid);

	rsmseglock_acquire(seg);
	/*
	 * wait for republish to complete
	 */
	while ((seg->s_state == RSM_STATE_EXPORT) &&
	    (seg->s_flags & RSM_REPUBLISH_WAIT)) {
		if (cv_wait_sig(&seg->s_cv, &seg->s_lock) == 0) {
			DBG_PRINTF((category, RSM_ERR,
			    "rsm_unpublish done: cv_wait INTR repubing"));
			rsmseglock_release(seg);
			return (RSMERR_INTERRUPTED);
		}
	}

	if ((seg->s_state != RSM_STATE_EXPORT) &&
	    (seg->s_state != RSM_STATE_EXPORT_QUIESCED)) {
		DBG_PRINTF((category, RSM_ERR,
		    "rsm_unpublish done: invalid state"));
		rsmseglock_release(seg);
		return (RSMERR_SEG_NOT_PUBLISHED);
	}

	/*
	 * check for putv/get surrogate segment which was not published
	 * to the driver.
	 *
	 * Be certain to see if there is an ACL first!  If this segment was
	 * not published with an ACL, acl will be a null pointer.  Check
	 * that before dereferencing it.
	 */
	acl = seg->s_acl;
	if (acl != (rsmapi_access_entry_t *)NULL) {
		if (acl[0].ae_node == my_nodeid && acl[0].ae_permission == 0)
			goto bypass;
	}

	/* The RSMPI unpublish/destroy has been done if seg is QUIESCED */
	if (seg->s_state == RSM_STATE_EXPORT_QUIESCED)
		goto bypass;

	adapter = seg->s_adapter;
	for (;;) {
		if (seg->s_state != RSM_STATE_EXPORT) {
			rsmseglock_release(seg);
			DBG_PRINTF((category, RSM_ERR,
			    "rsm_unpublish done: bad state %x\n",
			    seg->s_state));
			return (RSMERR_SEG_NOT_PUBLISHED);
		}

		/* unpublish from adapter */
		e = adapter->rsmpi_ops->rsm_unpublish(seg->s_handle.out);

		if (e == RSM_SUCCESS) {
			break;
		}

		if (e == RSMERR_SEG_IN_USE && mode == 1) {
			/*
			 * wait for unpublish to succeed, it's busy.
			 */
			seg->s_flags |= RSM_EXPORT_WAIT;

			/* wait for a max of 1 ms - this is an empirical */
			/* value that was found by some minimal testing  */
			/* can be fine tuned when we have better numbers */
			/* A long term fix would be to send cv_signal	 */
			/* from the intr callback routine		 */
			/* currently nobody signals this wait		 */
			(void) cv_reltimedwait(&seg->s_cv, &seg->s_lock,
			    drv_usectohz(1000), TR_CLOCK_TICK);

			DBG_PRINTF((category, RSM_ERR,
			    "rsm_unpublish: SEG_IN_USE\n"));

			seg->s_flags &= ~RSM_EXPORT_WAIT;
		} else {
			if (mode == 1) {
				DBG_PRINTF((category, RSM_ERR,
				    "rsm:rsmpi unpublish err %x\n", e));
				seg->s_state = RSM_STATE_BIND;
			}
			rsmseglock_release(seg);
			return (e);
		}
	}

	/* Free segment */
	e = adapter->rsmpi_ops->rsm_seg_destroy(seg->s_handle.out);

	if (e != RSM_SUCCESS) {
		DBG_PRINTF((category, RSM_ERR,
		    "rsm_unpublish: rsmpi destroy key=%x failed %x\n",
		    seg->s_key, e));
	}

bypass:
	acl = seg->s_acl;
	rsmpi_acl = seg->s_acl_in;
	acl_len = seg->s_acl_len;

	seg->s_acl = NULL;
	seg->s_acl_in = NULL;
	seg->s_acl_len = 0;

	if (seg->s_state == RSM_STATE_EXPORT) {
		seg->s_state = RSM_STATE_BIND;
	} else if (seg->s_state == RSM_STATE_EXPORT_QUIESCED) {
		seg->s_state = RSM_STATE_BIND_QUIESCED;
		cv_broadcast(&seg->s_cv);
	}

	rsmseglock_release(seg);

	rsmacl_free(acl, acl_len);
	rsmpiacl_free(rsmpi_acl, acl_len);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_unpublish done\n"));

	return (DDI_SUCCESS);
}

/*
 * Called from rsm_unpublish to force an unload and disconnection of all
 * importers of the unpublished segment.
 *
 * First build the list of segments requiring a force disconnect, then
 * send a request for each.
 */
static void
rsm_send_importer_disconnects(rsm_memseg_id_t ex_segid,
    rsm_node_id_t ex_nodeid)
{
	rsmipc_request_t 	request;
	importing_token_t	*prev_token, *token, *tmp_token, *tokp;
	importing_token_t	*force_disconnect_list = NULL;
	int			index;

	DBG_PRINTF((RSM_KERNEL_AGENT | RSM_EXPORT, RSM_DEBUG_VERBOSE,
	    "rsm_send_importer_disconnects enter\n"));

	index = rsmhash(ex_segid);

	mutex_enter(&importer_list.lock);

	prev_token = NULL;
	token = importer_list.bucket[index];

	while (token != NULL) {
		if (token->key == ex_segid) {
			/*
			 * take it off the importer list and add it
			 * to the force disconnect list.
			 */
			if (prev_token == NULL)
				importer_list.bucket[index] = token->next;
			else
				prev_token->next = token->next;
			tmp_token = token;
			token = token->next;
			if (force_disconnect_list == NULL) {
				force_disconnect_list = tmp_token;
				tmp_token->next = NULL;
			} else {
				tokp = force_disconnect_list;
				/*
				 * make sure that the tmp_token's node
				 * is not already on the force disconnect
				 * list.
				 */
				while (tokp != NULL) {
					if (tokp->importing_node ==
					    tmp_token->importing_node) {
						break;
					}
					tokp = tokp->next;
				}
				if (tokp == NULL) {
					tmp_token->next =
					    force_disconnect_list;
					force_disconnect_list = tmp_token;
				} else {
					kmem_free((void *)tmp_token,
					    sizeof (*token));
				}
			}

		} else {
			prev_token = token;
			token = token->next;
		}
	}
	mutex_exit(&importer_list.lock);

	token = force_disconnect_list;
	while (token != NULL) {
		if (token->importing_node == my_nodeid) {
			rsm_force_unload(ex_nodeid, ex_segid,
			    DISCONNECT);
		} else {
			request.rsmipc_hdr.rsmipc_type =
			    RSMIPC_MSG_DISCONNECT;
			request.rsmipc_key = token->key;
			for (;;) {
				if (rsmipc_send(token->importing_node,
				    &request,
				    RSM_NO_REPLY) == RSM_SUCCESS) {
					break;
				} else {
					delay(drv_usectohz(10000));
				}
			}
		}
		tmp_token = token;
		token = token->next;
		kmem_free((void *)tmp_token, sizeof (*token));
	}

	DBG_PRINTF((RSM_KERNEL_AGENT | RSM_EXPORT, RSM_DEBUG_VERBOSE,
	    "rsm_send_importer_disconnects done\n"));
}

/*
 * This function is used as a callback for unlocking the pages locked
 * down by a process which then does a fork or an exec.
 * It marks the export segments corresponding to umem cookie given by
 * the *arg to be in a ZOMBIE state(by calling rsmseg_close to be
 * destroyed later when an rsm_close occurs).
 */
static void
rsm_export_force_destroy(ddi_umem_cookie_t *ck)
{
	rsmresource_blk_t *blk;
	rsmresource_t *p;
	rsmseg_t *eseg = NULL;
	int i, j;
	int found = 0;

	DBG_PRINTF((RSM_KERNEL_AGENT | RSM_EXPORT, RSM_DEBUG_VERBOSE,
	    "rsm_export_force_destroy enter\n"));

	/*
	 * Walk the resource list and locate the export segment (either
	 * in the BIND or the EXPORT state) which corresponds to the
	 * ddi_umem_cookie_t being freed up, and call rsmseg_close.
	 * Change the state to ZOMBIE by calling rsmseg_close with the
	 * force_flag argument (the second argument) set to 1. Also,
	 * unpublish and unbind the segment, but don't free it. Free it
	 * only on a rsm_close call for the segment.
	 */
	rw_enter(&rsm_resource.rsmrc_lock, RW_READER);

	for (i = 0; i < rsm_resource.rsmrc_len; i++) {
		blk = rsm_resource.rsmrc_root[i];
		if (blk == NULL) {
			continue;
		}

		for (j = 0; j < RSMRC_BLKSZ; j++) {
			p = blk->rsmrcblk_blks[j];
			if ((p != NULL) && (p != RSMRC_RESERVED) &&
			    (p->rsmrc_type == RSM_RESOURCE_EXPORT_SEGMENT)) {
				eseg = (rsmseg_t *)p;
				if (eseg->s_cookie != ck)
					continue; /* continue searching */
				/*
				 * Found the segment, set flag to indicate
				 * force destroy processing is in progress
				 */
				rsmseglock_acquire(eseg);
				eseg->s_flags |= RSM_FORCE_DESTROY_WAIT;
				rsmseglock_release(eseg);
				found = 1;
				break;
			}
		}

		if (found)
			break;
	}

	rw_exit(&rsm_resource.rsmrc_lock);

	if (found) {
		ASSERT(eseg != NULL);
		/* call rsmseg_close with force flag set to 1 */
		rsmseg_close(eseg, 1);
		/*
		 * force destroy processing done, clear flag and signal any
		 * thread waiting in rsmseg_close.
		 */
		rsmseglock_acquire(eseg);
		eseg->s_flags &= ~RSM_FORCE_DESTROY_WAIT;
		cv_broadcast(&eseg->s_cv);
		rsmseglock_release(eseg);
	}

	DBG_PRINTF((RSM_KERNEL_AGENT | RSM_EXPORT, RSM_DEBUG_VERBOSE,
	    "rsm_export_force_destroy done\n"));
}

/* ******************************* Remote Calls *********************** */
static void
rsm_intr_segconnect(rsm_node_id_t src, rsmipc_request_t *req)
{
	rsmipc_reply_t reply;
	DBG_DEFINE(category,
	    RSM_KERNEL_AGENT | RSM_FUNC_ALL | RSM_INTR_CALLBACK);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
	    "rsm_intr_segconnect enter\n"));

	reply.rsmipc_status = (short)rsmsegacl_validate(req, src, &reply);

	reply.rsmipc_hdr.rsmipc_type = RSMIPC_MSG_REPLY;
	reply.rsmipc_hdr.rsmipc_cookie = req->rsmipc_hdr.rsmipc_cookie;

	(void) rsmipc_send(src, NULL, &reply);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
	    "rsm_intr_segconnect done\n"));
}


/*
 * When an exported segment is unpublished the exporter sends an ipc
 * message (RSMIPC_MSG_DISCONNECT) to all importers.  The recv ipc dispatcher
 * calls this function.  The import list is scanned; segments which match the
 * exported segment id are unloaded and disconnected.
 *
 * Will also be called from rsm_rebind with disconnect_flag FALSE.
 *
 */
static void
rsm_force_unload(rsm_node_id_t src_nodeid,
    rsm_memseg_id_t ex_segid,
    boolean_t disconnect_flag)

{
	rsmresource_t	*p = NULL;
	rsmhash_table_t *rhash = &rsm_import_segs;
	uint_t		index;
	DBG_DEFINE(category,
	    RSM_KERNEL_AGENT | RSM_FUNC_ALL | RSM_INTR_CALLBACK);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_force_unload enter\n"));

	index = rsmhash(ex_segid);

	rw_enter(&rhash->rsmhash_rw, RW_READER);

	p = rsmhash_getbkt(rhash, index);

	for (; p; p = p->rsmrc_next) {
		rsmseg_t *seg = (rsmseg_t *)p;
		if ((seg->s_segid == ex_segid) && (seg->s_node == src_nodeid)) {
			/*
			 * In order to make rsmseg_unload and rsm_force_unload
			 * thread safe, acquire the segment lock here.
			 * rsmseg_unload is responsible for releasing the lock.
			 * rsmseg_unload releases the lock just before a call
			 * to rsmipc_send or in case of an early exit which
			 * occurs if the segment was in the state
			 * RSM_STATE_CONNECTING or RSM_STATE_NEW.
			 */
			rsmseglock_acquire(seg);
			if (disconnect_flag)
				seg->s_flags |= RSM_FORCE_DISCONNECT;
			rsmseg_unload(seg);
		}
	}
	rw_exit(&rhash->rsmhash_rw);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_force_unload done\n"));
}

static void
rsm_intr_reply(rsmipc_msghdr_t *msg)
{
	/*
	 * Find slot for cookie in reply.
	 * Match sequence with sequence in cookie
	 * If no match; return
	 * Try to grap lock of slot, if locked return
	 * copy data into reply slot area
	 * signal waiter
	 */
	rsmipc_slot_t 	*slot;
	rsmipc_cookie_t	*cookie;
	void *data = (void *) msg;
	size_t size = sizeof (rsmipc_reply_t);
	DBG_DEFINE(category,
	    RSM_KERNEL_AGENT | RSM_FUNC_ALL | RSM_INTR_CALLBACK);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_intr_reply enter\n"));

	cookie = &msg->rsmipc_cookie;
	if (cookie->ic.index >= RSMIPC_SZ) {
		DBG_PRINTF((category, RSM_ERR,
		    "rsm: rsm_intr_reply bad cookie %d\n", cookie->ic.index));
		return;
	}

	ASSERT(cookie->ic.index < RSMIPC_SZ);
	slot = &rsm_ipc.slots[cookie->ic.index];
	mutex_enter(&slot->rsmipc_lock);
	if (slot->rsmipc_cookie.value == cookie->value) {
		/* found a match */
		if (RSMIPC_GET(slot, RSMIPC_PENDING)) {
			bcopy(data, slot->rsmipc_data, size);
			RSMIPC_CLEAR(slot, RSMIPC_PENDING);
			cv_signal(&slot->rsmipc_cv);
		}
	} else {
		DBG_PRINTF((category, RSM_DEBUG,
		    "rsm: rsm_intr_reply mismatched reply %d\n",
		    cookie->ic.index));
	}
	mutex_exit(&slot->rsmipc_lock);
	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_intr_reply done\n"));
}

/*
 * This function gets dispatched on the worker thread when we receive
 * the SQREADY message. This function sends the SQREADY_ACK message.
 */
static void
rsm_sqready_ack_deferred(void *arg)
{
	path_t	*path = (path_t *)arg;
	DBG_DEFINE(category,
	    RSM_KERNEL_AGENT | RSM_FUNC_ALL | RSM_INTR_CALLBACK);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
	    "rsm_sqready_ack_deferred enter\n"));

	mutex_enter(&path->mutex);

	/*
	 * If path is not active no point in sending the ACK
	 * because the whole SQREADY protocol will again start
	 * when the path becomes active.
	 */
	if (path->state != RSMKA_PATH_ACTIVE) {
		/*
		 * decrement the path refcnt incremented in rsm_proc_sqready
		 */
		PATH_RELE_NOLOCK(path);
		mutex_exit(&path->mutex);
		DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
		    "rsm_sqready_ack_deferred done:!ACTIVE\n"));
		return;
	}

	/* send an SQREADY_ACK message */
	(void) rsmipc_send_controlmsg(path, RSMIPC_MSG_SQREADY_ACK);

	/* initialize credits to the max level */
	path->sendq_token.msgbuf_avail = RSMIPC_MAX_MESSAGES;

	/* wake up any send that is waiting for credits */
	cv_broadcast(&path->sendq_token.sendq_cv);

	/*
	 * decrement the path refcnt since we incremented it in
	 * rsm_proc_sqready
	 */
	PATH_RELE_NOLOCK(path);

	mutex_exit(&path->mutex);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
	    "rsm_sqready_ack_deferred done\n"));
}

/*
 * Process the SQREADY message
 */
static void
rsm_proc_sqready(rsmipc_controlmsg_t *msg, rsm_addr_t src_hwaddr,
    rsm_intr_hand_arg_t arg)
{
	rsmipc_msghdr_t		*msghdr = (rsmipc_msghdr_t *)msg;
	srv_handler_arg_t	*hdlr_argp = (srv_handler_arg_t *)arg;
	path_t			*path;
	DBG_DEFINE(category,
	    RSM_KERNEL_AGENT | RSM_FUNC_ALL | RSM_INTR_CALLBACK);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_proc_sqready enter\n"));

	/* look up the path - incr the path refcnt */
	path = rsm_find_path(hdlr_argp->adapter_name,
	    hdlr_argp->adapter_instance, src_hwaddr);

	/*
	 * No path exists or path is not active - drop the message
	 */
	if (path == NULL) {
		DBG_PRINTF((category, RSM_DEBUG,
		    "rsm_proc_sqready done: msg dropped no path\n"));
		return;
	}

	mutex_exit(&path->mutex);

	/* drain any tasks from the previous incarnation */
	taskq_wait(path->recv_taskq);

	mutex_enter(&path->mutex);
	/*
	 * If we'd sent an SQREADY message and were waiting for SQREADY_ACK
	 * in the meanwhile we received an SQREADY message, blindly reset
	 * the WAIT_FOR_SQACK flag because we'll just send SQREADY_ACK
	 * and forget about the SQREADY that we sent.
	 */
	path->flags &= ~RSMKA_WAIT_FOR_SQACK;

	if (path->state != RSMKA_PATH_ACTIVE) {
		/* decr refcnt and drop the mutex */
		PATH_RELE_NOLOCK(path);
		mutex_exit(&path->mutex);
		DBG_PRINTF((category, RSM_DEBUG,
		    "rsm_proc_sqready done: msg dropped path !ACTIVE\n"));
		return;
	}

	DBG_PRINTF((category, RSM_DEBUG, "rsm_proc_sqready:path=%lx "
	    " src=%lx:%llx\n", path, msghdr->rsmipc_src, src_hwaddr));

	/*
	 * The sender's local incarnation number is our remote incarnation
	 * number save it in the path data structure
	 */
	path->remote_incn = msg->rsmipc_local_incn;
	path->sendq_token.msgbuf_avail = 0;
	path->procmsg_cnt = 0;

	/*
	 * path is active - dispatch task to send SQREADY_ACK - remember
	 * RSMPI calls can't be done in interrupt context
	 *
	 * We can use the recv_taskq to send because the remote endpoint
	 * cannot start sending messages till it receives SQREADY_ACK hence
	 * at this point there are no tasks on recv_taskq.
	 *
	 * The path refcnt will be decremented in rsm_sqready_ack_deferred.
	 */
	(void) taskq_dispatch(path->recv_taskq,
	    rsm_sqready_ack_deferred, path, KM_NOSLEEP);

	mutex_exit(&path->mutex);


	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_proc_sqready done\n"));
}

/*
 * Process the SQREADY_ACK message
 */
static void
rsm_proc_sqready_ack(rsmipc_controlmsg_t *msg, rsm_addr_t src_hwaddr,
    rsm_intr_hand_arg_t arg)
{
	rsmipc_msghdr_t		*msghdr = (rsmipc_msghdr_t *)msg;
	srv_handler_arg_t	*hdlr_argp = (srv_handler_arg_t *)arg;
	path_t			*path;
	DBG_DEFINE(category,
	    RSM_KERNEL_AGENT | RSM_FUNC_ALL | RSM_INTR_CALLBACK);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
	    "rsm_proc_sqready_ack enter\n"));

	/* look up the path - incr the path refcnt */
	path = rsm_find_path(hdlr_argp->adapter_name,
	    hdlr_argp->adapter_instance, src_hwaddr);

	/*
	 * drop the message if - no path exists or path is not active
	 * or if its not waiting for SQREADY_ACK message
	 */
	if (path == NULL) {
		DBG_PRINTF((category, RSM_DEBUG,
		    "rsm_proc_sqready_ack done: msg dropped no path\n"));
		return;
	}

	if ((path->state != RSMKA_PATH_ACTIVE) ||
	    !(path->flags & RSMKA_WAIT_FOR_SQACK)) {
		/* decrement the refcnt */
		PATH_RELE_NOLOCK(path);
		mutex_exit(&path->mutex);
		DBG_PRINTF((category, RSM_DEBUG,
		    "rsm_proc_sqready_ack done: msg dropped\n"));
		return;
	}

	/*
	 * Check if this message is in response to the last RSMIPC_MSG_SQREADY
	 * sent, if not drop it.
	 */
	if (path->local_incn != msghdr->rsmipc_incn) {
		/* decrement the refcnt */
		PATH_RELE_NOLOCK(path);
		mutex_exit(&path->mutex);
		DBG_PRINTF((category, RSM_DEBUG,
		    "rsm_proc_sqready_ack done: msg old incn %lld\n",
		    msghdr->rsmipc_incn));
		return;
	}

	DBG_PRINTF((category, RSM_DEBUG, "rsm_proc_sqready_ack:path=%lx "
	    " src=%lx:%llx\n", path, msghdr->rsmipc_src, src_hwaddr));

	/*
	 * clear the WAIT_FOR_SQACK flag since we have recvd the ack
	 */
	path->flags &= ~RSMKA_WAIT_FOR_SQACK;

	/* save the remote sendq incn number */
	path->remote_incn = msg->rsmipc_local_incn;

	/* initialize credits to the max level */
	path->sendq_token.msgbuf_avail = RSMIPC_MAX_MESSAGES;

	/* wake up any send that is waiting for credits */
	cv_broadcast(&path->sendq_token.sendq_cv);

	/* decrement the refcnt */
	PATH_RELE_NOLOCK(path);

	mutex_exit(&path->mutex);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
	    "rsm_proc_sqready_ack done\n"));
}

/*
 * process the RSMIPC_MSG_CREDIT message
 */
static void
rsm_add_credits(rsmipc_controlmsg_t *msg, rsm_addr_t src_hwaddr,
    rsm_intr_hand_arg_t arg)
{
	rsmipc_msghdr_t		*msghdr = (rsmipc_msghdr_t *)msg;
	srv_handler_arg_t	*hdlr_argp = (srv_handler_arg_t *)arg;
	path_t			*path;
	DBG_DEFINE(category,
	    RSM_KERNEL_AGENT | RSM_FUNC_ALL |
	    RSM_INTR_CALLBACK | RSM_FLOWCONTROL);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_add_credits enter\n"));

	/* look up the path - incr the path refcnt */
	path = rsm_find_path(hdlr_argp->adapter_name,
	    hdlr_argp->adapter_instance, src_hwaddr);

	if (path == NULL) {
		DBG_PRINTF((category, RSM_DEBUG,
		    "rsm_add_credits enter: path not found\n"));
		return;
	}

	/* the path is not active - discard credits */
	if (path->state != RSMKA_PATH_ACTIVE) {
		PATH_RELE_NOLOCK(path);
		mutex_exit(&path->mutex);
		DBG_PRINTF((category, RSM_DEBUG,
		    "rsm_add_credits enter:path=%lx !ACTIVE\n", path));
		return;
	}

	/*
	 * Check if these credits are for current incarnation of the path.
	 */
	if (path->local_incn != msghdr->rsmipc_incn) {
		/* decrement the refcnt */
		PATH_RELE_NOLOCK(path);
		mutex_exit(&path->mutex);
		DBG_PRINTF((category, RSM_DEBUG,
		    "rsm_add_credits enter: old incn %lld\n",
		    msghdr->rsmipc_incn));
		return;
	}

	DBG_PRINTF((category, RSM_DEBUG,
	    "rsm_add_credits:path=%lx new-creds=%d "
	    "curr credits=%d src=%lx:%llx\n", path, msg->rsmipc_credits,
	    path->sendq_token.msgbuf_avail, msghdr->rsmipc_src,
	    src_hwaddr));


	/* add credits to the path's sendq */
	path->sendq_token.msgbuf_avail += msg->rsmipc_credits;

	ASSERT(path->sendq_token.msgbuf_avail <= RSMIPC_MAX_MESSAGES);

	/* wake up any send that is waiting for credits */
	cv_broadcast(&path->sendq_token.sendq_cv);

	/* decrement the refcnt */
	PATH_RELE_NOLOCK(path);

	mutex_exit(&path->mutex);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_add_credits done\n"));
}

static void
rsm_intr_event(rsmipc_request_t *msg)
{
	rsmseg_t	*seg;
	rsmresource_t	*p;
	rsm_node_id_t	src_node;
	DBG_DEFINE(category,
	    RSM_KERNEL_AGENT | RSM_FUNC_ALL | RSM_INTR_CALLBACK);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_intr_event enter\n"));

	src_node = msg->rsmipc_hdr.rsmipc_src;

	if ((seg = msg->rsmipc_segment_cookie) != NULL) {
		/* This is for an import segment */
		uint_t hashval = rsmhash(msg->rsmipc_key);

		rw_enter(&rsm_import_segs.rsmhash_rw, RW_READER);

		p = (rsmresource_t *)rsmhash_getbkt(&rsm_import_segs, hashval);

		for (; p; p = p->rsmrc_next) {
			if ((p->rsmrc_key == msg->rsmipc_key) &&
			    (p->rsmrc_node == src_node)) {
				seg = (rsmseg_t *)p;
				rsmseglock_acquire(seg);

				atomic_add_32(&seg->s_pollevent, 1);

				if (seg->s_pollflag & RSM_SEGMENT_POLL)
					pollwakeup(&seg->s_poll, POLLRDNORM);

				rsmseglock_release(seg);
			}
		}

		rw_exit(&rsm_import_segs.rsmhash_rw);
	} else {
		/* This is for an export segment */
		seg = rsmexport_lookup(msg->rsmipc_key);
		if (!seg) {
			DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
			    "rsm_intr_event done: exp seg not found\n"));
			return;
		}

		ASSERT(rsmseglock_held(seg));

		atomic_add_32(&seg->s_pollevent, 1);

		/*
		 * We must hold the segment lock here, or else the segment
		 * can be freed while pollwakeup is using it. This implies
		 * that we MUST NOT grab the segment lock during rsm_chpoll,
		 * as outlined in the chpoll(2) man page.
		 */
		if (seg->s_pollflag & RSM_SEGMENT_POLL)
			pollwakeup(&seg->s_poll, POLLRDNORM);

		rsmseglock_release(seg);
	}

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_intr_event done\n"));
}

/*
 * The exporter did a republish and changed the ACL - this change is only
 * visible to new importers.
 */
static void
importer_update(rsm_node_id_t src_node, rsm_memseg_id_t key,
    rsm_permission_t perm)
{

	rsmresource_t	*p;
	rsmseg_t	*seg;
	uint_t		hashval = rsmhash(key);
	DBG_DEFINE(category,
	    RSM_KERNEL_AGENT | RSM_FUNC_ALL | RSM_INTR_CALLBACK);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "importer_update enter\n"));

	rw_enter(&rsm_import_segs.rsmhash_rw, RW_READER);

	p = (rsmresource_t *)rsmhash_getbkt(&rsm_import_segs, hashval);

	for (; p; p = p->rsmrc_next) {
		/*
		 * find the importer and update the permission in the shared
		 * data structure. Any new importers will use the new perms
		 */
		if ((p->rsmrc_key == key) && (p->rsmrc_node == src_node)) {
			seg = (rsmseg_t *)p;

			rsmseglock_acquire(seg);
			rsmsharelock_acquire(seg);
			seg->s_share->rsmsi_mode = perm;
			rsmsharelock_release(seg);
			rsmseglock_release(seg);

			break;
		}
	}

	rw_exit(&rsm_import_segs.rsmhash_rw);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "importer_update done\n"));
}

void
rsm_suspend_complete(rsm_node_id_t src_node, int flag)
{
	int		done = 1; /* indicate all SUSPENDS have been acked */
	list_element_t	*elem;
	DBG_DEFINE(category,
	    RSM_KERNEL_AGENT | RSM_FUNC_ALL | RSM_INTR_CALLBACK);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
	    "rsm_suspend_complete enter\n"));

	mutex_enter(&rsm_suspend_list.list_lock);

	if (rsm_suspend_list.list_head == NULL) {
		DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
		    "rsm_suspend_complete done: suspend_list is empty\n"));
		mutex_exit(&rsm_suspend_list.list_lock);
		return;
	}

	elem = rsm_suspend_list.list_head;
	while (elem != NULL) {
		if (elem->nodeid == src_node) {
			/* clear the pending flag for the node */
			elem->flags &= ~RSM_SUSPEND_ACKPENDING;
			elem->flags |= flag;
		}

		if (done && (elem->flags & RSM_SUSPEND_ACKPENDING))
			done = 0; /* still some nodes have not yet ACKED */

		elem = elem->next;
	}

	mutex_exit(&rsm_suspend_list.list_lock);

	if (!done) {
		DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
		    "rsm_suspend_complete done: acks pending\n"));
		return;
	}
	/*
	 * Now that we are done with suspending all the remote importers
	 * time to quiesce the local exporters
	 */
	exporter_quiesce();

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
	    "rsm_suspend_complete done\n"));
}

static void
exporter_quiesce()
{
	int		i, e;
	rsmresource_t	*current;
	rsmseg_t	*seg;
	adapter_t	*adapter;
	DBG_DEFINE(category,
	    RSM_KERNEL_AGENT | RSM_FUNC_ALL | RSM_INTR_CALLBACK);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "exporter_quiesce enter\n"));
	/*
	 * The importers send a SUSPEND_COMPLETE to the exporter node
	 *	Unpublish, unbind the export segment and
	 *	move the segments to the EXPORT_QUIESCED state
	 */

	rw_enter(&rsm_export_segs.rsmhash_rw, RW_READER);

	for (i = 0; i < rsm_hash_size; i++) {
		current = rsm_export_segs.bucket[i];
		while (current != NULL) {
			seg = (rsmseg_t *)current;
			rsmseglock_acquire(seg);
			if (current->rsmrc_state ==
			    RSM_STATE_EXPORT_QUIESCING) {
				adapter = seg->s_adapter;
				/*
				 * some local memory handles are not published
				 * check if it was published
				 */
				if ((seg->s_acl == NULL) ||
				    (seg->s_acl[0].ae_node != my_nodeid) ||
				    (seg->s_acl[0].ae_permission != 0)) {

					e = adapter->rsmpi_ops->rsm_unpublish(
					    seg->s_handle.out);
					DBG_PRINTF((category, RSM_DEBUG,
					    "exporter_quiesce:unpub %d\n", e));

					e = adapter->rsmpi_ops->rsm_seg_destroy(
					    seg->s_handle.out);

					DBG_PRINTF((category, RSM_DEBUG,
					    "exporter_quiesce:destroy %d\n",
					    e));
				}

				(void) rsm_unbind_pages(seg);
				seg->s_state = RSM_STATE_EXPORT_QUIESCED;
				cv_broadcast(&seg->s_cv);
			}
			rsmseglock_release(seg);
			current = current->rsmrc_next;
		}
	}
	rw_exit(&rsm_export_segs.rsmhash_rw);

	/*
	 * All the local segments we are done with the pre-del processing
	 * - time to move to PREDEL_COMPLETED.
	 */

	mutex_enter(&rsm_drv_data.drv_lock);

	ASSERT(rsm_drv_data.drv_state == RSM_DRV_PREDEL_STARTED);

	rsm_drv_data.drv_state = RSM_DRV_PREDEL_COMPLETED;

	cv_broadcast(&rsm_drv_data.drv_cv);

	mutex_exit(&rsm_drv_data.drv_lock);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "exporter_quiesce done\n"));
}

static void
importer_suspend(rsm_node_id_t src_node)
{
	int		i;
	int		susp_flg; /* true means already suspended */
	int		num_importers;
	rsmresource_t	*p = NULL, *curp;
	rsmhash_table_t *rhash = &rsm_import_segs;
	rsmseg_t	*seg;
	rsmipc_request_t request;
	DBG_DEFINE(category,
	    RSM_KERNEL_AGENT | RSM_FUNC_ALL | RSM_INTR_CALLBACK);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "importer_suspend enter\n"));

	rw_enter(&rhash->rsmhash_rw, RW_READER);
	for (i = 0; i < rsm_hash_size; i++) {
		p = rhash->bucket[i];

		/*
		 * Suspend all importers with same <node, key> pair.
		 * After the last one of the shared importers has been
		 * suspended - suspend the shared mappings/connection.
		 */
		for (; p; p = p->rsmrc_next) {
			rsmseg_t *first = (rsmseg_t *)p;
			if ((first->s_node != src_node) ||
			    (first->s_state == RSM_STATE_DISCONNECT))
				continue; /* go to next entry */
			/*
			 * search the rest of the bucket for
			 * other siblings (imprtrs with the same key)
			 * of "first" and suspend them.
			 * All importers with same key fall in
			 * the same bucket.
			 */
			num_importers = 0;
			for (curp = p; curp; curp = curp->rsmrc_next) {
				seg = (rsmseg_t *)curp;

				rsmseglock_acquire(seg);

				if ((seg->s_node != first->s_node) ||
				    (seg->s_key != first->s_key) ||
				    (seg->s_state == RSM_STATE_DISCONNECT)) {
					/*
					 * either not a peer segment or its a
					 * disconnected segment - skip it
					 */
					rsmseglock_release(seg);
					continue;
				}

				rsmseg_suspend(seg, &susp_flg);

				if (susp_flg) { /* seg already suspended */
					rsmseglock_release(seg);
					break; /* the inner for loop */
				}

				num_importers++;
				rsmsharelock_acquire(seg);
				/*
				 * we've processed all importers that are
				 * siblings of "first"
				 */
				if (num_importers ==
				    seg->s_share->rsmsi_refcnt) {
					rsmsharelock_release(seg);
					rsmseglock_release(seg);
					break;
				}
				rsmsharelock_release(seg);
				rsmseglock_release(seg);
			}

			/*
			 * All the importers with the same key and
			 * nodeid as "first" have been suspended.
			 * Now suspend the shared connect/mapping.
			 * This is done only once.
			 */
			if (!susp_flg) {
				rsmsegshare_suspend(seg);
			}
		}
	}

	rw_exit(&rhash->rsmhash_rw);

	/* send an ACK for SUSPEND message */
	request.rsmipc_hdr.rsmipc_type = RSMIPC_MSG_SUSPEND_DONE;
	(void) rsmipc_send(src_node, &request, RSM_NO_REPLY);


	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "importer_suspend done\n"));

}

static void
rsmseg_suspend(rsmseg_t *seg, int *susp_flg)
{
	int		recheck_state;
	rsmcookie_t	*hdl;
	DBG_DEFINE(category,
	    RSM_KERNEL_AGENT | RSM_FUNC_ALL | RSM_INTR_CALLBACK);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
	    "rsmseg_suspend enter: key=%u\n", seg->s_key));

	*susp_flg = 0;

	ASSERT(rsmseglock_held(seg));
	/* wait if putv/getv is in progress */
	while (seg->s_rdmacnt > 0)
		cv_wait(&seg->s_cv, &seg->s_lock);

	do {
		recheck_state = 0;

		DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
		    "rsmseg_suspend:segment %x state=%d\n",
		    seg->s_key, seg->s_state));

		switch (seg->s_state) {
		case RSM_STATE_NEW:
			/* not a valid state */
			break;
		case RSM_STATE_CONNECTING:
			seg->s_state = RSM_STATE_ABORT_CONNECT;
			break;
		case RSM_STATE_ABORT_CONNECT:
			break;
		case RSM_STATE_CONNECT:
			seg->s_handle.in = NULL;
			seg->s_state = RSM_STATE_CONN_QUIESCE;
			break;
		case RSM_STATE_MAPPING:
			/* wait until segment leaves the mapping state */
			while (seg->s_state == RSM_STATE_MAPPING)
				cv_wait(&seg->s_cv, &seg->s_lock);
			recheck_state = 1;
			break;
		case RSM_STATE_ACTIVE:
			/* unload the mappings */
			if (seg->s_ckl != NULL) {
				hdl = seg->s_ckl;
				for (; hdl != NULL; hdl = hdl->c_next) {
					(void) devmap_unload(hdl->c_dhp,
					    hdl->c_off, hdl->c_len);
				}
			}
			seg->s_mapinfo = NULL;
			seg->s_state = RSM_STATE_MAP_QUIESCE;
			break;
		case RSM_STATE_CONN_QUIESCE:
			/* FALLTHRU */
		case RSM_STATE_MAP_QUIESCE:
			/* rsmseg_suspend already done for seg */
			*susp_flg = 1;
			break;
		case RSM_STATE_DISCONNECT:
			break;
		default:
			ASSERT(0); /* invalid state */
		}
	} while (recheck_state);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmseg_suspend done\n"));
}

static void
rsmsegshare_suspend(rsmseg_t *seg)
{
	int			e;
	adapter_t		*adapter;
	rsm_import_share_t	*sharedp;
	DBG_DEFINE(category,
	    RSM_KERNEL_AGENT | RSM_FUNC_ALL | RSM_INTR_CALLBACK);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
	    "rsmsegshare_suspend enter\n"));

	rsmseglock_acquire(seg);
	rsmsharelock_acquire(seg);

	sharedp = seg->s_share;
	adapter = seg->s_adapter;
	switch (sharedp->rsmsi_state) {
	case RSMSI_STATE_NEW:
		break;
	case RSMSI_STATE_CONNECTING:
		sharedp->rsmsi_state = RSMSI_STATE_ABORT_CONNECT;
		break;
	case RSMSI_STATE_ABORT_CONNECT:
		break;
	case RSMSI_STATE_CONNECTED:
		/* do the rsmpi disconnect */
		if (sharedp->rsmsi_node != my_nodeid) {
			e = adapter->rsmpi_ops->
			    rsm_disconnect(sharedp->rsmsi_handle);

			DBG_PRINTF((category, RSM_DEBUG,
			    "rsm:rsmpi disconnect seg=%x:err=%d\n",
			    sharedp->rsmsi_segid, e));
		}

		sharedp->rsmsi_handle = NULL;

		sharedp->rsmsi_state = RSMSI_STATE_CONN_QUIESCE;
		break;
	case RSMSI_STATE_CONN_QUIESCE:
		break;
	case RSMSI_STATE_MAPPED:
		/* do the rsmpi unmap and disconnect */
		if (sharedp->rsmsi_node != my_nodeid) {
			e = adapter->rsmpi_ops->rsm_unmap(seg->s_handle.in);

			DBG_PRINTF((category, RSM_DEBUG,
			    "rsmshare_suspend: rsmpi unmap %d\n", e));

			e = adapter->rsmpi_ops->
			    rsm_disconnect(sharedp->rsmsi_handle);
			DBG_PRINTF((category, RSM_DEBUG,
			    "rsm:rsmpi disconnect seg=%x:err=%d\n",
			    sharedp->rsmsi_segid, e));
		}

		sharedp->rsmsi_handle = NULL;

		sharedp->rsmsi_state = RSMSI_STATE_MAP_QUIESCE;
		break;
	case RSMSI_STATE_MAP_QUIESCE:
		break;
	case RSMSI_STATE_DISCONNECTED:
		break;
	default:
		ASSERT(0); /* invalid state */
	}

	rsmsharelock_release(seg);
	rsmseglock_release(seg);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
	    "rsmsegshare_suspend done\n"));
}

/*
 * This should get called on receiving a RESUME message or from
 * the pathmanger if the node undergoing DR dies.
 */
static void
importer_resume(rsm_node_id_t src_node)
{
	int		i;
	rsmresource_t	*p = NULL;
	rsmhash_table_t *rhash = &rsm_import_segs;
	void		*cookie;
	DBG_DEFINE(category,
	    RSM_KERNEL_AGENT | RSM_FUNC_ALL | RSM_INTR_CALLBACK);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "importer_resume enter\n"));

	rw_enter(&rhash->rsmhash_rw, RW_READER);

	for (i = 0; i < rsm_hash_size; i++) {
		p = rhash->bucket[i];

		for (; p; p = p->rsmrc_next) {
			rsmseg_t *seg = (rsmseg_t *)p;

			rsmseglock_acquire(seg);

			/* process only importers of node undergoing DR */
			if (seg->s_node != src_node) {
				rsmseglock_release(seg);
				continue;
			}

			if (rsmseg_resume(seg, &cookie) != RSM_SUCCESS) {
				rsmipc_request_t	request;
				/*
				 * rsmpi map/connect failed
				 * inform the exporter so that it can
				 * remove the importer.
				 */
				request.rsmipc_hdr.rsmipc_type =
				    RSMIPC_MSG_NOTIMPORTING;
				request.rsmipc_key = seg->s_segid;
				request.rsmipc_segment_cookie = cookie;
				rsmseglock_release(seg);
				(void) rsmipc_send(seg->s_node, &request,
				    RSM_NO_REPLY);
			} else {
				rsmseglock_release(seg);
			}
		}
	}

	rw_exit(&rhash->rsmhash_rw);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "importer_resume done\n"));
}

static int
rsmseg_resume(rsmseg_t *seg, void **cookie)
{
	int			e;
	int			retc;
	off_t			dev_offset;
	size_t			maplen;
	uint_t			maxprot;
	rsm_mapinfo_t		*p;
	rsmcookie_t		*hdl;
	rsm_import_share_t	*sharedp;
	DBG_DEFINE(category,
	    RSM_KERNEL_AGENT | RSM_FUNC_ALL | RSM_INTR_CALLBACK);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
	    "rsmseg_resume enter: key=%u\n", seg->s_key));

	*cookie = NULL;

	ASSERT(rsmseglock_held(seg));

	if ((seg->s_state != RSM_STATE_CONN_QUIESCE) &&
	    (seg->s_state != RSM_STATE_MAP_QUIESCE)) {
		return (RSM_SUCCESS);
	}

	sharedp = seg->s_share;

	rsmsharelock_acquire(seg);

	/* resume the shared connection and/or mapping */
	retc = rsmsegshare_resume(seg);

	if (seg->s_state == RSM_STATE_CONN_QUIESCE) {
		/* shared state can either be connected or mapped */
		if ((sharedp->rsmsi_state == RSMSI_STATE_CONNECTED) ||
		    (sharedp->rsmsi_state == RSMSI_STATE_MAPPED)) {
			ASSERT(retc == RSM_SUCCESS);
			seg->s_handle.in = sharedp->rsmsi_handle;
			rsmsharelock_release(seg);
			seg->s_state = RSM_STATE_CONNECT;

		} else { /* error in rsmpi connect during resume */
			seg->s_handle.in = NULL;
			seg->s_state = RSM_STATE_DISCONNECT;

			sharedp->rsmsi_refcnt--;
			cookie = (void *)sharedp->rsmsi_cookie;

			if (sharedp->rsmsi_refcnt == 0) {
				ASSERT(sharedp->rsmsi_mapcnt == 0);
				rsmsharelock_release(seg);

				/* clean up the shared data structure */
				mutex_destroy(&sharedp->rsmsi_lock);
				cv_destroy(&sharedp->rsmsi_cv);
				kmem_free((void *)(sharedp),
				    sizeof (rsm_import_share_t));

			} else {
				rsmsharelock_release(seg);
			}
			/*
			 * The following needs to be done after any
			 * rsmsharelock calls which use seg->s_share.
			 */
			seg->s_share = NULL;
		}

		/* signal any waiting segment */
		cv_broadcast(&seg->s_cv);

		DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
		    "rsmseg_resume done:state=%d\n", seg->s_state));
		return (retc);
	}

	ASSERT(seg->s_state == RSM_STATE_MAP_QUIESCE);

	/* Setup protections for remap */
	maxprot = PROT_USER;
	if (seg->s_mode & RSM_PERM_READ) {
		maxprot |= PROT_READ;
	}
	if (seg->s_mode & RSM_PERM_WRITE) {
		maxprot |= PROT_WRITE;
	}

	if (sharedp->rsmsi_state != RSMSI_STATE_MAPPED) {
		/* error in rsmpi connect or map during resume */

		/* remap to trash page */
		ASSERT(seg->s_ckl != NULL);

		for (hdl = seg->s_ckl; hdl != NULL; hdl = hdl->c_next) {
			e = devmap_umem_remap(hdl->c_dhp, rsm_dip,
			    remap_cookie, hdl->c_off, hdl->c_len,
			    maxprot, 0, NULL);

			DBG_PRINTF((category, RSM_ERR,
			    "rsmseg_resume:remap=%d\n", e));
		}

		seg->s_handle.in = NULL;
		seg->s_state = RSM_STATE_DISCONNECT;

		sharedp->rsmsi_refcnt--;

		sharedp->rsmsi_mapcnt--;
		seg->s_mapinfo = NULL;

		if (sharedp->rsmsi_refcnt == 0) {
			ASSERT(sharedp->rsmsi_mapcnt == 0);
			rsmsharelock_release(seg);

			/* clean up the shared data structure */
			mutex_destroy(&sharedp->rsmsi_lock);
			cv_destroy(&sharedp->rsmsi_cv);
			kmem_free((void *)(sharedp),
			    sizeof (rsm_import_share_t));

		} else {
			rsmsharelock_release(seg);
		}
		/*
		 * The following needs to be done after any
		 * rsmsharelock calls which use seg->s_share.
		 */
		seg->s_share = NULL;

		/* signal any waiting segment */
		cv_broadcast(&seg->s_cv);

		DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
		    "rsmseg_resume done:seg=%x,err=%d\n",
		    seg->s_key, retc));
		return (retc);

	}

	seg->s_handle.in = sharedp->rsmsi_handle;

	if (seg->s_node == my_nodeid) { /* loopback */
		ASSERT(seg->s_mapinfo == NULL);

		for (hdl = seg->s_ckl; hdl != NULL; hdl = hdl->c_next) {
			e = devmap_umem_remap(hdl->c_dhp,
			    rsm_dip, seg->s_cookie,
			    hdl->c_off, hdl->c_len,
			    maxprot, 0, NULL);

			DBG_PRINTF((category, RSM_ERR,
			    "rsmseg_resume:remap=%d\n", e));
		}
	} else { /* remote exporter */
		/* remap to the new rsmpi maps */
		seg->s_mapinfo = sharedp->rsmsi_mapinfo;

		for (hdl = seg->s_ckl; hdl != NULL; hdl = hdl->c_next) {
			p = rsm_get_mapinfo(seg, hdl->c_off, hdl->c_len,
			    &dev_offset, &maplen);
			e = devmap_devmem_remap(hdl->c_dhp,
			    p->dip, p->dev_register, dev_offset,
			    maplen, maxprot, 0, NULL);

			DBG_PRINTF((category, RSM_ERR,
			    "rsmseg_resume:remap=%d\n", e));
		}
	}

	rsmsharelock_release(seg);

	seg->s_state = RSM_STATE_ACTIVE;
	cv_broadcast(&seg->s_cv);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmseg_resume done\n"));

	return (retc);
}

static int
rsmsegshare_resume(rsmseg_t *seg)
{
	int			e = RSM_SUCCESS;
	adapter_t		*adapter;
	rsm_import_share_t	*sharedp;
	DBG_DEFINE(category,
	    RSM_KERNEL_AGENT | RSM_FUNC_ALL | RSM_INTR_CALLBACK);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmsegshare_resume enter\n"));

	ASSERT(rsmseglock_held(seg));
	ASSERT(rsmsharelock_held(seg));

	sharedp = seg->s_share;

	/*
	 * If we are not in a xxxx_QUIESCE state that means shared
	 * connect/mapping processing has been already been done
	 * so return success.
	 */
	if ((sharedp->rsmsi_state != RSMSI_STATE_CONN_QUIESCE) &&
	    (sharedp->rsmsi_state != RSMSI_STATE_MAP_QUIESCE)) {
		return (RSM_SUCCESS);
	}

	adapter = seg->s_adapter;

	if (sharedp->rsmsi_node != my_nodeid) {
		rsm_addr_t	hwaddr;
		hwaddr = get_remote_hwaddr(adapter, sharedp->rsmsi_node);

		e = adapter->rsmpi_ops->rsm_connect(
		    adapter->rsmpi_handle, hwaddr,
		    sharedp->rsmsi_segid, &sharedp->rsmsi_handle);

		DBG_PRINTF((category, RSM_DEBUG,
		    "rsmsegshare_resume:rsmpi connect seg=%x:err=%d\n",
		    sharedp->rsmsi_segid, e));

		if (e != RSM_SUCCESS) {
			/* when do we send the NOT_IMPORTING message */
			sharedp->rsmsi_handle = NULL;
			sharedp->rsmsi_state = RSMSI_STATE_DISCONNECTED;
			/* signal any waiting segment */
			cv_broadcast(&sharedp->rsmsi_cv);
			return (e);
		}
	}

	if (sharedp->rsmsi_state == RSMSI_STATE_CONN_QUIESCE) {
		sharedp->rsmsi_state = RSMSI_STATE_CONNECTED;
		/* signal any waiting segment */
		cv_broadcast(&sharedp->rsmsi_cv);
		return (e);
	}

	ASSERT(sharedp->rsmsi_state == RSMSI_STATE_MAP_QUIESCE);

	/* do the rsmpi map of the whole segment here */
	if (sharedp->rsmsi_node != my_nodeid) {
		size_t mapped_len;
		rsm_mapinfo_t *p;

		/*
		 * We need to do rsmpi maps with <off, lens> identical to
		 * the old mapinfo list because the segment mapping handles
		 * dhp and such need the fragmentation of rsmpi maps to be
		 * identical to what it was during the mmap of the segment
		 */
		p = sharedp->rsmsi_mapinfo;

		while (p != NULL) {
			mapped_len = 0;

			e = adapter->rsmpi_ops->rsm_map(
			    sharedp->rsmsi_handle, p->start_offset,
			    p->individual_len, &mapped_len,
			    &p->dip, &p->dev_register, &p->dev_offset,
			    NULL, NULL);

			if (e != 0) {
				DBG_PRINTF((category, RSM_ERR,
				    "rsmsegshare_resume: rsmpi map err=%d\n",
				    e));
				break;
			}

			if (mapped_len != p->individual_len) {
				DBG_PRINTF((category, RSM_ERR,
				    "rsmsegshare_resume: rsmpi maplen"
				    "< reqlen=%lx\n", mapped_len));
				e = RSMERR_BAD_LENGTH;
				break;
			}

			p = p->next;

		}


		if (e != RSM_SUCCESS) { /* rsmpi map failed */
			int	err;
			/* Check if this is the first rsm_map */
			if (p != sharedp->rsmsi_mapinfo) {
				/*
				 * A single rsm_unmap undoes multiple rsm_maps.
				 */
				(void) seg->s_adapter->rsmpi_ops->
				    rsm_unmap(sharedp->rsmsi_handle);
			}

			rsm_free_mapinfo(sharedp->rsmsi_mapinfo);
			sharedp->rsmsi_mapinfo = NULL;

			err = adapter->rsmpi_ops->
			    rsm_disconnect(sharedp->rsmsi_handle);

			DBG_PRINTF((category, RSM_DEBUG,
			    "rsmsegshare_resume:disconn seg=%x:err=%d\n",
			    sharedp->rsmsi_segid, err));

			sharedp->rsmsi_handle = NULL;
			sharedp->rsmsi_state = RSMSI_STATE_DISCONNECTED;

			/* signal the waiting segments */
			cv_broadcast(&sharedp->rsmsi_cv);
			DBG_PRINTF((category, RSM_DEBUG,
			    "rsmsegshare_resume done: rsmpi map err\n"));
			return (e);
		}
	}

	sharedp->rsmsi_state = RSMSI_STATE_MAPPED;

	/* signal any waiting segment */
	cv_broadcast(&sharedp->rsmsi_cv);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmsegshare_resume done\n"));

	return (e);
}

/*
 * this is the routine that gets called by recv_taskq which is the
 * thread that processes messages that are flow-controlled.
 */
static void
rsm_intr_proc_deferred(void *arg)
{
	path_t			*path = (path_t *)arg;
	rsmipc_request_t	*msg;
	rsmipc_msghdr_t		*msghdr;
	rsm_node_id_t		src_node;
	msgbuf_elem_t		*head;
	int			e;
	DBG_DEFINE(category,
	    RSM_KERNEL_AGENT | RSM_FUNC_ALL | RSM_INTR_CALLBACK);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
	    "rsm_intr_proc_deferred enter\n"));

	mutex_enter(&path->mutex);

	/* use the head of the msgbuf_queue */
	head = rsmka_gethead_msgbuf(path);

	mutex_exit(&path->mutex);

	msg = (rsmipc_request_t *)&(head->msg);
	msghdr = (rsmipc_msghdr_t *)msg;

	src_node = msghdr->rsmipc_src;

	/*
	 * messages that need to send a reply should check the message version
	 * before processing the message. And all messages that need to
	 * send a reply should be processed here by the worker thread.
	 */
	switch (msghdr->rsmipc_type) {
	case RSMIPC_MSG_SEGCONNECT:
		if (msghdr->rsmipc_version != RSM_VERSION) {
			rsmipc_reply_t reply;
			reply.rsmipc_status = RSMERR_BAD_DRIVER_VERSION;
			reply.rsmipc_hdr.rsmipc_type = RSMIPC_MSG_REPLY;
			reply.rsmipc_hdr.rsmipc_cookie = msghdr->rsmipc_cookie;
			(void) rsmipc_send(msghdr->rsmipc_src, NULL, &reply);
		} else {
			rsm_intr_segconnect(src_node, msg);
		}
		break;
	case RSMIPC_MSG_DISCONNECT:
		rsm_force_unload(src_node, msg->rsmipc_key, DISCONNECT);
		break;
	case RSMIPC_MSG_SUSPEND:
		importer_suspend(src_node);
		break;
	case RSMIPC_MSG_SUSPEND_DONE:
		rsm_suspend_complete(src_node, 0);
		break;
	case RSMIPC_MSG_RESUME:
		importer_resume(src_node);
		break;
	default:
		ASSERT(0);
	}

	mutex_enter(&path->mutex);

	rsmka_dequeue_msgbuf(path);

	/* incr procmsg_cnt can be at most RSMIPC_MAX_MESSAGES */
	if (path->procmsg_cnt < RSMIPC_MAX_MESSAGES)
		path->procmsg_cnt++;

	ASSERT(path->procmsg_cnt <= RSMIPC_MAX_MESSAGES);

	/* No need to send credits if path is going down */
	if ((path->state == RSMKA_PATH_ACTIVE) &&
	    (path->procmsg_cnt >= RSMIPC_LOTSFREE_MSGBUFS)) {
		/*
		 * send credits and reset procmsg_cnt if success otherwise
		 * credits will be sent after processing the next message
		 */
		e = rsmipc_send_controlmsg(path, RSMIPC_MSG_CREDIT);
		if (e == 0)
			path->procmsg_cnt = 0;
		else
			DBG_PRINTF((category, RSM_ERR,
			    "rsm_intr_proc_deferred:send credits err=%d\n", e));
	}

	/*
	 * decrement the path refcnt since we incremented it in
	 * rsm_intr_callback_dispatch
	 */
	PATH_RELE_NOLOCK(path);

	mutex_exit(&path->mutex);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
	    "rsm_intr_proc_deferred done\n"));
}

/*
 * Flow-controlled messages are enqueued and dispatched onto a taskq here
 */
static void
rsm_intr_callback_dispatch(void *data, rsm_addr_t src_hwaddr,
    rsm_intr_hand_arg_t arg)
{
	srv_handler_arg_t	*hdlr_argp = (srv_handler_arg_t *)arg;
	path_t			*path;
	rsmipc_msghdr_t *msghdr = (rsmipc_msghdr_t *)data;
	DBG_DEFINE(category,
	    RSM_KERNEL_AGENT | RSM_FUNC_ALL | RSM_INTR_CALLBACK);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
	    "rsm_intr_callback_dispatch enter\n"));
	ASSERT(data && hdlr_argp);

	/* look up the path - incr the path refcnt */
	path = rsm_find_path(hdlr_argp->adapter_name,
	    hdlr_argp->adapter_instance, src_hwaddr);

	/* the path has been removed - drop this message */
	if (path == NULL) {
		DBG_PRINTF((category, RSM_DEBUG,
		    "rsm_intr_callback_dispatch done: msg dropped\n"));
		return;
	}
	/* the path is not active - don't accept new messages */
	if (path->state != RSMKA_PATH_ACTIVE) {
		PATH_RELE_NOLOCK(path);
		mutex_exit(&path->mutex);
		DBG_PRINTF((category, RSM_DEBUG,
		    "rsm_intr_callback_dispatch done: msg dropped"
		    " path=%lx !ACTIVE\n", path));
		return;
	}

	/*
	 * Check if this message was sent to an older incarnation
	 * of the path/sendq.
	 */
	if (path->local_incn != msghdr->rsmipc_incn) {
		/* decrement the refcnt */
		PATH_RELE_NOLOCK(path);
		mutex_exit(&path->mutex);
		DBG_PRINTF((category, RSM_DEBUG,
		    "rsm_intr_callback_dispatch done: old incn %lld\n",
		    msghdr->rsmipc_incn));
		return;
	}

	/* copy and enqueue msg on the path's msgbuf queue */
	rsmka_enqueue_msgbuf(path, data);

	/*
	 * schedule task to process messages - ignore retval from
	 * task_dispatch because we sender cannot send more than
	 * what receiver can handle.
	 */
	(void) taskq_dispatch(path->recv_taskq,
	    rsm_intr_proc_deferred, path, KM_NOSLEEP);

	mutex_exit(&path->mutex);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
	    "rsm_intr_callback_dispatch done\n"));
}

/*
 * This procedure is called from rsm_srv_func when a remote node creates a
 * a send queue.  This event is used as a hint that an  earlier failed
 * attempt to create a send queue to that remote node may now succeed and
 * should be retried.  Indication of an earlier failed attempt is provided
 * by the RSMKA_SQCREATE_PENDING flag.
 */
static void
rsm_sqcreateop_callback(rsm_addr_t src_hwaddr, rsm_intr_hand_arg_t arg)
{
	srv_handler_arg_t	*hdlr_argp = (srv_handler_arg_t *)arg;
	path_t			*path;
	DBG_DEFINE(category,
	    RSM_KERNEL_AGENT | RSM_FUNC_ALL | RSM_INTR_CALLBACK);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
	    "rsm_sqcreateop_callback enter\n"));

	/* look up the path - incr the path refcnt */
	path = rsm_find_path(hdlr_argp->adapter_name,
	    hdlr_argp->adapter_instance, src_hwaddr);

	if (path == NULL) {
		DBG_PRINTF((category, RSM_DEBUG,
		    "rsm_sqcreateop_callback done: no path\n"));
		return;
	}

	if ((path->state == RSMKA_PATH_UP) &&
	    (path->flags & RSMKA_SQCREATE_PENDING)) {
		/*
		 * previous attempt to create sendq had failed, retry
		 * it and move to RSMKA_PATH_ACTIVE state if successful.
		 * the refcnt will be decremented in the do_deferred_work
		 */
		(void) rsmka_do_path_active(path, RSMKA_NO_SLEEP);
	} else {
		/* decrement the refcnt */
		PATH_RELE_NOLOCK(path);
	}
	mutex_exit(&path->mutex);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
	    "rsm_sqcreateop_callback done\n"));
}

static void
rsm_intr_callback(void *data, rsm_addr_t src_hwaddr, rsm_intr_hand_arg_t arg)
{
	rsmipc_msghdr_t *msghdr = (rsmipc_msghdr_t *)data;
	rsmipc_request_t *msg = (rsmipc_request_t *)data;
	rsmipc_controlmsg_t *ctrlmsg = (rsmipc_controlmsg_t *)data;
	rsm_node_id_t src_node;
	DBG_DEFINE(category,
	    RSM_KERNEL_AGENT | RSM_FUNC_ALL | RSM_INTR_CALLBACK);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_intr_callback enter:"
	    "src=%d, type=%d\n", msghdr->rsmipc_src,
	    msghdr->rsmipc_type));

	/*
	 * Check for the version number in the msg header. If it is not
	 * RSM_VERSION, drop the message. In the future, we need to manage
	 * incompatible version numbers in some way
	 */
	if (msghdr->rsmipc_version != RSM_VERSION) {
		DBG_PRINTF((category, RSM_ERR, "wrong KA version\n"));
		/*
		 * Drop requests that don't have a reply right here
		 * Request with reply will send a BAD_VERSION reply
		 * when they get processed by the worker thread.
		 */
		if (msghdr->rsmipc_type != RSMIPC_MSG_SEGCONNECT) {
			return;
		}

	}

	src_node = msghdr->rsmipc_src;

	switch (msghdr->rsmipc_type) {
	case RSMIPC_MSG_SEGCONNECT:
	case RSMIPC_MSG_DISCONNECT:
	case RSMIPC_MSG_SUSPEND:
	case RSMIPC_MSG_SUSPEND_DONE:
	case RSMIPC_MSG_RESUME:
		/*
		 * These message types are handled by a worker thread using
		 * the flow-control algorithm.
		 * Any message processing that does one or more of the
		 * following should be handled in a worker thread.
		 *	- allocates resources and might sleep
		 *	- makes RSMPI calls down to the interconnect driver
		 *	this by defn include requests with reply.
		 *	- takes a long duration of time
		 */
		rsm_intr_callback_dispatch(data, src_hwaddr, arg);
		break;
	case RSMIPC_MSG_NOTIMPORTING:
		importer_list_rm(src_node, msg->rsmipc_key,
		    msg->rsmipc_segment_cookie);
		break;
	case RSMIPC_MSG_SQREADY:
		rsm_proc_sqready(data, src_hwaddr, arg);
		break;
	case RSMIPC_MSG_SQREADY_ACK:
		rsm_proc_sqready_ack(data, src_hwaddr, arg);
		break;
	case RSMIPC_MSG_CREDIT:
		rsm_add_credits(ctrlmsg, src_hwaddr, arg);
		break;
	case RSMIPC_MSG_REPLY:
		rsm_intr_reply(msghdr);
		break;
	case RSMIPC_MSG_BELL:
		rsm_intr_event(msg);
		break;
	case RSMIPC_MSG_IMPORTING:
		importer_list_add(src_node, msg->rsmipc_key,
		    msg->rsmipc_adapter_hwaddr,
		    msg->rsmipc_segment_cookie);
		break;
	case RSMIPC_MSG_REPUBLISH:
		importer_update(src_node, msg->rsmipc_key, msg->rsmipc_perm);
		break;
	default:
		DBG_PRINTF((category, RSM_DEBUG,
		    "rsm_intr_callback: bad msg %lx type %d data %lx\n",
		    (size_t)msg, (int)(msghdr->rsmipc_type), (size_t)data));
	}

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_intr_callback done\n"));

}

rsm_intr_hand_ret_t rsm_srv_func(rsm_controller_object_t *chd,
    rsm_intr_q_op_t opcode, rsm_addr_t src,
    void *data, size_t size, rsm_intr_hand_arg_t arg)
{
	DBG_DEFINE(category,
	    RSM_KERNEL_AGENT | RSM_FUNC_ALL | RSM_INTR_CALLBACK);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_srv_func enter\n"));

	switch (opcode) {
	case RSM_INTR_Q_OP_CREATE:
		DBG_PRINTF((category, RSM_DEBUG, "rsm_srv_func:OP_CREATE\n"));
		rsm_sqcreateop_callback(src, arg);
		break;
	case RSM_INTR_Q_OP_DESTROY:
		DBG_PRINTF((category, RSM_DEBUG, "rsm_srv_func:OP_DESTROY\n"));
		break;
	case RSM_INTR_Q_OP_RECEIVE:
		rsm_intr_callback(data, src, arg);
		break;
	default:
		DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
		    "rsm_srv_func: unknown opcode = %x\n", opcode));
	}

	chd = chd;
	size = size;

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_srv_func done\n"));

	return (RSM_INTR_HAND_CLAIMED);
}

/* *************************** IPC slots ************************* */
static rsmipc_slot_t *
rsmipc_alloc()
{
	int i;
	rsmipc_slot_t *slot;
	DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_FUNC_ALL);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmipc_alloc enter\n"));

	/* try to find a free slot, if not wait */
	mutex_enter(&rsm_ipc.lock);

	while (rsm_ipc.count == 0) {
		rsm_ipc.wanted = 1;
		cv_wait(&rsm_ipc.cv, &rsm_ipc.lock);
	}

	/* An empty slot is available, find it */
	slot = &rsm_ipc.slots[0];
	for (i = 0; i < RSMIPC_SZ; i++, slot++) {
		if (RSMIPC_GET(slot, RSMIPC_FREE)) {
			RSMIPC_CLEAR(slot, RSMIPC_FREE);
			break;
		}
	}

	ASSERT(i < RSMIPC_SZ);
	rsm_ipc.count--;	/* one less is available */
	rsm_ipc.sequence++; /* new sequence */

	slot->rsmipc_cookie.ic.sequence = (uint_t)rsm_ipc.sequence;
	slot->rsmipc_cookie.ic.index = (uint_t)i;

	mutex_exit(&rsm_ipc.lock);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmipc_alloc done\n"));

	return (slot);
}

static void
rsmipc_free(rsmipc_slot_t *slot)
{
	DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_FUNC_ALL);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmipc_free enter\n"));

	ASSERT(MUTEX_HELD(&slot->rsmipc_lock));
	ASSERT(&rsm_ipc.slots[slot->rsmipc_cookie.ic.index] == slot);

	mutex_enter(&rsm_ipc.lock);

	RSMIPC_SET(slot, RSMIPC_FREE);

	slot->rsmipc_cookie.ic.sequence = 0;

	mutex_exit(&slot->rsmipc_lock);
	rsm_ipc.count++;
	ASSERT(rsm_ipc.count <= RSMIPC_SZ);
	if (rsm_ipc.wanted) {
		rsm_ipc.wanted = 0;
		cv_broadcast(&rsm_ipc.cv);
	}

	mutex_exit(&rsm_ipc.lock);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmipc_free done\n"));
}

static int
rsmipc_send(rsm_node_id_t dest, rsmipc_request_t *req, rsmipc_reply_t *reply)
{
	int		e = 0;
	int		credit_check = 0;
	int		retry_cnt = 0;
	int		min_retry_cnt = 10;
	rsm_send_t	is;
	rsmipc_slot_t	*rslot;
	adapter_t	*adapter;
	path_t		*path;
	sendq_token_t	*sendq_token;
	sendq_token_t	*used_sendq_token = NULL;
	rsm_send_q_handle_t	ipc_handle;
	DBG_DEFINE(category,
	    RSM_KERNEL_AGENT | RSM_FUNC_ALL | RSM_INTR_CALLBACK);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmipc_send enter:dest=%d",
	    dest));

	/*
	 * Check if this is a local case
	 */
	if (dest == my_nodeid) {
		switch (req->rsmipc_hdr.rsmipc_type) {
		case RSMIPC_MSG_SEGCONNECT:
			reply->rsmipc_status = (short)rsmsegacl_validate(
			    req, dest, reply);
			break;
		case RSMIPC_MSG_BELL:
			req->rsmipc_hdr.rsmipc_src = dest;
			rsm_intr_event(req);
			break;
		case RSMIPC_MSG_IMPORTING:
			importer_list_add(dest, req->rsmipc_key,
			    req->rsmipc_adapter_hwaddr,
			    req->rsmipc_segment_cookie);
			break;
		case RSMIPC_MSG_NOTIMPORTING:
			importer_list_rm(dest, req->rsmipc_key,
			    req->rsmipc_segment_cookie);
			break;
		case RSMIPC_MSG_REPUBLISH:
			importer_update(dest, req->rsmipc_key,
			    req->rsmipc_perm);
			break;
		case RSMIPC_MSG_SUSPEND:
			importer_suspend(dest);
			break;
		case RSMIPC_MSG_SUSPEND_DONE:
			rsm_suspend_complete(dest, 0);
			break;
		case RSMIPC_MSG_RESUME:
			importer_resume(dest);
			break;
		default:
			ASSERT(0);
		}
		DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
		    "rsmipc_send done\n"));
		return (0);
	}

	if (dest >= MAX_NODES) {
		DBG_PRINTF((category, RSM_ERR,
		    "rsm: rsmipc_send bad node number %x\n", dest));
		return (RSMERR_REMOTE_NODE_UNREACHABLE);
	}

	/*
	 * Oh boy! we are going remote.
	 */

	/*
	 * identify if we need to have credits to send this message
	 * - only selected requests are flow controlled
	 */
	if (req != NULL) {
		DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
		    "rsmipc_send:request type=%d\n",
		    req->rsmipc_hdr.rsmipc_type));

		switch (req->rsmipc_hdr.rsmipc_type) {
		case RSMIPC_MSG_SEGCONNECT:
		case RSMIPC_MSG_DISCONNECT:
		case RSMIPC_MSG_IMPORTING:
		case RSMIPC_MSG_SUSPEND:
		case RSMIPC_MSG_SUSPEND_DONE:
		case RSMIPC_MSG_RESUME:
			credit_check = 1;
			break;
		default:
			credit_check = 0;
		}
	}

again:
	if (retry_cnt++ == min_retry_cnt) {
		/* backoff before further retries for 10ms */
		delay(drv_usectohz(10000));
		retry_cnt = 0; /* reset retry_cnt */
	}
	sendq_token = rsmka_get_sendq_token(dest, used_sendq_token);
	if (sendq_token == NULL) {
		DBG_PRINTF((category, RSM_ERR,
		    "rsm: rsmipc_send no device to reach node %d\n", dest));
		return (RSMERR_REMOTE_NODE_UNREACHABLE);
	}

	if ((sendq_token == used_sendq_token) &&
	    ((e == RSMERR_CONN_ABORTED) || (e == RSMERR_TIMEOUT) ||
	    (e == RSMERR_COMM_ERR_MAYBE_DELIVERED))) {
		rele_sendq_token(sendq_token);
		DBG_PRINTF((category, RSM_DEBUG, "rsmipc_send done=%d\n", e));
		return (RSMERR_CONN_ABORTED);
	} else
		used_sendq_token = sendq_token;

/* lint -save -e413 */
	path = SQ_TOKEN_TO_PATH(sendq_token);
	adapter = path->local_adapter;
/* lint -restore */
	ipc_handle = sendq_token->rsmpi_sendq_handle;

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
	    "rsmipc_send: path=%lx sendq_hdl=%lx\n", path, ipc_handle));

	if (reply == NULL) {
		/* Send request without ack */
		/*
		 * Set the rsmipc_version number in the msghdr for KA
		 * communication versioning
		 */
		req->rsmipc_hdr.rsmipc_version = RSM_VERSION;
		req->rsmipc_hdr.rsmipc_src = my_nodeid;
		/*
		 * remote endpoints incn should match the value in our
		 * path's remote_incn field. No need to grab any lock
		 * since we have refcnted the path in rsmka_get_sendq_token
		 */
		req->rsmipc_hdr.rsmipc_incn = path->remote_incn;

		is.is_data = (void *)req;
		is.is_size = sizeof (*req);
		is.is_flags = RSM_INTR_SEND_DELIVER | RSM_INTR_SEND_SLEEP;
		is.is_wait = 0;

		if (credit_check) {
			mutex_enter(&path->mutex);
			/*
			 * wait till we recv credits or path goes down. If path
			 * goes down rsm_send will fail and we handle the error
			 * then
			 */
			while ((sendq_token->msgbuf_avail == 0) &&
			    (path->state == RSMKA_PATH_ACTIVE)) {
				e = cv_wait_sig(&sendq_token->sendq_cv,
				    &path->mutex);
				if (e == 0) {
					mutex_exit(&path->mutex);
					no_reply_cnt++;
					rele_sendq_token(sendq_token);
					DBG_PRINTF((category, RSM_DEBUG,
					    "rsmipc_send done: "
					    "cv_wait INTERRUPTED"));
					return (RSMERR_INTERRUPTED);
				}
			}

			/*
			 * path is not active retry on another path.
			 */
			if (path->state != RSMKA_PATH_ACTIVE) {
				mutex_exit(&path->mutex);
				rele_sendq_token(sendq_token);
				e = RSMERR_CONN_ABORTED;
				DBG_PRINTF((category, RSM_ERR,
				    "rsm: rsmipc_send: path !ACTIVE"));
				goto again;
			}

			ASSERT(sendq_token->msgbuf_avail > 0);

			/*
			 * reserve a msgbuf
			 */
			sendq_token->msgbuf_avail--;

			mutex_exit(&path->mutex);

			e = adapter->rsmpi_ops->rsm_send(ipc_handle, &is,
			    NULL);

			if (e != RSM_SUCCESS) {
				mutex_enter(&path->mutex);
				/*
				 * release the reserved msgbuf since
				 * the send failed
				 */
				sendq_token->msgbuf_avail++;
				cv_broadcast(&sendq_token->sendq_cv);
				mutex_exit(&path->mutex);
			}
		} else
			e = adapter->rsmpi_ops->rsm_send(ipc_handle, &is,
			    NULL);

		no_reply_cnt++;
		rele_sendq_token(sendq_token);
		if (e != RSM_SUCCESS) {
			DBG_PRINTF((category, RSM_ERR,
			    "rsm: rsmipc_send no reply send"
			    " err = %d no reply count = %d\n",
			    e, no_reply_cnt));
			ASSERT(e != RSMERR_QUEUE_FENCE_UP &&
			    e != RSMERR_BAD_BARRIER_HNDL);
			atomic_add_64(&rsm_ipcsend_errcnt, 1);
			goto again;
		} else {
			DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
			    "rsmipc_send done\n"));
			return (e);
		}

	}

	if (req == NULL) {
		/* Send reply - No flow control is done for reply */
		/*
		 * Set the version in the msg header for KA communication
		 * versioning
		 */
		reply->rsmipc_hdr.rsmipc_version = RSM_VERSION;
		reply->rsmipc_hdr.rsmipc_src = my_nodeid;
		/* incn number is not used for reply msgs currently */
		reply->rsmipc_hdr.rsmipc_incn = path->remote_incn;

		is.is_data = (void *)reply;
		is.is_size = sizeof (*reply);
		is.is_flags = RSM_INTR_SEND_DELIVER | RSM_INTR_SEND_SLEEP;
		is.is_wait = 0;
		e = adapter->rsmpi_ops->rsm_send(ipc_handle, &is, NULL);
		rele_sendq_token(sendq_token);
		if (e != RSM_SUCCESS) {
			DBG_PRINTF((category, RSM_ERR,
			    "rsm: rsmipc_send reply send"
			    " err = %d\n", e));
			atomic_add_64(&rsm_ipcsend_errcnt, 1);
			goto again;
		} else {
			DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
			    "rsmipc_send done\n"));
			return (e);
		}
	}

	/* Reply needed */
	rslot = rsmipc_alloc(); /* allocate a new ipc slot */

	mutex_enter(&rslot->rsmipc_lock);

	rslot->rsmipc_data = (void *)reply;
	RSMIPC_SET(rslot, RSMIPC_PENDING);

	while (RSMIPC_GET(rslot, RSMIPC_PENDING)) {
		/*
		 * Set the rsmipc_version number in the msghdr for KA
		 * communication versioning
		 */
		req->rsmipc_hdr.rsmipc_version = RSM_VERSION;
		req->rsmipc_hdr.rsmipc_src = my_nodeid;
		req->rsmipc_hdr.rsmipc_cookie = rslot->rsmipc_cookie;
		/*
		 * remote endpoints incn should match the value in our
		 * path's remote_incn field. No need to grab any lock
		 * since we have refcnted the path in rsmka_get_sendq_token
		 */
		req->rsmipc_hdr.rsmipc_incn = path->remote_incn;

		is.is_data = (void *)req;
		is.is_size = sizeof (*req);
		is.is_flags = RSM_INTR_SEND_DELIVER | RSM_INTR_SEND_SLEEP;
		is.is_wait = 0;
		if (credit_check) {

			mutex_enter(&path->mutex);
			/*
			 * wait till we recv credits or path goes down. If path
			 * goes down rsm_send will fail and we handle the error
			 * then.
			 */
			while ((sendq_token->msgbuf_avail == 0) &&
			    (path->state == RSMKA_PATH_ACTIVE)) {
				e = cv_wait_sig(&sendq_token->sendq_cv,
				    &path->mutex);
				if (e == 0) {
					mutex_exit(&path->mutex);
					RSMIPC_CLEAR(rslot, RSMIPC_PENDING);
					rsmipc_free(rslot);
					rele_sendq_token(sendq_token);
					DBG_PRINTF((category, RSM_DEBUG,
					    "rsmipc_send done: "
					    "cv_wait INTERRUPTED"));
					return (RSMERR_INTERRUPTED);
				}
			}

			/*
			 * path is not active retry on another path.
			 */
			if (path->state != RSMKA_PATH_ACTIVE) {
				mutex_exit(&path->mutex);
				RSMIPC_CLEAR(rslot, RSMIPC_PENDING);
				rsmipc_free(rslot);
				rele_sendq_token(sendq_token);
				e = RSMERR_CONN_ABORTED;
				DBG_PRINTF((category, RSM_ERR,
				    "rsm: rsmipc_send: path !ACTIVE"));
				goto again;
			}

			ASSERT(sendq_token->msgbuf_avail > 0);

			/*
			 * reserve a msgbuf
			 */
			sendq_token->msgbuf_avail--;

			mutex_exit(&path->mutex);

			e = adapter->rsmpi_ops->rsm_send(ipc_handle, &is,
			    NULL);

			if (e != RSM_SUCCESS) {
				mutex_enter(&path->mutex);
				/*
				 * release the reserved msgbuf since
				 * the send failed
				 */
				sendq_token->msgbuf_avail++;
				cv_broadcast(&sendq_token->sendq_cv);
				mutex_exit(&path->mutex);
			}
		} else
			e = adapter->rsmpi_ops->rsm_send(ipc_handle, &is,
			    NULL);

		if (e != RSM_SUCCESS) {
			DBG_PRINTF((category, RSM_ERR,
			    "rsm: rsmipc_send rsmpi send err = %d\n", e));
			RSMIPC_CLEAR(rslot, RSMIPC_PENDING);
			rsmipc_free(rslot);
			rele_sendq_token(sendq_token);
			atomic_add_64(&rsm_ipcsend_errcnt, 1);
			goto again;
		}

		/* wait for a reply signal, a SIGINT, or 5 sec. timeout */
		e = cv_reltimedwait_sig(&rslot->rsmipc_cv, &rslot->rsmipc_lock,
		    drv_usectohz(5000000), TR_CLOCK_TICK);
		if (e < 0) {
			/* timed out - retry */
			e = RSMERR_TIMEOUT;
		} else if (e == 0) {
			/* signalled - return error */
			e = RSMERR_INTERRUPTED;
			break;
		} else {
			e = RSM_SUCCESS;
		}
	}

	RSMIPC_CLEAR(rslot, RSMIPC_PENDING);
	rsmipc_free(rslot);
	rele_sendq_token(sendq_token);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmipc_send done=%d\n", e));
	return (e);
}

static int
rsm_send_notimporting(rsm_node_id_t dest, rsm_memseg_id_t segid,  void *cookie)
{
	rsmipc_request_t request;

	/*
	 *  inform the exporter to delete this importer
	 */
	request.rsmipc_hdr.rsmipc_type = RSMIPC_MSG_NOTIMPORTING;
	request.rsmipc_key = segid;
	request.rsmipc_segment_cookie = cookie;
	return (rsmipc_send(dest, &request, RSM_NO_REPLY));
}

static void
rsm_send_republish(rsm_memseg_id_t segid, rsmapi_access_entry_t	*acl,
    int acl_len, rsm_permission_t default_permission)
{
	int			i;
	importing_token_t	*token;
	rsmipc_request_t	request;
	republish_token_t	*republish_list = NULL;
	republish_token_t	*rp;
	rsm_permission_t	permission;
	int			index;

	/*
	 * send the new access mode to all the nodes that have imported
	 * this segment.
	 * If the new acl does not have a node that was present in
	 * the old acl a access permission of 0 is sent.
	 */

	index = rsmhash(segid);

	/*
	 * create a list of node/permissions to send the republish message
	 */
	mutex_enter(&importer_list.lock);

	token = importer_list.bucket[index];
	while (token != NULL) {
		if (segid == token->key) {
			permission = default_permission;

			for (i = 0; i < acl_len; i++) {
				if (token->importing_node == acl[i].ae_node) {
					permission = acl[i].ae_permission;
					break;
				}
			}
			rp = kmem_zalloc(sizeof (republish_token_t), KM_SLEEP);

			rp->key = segid;
			rp->importing_node = token->importing_node;
			rp->permission = permission;
			rp->next = republish_list;
			republish_list = rp;
		}
		token = token->next;
	}

	mutex_exit(&importer_list.lock);

	request.rsmipc_hdr.rsmipc_type = RSMIPC_MSG_REPUBLISH;
	request.rsmipc_key = segid;

	while (republish_list != NULL) {
		request.rsmipc_perm = republish_list->permission;
		(void) rsmipc_send(republish_list->importing_node,
		    &request, RSM_NO_REPLY);
		rp = republish_list;
		republish_list = republish_list->next;
		kmem_free(rp, sizeof (republish_token_t));
	}
}

static void
rsm_send_suspend()
{
	int			i, e;
	rsmipc_request_t 	request;
	list_element_t		*tokp;
	list_element_t		*head = NULL;
	importing_token_t	*token;
	DBG_PRINTF((RSM_KERNEL_AGENT | RSM_EXPORT, RSM_DEBUG_VERBOSE,
	    "rsm_send_suspend enter\n"));

	/*
	 * create a list of node to send the suspend message
	 *
	 * Currently the whole importer list is scanned and we obtain
	 * all the nodes - this basically gets all nodes that at least
	 * import one segment from the local node.
	 *
	 * no need to grab the rsm_suspend_list lock here since we are
	 * single threaded when suspend is called.
	 */

	mutex_enter(&importer_list.lock);
	for (i = 0; i < rsm_hash_size; i++) {

		token = importer_list.bucket[i];

		while (token != NULL) {

			tokp = head;

			/*
			 * make sure that the token's node
			 * is not already on the suspend list
			 */
			while (tokp != NULL) {
				if (tokp->nodeid == token->importing_node) {
					break;
				}
				tokp = tokp->next;
			}

			if (tokp == NULL) { /* not in suspend list */
				tokp = kmem_zalloc(sizeof (list_element_t),
				    KM_SLEEP);
				tokp->nodeid = token->importing_node;
				tokp->next = head;
				head = tokp;
			}

			token = token->next;
		}
	}
	mutex_exit(&importer_list.lock);

	if (head == NULL) { /* no importers so go ahead and quiesce segments */
		exporter_quiesce();
		return;
	}

	mutex_enter(&rsm_suspend_list.list_lock);
	ASSERT(rsm_suspend_list.list_head == NULL);
	/*
	 * update the suspend list righaway so that if a node dies the
	 * pathmanager can set the NODE dead flag
	 */
	rsm_suspend_list.list_head = head;
	mutex_exit(&rsm_suspend_list.list_lock);

	tokp = head;

	while (tokp != NULL) {
		request.rsmipc_hdr.rsmipc_type = RSMIPC_MSG_SUSPEND;
		e = rsmipc_send(tokp->nodeid, &request, RSM_NO_REPLY);
		/*
		 * Error in rsmipc_send currently happens due to inaccessibility
		 * of the remote node.
		 */
		if (e == RSM_SUCCESS) { /* send failed - don't wait for ack */
			tokp->flags |= RSM_SUSPEND_ACKPENDING;
		}

		tokp = tokp->next;
	}

	DBG_PRINTF((RSM_KERNEL_AGENT | RSM_EXPORT, RSM_DEBUG_VERBOSE,
	    "rsm_send_suspend done\n"));

}

static void
rsm_send_resume()
{
	rsmipc_request_t 	request;
	list_element_t		*elem, *head;

	/*
	 * save the suspend list so that we know where to send
	 * the resume messages and make the suspend list head
	 * NULL.
	 */
	mutex_enter(&rsm_suspend_list.list_lock);
	head = rsm_suspend_list.list_head;
	rsm_suspend_list.list_head = NULL;
	mutex_exit(&rsm_suspend_list.list_lock);

	while (head != NULL) {
		elem = head;
		head = head->next;

		request.rsmipc_hdr.rsmipc_type = RSMIPC_MSG_RESUME;

		(void) rsmipc_send(elem->nodeid, &request, RSM_NO_REPLY);

		kmem_free((void *)elem, sizeof (list_element_t));

	}

}

/*
 * This function takes path and sends a message using the sendq
 * corresponding to it. The RSMIPC_MSG_SQREADY, RSMIPC_MSG_SQREADY_ACK
 * and RSMIPC_MSG_CREDIT are sent using this function.
 */
int
rsmipc_send_controlmsg(path_t *path, int msgtype)
{
	int			e;
	int			retry_cnt = 0;
	int			min_retry_cnt = 10;
	adapter_t		*adapter;
	rsm_send_t		is;
	rsm_send_q_handle_t	ipc_handle;
	rsmipc_controlmsg_t	msg;
	DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_FUNC_ALL | RSM_FLOWCONTROL);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
	    "rsmipc_send_controlmsg enter\n"));

	ASSERT(MUTEX_HELD(&path->mutex));

	adapter = path->local_adapter;

	DBG_PRINTF((category, RSM_DEBUG, "rsmipc_send_controlmsg:path=%lx "
	    "msgtype=%d %lx:%llx->%lx:%llx procmsg=%d\n", path, msgtype,
	    my_nodeid, adapter->hwaddr, path->remote_node,
	    path->remote_hwaddr, path->procmsg_cnt));

	if (path->state != RSMKA_PATH_ACTIVE) {
		DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
		    "rsmipc_send_controlmsg done: ! RSMKA_PATH_ACTIVE"));
		return (1);
	}

	ipc_handle = path->sendq_token.rsmpi_sendq_handle;

	msg.rsmipc_hdr.rsmipc_version = RSM_VERSION;
	msg.rsmipc_hdr.rsmipc_src = my_nodeid;
	msg.rsmipc_hdr.rsmipc_type = msgtype;
	msg.rsmipc_hdr.rsmipc_incn = path->remote_incn;

	if (msgtype == RSMIPC_MSG_CREDIT)
		msg.rsmipc_credits = path->procmsg_cnt;

	msg.rsmipc_local_incn = path->local_incn;

	msg.rsmipc_adapter_hwaddr = adapter->hwaddr;
	/* incr the sendq, path refcnt */
	PATH_HOLD_NOLOCK(path);
	SENDQ_TOKEN_HOLD(path);

	do {
		/* drop the path lock before doing the rsm_send */
		mutex_exit(&path->mutex);

		is.is_data = (void *)&msg;
		is.is_size = sizeof (msg);
		is.is_flags = RSM_INTR_SEND_DELIVER | RSM_INTR_SEND_SLEEP;
		is.is_wait = 0;

		e = adapter->rsmpi_ops->rsm_send(ipc_handle, &is, NULL);

		ASSERT(e != RSMERR_QUEUE_FENCE_UP &&
		    e != RSMERR_BAD_BARRIER_HNDL);

		mutex_enter(&path->mutex);

		if (e == RSM_SUCCESS) {
			break;
		}
		/* error counter for statistics */
		atomic_add_64(&rsm_ctrlmsg_errcnt, 1);

		DBG_PRINTF((category, RSM_ERR,
		    "rsmipc_send_controlmsg:rsm_send error=%d", e));

		if (++retry_cnt == min_retry_cnt) { /* backoff before retry */
			(void) cv_reltimedwait(&path->sendq_token.sendq_cv,
			    &path->mutex, drv_usectohz(10000), TR_CLOCK_TICK);
			retry_cnt = 0;
		}
	} while (path->state == RSMKA_PATH_ACTIVE);

	/* decrement the sendq,path refcnt that we incr before rsm_send */
	SENDQ_TOKEN_RELE(path);
	PATH_RELE_NOLOCK(path);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
	    "rsmipc_send_controlmsg done=%d", e));
	return (e);
}

/*
 * Called from rsm_force_unload and path_importer_disconnect. The memory
 * mapping for the imported segment is removed and the segment is
 * disconnected at the interconnect layer if disconnect_flag is TRUE.
 * rsm_force_unload will get disconnect_flag TRUE from rsm_intr_callback
 * and FALSE from rsm_rebind.
 *
 * When subsequent accesses cause page faulting, the dummy page is mapped
 * to resolve the fault, and the mapping generation number is incremented
 * so that the application can be notified on a close barrier operation.
 *
 * It is important to note that the caller of rsmseg_unload is responsible for
 * acquiring the segment lock before making a call to rsmseg_unload. This is
 * required to make the caller and rsmseg_unload thread safe. The segment lock
 * will be released by the rsmseg_unload function.
 */
void
rsmseg_unload(rsmseg_t *im_seg)
{
	rsmcookie_t		*hdl;
	void			*shared_cookie;
	rsmipc_request_t	request;
	uint_t			maxprot;

	DBG_DEFINE(category,
	    RSM_KERNEL_AGENT | RSM_FUNC_ALL | RSM_INTR_CALLBACK);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmseg_unload enter\n"));

	ASSERT(im_seg->s_hdr.rsmrc_type == RSM_RESOURCE_IMPORT_SEGMENT);

	/* wait until segment leaves the mapping state */
	while (im_seg->s_state == RSM_STATE_MAPPING)
		cv_wait(&im_seg->s_cv, &im_seg->s_lock);
	/*
	 * An unload is only necessary if the segment is connected. However,
	 * if the segment was on the import list in state RSM_STATE_CONNECTING
	 * then a connection was in progress. Change to RSM_STATE_NEW
	 * here to cause an early exit from the connection process.
	 */
	if (im_seg->s_state == RSM_STATE_NEW) {
		rsmseglock_release(im_seg);
		DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
		    "rsmseg_unload done: RSM_STATE_NEW\n"));
		return;
	} else if (im_seg->s_state == RSM_STATE_CONNECTING) {
		im_seg->s_state = RSM_STATE_ABORT_CONNECT;
		rsmsharelock_acquire(im_seg);
		im_seg->s_share->rsmsi_state = RSMSI_STATE_ABORT_CONNECT;
		rsmsharelock_release(im_seg);
		rsmseglock_release(im_seg);
		DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
		    "rsmseg_unload done: RSM_STATE_CONNECTING\n"));
		return;
	}

	if (im_seg->s_flags & RSM_FORCE_DISCONNECT) {
		if (im_seg->s_ckl != NULL) {
			int e;
			/* Setup protections for remap */
			maxprot = PROT_USER;
			if (im_seg->s_mode & RSM_PERM_READ) {
				maxprot |= PROT_READ;
			}
			if (im_seg->s_mode & RSM_PERM_WRITE) {
				maxprot |= PROT_WRITE;
			}
			hdl = im_seg->s_ckl;
			for (; hdl != NULL; hdl = hdl->c_next) {
				e = devmap_umem_remap(hdl->c_dhp, rsm_dip,
				    remap_cookie,
				    hdl->c_off, hdl->c_len,
				    maxprot, 0, NULL);

				DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
				    "remap returns %d\n", e));
			}
		}

		(void) rsm_closeconnection(im_seg, &shared_cookie);

		if (shared_cookie != NULL) {
			/*
			 * inform the exporting node so this import
			 * can be deleted from the list of importers.
			 */
			request.rsmipc_hdr.rsmipc_type =
			    RSMIPC_MSG_NOTIMPORTING;
			request.rsmipc_key = im_seg->s_segid;
			request.rsmipc_segment_cookie = shared_cookie;
			rsmseglock_release(im_seg);
			(void) rsmipc_send(im_seg->s_node, &request,
			    RSM_NO_REPLY);
		} else {
			rsmseglock_release(im_seg);
		}
	}
	else
		rsmseglock_release(im_seg);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmseg_unload done\n"));

}

/* ****************************** Importer Calls ************************ */

static int
rsm_access(uid_t owner, gid_t group, int perm, int mode, const struct cred *cr)
{
	int shifts = 0;

	if (crgetuid(cr) != owner) {
		shifts += 3;
		if (!groupmember(group, cr))
			shifts += 3;
	}

	mode &= ~(perm << shifts);

	if (mode == 0)
		return (0);

	return (secpolicy_rsm_access(cr, owner, mode));
}


static int
rsm_connect(rsmseg_t *seg, rsm_ioctlmsg_t *msg, cred_t *cred,
    intptr_t dataptr, int mode)
{
	int e;
	int			recheck_state = 0;
	void			*shared_cookie;
	rsmipc_request_t	request;
	rsmipc_reply_t		reply;
	rsm_permission_t	access;
	adapter_t		*adapter;
	rsm_addr_t		addr = 0;
	rsm_import_share_t	*sharedp;
	DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_IMPORT);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_connect enter\n"));

	adapter = rsm_getadapter(msg, mode);
	if (adapter == NULL) {
		DBG_PRINTF((category, RSM_ERR,
		    "rsm_connect done:ENODEV adapter=NULL\n"));
		return (RSMERR_CTLR_NOT_PRESENT);
	}

	if ((adapter == &loopback_adapter) && (msg->nodeid != my_nodeid)) {
		rsmka_release_adapter(adapter);
		DBG_PRINTF((category, RSM_ERR,
		    "rsm_connect done:ENODEV loopback\n"));
		return (RSMERR_CTLR_NOT_PRESENT);
	}


	ASSERT(seg->s_hdr.rsmrc_type == RSM_RESOURCE_IMPORT_SEGMENT);
	ASSERT(seg->s_state == RSM_STATE_NEW);

	/*
	 * Translate perm to access
	 */
	if (msg->perm & ~RSM_PERM_RDWR) {
		rsmka_release_adapter(adapter);
		DBG_PRINTF((category, RSM_ERR,
		    "rsm_connect done:EINVAL invalid perms\n"));
		return (RSMERR_BAD_PERMS);
	}
	access = 0;
	if (msg->perm & RSM_PERM_READ)
		access |= RSM_ACCESS_READ;
	if (msg->perm & RSM_PERM_WRITE)
		access |= RSM_ACCESS_WRITE;

	seg->s_node = msg->nodeid;

	/*
	 * Adding to the import list locks the segment; release the segment
	 * lock so we can get the reply for the send.
	 */
	e = rsmimport_add(seg, msg->key);
	if (e) {
		rsmka_release_adapter(adapter);
		DBG_PRINTF((category, RSM_ERR,
		    "rsm_connect done:rsmimport_add failed %d\n", e));
		return (e);
	}
	seg->s_state = RSM_STATE_CONNECTING;

	/*
	 * Set the s_adapter field here so as to have a valid comparison of
	 * the adapter and the s_adapter value during rsmshare_get. For
	 * any error, set s_adapter to NULL before doing a release_adapter
	 */
	seg->s_adapter = adapter;

	rsmseglock_release(seg);

	/*
	 * get the pointer to the shared data structure; the
	 * shared data is locked and refcount has been incremented
	 */
	sharedp = rsmshare_get(msg->key, msg->nodeid, adapter, seg);

	ASSERT(rsmsharelock_held(seg));

	do {
		/* flag indicates whether we need to recheck the state */
		recheck_state = 0;
		DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
		    "rsm_connect:RSMSI_STATE=%d\n", sharedp->rsmsi_state));
		switch (sharedp->rsmsi_state) {
		case RSMSI_STATE_NEW:
			sharedp->rsmsi_state = RSMSI_STATE_CONNECTING;
			break;
		case RSMSI_STATE_CONNECTING:
			/* FALLTHRU */
		case RSMSI_STATE_CONN_QUIESCE:
			/* FALLTHRU */
		case RSMSI_STATE_MAP_QUIESCE:
			/* wait for the state to change */
			while ((sharedp->rsmsi_state ==
			    RSMSI_STATE_CONNECTING) ||
			    (sharedp->rsmsi_state ==
			    RSMSI_STATE_CONN_QUIESCE) ||
			    (sharedp->rsmsi_state ==
			    RSMSI_STATE_MAP_QUIESCE)) {
				if (cv_wait_sig(&sharedp->rsmsi_cv,
				    &sharedp->rsmsi_lock) == 0) {
					/* signalled - clean up and return */
					rsmsharelock_release(seg);
					rsmimport_rm(seg);
					seg->s_adapter = NULL;
					rsmka_release_adapter(adapter);
					seg->s_state = RSM_STATE_NEW;
					DBG_PRINTF((category, RSM_ERR,
					    "rsm_connect done: INTERRUPTED\n"));
					return (RSMERR_INTERRUPTED);
				}
			}
			/*
			 * the state changed, loop back and check what it is
			 */
			recheck_state = 1;
			break;
		case RSMSI_STATE_ABORT_CONNECT:
			/* exit the loop and clean up further down */
			break;
		case RSMSI_STATE_CONNECTED:
			/* already connected, good - fall through */
		case RSMSI_STATE_MAPPED:
			/* already mapped, wow - fall through */
			/* access validation etc is done further down */
			break;
		case RSMSI_STATE_DISCONNECTED:
			/* disconnected - so reconnect now */
			sharedp->rsmsi_state = RSMSI_STATE_CONNECTING;
			break;
		default:
			ASSERT(0); /* Invalid State */
		}
	} while (recheck_state);

	if (sharedp->rsmsi_state == RSMSI_STATE_CONNECTING) {
		/* we are the first to connect */
		rsmsharelock_release(seg);

		if (msg->nodeid != my_nodeid) {
			addr = get_remote_hwaddr(adapter, msg->nodeid);

			if ((int64_t)addr < 0) {
				rsmsharelock_acquire(seg);
				rsmsharecv_signal(seg, RSMSI_STATE_CONNECTING,
				    RSMSI_STATE_NEW);
				rsmsharelock_release(seg);
				rsmimport_rm(seg);
				seg->s_adapter = NULL;
				rsmka_release_adapter(adapter);
				seg->s_state = RSM_STATE_NEW;
				DBG_PRINTF((category, RSM_ERR,
				    "rsm_connect done: hwaddr<0\n"));
				return (RSMERR_INTERNAL_ERROR);
			}
		} else {
			addr = adapter->hwaddr;
		}

		/*
		 * send request to node [src, dest, key, msgid] and get back
		 * [status, msgid, cookie]
		 */
		request.rsmipc_key = msg->key;
		/*
		 * we need the s_mode of the exporter so pass
		 * RSM_ACCESS_TRUSTED
		 */
		request.rsmipc_perm = RSM_ACCESS_TRUSTED;
		request.rsmipc_hdr.rsmipc_type = RSMIPC_MSG_SEGCONNECT;
		request.rsmipc_adapter_hwaddr = addr;
		request.rsmipc_segment_cookie = sharedp;

		e = (int)rsmipc_send(msg->nodeid, &request, &reply);
		if (e) {
			rsmsharelock_acquire(seg);
			rsmsharecv_signal(seg, RSMSI_STATE_CONNECTING,
			    RSMSI_STATE_NEW);
			rsmsharelock_release(seg);
			rsmimport_rm(seg);
			seg->s_adapter = NULL;
			rsmka_release_adapter(adapter);
			seg->s_state = RSM_STATE_NEW;
			DBG_PRINTF((category, RSM_ERR,
			    "rsm_connect done:rsmipc_send failed %d\n", e));
			return (e);
		}

		if (reply.rsmipc_status != RSM_SUCCESS) {
			rsmsharelock_acquire(seg);
			rsmsharecv_signal(seg, RSMSI_STATE_CONNECTING,
			    RSMSI_STATE_NEW);
			rsmsharelock_release(seg);
			rsmimport_rm(seg);
			seg->s_adapter = NULL;
			rsmka_release_adapter(adapter);
			seg->s_state = RSM_STATE_NEW;
			DBG_PRINTF((category, RSM_ERR,
			    "rsm_connect done:rsmipc_send reply err %d\n",
			    reply.rsmipc_status));
			return (reply.rsmipc_status);
		}

		rsmsharelock_acquire(seg);
		/* store the information recvd into the shared data struct */
		sharedp->rsmsi_mode = reply.rsmipc_mode;
		sharedp->rsmsi_uid = reply.rsmipc_uid;
		sharedp->rsmsi_gid = reply.rsmipc_gid;
		sharedp->rsmsi_seglen = reply.rsmipc_seglen;
		sharedp->rsmsi_cookie = sharedp;
	}

	rsmsharelock_release(seg);

	/*
	 * Get the segment lock and check for a force disconnect
	 * from the export side which would have changed the state
	 * back to RSM_STATE_NEW. Once the segment lock is acquired a
	 * force disconnect will be held off until the connection
	 * has completed.
	 */
	rsmseglock_acquire(seg);
	rsmsharelock_acquire(seg);
	ASSERT(seg->s_state == RSM_STATE_CONNECTING ||
	    seg->s_state == RSM_STATE_ABORT_CONNECT);

	shared_cookie = sharedp->rsmsi_cookie;

	if ((seg->s_state == RSM_STATE_ABORT_CONNECT) ||
	    (sharedp->rsmsi_state == RSMSI_STATE_ABORT_CONNECT)) {
		seg->s_state = RSM_STATE_NEW;
		seg->s_adapter = NULL;
		rsmsharelock_release(seg);
		rsmseglock_release(seg);
		rsmimport_rm(seg);
		rsmka_release_adapter(adapter);

		rsmsharelock_acquire(seg);
		if (!(sharedp->rsmsi_flags & RSMSI_FLAGS_ABORTDONE)) {
			/*
			 * set a flag indicating abort handling has been
			 * done
			 */
			sharedp->rsmsi_flags |= RSMSI_FLAGS_ABORTDONE;
			rsmsharelock_release(seg);
			/* send a message to exporter - only once */
			(void) rsm_send_notimporting(msg->nodeid,
			    msg->key, shared_cookie);
			rsmsharelock_acquire(seg);
			/*
			 * wake up any waiting importers and inform that
			 * connection has been aborted
			 */
			cv_broadcast(&sharedp->rsmsi_cv);
		}
		rsmsharelock_release(seg);

		DBG_PRINTF((category, RSM_ERR,
		    "rsm_connect done: RSM_STATE_ABORT_CONNECT\n"));
		return (RSMERR_INTERRUPTED);
	}


	/*
	 * We need to verify that this process has access
	 */
	e = rsm_access(sharedp->rsmsi_uid, sharedp->rsmsi_gid,
	    access & sharedp->rsmsi_mode,
	    (int)(msg->perm & RSM_PERM_RDWR), cred);
	if (e) {
		rsmsharelock_release(seg);
		seg->s_state = RSM_STATE_NEW;
		seg->s_adapter = NULL;
		rsmseglock_release(seg);
		rsmimport_rm(seg);
		rsmka_release_adapter(adapter);
		/*
		 * No need to lock segment it has been removed
		 * from the hash table
		 */
		rsmsharelock_acquire(seg);
		if (sharedp->rsmsi_state == RSMSI_STATE_CONNECTING) {
			rsmsharelock_release(seg);
			/* this is the first importer */

			(void) rsm_send_notimporting(msg->nodeid, msg->key,
			    shared_cookie);
			rsmsharelock_acquire(seg);
			sharedp->rsmsi_state = RSMSI_STATE_NEW;
			cv_broadcast(&sharedp->rsmsi_cv);
		}
		rsmsharelock_release(seg);

		DBG_PRINTF((category, RSM_ERR,
		    "rsm_connect done: ipcaccess failed\n"));
		return (RSMERR_PERM_DENIED);
	}

	/* update state and cookie */
	seg->s_segid = sharedp->rsmsi_segid;
	seg->s_len = sharedp->rsmsi_seglen;
	seg->s_mode = access & sharedp->rsmsi_mode;
	seg->s_pid = ddi_get_pid();
	seg->s_mapinfo = NULL;

	if (seg->s_node != my_nodeid) {
		if (sharedp->rsmsi_state == RSMSI_STATE_CONNECTING) {
			e = adapter->rsmpi_ops->rsm_connect(
			    adapter->rsmpi_handle,
			    addr, seg->s_segid, &sharedp->rsmsi_handle);

			if (e != RSM_SUCCESS) {
				seg->s_state = RSM_STATE_NEW;
				seg->s_adapter = NULL;
				rsmsharelock_release(seg);
				rsmseglock_release(seg);
				rsmimport_rm(seg);
				rsmka_release_adapter(adapter);
				/*
				 *  inform the exporter to delete this importer
				 */
				(void) rsm_send_notimporting(msg->nodeid,
				    msg->key, shared_cookie);

				/*
				 * Now inform any waiting importers to
				 * retry connect. This needs to be done
				 * after sending notimporting so that
				 * the notimporting is sent before a waiting
				 * importer sends a segconnect while retrying
				 *
				 * No need to lock segment it has been removed
				 * from the hash table
				 */

				rsmsharelock_acquire(seg);
				sharedp->rsmsi_state = RSMSI_STATE_NEW;
				cv_broadcast(&sharedp->rsmsi_cv);
				rsmsharelock_release(seg);

				DBG_PRINTF((category, RSM_ERR,
				    "rsm_connect error %d\n", e));
				if (e == RSMERR_SEG_NOT_PUBLISHED_TO_RSM_ADDR)
					return (
					    RSMERR_SEG_NOT_PUBLISHED_TO_NODE);
				else if ((e == RSMERR_RSM_ADDR_UNREACHABLE) ||
				    (e == RSMERR_UNKNOWN_RSM_ADDR))
					return (RSMERR_REMOTE_NODE_UNREACHABLE);
				else
					return (e);
			}

		}
		seg->s_handle.in = sharedp->rsmsi_handle;

	}

	seg->s_state = RSM_STATE_CONNECT;


	seg->s_flags &= ~RSM_IMPORT_DUMMY;	/* clear dummy flag */
	if (bar_va) {
		/* increment generation number on barrier page */
		atomic_add_16(bar_va + seg->s_hdr.rsmrc_num, 1);
		/* return user off into barrier page where status will be */
		msg->off = (int)seg->s_hdr.rsmrc_num;
		msg->gnum = bar_va[msg->off]; 	/* gnum race */
	} else {
		msg->off = 0;
		msg->gnum = 0;	/* gnum race */
	}

	msg->len = (int)sharedp->rsmsi_seglen;
	msg->rnum = seg->s_minor;
	rsmsharecv_signal(seg, RSMSI_STATE_CONNECTING, RSMSI_STATE_CONNECTED);
	rsmsharelock_release(seg);
	rsmseglock_release(seg);

	/* Return back to user the segment size & perm in case it's needed */

#ifdef _MULTI_DATAMODEL
	if ((mode & DATAMODEL_MASK) == DATAMODEL_ILP32) {
		rsm_ioctlmsg32_t msg32;

		if (msg->len > UINT_MAX)
			msg32.len = RSM_MAXSZ_PAGE_ALIGNED;
		else
			msg32.len = msg->len;
		msg32.off = msg->off;
		msg32.perm = msg->perm;
		msg32.gnum = msg->gnum;
		msg32.rnum = msg->rnum;

		DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
		    "rsm_connect done\n"));

		if (ddi_copyout((caddr_t)&msg32, (caddr_t)dataptr,
		    sizeof (msg32), mode))
			return (RSMERR_BAD_ADDR);
		else
			return (RSM_SUCCESS);
	}
#endif
	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_connect done\n"));

	if (ddi_copyout((caddr_t)msg, (caddr_t)dataptr, sizeof (*msg),
	    mode))
		return (RSMERR_BAD_ADDR);
	else
		return (RSM_SUCCESS);
}

static int
rsm_unmap(rsmseg_t *seg)
{
	int			err;
	adapter_t		*adapter;
	rsm_import_share_t	*sharedp;
	DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_IMPORT);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
	    "rsm_unmap enter %u\n", seg->s_segid));

	ASSERT(seg->s_hdr.rsmrc_type == RSM_RESOURCE_IMPORT_SEGMENT);

	/* assert seg is locked */
	ASSERT(rsmseglock_held(seg));
	ASSERT(seg->s_state != RSM_STATE_MAPPING);

	if ((seg->s_state != RSM_STATE_ACTIVE) &&
	    (seg->s_state != RSM_STATE_MAP_QUIESCE)) {
		/* segment unmap has already been done */
		DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_unmap done\n"));
		return (RSM_SUCCESS);
	}

	sharedp = seg->s_share;

	rsmsharelock_acquire(seg);

	/*
	 *	- shared data struct is in MAPPED or MAP_QUIESCE state
	 */

	ASSERT(sharedp->rsmsi_state == RSMSI_STATE_MAPPED ||
	    sharedp->rsmsi_state == RSMSI_STATE_MAP_QUIESCE);

	/*
	 * Unmap pages - previously rsm_memseg_import_unmap was called only if
	 * the segment cookie list was NULL; but it is always NULL when
	 * called from rsmmap_unmap and won't be NULL when called for
	 * a force disconnect - so the check for NULL cookie list was removed
	 */

	ASSERT(sharedp->rsmsi_mapcnt > 0);

	sharedp->rsmsi_mapcnt--;

	if (sharedp->rsmsi_mapcnt == 0) {
		if (sharedp->rsmsi_state == RSMSI_STATE_MAPPED) {
			/* unmap the shared RSMPI mapping */
			adapter = seg->s_adapter;
			if (seg->s_node != my_nodeid) {
				ASSERT(sharedp->rsmsi_handle != NULL);
				err = adapter->rsmpi_ops->
				    rsm_unmap(sharedp->rsmsi_handle);
				DBG_PRINTF((category, RSM_DEBUG,
				    "rsm_unmap: rsmpi unmap %d\n", err));
				rsm_free_mapinfo(sharedp->rsmsi_mapinfo);
				sharedp->rsmsi_mapinfo = NULL;
			}
			sharedp->rsmsi_state = RSMSI_STATE_CONNECTED;
		} else { /* MAP_QUIESCE --munmap()--> CONN_QUIESCE */
			sharedp->rsmsi_state = RSMSI_STATE_CONN_QUIESCE;
		}
	}

	rsmsharelock_release(seg);

	/*
	 * The s_cookie field is used to store the cookie returned from the
	 * ddi_umem_lock when binding the pages for an export segment. This
	 * is the primary use of the s_cookie field and does not normally
	 * pertain to any importing segment except in the loopback case.
	 * For the loopback case, the import segment and export segment are
	 * on the same node, the s_cookie field of the segment structure for
	 * the importer is initialized to the s_cookie field in the exported
	 * segment during the map operation and is used during the call to
	 * devmap_umem_setup for the import mapping.
	 * Thus, during unmap, we simply need to set s_cookie to NULL to
	 * indicate that the mapping no longer exists.
	 */
	seg->s_cookie = NULL;

	seg->s_mapinfo = NULL;

	if (seg->s_state == RSM_STATE_ACTIVE)
		seg->s_state = RSM_STATE_CONNECT;
	else
		seg->s_state = RSM_STATE_CONN_QUIESCE;

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_unmap done\n"));

	return (RSM_SUCCESS);
}

/*
 * cookie returned here if not null indicates that it is
 * the last importer and it can be used in the RSMIPC_NOT_IMPORTING
 * message.
 */
static int
rsm_closeconnection(rsmseg_t *seg, void **cookie)
{
	int			e;
	adapter_t		*adapter;
	rsm_import_share_t	*sharedp;
	DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_IMPORT);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
	    "rsm_closeconnection enter\n"));

	*cookie = (void *)NULL;

	ASSERT(seg->s_hdr.rsmrc_type == RSM_RESOURCE_IMPORT_SEGMENT);

	/* assert seg is locked */
	ASSERT(rsmseglock_held(seg));

	if (seg->s_state == RSM_STATE_DISCONNECT) {
		DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
		    "rsm_closeconnection done: already disconnected\n"));
		return (RSM_SUCCESS);
	}

	/* wait for all putv/getv ops to get done */
	while (seg->s_rdmacnt > 0) {
		cv_wait(&seg->s_cv, &seg->s_lock);
	}

	(void) rsm_unmap(seg);

	ASSERT(seg->s_state == RSM_STATE_CONNECT ||
	    seg->s_state == RSM_STATE_CONN_QUIESCE);

	adapter = seg->s_adapter;
	sharedp = seg->s_share;

	ASSERT(sharedp != NULL);

	rsmsharelock_acquire(seg);

	/*
	 * Disconnect on adapter
	 *
	 * The current algorithm is stateless, I don't have to contact
	 * server when I go away. He only gives me permissions. Of course,
	 * the adapters will talk to terminate the connect.
	 *
	 * disconnect is needed only if we are CONNECTED not in CONN_QUIESCE
	 */
	if ((sharedp->rsmsi_state == RSMSI_STATE_CONNECTED) &&
	    (sharedp->rsmsi_node != my_nodeid)) {

		if (sharedp->rsmsi_refcnt == 1) {
			/* this is the last importer */
			ASSERT(sharedp->rsmsi_mapcnt == 0);

			e = adapter->rsmpi_ops->
			    rsm_disconnect(sharedp->rsmsi_handle);
			if (e != RSM_SUCCESS) {
				DBG_PRINTF((category, RSM_DEBUG,
				    "rsm:disconnect failed seg=%x:err=%d\n",
				    seg->s_key, e));
			}
		}
	}

	seg->s_handle.in = NULL;

	sharedp->rsmsi_refcnt--;

	if (sharedp->rsmsi_refcnt == 0) {
		*cookie = (void *)sharedp->rsmsi_cookie;
		sharedp->rsmsi_state = RSMSI_STATE_DISCONNECTED;
		sharedp->rsmsi_handle = NULL;
		rsmsharelock_release(seg);

		/* clean up the shared data structure */
		mutex_destroy(&sharedp->rsmsi_lock);
		cv_destroy(&sharedp->rsmsi_cv);
		kmem_free((void *)(sharedp), sizeof (rsm_import_share_t));

	} else {
		rsmsharelock_release(seg);
	}

	/* increment generation number on barrier page */
	if (bar_va) {
		atomic_add_16(bar_va + seg->s_hdr.rsmrc_num, 1);
	}

	/*
	 * The following needs to be done after any
	 * rsmsharelock calls which use seg->s_share.
	 */
	seg->s_share = NULL;

	seg->s_state = RSM_STATE_DISCONNECT;
	/* signal anyone waiting in the CONN_QUIESCE state */
	cv_broadcast(&seg->s_cv);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
	    "rsm_closeconnection done\n"));

	return (RSM_SUCCESS);
}

int
rsm_disconnect(rsmseg_t *seg)
{
	rsmipc_request_t	request;
	void			*shared_cookie;
	DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_IMPORT);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_disconnect enter\n"));

	ASSERT(seg->s_hdr.rsmrc_type == RSM_RESOURCE_IMPORT_SEGMENT);

	/* assert seg isn't locked */
	ASSERT(!rsmseglock_held(seg));


	/* Remove segment from imported list */
	rsmimport_rm(seg);

	/* acquire the segment */
	rsmseglock_acquire(seg);

	/* wait until segment leaves the mapping state */
	while (seg->s_state == RSM_STATE_MAPPING)
		cv_wait(&seg->s_cv, &seg->s_lock);

	if (seg->s_state == RSM_STATE_DISCONNECT) {
		seg->s_state = RSM_STATE_NEW;
		rsmseglock_release(seg);
		DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
		    "rsm_disconnect done: already disconnected\n"));
		return (RSM_SUCCESS);
	}

	(void) rsm_closeconnection(seg, &shared_cookie);

	/* update state */
	seg->s_state = RSM_STATE_NEW;

	if (shared_cookie != NULL) {
		/*
		 *  This is the last importer so inform the exporting node
		 *  so this import can be deleted from the list of importers.
		 */
		request.rsmipc_hdr.rsmipc_type = RSMIPC_MSG_NOTIMPORTING;
		request.rsmipc_key = seg->s_segid;
		request.rsmipc_segment_cookie = shared_cookie;
		rsmseglock_release(seg);
		(void) rsmipc_send(seg->s_node, &request, RSM_NO_REPLY);
	} else {
		rsmseglock_release(seg);
	}

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_disconnect done\n"));

	return (DDI_SUCCESS);
}

/*ARGSUSED*/
static int
rsm_chpoll(dev_t dev, short events, int anyyet, short *reventsp,
    struct pollhead **phpp)
{
	minor_t		rnum;
	rsmresource_t	*res;
	rsmseg_t 	*seg;
	DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_FUNC_ALL | RSM_DDI);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_chpoll enter\n"));

	/* find minor, no lock */
	rnum = getminor(dev);
	res = rsmresource_lookup(rnum, RSM_NOLOCK);

	/* poll is supported only for export/import segments */
	if ((res == NULL) || (res == RSMRC_RESERVED) ||
	    (res->rsmrc_type == RSM_RESOURCE_BAR)) {
		return (ENXIO);
	}

	*reventsp = 0;

	/*
	 * An exported segment must be in state RSM_STATE_EXPORT; an
	 * imported segment must be in state RSM_STATE_ACTIVE.
	 */
	seg = (rsmseg_t *)res;

	if (seg->s_pollevent) {
		*reventsp = POLLRDNORM;
	} else if (!anyyet) {
		/* cannot take segment lock here */
		*phpp = &seg->s_poll;
		seg->s_pollflag |= RSM_SEGMENT_POLL;
	}
	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_chpoll done\n"));
	return (0);
}



/* ************************* IOCTL Commands ********************* */

static rsmseg_t *
rsmresource_seg(rsmresource_t *res, minor_t rnum, cred_t *credp,
    rsm_resource_type_t type)
{
	/* get segment from resource handle */
	rsmseg_t *seg;
	DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_FUNC_ALL | RSM_IOCTL);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmresource_seg enter\n"));


	if (res != RSMRC_RESERVED) {
		seg = (rsmseg_t *)res;
	} else {
		/* Allocate segment now and bind it */
		seg = rsmseg_alloc(rnum, credp);

		/*
		 * if DR pre-processing is going on or DR is in progress
		 * then the new export segments should be in the NEW_QSCD state
		 */
		if (type == RSM_RESOURCE_EXPORT_SEGMENT) {
			mutex_enter(&rsm_drv_data.drv_lock);
			if ((rsm_drv_data.drv_state ==
			    RSM_DRV_PREDEL_STARTED) ||
			    (rsm_drv_data.drv_state ==
			    RSM_DRV_PREDEL_COMPLETED) ||
			    (rsm_drv_data.drv_state ==
			    RSM_DRV_DR_IN_PROGRESS)) {
				seg->s_state = RSM_STATE_NEW_QUIESCED;
			}
			mutex_exit(&rsm_drv_data.drv_lock);
		}

		rsmresource_insert(rnum, (rsmresource_t *)seg, type);
	}

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmresource_seg done\n"));

	return (seg);
}

static int
rsmexport_ioctl(rsmseg_t *seg, rsm_ioctlmsg_t *msg, int cmd, intptr_t arg,
    int mode, cred_t *credp)
{
	int error;
	DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_EXPORT | RSM_IOCTL);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmexport_ioctl enter\n"));

	arg = arg;
	credp = credp;

	ASSERT(seg != NULL);

	switch (cmd) {
	case RSM_IOCTL_BIND:
		error = rsm_bind(seg, msg, arg, mode);
		break;
	case RSM_IOCTL_REBIND:
		error = rsm_rebind(seg, msg);
		break;
	case RSM_IOCTL_UNBIND:
		error = ENOTSUP;
		break;
	case RSM_IOCTL_PUBLISH:
		error = rsm_publish(seg, msg, arg, mode);
		break;
	case RSM_IOCTL_REPUBLISH:
		error = rsm_republish(seg, msg, mode);
		break;
	case RSM_IOCTL_UNPUBLISH:
		error = rsm_unpublish(seg, 1);
		break;
	default:
		error = EINVAL;
		break;
	}

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmexport_ioctl done: %d\n",
	    error));

	return (error);
}
static int
rsmimport_ioctl(rsmseg_t *seg, rsm_ioctlmsg_t *msg, int cmd, intptr_t arg,
    int mode, cred_t *credp)
{
	int error;
	DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_IMPORT | RSM_IOCTL);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmimport_ioctl enter\n"));

	ASSERT(seg);

	switch (cmd) {
	case RSM_IOCTL_CONNECT:
		error = rsm_connect(seg, msg, credp, arg, mode);
		break;
	default:
		error = EINVAL;
		break;
	}

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmimport_ioctl done: %d\n",
	    error));
	return (error);
}

static int
rsmbar_ioctl(rsmseg_t *seg, rsm_ioctlmsg_t *msg, int cmd, intptr_t arg,
    int mode)
{
	int e;
	adapter_t *adapter;
	DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_IMPORT | RSM_IOCTL);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmbar_ioctl enter\n"));


	if ((seg->s_flags & RSM_IMPORT_DUMMY) != 0) {
		DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
		    "rsmbar_ioctl done: RSM_IMPORT_DUMMY\n"));
		return (RSMERR_CONN_ABORTED);
	} else if (seg->s_node == my_nodeid) {
		DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
		    "rsmbar_ioctl done: loopback\n"));
		return (RSM_SUCCESS);
	}

	adapter = seg->s_adapter;

	switch (cmd) {
	case RSM_IOCTL_BAR_CHECK:
		DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
		    "rsmbar_ioctl done: RSM_BAR_CHECK %d\n", bar_va));
		return (bar_va ? RSM_SUCCESS : EINVAL);
	case RSM_IOCTL_BAR_OPEN:
		e = adapter->rsmpi_ops->
		    rsm_open_barrier_ctrl(adapter->rsmpi_handle, &msg->bar);
		break;
	case RSM_IOCTL_BAR_ORDER:
		e = adapter->rsmpi_ops->rsm_order_barrier(&msg->bar);
		break;
	case RSM_IOCTL_BAR_CLOSE:
		e = adapter->rsmpi_ops->rsm_close_barrier(&msg->bar);
		break;
	default:
		e = EINVAL;
		break;
	}

	if (e == RSM_SUCCESS) {
#ifdef _MULTI_DATAMODEL
		if ((mode & DATAMODEL_MASK) == DATAMODEL_ILP32) {
			rsm_ioctlmsg32_t msg32;
			int i;

			for (i = 0; i < 4; i++) {
				msg32.bar.comp[i].u64 = msg->bar.comp[i].u64;
			}

			DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
			    "rsmbar_ioctl done\n"));
			if (ddi_copyout((caddr_t)&msg32, (caddr_t)arg,
			    sizeof (msg32), mode))
				return (RSMERR_BAD_ADDR);
			else
				return (RSM_SUCCESS);
		}
#endif
		DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
		    "rsmbar_ioctl done\n"));
		if (ddi_copyout((caddr_t)&msg->bar, (caddr_t)arg,
		    sizeof (*msg), mode))
			return (RSMERR_BAD_ADDR);
		else
			return (RSM_SUCCESS);
	}

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
	    "rsmbar_ioctl done: error=%d\n", e));

	return (e);
}

/*
 * Ring the doorbell of the export segment to which this segment is
 * connected.
 */
static int
exportbell_ioctl(rsmseg_t *seg, int cmd /*ARGSUSED*/)
{
	int e = 0;
	rsmipc_request_t request;

	DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_IMPORT | RSM_IOCTL);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "exportbell_ioctl enter\n"));

	request.rsmipc_key = seg->s_segid;
	request.rsmipc_hdr.rsmipc_type = RSMIPC_MSG_BELL;
	request.rsmipc_segment_cookie = NULL;
	e = rsmipc_send(seg->s_node, &request, RSM_NO_REPLY);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
	    "exportbell_ioctl done: %d\n", e));

	return (e);
}

/*
 * Ring the doorbells of all segments importing this segment
 */
static int
importbell_ioctl(rsmseg_t *seg, int cmd /*ARGSUSED*/)
{
	importing_token_t	*token = NULL;
	rsmipc_request_t	request;
	int			index;

	DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_EXPORT | RSM_IOCTL);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "importbell_ioctl enter\n"));

	ASSERT(seg->s_state != RSM_STATE_NEW &&
	    seg->s_state != RSM_STATE_NEW_QUIESCED);

	request.rsmipc_key = seg->s_segid;
	request.rsmipc_hdr.rsmipc_type = RSMIPC_MSG_BELL;

	index = rsmhash(seg->s_segid);

	token = importer_list.bucket[index];

	while (token != NULL) {
		if (seg->s_key == token->key) {
			request.rsmipc_segment_cookie =
			    token->import_segment_cookie;
			(void) rsmipc_send(token->importing_node,
			    &request, RSM_NO_REPLY);
		}
		token = token->next;
	}

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
	    "importbell_ioctl done\n"));
	return (RSM_SUCCESS);
}

static int
rsm_consumeevent_copyin(caddr_t arg, rsm_consume_event_msg_t *msgp,
    rsm_poll_event_t **eventspp, int mode)
{
	rsm_poll_event_t	*evlist = NULL;
	size_t			evlistsz;
	DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_IOCTL);

#ifdef _MULTI_DATAMODEL
	if ((mode & DATAMODEL_MASK) == DATAMODEL_ILP32) {
		int i;
		rsm_consume_event_msg32_t cemsg32 = {0};
		rsm_poll_event32_t	event32[RSM_MAX_POLLFDS];
		rsm_poll_event32_t	*evlist32;
		size_t			evlistsz32;

		/* copyin the ioctl message */
		if (ddi_copyin(arg, (caddr_t)&cemsg32,
		    sizeof (rsm_consume_event_msg32_t), mode)) {
			DBG_PRINTF((category, RSM_ERR,
			    "consumeevent_copyin msgp: RSMERR_BAD_ADDR\n"));
			return (RSMERR_BAD_ADDR);
		}
		msgp->seglist = (caddr_t)(uintptr_t)cemsg32.seglist;
		msgp->numents = (int)cemsg32.numents;

		evlistsz32 = sizeof (rsm_poll_event32_t) * msgp->numents;
		/*
		 * If numents is large alloc events list on heap otherwise
		 * use the address of array that was passed in.
		 */
		if (msgp->numents > RSM_MAX_POLLFDS) {
			if (msgp->numents > max_segs) { /* validate numents */
				DBG_PRINTF((category, RSM_ERR,
				    "consumeevent_copyin: "
				    "RSMERR_BAD_ARGS_ERRORS\n"));
				return (RSMERR_BAD_ARGS_ERRORS);
			}
			evlist32 = kmem_zalloc(evlistsz32, KM_SLEEP);
		} else {
			evlist32 = event32;
		}

		/* copyin the seglist into the rsm_poll_event32_t array */
		if (ddi_copyin((caddr_t)msgp->seglist, (caddr_t)evlist32,
		    evlistsz32, mode)) {
			if ((msgp->numents > RSM_MAX_POLLFDS) && evlist32) {
				kmem_free(evlist32, evlistsz32);
			}
			DBG_PRINTF((category, RSM_ERR,
			    "consumeevent_copyin evlist: RSMERR_BAD_ADDR\n"));
			return (RSMERR_BAD_ADDR);
		}

		/* evlist and evlistsz are based on rsm_poll_event_t type */
		evlistsz = sizeof (rsm_poll_event_t)* msgp->numents;

		if (msgp->numents > RSM_MAX_POLLFDS) {
			evlist = kmem_zalloc(evlistsz, KM_SLEEP);
			*eventspp = evlist;
		} else {
			evlist = *eventspp;
		}
		/*
		 * copy the rsm_poll_event32_t array to the rsm_poll_event_t
		 * array
		 */
		for (i = 0; i < msgp->numents; i++) {
			evlist[i].rnum = evlist32[i].rnum;
			evlist[i].fdsidx = evlist32[i].fdsidx;
			evlist[i].revent = evlist32[i].revent;
		}
		/* free the temp 32-bit event list */
		if ((msgp->numents > RSM_MAX_POLLFDS) && evlist32) {
			kmem_free(evlist32, evlistsz32);
		}

		return (RSM_SUCCESS);
	}
#endif
	/* copyin the ioctl message */
	if (ddi_copyin(arg, (caddr_t)msgp, sizeof (rsm_consume_event_msg_t),
	    mode)) {
		DBG_PRINTF((category, RSM_ERR,
		    "consumeevent_copyin msgp: RSMERR_BAD_ADDR\n"));
		return (RSMERR_BAD_ADDR);
	}
	/*
	 * If numents is large alloc events list on heap otherwise
	 * use the address of array that was passed in.
	 */
	if (msgp->numents > RSM_MAX_POLLFDS) {
		if (msgp->numents > max_segs) { /* validate numents */
			DBG_PRINTF((category, RSM_ERR,
			    "consumeevent_copyin: RSMERR_BAD_ARGS_ERRORS\n"));
			return (RSMERR_BAD_ARGS_ERRORS);
		}
		evlistsz = sizeof (rsm_poll_event_t)*msgp->numents;
		evlist = kmem_zalloc(evlistsz, KM_SLEEP);
		*eventspp  = evlist;
	}

	/* copyin the seglist */
	if (ddi_copyin((caddr_t)msgp->seglist, (caddr_t)(*eventspp),
	    sizeof (rsm_poll_event_t)*msgp->numents, mode)) {
		if (evlist) {
			kmem_free(evlist, evlistsz);
			*eventspp = NULL;
		}
		DBG_PRINTF((category, RSM_ERR,
		    "consumeevent_copyin evlist: RSMERR_BAD_ADDR\n"));
		return (RSMERR_BAD_ADDR);
	}

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
	    "consumeevent_copyin done\n"));
	return (RSM_SUCCESS);
}

static int
rsm_consumeevent_copyout(rsm_consume_event_msg_t *msgp,
    rsm_poll_event_t *eventsp, int mode)
{
	size_t			evlistsz;
	int			err = RSM_SUCCESS;
	DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_IOCTL);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
	    "consumeevent_copyout enter: numents(%d) eventsp(%p)\n",
	    msgp->numents, eventsp));

#ifdef _MULTI_DATAMODEL
	if ((mode & DATAMODEL_MASK) == DATAMODEL_ILP32) {
		int i;
		rsm_poll_event32_t	event32[RSM_MAX_POLLFDS];
		rsm_poll_event32_t	*evlist32;
		size_t			evlistsz32;

		evlistsz32 = sizeof (rsm_poll_event32_t)*msgp->numents;
		if (msgp->numents > RSM_MAX_POLLFDS) {
			evlist32 = kmem_zalloc(evlistsz32, KM_SLEEP);
		} else {
			evlist32 = event32;
		}

		/*
		 * copy the rsm_poll_event_t array to the rsm_poll_event32_t
		 * array
		 */
		for (i = 0; i < msgp->numents; i++) {
			evlist32[i].rnum = eventsp[i].rnum;
			evlist32[i].fdsidx = eventsp[i].fdsidx;
			evlist32[i].revent = eventsp[i].revent;
		}

		if (ddi_copyout((caddr_t)evlist32, (caddr_t)msgp->seglist,
		    evlistsz32, mode)) {
			err = RSMERR_BAD_ADDR;
		}

		if (msgp->numents > RSM_MAX_POLLFDS) {
			if (evlist32) {	/* free the temp 32-bit event list */
				kmem_free(evlist32, evlistsz32);
			}
			/*
			 * eventsp and evlistsz are based on rsm_poll_event_t
			 * type
			 */
			evlistsz = sizeof (rsm_poll_event_t)*msgp->numents;
			/* event list on the heap and needs to be freed here */
			if (eventsp) {
				kmem_free(eventsp, evlistsz);
			}
		}

		DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
		    "consumeevent_copyout done: err=%d\n", err));
		return (err);
	}
#endif
	evlistsz = sizeof (rsm_poll_event_t)*msgp->numents;

	if (ddi_copyout((caddr_t)eventsp, (caddr_t)msgp->seglist, evlistsz,
	    mode)) {
		err = RSMERR_BAD_ADDR;
	}

	if ((msgp->numents > RSM_MAX_POLLFDS) && eventsp) {
		/* event list on the heap and needs to be freed here */
		kmem_free(eventsp, evlistsz);
	}

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
	    "consumeevent_copyout done: err=%d\n", err));
	return (err);
}

static int
rsm_consumeevent_ioctl(caddr_t arg, int mode)
{
	int	rc;
	int	i;
	minor_t	rnum;
	rsm_consume_event_msg_t	msg = {0};
	rsmseg_t		*seg;
	rsm_poll_event_t	*event_list;
	rsm_poll_event_t	events[RSM_MAX_POLLFDS];
	DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_IOCTL);

	event_list = events;

	if ((rc = rsm_consumeevent_copyin(arg, &msg, &event_list, mode)) !=
	    RSM_SUCCESS) {
		return (rc);
	}

	for (i = 0; i < msg.numents; i++) {
		rnum = event_list[i].rnum;
		event_list[i].revent = 0;
		/* get the segment structure */
		seg = (rsmseg_t *)rsmresource_lookup(rnum, RSM_LOCK);
		if (seg) {
			DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
			    "consumeevent_ioctl: rnum(%d) seg(%p)\n", rnum,
			    seg));
			if (seg->s_pollevent) {
				/* consume the event */
				atomic_add_32(&seg->s_pollevent, -1);
				event_list[i].revent = POLLRDNORM;
			}
			rsmseglock_release(seg);
		}
	}

	if ((rc = rsm_consumeevent_copyout(&msg, event_list, mode)) !=
	    RSM_SUCCESS) {
		return (rc);
	}

	return (RSM_SUCCESS);
}

static int
iovec_copyin(caddr_t user_vec, rsmka_iovec_t *iovec, int count, int mode)
{
	int size;
	DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_IMPORT | RSM_IOCTL);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "iovec_copyin enter\n"));

#ifdef _MULTI_DATAMODEL
	if ((mode & DATAMODEL_MASK) == DATAMODEL_ILP32) {
		rsmka_iovec32_t	*iovec32, *iovec32_base;
		int i;

		size = count * sizeof (rsmka_iovec32_t);
		iovec32_base = iovec32 = kmem_zalloc(size, KM_SLEEP);
		if (ddi_copyin((caddr_t)user_vec,
		    (caddr_t)iovec32, size, mode)) {
			kmem_free(iovec32, size);
			DBG_PRINTF((category, RSM_DEBUG,
			    "iovec_copyin: returning RSMERR_BAD_ADDR\n"));
			return (RSMERR_BAD_ADDR);
		}

		for (i = 0; i < count; i++, iovec++, iovec32++) {
			iovec->io_type = (int)iovec32->io_type;
			if (iovec->io_type == RSM_HANDLE_TYPE)
				iovec->local.segid = (rsm_memseg_id_t)
				    iovec32->local;
			else
				iovec->local.vaddr =
				    (caddr_t)(uintptr_t)iovec32->local;
			iovec->local_offset = (size_t)iovec32->local_offset;
			iovec->remote_offset = (size_t)iovec32->remote_offset;
			iovec->transfer_len = (size_t)iovec32->transfer_len;

		}
		kmem_free(iovec32_base, size);
		DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
		    "iovec_copyin done\n"));
		return (DDI_SUCCESS);
	}
#endif

	size = count * sizeof (rsmka_iovec_t);
	if (ddi_copyin((caddr_t)user_vec, (caddr_t)iovec, size, mode)) {
		DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
		    "iovec_copyin done: RSMERR_BAD_ADDR\n"));
		return (RSMERR_BAD_ADDR);
	}

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "iovec_copyin done\n"));

	return (DDI_SUCCESS);
}


static int
sgio_copyin(caddr_t arg, rsmka_scat_gath_t *sg_io, int mode)
{
	DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_IMPORT | RSM_IOCTL);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "sgio_copyin enter\n"));

#ifdef _MULTI_DATAMODEL
	if ((mode & DATAMODEL_MASK) == DATAMODEL_ILP32) {
		rsmka_scat_gath32_t sg_io32;

		if (ddi_copyin(arg, (caddr_t)&sg_io32, sizeof (sg_io32),
		    mode)) {
			DBG_PRINTF((category, RSM_DEBUG,
			    "sgio_copyin done: returning EFAULT\n"));
			return (RSMERR_BAD_ADDR);
		}
		sg_io->local_nodeid = (rsm_node_id_t)sg_io32.local_nodeid;
		sg_io->io_request_count =  (size_t)sg_io32.io_request_count;
		sg_io->io_residual_count = (size_t)sg_io32.io_residual_count;
		sg_io->flags = (size_t)sg_io32.flags;
		sg_io->remote_handle = (rsm_memseg_import_handle_t)
		    (uintptr_t)sg_io32.remote_handle;
		sg_io->iovec = (rsmka_iovec_t *)(uintptr_t)sg_io32.iovec;
		DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
		    "sgio_copyin done\n"));
		return (DDI_SUCCESS);
	}
#endif
	if (ddi_copyin(arg, (caddr_t)sg_io, sizeof (rsmka_scat_gath_t),
	    mode)) {
		DBG_PRINTF((category, RSM_DEBUG,
		    "sgio_copyin done: returning EFAULT\n"));
		return (RSMERR_BAD_ADDR);
	}
	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "sgio_copyin done\n"));
	return (DDI_SUCCESS);
}

static int
sgio_resid_copyout(caddr_t arg, rsmka_scat_gath_t *sg_io, int mode)
{
	DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_IMPORT | RSM_IOCTL);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
	    "sgio_resid_copyout enter\n"));

#ifdef _MULTI_DATAMODEL
	if ((mode & DATAMODEL_MASK) == DATAMODEL_ILP32) {
		rsmka_scat_gath32_t sg_io32;

		sg_io32.io_residual_count = sg_io->io_residual_count;
		sg_io32.flags = sg_io->flags;

		if (ddi_copyout((caddr_t)&sg_io32.io_residual_count,
		    (caddr_t)&((rsmka_scat_gath32_t *)arg)->io_residual_count,
		    sizeof (uint32_t), mode)) {

			DBG_PRINTF((category, RSM_ERR,
			    "sgio_resid_copyout error: rescnt\n"));
			return (RSMERR_BAD_ADDR);
		}

		if (ddi_copyout((caddr_t)&sg_io32.flags,
		    (caddr_t)&((rsmka_scat_gath32_t *)arg)->flags,
		    sizeof (uint32_t), mode)) {

			DBG_PRINTF((category, RSM_ERR,
			    "sgio_resid_copyout error: flags\n"));
			return (RSMERR_BAD_ADDR);
		}
		DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
		    "sgio_resid_copyout done\n"));
		return (DDI_SUCCESS);
	}
#endif
	if (ddi_copyout((caddr_t)&sg_io->io_residual_count,
	    (caddr_t)&((rsmka_scat_gath_t *)arg)->io_residual_count,
	    sizeof (ulong_t), mode)) {

		DBG_PRINTF((category, RSM_ERR,
		    "sgio_resid_copyout error:rescnt\n"));
		return (RSMERR_BAD_ADDR);
	}

	if (ddi_copyout((caddr_t)&sg_io->flags,
	    (caddr_t)&((rsmka_scat_gath_t *)arg)->flags,
	    sizeof (uint_t), mode)) {

		DBG_PRINTF((category, RSM_ERR,
		    "sgio_resid_copyout error:flags\n"));
		return (RSMERR_BAD_ADDR);
	}

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "sgio_resid_copyout done\n"));
	return (DDI_SUCCESS);
}


static int
rsm_iovec_ioctl(dev_t dev, caddr_t arg, int cmd, int mode, cred_t *credp)
{
	rsmka_scat_gath_t	sg_io;
	rsmka_iovec_t		ka_iovec_arr[RSM_MAX_IOVLEN];
	rsmka_iovec_t		*ka_iovec;
	rsmka_iovec_t		*ka_iovec_start;
	rsmpi_scat_gath_t	rsmpi_sg_io;
	rsmpi_iovec_t		iovec_arr[RSM_MAX_IOVLEN];
	rsmpi_iovec_t		*iovec;
	rsmpi_iovec_t		*iovec_start = NULL;
	rsmapi_access_entry_t	*acl;
	rsmresource_t		*res;
	minor_t			rnum;
	rsmseg_t		*im_seg, *ex_seg;
	int			e;
	int			error = 0;
	uint_t			i;
	uint_t			iov_proc = 0; /* num of iovecs processed */
	size_t			size = 0;
	size_t			ka_size;

	DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_IMPORT | RSM_IOCTL);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_iovec_ioctl enter\n"));

	credp = credp;

	/*
	 * Copyin the scatter/gather structure  and build new structure
	 * for rsmpi.
	 */
	e = sgio_copyin(arg, &sg_io, mode);
	if (e != DDI_SUCCESS) {
		DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
		    "rsm_iovec_ioctl done: sgio_copyin %d\n", e));
		return (e);
	}

	if (sg_io.io_request_count > RSM_MAX_SGIOREQS) {
		DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
		    "rsm_iovec_ioctl done: request_count(%d) too large\n",
		    sg_io.io_request_count));
		return (RSMERR_BAD_SGIO);
	}

	rsmpi_sg_io.io_request_count = sg_io.io_request_count;
	rsmpi_sg_io.io_residual_count = sg_io.io_request_count;
	rsmpi_sg_io.io_segflg = 0;

	/* Allocate memory and copyin io vector array  */
	if (sg_io.io_request_count > RSM_MAX_IOVLEN) {
		ka_size =  sg_io.io_request_count * sizeof (rsmka_iovec_t);
		ka_iovec_start = ka_iovec = kmem_zalloc(ka_size, KM_SLEEP);
	} else {
		ka_iovec_start = ka_iovec = ka_iovec_arr;
	}
	e = iovec_copyin((caddr_t)sg_io.iovec, ka_iovec,
	    sg_io.io_request_count, mode);
	if (e != DDI_SUCCESS) {
		if (sg_io.io_request_count > RSM_MAX_IOVLEN)
			kmem_free(ka_iovec, ka_size);
		DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
		    "rsm_iovec_ioctl done: iovec_copyin %d\n", e));
		return (e);
	}

	/* get the import segment descriptor */
	rnum = getminor(dev);
	res = rsmresource_lookup(rnum, RSM_LOCK);

	/*
	 * The following sequence of locking may (or MAY NOT) cause a
	 * deadlock but this is currently not addressed here since the
	 * implementation will be changed to incorporate the use of
	 * reference counting for both the import and the export segments.
	 */

	/* rsmseglock_acquire(im_seg) done in rsmresource_lookup */

	im_seg = (rsmseg_t *)res;

	if (im_seg == NULL) {
		if (sg_io.io_request_count > RSM_MAX_IOVLEN)
			kmem_free(ka_iovec, ka_size);
		DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
		    "rsm_iovec_ioctl done: rsmresource_lookup failed\n"));
		return (EINVAL);
	}
	/* putv/getv supported is supported only on import segments */
	if (im_seg->s_type != RSM_RESOURCE_IMPORT_SEGMENT) {
		rsmseglock_release(im_seg);
		if (sg_io.io_request_count > RSM_MAX_IOVLEN)
			kmem_free(ka_iovec, ka_size);
		DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
		    "rsm_iovec_ioctl done: not an import segment\n"));
		return (EINVAL);
	}

	/*
	 * wait for a remote DR to complete ie. for segments to get UNQUIESCED
	 * as well as wait for a local DR to complete.
	 */
	while ((im_seg->s_state == RSM_STATE_CONN_QUIESCE) ||
	    (im_seg->s_state == RSM_STATE_MAP_QUIESCE) ||
	    (im_seg->s_flags & RSM_DR_INPROGRESS)) {
		if (cv_wait_sig(&im_seg->s_cv, &im_seg->s_lock) == 0) {
			DBG_PRINTF((category, RSM_DEBUG,
			    "rsm_iovec_ioctl done: cv_wait INTR"));
			rsmseglock_release(im_seg);
			return (RSMERR_INTERRUPTED);
		}
	}

	if ((im_seg->s_state != RSM_STATE_CONNECT) &&
	    (im_seg->s_state != RSM_STATE_ACTIVE)) {

		ASSERT(im_seg->s_state == RSM_STATE_DISCONNECT ||
		    im_seg->s_state == RSM_STATE_NEW);

		DBG_PRINTF((category, RSM_DEBUG,
		    "rsm_iovec_ioctl done: im_seg not conn/map"));
		rsmseglock_release(im_seg);
		e = RSMERR_BAD_SGIO;
		goto out;
	}

	im_seg->s_rdmacnt++;
	rsmseglock_release(im_seg);

	/*
	 * Allocate and set up the io vector for rsmpi
	 */
	if (sg_io.io_request_count > RSM_MAX_IOVLEN) {
		size = sg_io.io_request_count * sizeof (rsmpi_iovec_t);
		iovec_start = iovec = kmem_zalloc(size, KM_SLEEP);
	} else {
		iovec_start = iovec = iovec_arr;
	}

	rsmpi_sg_io.iovec = iovec;
	for (iov_proc = 0; iov_proc < sg_io.io_request_count; iov_proc++) {
		if (ka_iovec->io_type == RSM_HANDLE_TYPE) {
			ex_seg = rsmexport_lookup(ka_iovec->local.segid);

			if (ex_seg == NULL) {
				e = RSMERR_BAD_SGIO;
				break;
			}
			ASSERT(ex_seg->s_state == RSM_STATE_EXPORT);

			acl = ex_seg->s_acl;
			if (acl[0].ae_permission == 0) {
				struct buf *xbuf;
				dev_t sdev = 0;

				xbuf = ddi_umem_iosetup(ex_seg->s_cookie,
				    0, ex_seg->s_len, B_WRITE,
				    sdev, 0, NULL, DDI_UMEM_SLEEP);

				ASSERT(xbuf != NULL);

				iovec->local_mem.ms_type = RSM_MEM_BUF;
				iovec->local_mem.ms_memory.bp = xbuf;
			} else {
				iovec->local_mem.ms_type = RSM_MEM_HANDLE;
				iovec->local_mem.ms_memory.handle =
				    ex_seg->s_handle.out;
			}
			ex_seg->s_rdmacnt++; /* refcnt the handle */
			rsmseglock_release(ex_seg);
		} else {
			iovec->local_mem.ms_type = RSM_MEM_VADDR;
			iovec->local_mem.ms_memory.vr.vaddr =
			    ka_iovec->local.vaddr;
		}

		iovec->local_offset = ka_iovec->local_offset;
		iovec->remote_handle = im_seg->s_handle.in;
		iovec->remote_offset = ka_iovec->remote_offset;
		iovec->transfer_length = ka_iovec->transfer_len;
		iovec++;
		ka_iovec++;
	}

	if (iov_proc <  sg_io.io_request_count) {
		/* error while processing handle */
		rsmseglock_acquire(im_seg);
		im_seg->s_rdmacnt--;   /* decrement the refcnt for importseg */
		if (im_seg->s_rdmacnt == 0) {
			cv_broadcast(&im_seg->s_cv);
		}
		rsmseglock_release(im_seg);
		goto out;
	}

	/* call rsmpi */
	if (cmd == RSM_IOCTL_PUTV)
		e = im_seg->s_adapter->rsmpi_ops->rsm_memseg_import_putv(
		    im_seg->s_adapter->rsmpi_handle,
		    &rsmpi_sg_io);
	else if (cmd == RSM_IOCTL_GETV)
		e = im_seg->s_adapter->rsmpi_ops->rsm_memseg_import_getv(
		    im_seg->s_adapter->rsmpi_handle,
		    &rsmpi_sg_io);
	else {
		e = EINVAL;
		DBG_PRINTF((category, RSM_DEBUG,
		    "iovec_ioctl: bad command = %x\n", cmd));
	}


	DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
	    "rsm_iovec_ioctl RSMPI oper done %d\n", e));

	sg_io.io_residual_count = rsmpi_sg_io.io_residual_count;

	/*
	 * Check for implicit signal post flag and do the signal
	 * post if needed
	 */
	if (sg_io.flags & RSM_IMPLICIT_SIGPOST &&
	    e == RSM_SUCCESS) {
		rsmipc_request_t request;

		request.rsmipc_key = im_seg->s_segid;
		request.rsmipc_hdr.rsmipc_type = RSMIPC_MSG_BELL;
		request.rsmipc_segment_cookie = NULL;
		e = rsmipc_send(im_seg->s_node, &request, RSM_NO_REPLY);
		/*
		 * Reset the implicit signal post flag to 0 to indicate
		 * that the signal post has been done and need not be
		 * done in the RSMAPI library
		 */
		sg_io.flags &= ~RSM_IMPLICIT_SIGPOST;
	}

	rsmseglock_acquire(im_seg);
	im_seg->s_rdmacnt--;
	if (im_seg->s_rdmacnt == 0) {
		cv_broadcast(&im_seg->s_cv);
	}
	rsmseglock_release(im_seg);
	error = sgio_resid_copyout(arg, &sg_io, mode);
out:
	iovec = iovec_start;
	ka_iovec = ka_iovec_start;
	for (i = 0; i < iov_proc; i++) {
		if (ka_iovec->io_type == RSM_HANDLE_TYPE) {
			ex_seg = rsmexport_lookup(ka_iovec->local.segid);

			ASSERT(ex_seg != NULL);
			ASSERT(ex_seg->s_state == RSM_STATE_EXPORT);

			ex_seg->s_rdmacnt--; /* unrefcnt the handle */
			if (ex_seg->s_rdmacnt == 0) {
				cv_broadcast(&ex_seg->s_cv);
			}
			rsmseglock_release(ex_seg);
		}

		ASSERT(iovec != NULL); /* true if iov_proc > 0 */

		/*
		 * At present there is no dependency on the existence of xbufs
		 * created by ddi_umem_iosetup for each of the iovecs. So we
		 * can these xbufs here.
		 */
		if (iovec->local_mem.ms_type == RSM_MEM_BUF) {
			freerbuf(iovec->local_mem.ms_memory.bp);
		}

		iovec++;
		ka_iovec++;
	}

	if (sg_io.io_request_count > RSM_MAX_IOVLEN) {
		if (iovec_start)
			kmem_free(iovec_start, size);
		kmem_free(ka_iovec_start, ka_size);
	}

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
	    "rsm_iovec_ioctl done %d\n", e));
	/* if RSMPI call fails return that else return copyout's retval */
	return ((e != RSM_SUCCESS) ? e : error);

}


static int
rsmaddr_ioctl(int cmd, rsm_ioctlmsg_t *msg, int mode)
{
	adapter_t	*adapter;
	rsm_addr_t	addr;
	rsm_node_id_t	node;
	int		rval = DDI_SUCCESS;
	DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_FUNC_ALL | RSM_IOCTL);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmaddr_ioctl enter\n"));

	adapter =  rsm_getadapter(msg, mode);
	if (adapter == NULL) {
		DBG_PRINTF((category, RSM_DEBUG,
		    "rsmaddr_ioctl done: adapter not found\n"));
		return (RSMERR_CTLR_NOT_PRESENT);
	}

	switch (cmd) {
	case RSM_IOCTL_MAP_TO_ADDR: /* nodeid to hwaddr mapping */
		/* returns the hwaddr in msg->hwaddr */
		if (msg->nodeid == my_nodeid) {
			msg->hwaddr = adapter->hwaddr;
		} else {
			addr = get_remote_hwaddr(adapter, msg->nodeid);
			if ((int64_t)addr < 0) {
				rval = RSMERR_INTERNAL_ERROR;
			} else {
				msg->hwaddr = addr;
			}
		}
		break;
	case RSM_IOCTL_MAP_TO_NODEID: /* hwaddr to nodeid mapping */
		/* returns the nodeid in msg->nodeid */
		if (msg->hwaddr == adapter->hwaddr) {
			msg->nodeid = my_nodeid;
		} else {
			node = get_remote_nodeid(adapter, msg->hwaddr);
			if ((int)node < 0) {
				rval = RSMERR_INTERNAL_ERROR;
			} else {
				msg->nodeid = (rsm_node_id_t)node;
			}
		}
		break;
	default:
		rval = EINVAL;
		break;
	}

	rsmka_release_adapter(adapter);
	DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
	    "rsmaddr_ioctl done: %d\n", rval));
	return (rval);
}

static int
rsm_ddi_copyin(caddr_t arg, rsm_ioctlmsg_t *msg, int mode)
{
	DBG_DEFINE(category,
	    RSM_KERNEL_AGENT | RSM_FUNC_ALL | RSM_IOCTL | RSM_DDI);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_ddi_copyin enter\n"));

#ifdef _MULTI_DATAMODEL

	if ((mode & DATAMODEL_MASK) == DATAMODEL_ILP32) {
		rsm_ioctlmsg32_t msg32;
		int i;

		if (ddi_copyin(arg, (caddr_t)&msg32, sizeof (msg32), mode)) {
			DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
			    "rsm_ddi_copyin done: EFAULT\n"));
			return (RSMERR_BAD_ADDR);
		}
		msg->len = msg32.len;
		msg->vaddr = (caddr_t)(uintptr_t)msg32.vaddr;
		msg->arg = (caddr_t)(uintptr_t)msg32.arg;
		msg->key = msg32.key;
		msg->acl_len = msg32.acl_len;
		msg->acl = (rsmapi_access_entry_t *)(uintptr_t)msg32.acl;
		msg->cnum = msg32.cnum;
		msg->cname = (caddr_t)(uintptr_t)msg32.cname;
		msg->cname_len = msg32.cname_len;
		msg->nodeid = msg32.nodeid;
		msg->hwaddr = msg32.hwaddr;
		msg->perm = msg32.perm;
		for (i = 0; i < 4; i++) {
			msg->bar.comp[i].u64 = msg32.bar.comp[i].u64;
		}
		DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
		    "rsm_ddi_copyin done\n"));
		return (RSM_SUCCESS);
	}
#endif
	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_ddi_copyin done\n"));
	if (ddi_copyin(arg, (caddr_t)msg, sizeof (*msg), mode))
		return (RSMERR_BAD_ADDR);
	else
		return (RSM_SUCCESS);
}

static int
rsmattr_ddi_copyout(adapter_t *adapter, caddr_t arg, int mode)
{
	rsmka_int_controller_attr_t	rsm_cattr;
	DBG_DEFINE(category,
	    RSM_KERNEL_AGENT | RSM_FUNC_ALL | RSM_IOCTL | RSM_DDI);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
	    "rsmattr_ddi_copyout enter\n"));
	/*
	 * need to copy appropriate data from rsm_controller_attr_t
	 * to rsmka_int_controller_attr_t
	 */
#ifdef	_MULTI_DATAMODEL
	if ((mode & DATAMODEL_MASK) == DATAMODEL_ILP32) {
		rsmka_int_controller_attr32_t rsm_cattr32;

		rsm_cattr32.attr_direct_access_sizes =
		    adapter->rsm_attr.attr_direct_access_sizes;
		rsm_cattr32.attr_atomic_sizes =
		    adapter->rsm_attr.attr_atomic_sizes;
		rsm_cattr32.attr_page_size =
		    adapter->rsm_attr.attr_page_size;
		if (adapter->rsm_attr.attr_max_export_segment_size >
		    UINT_MAX)
			rsm_cattr32.attr_max_export_segment_size =
			    RSM_MAXSZ_PAGE_ALIGNED;
		else
			rsm_cattr32.attr_max_export_segment_size =
			    adapter->rsm_attr.attr_max_export_segment_size;
		if (adapter->rsm_attr.attr_tot_export_segment_size >
		    UINT_MAX)
			rsm_cattr32.attr_tot_export_segment_size =
			    RSM_MAXSZ_PAGE_ALIGNED;
		else
			rsm_cattr32.attr_tot_export_segment_size =
			    adapter->rsm_attr.attr_tot_export_segment_size;
		if (adapter->rsm_attr.attr_max_export_segments >
		    UINT_MAX)
			rsm_cattr32.attr_max_export_segments =
			    UINT_MAX;
		else
			rsm_cattr32.attr_max_export_segments =
			    adapter->rsm_attr.attr_max_export_segments;
		if (adapter->rsm_attr.attr_max_import_map_size >
		    UINT_MAX)
			rsm_cattr32.attr_max_import_map_size =
			    RSM_MAXSZ_PAGE_ALIGNED;
		else
			rsm_cattr32.attr_max_import_map_size =
			    adapter->rsm_attr.attr_max_import_map_size;
		if (adapter->rsm_attr.attr_tot_import_map_size >
		    UINT_MAX)
			rsm_cattr32.attr_tot_import_map_size =
			    RSM_MAXSZ_PAGE_ALIGNED;
		else
			rsm_cattr32.attr_tot_import_map_size =
			    adapter->rsm_attr.attr_tot_import_map_size;
		if (adapter->rsm_attr.attr_max_import_segments >
		    UINT_MAX)
			rsm_cattr32.attr_max_import_segments =
			    UINT_MAX;
		else
			rsm_cattr32.attr_max_import_segments =
			    adapter->rsm_attr.attr_max_import_segments;
		rsm_cattr32.attr_controller_addr =
		    adapter->rsm_attr.attr_controller_addr;

		DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
		    "rsmattr_ddi_copyout done\n"));
		if (ddi_copyout((caddr_t)&rsm_cattr32, arg,
		    sizeof (rsmka_int_controller_attr32_t), mode)) {
			return (RSMERR_BAD_ADDR);
		}
		else
			return (RSM_SUCCESS);
	}
#endif
	rsm_cattr.attr_direct_access_sizes =
	    adapter->rsm_attr.attr_direct_access_sizes;
	rsm_cattr.attr_atomic_sizes =
	    adapter->rsm_attr.attr_atomic_sizes;
	rsm_cattr.attr_page_size =
	    adapter->rsm_attr.attr_page_size;
	rsm_cattr.attr_max_export_segment_size =
	    adapter->rsm_attr.attr_max_export_segment_size;
	rsm_cattr.attr_tot_export_segment_size =
	    adapter->rsm_attr.attr_tot_export_segment_size;
	rsm_cattr.attr_max_export_segments =
	    adapter->rsm_attr.attr_max_export_segments;
	rsm_cattr.attr_max_import_map_size =
	    adapter->rsm_attr.attr_max_import_map_size;
	rsm_cattr.attr_tot_import_map_size =
	    adapter->rsm_attr.attr_tot_import_map_size;
	rsm_cattr.attr_max_import_segments =
	    adapter->rsm_attr.attr_max_import_segments;
	rsm_cattr.attr_controller_addr =
	    adapter->rsm_attr.attr_controller_addr;
	DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
	    "rsmattr_ddi_copyout done\n"));
	if (ddi_copyout((caddr_t)&rsm_cattr, arg,
	    sizeof (rsmka_int_controller_attr_t), mode)) {
		return (RSMERR_BAD_ADDR);
	}
	else
		return (RSM_SUCCESS);
}

/*ARGSUSED*/
static int
rsm_ioctl(dev_t dev, int cmd, intptr_t arg, int mode, cred_t *credp,
    int *rvalp)
{
	rsmseg_t *seg;
	rsmresource_t	*res;
	minor_t		rnum;
	rsm_ioctlmsg_t msg = {0};
	int error;
	adapter_t *adapter;
	DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_FUNC_ALL | RSM_IOCTL);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_ioctl enter\n"));

	if (cmd == RSM_IOCTL_CONSUMEEVENT) {
		error = rsm_consumeevent_ioctl((caddr_t)arg, mode);
		DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
		    "rsm_ioctl RSM_IOCTL_CONSUMEEVENT done: %d\n", error));
		return (error);
	}

	/* topology cmd does not use the arg common to other cmds */
	if (RSM_IOCTL_CMDGRP(cmd) == RSM_IOCTL_TOPOLOGY) {
		error = rsmka_topology_ioctl((caddr_t)arg, cmd, mode);
		DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
		    "rsm_ioctl done: %d\n", error));
		return (error);
	}

	if (RSM_IOCTL_CMDGRP(cmd) == RSM_IOCTL_IOVEC) {
		error = rsm_iovec_ioctl(dev, (caddr_t)arg, cmd, mode, credp);
		DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
		    "rsm_ioctl done: %d\n", error));
		return (error);
	}

	/*
	 * try to load arguments
	 */
	if (cmd != RSM_IOCTL_RING_BELL &&
	    rsm_ddi_copyin((caddr_t)arg, &msg, mode)) {
		DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
		    "rsm_ioctl done: EFAULT\n"));
		return (RSMERR_BAD_ADDR);
	}

	if (cmd == RSM_IOCTL_ATTR) {
		adapter =  rsm_getadapter(&msg, mode);
		if (adapter == NULL) {
			DBG_PRINTF((category, RSM_DEBUG,
			    "rsm_ioctl done: ENODEV\n"));
			return (RSMERR_CTLR_NOT_PRESENT);
		}
		error = rsmattr_ddi_copyout(adapter, msg.arg, mode);
		rsmka_release_adapter(adapter);
		DBG_PRINTF((category, RSM_DEBUG,
		    "rsm_ioctl:after copyout %d\n", error));
		return (error);
	}

	if (cmd == RSM_IOCTL_BAR_INFO) {
		/* Return library off,len of barrier page */
		msg.off = barrier_offset;
		msg.len = (int)barrier_size;
#ifdef _MULTI_DATAMODEL
		if ((mode & DATAMODEL_MASK) == DATAMODEL_ILP32) {
			rsm_ioctlmsg32_t msg32;

			if (msg.len > UINT_MAX)
				msg.len = RSM_MAXSZ_PAGE_ALIGNED;
			else
				msg32.len = (int32_t)msg.len;
			msg32.off = (int32_t)msg.off;
			DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
			    "rsm_ioctl done\n"));
			if (ddi_copyout((caddr_t)&msg32, (caddr_t)arg,
			    sizeof (msg32), mode))
				return (RSMERR_BAD_ADDR);
			else
				return (RSM_SUCCESS);
		}
#endif
		DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
		    "rsm_ioctl done\n"));
		if (ddi_copyout((caddr_t)&msg, (caddr_t)arg,
		    sizeof (msg), mode))
			return (RSMERR_BAD_ADDR);
		else
			return (RSM_SUCCESS);
	}

	if (RSM_IOCTL_CMDGRP(cmd) == RSM_IOCTL_MAP_ADDR) {
		/* map the nodeid or hwaddr */
		error = rsmaddr_ioctl(cmd, &msg, mode);
		if (error == RSM_SUCCESS) {
#ifdef _MULTI_DATAMODEL
			if ((mode & DATAMODEL_MASK) == DATAMODEL_ILP32) {
				rsm_ioctlmsg32_t msg32;

				msg32.hwaddr = (uint64_t)msg.hwaddr;
				msg32.nodeid = (uint32_t)msg.nodeid;

				DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
				    "rsm_ioctl done\n"));
				if (ddi_copyout((caddr_t)&msg32, (caddr_t)arg,
				    sizeof (msg32), mode))
					return (RSMERR_BAD_ADDR);
				else
					return (RSM_SUCCESS);
			}
#endif
			DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
			    "rsm_ioctl done\n"));
			if (ddi_copyout((caddr_t)&msg, (caddr_t)arg,
			    sizeof (msg), mode))
				return (RSMERR_BAD_ADDR);
			else
				return (RSM_SUCCESS);
		}
		DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
		    "rsm_ioctl done: %d\n", error));
		return (error);
	}

	/* Find resource and look it in read mode */
	rnum = getminor(dev);
	res = rsmresource_lookup(rnum, RSM_NOLOCK);
	ASSERT(res != NULL);

	/*
	 * Find command group
	 */
	switch (RSM_IOCTL_CMDGRP(cmd)) {
	case RSM_IOCTL_EXPORT_SEG:
		/*
		 * Export list is searched during publish, loopback and
		 * remote lookup call.
		 */
		seg = rsmresource_seg(res, rnum, credp,
		    RSM_RESOURCE_EXPORT_SEGMENT);
		if (seg->s_type == RSM_RESOURCE_EXPORT_SEGMENT) {
			error = rsmexport_ioctl(seg, &msg, cmd, arg, mode,
			    credp);
		} else { /* export ioctl on an import/barrier resource */
			error = RSMERR_BAD_SEG_HNDL;
		}
		break;
	case RSM_IOCTL_IMPORT_SEG:
		/* Import list is searched during remote unmap call. */
		seg = rsmresource_seg(res, rnum, credp,
		    RSM_RESOURCE_IMPORT_SEGMENT);
		if (seg->s_type == RSM_RESOURCE_IMPORT_SEGMENT) {
			error = rsmimport_ioctl(seg, &msg, cmd, arg, mode,
			    credp);
		} else  { /* import ioctl on an export/barrier resource */
			error = RSMERR_BAD_SEG_HNDL;
		}
		break;
	case RSM_IOCTL_BAR:
		if (res != RSMRC_RESERVED &&
		    res->rsmrc_type == RSM_RESOURCE_IMPORT_SEGMENT) {
			error = rsmbar_ioctl((rsmseg_t *)res, &msg, cmd, arg,
			    mode);
		} else { /* invalid res value */
			error = RSMERR_BAD_SEG_HNDL;
		}
		break;
	case RSM_IOCTL_BELL:
		if (res != RSMRC_RESERVED) {
			if (res->rsmrc_type == RSM_RESOURCE_IMPORT_SEGMENT)
				error = exportbell_ioctl((rsmseg_t *)res, cmd);
			else if (res->rsmrc_type == RSM_RESOURCE_EXPORT_SEGMENT)
				error = importbell_ioctl((rsmseg_t *)res, cmd);
			else /* RSM_RESOURCE_BAR */
				error = RSMERR_BAD_SEG_HNDL;
		} else { /* invalid res value */
			error = RSMERR_BAD_SEG_HNDL;
		}
		break;
	default:
		error = EINVAL;
	}

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_ioctl done: %d\n",
	    error));
	return (error);
}


/* **************************** Segment Mapping Operations ********* */
static rsm_mapinfo_t *
rsm_get_mapinfo(rsmseg_t *seg, off_t off, size_t len, off_t *dev_offset,
    size_t *map_len)
{
	rsm_mapinfo_t	*p;
	/*
	 * Find the correct mapinfo structure to use during the mapping
	 * from the seg->s_mapinfo list.
	 * The seg->s_mapinfo list contains in reverse order the mappings
	 * as returned by the RSMPI rsm_map. In rsm_devmap, we need to
	 * access the correct entry within this list for the mapping
	 * requested.
	 *
	 * The algorithm for selecting a list entry is as follows:
	 *
	 * When start_offset of an entry <= off we have found the entry
	 * we were looking for. Adjust the dev_offset and map_len (needs
	 * to be PAGESIZE aligned).
	 */
	p = seg->s_mapinfo;
	for (; p; p = p->next) {
		if (p->start_offset <= off) {
			*dev_offset = p->dev_offset + off - p->start_offset;
			*map_len = (len > p->individual_len) ?
			    p->individual_len : ptob(btopr(len));
			return (p);
		}
		p = p->next;
	}

	return (NULL);
}

static void
rsm_free_mapinfo(rsm_mapinfo_t  *mapinfo)
{
	rsm_mapinfo_t *p;

	while (mapinfo != NULL) {
		p = mapinfo;
		mapinfo = mapinfo->next;
		kmem_free(p, sizeof (*p));
	}
}

static int
rsmmap_map(devmap_cookie_t dhp, dev_t dev, uint_t flags, offset_t off,
    size_t len, void **pvtp)
{
	rsmcookie_t	*p;
	rsmresource_t	*res;
	rsmseg_t	*seg;
	minor_t rnum;
	DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_FUNC_ALL | RSM_DDI);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmmap_map enter\n"));

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
	    "rsmmap_map: dhp = %x\n", dhp));

	flags = flags;

	rnum = getminor(dev);
	res = (rsmresource_t *)rsmresource_lookup(rnum, RSM_NOLOCK);
	ASSERT(res != NULL);

	seg = (rsmseg_t *)res;

	rsmseglock_acquire(seg);

	ASSERT(seg->s_hdr.rsmrc_type == RSM_RESOURCE_IMPORT_SEGMENT);

	/*
	 * Allocate structure and add cookie to segment list
	 */
	p = kmem_alloc(sizeof (*p), KM_SLEEP);

	p->c_dhp = dhp;
	p->c_off = off;
	p->c_len = len;
	p->c_next = seg->s_ckl;
	seg->s_ckl = p;

	*pvtp = (void *)seg;

	rsmseglock_release(seg);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmmap_map done\n"));
	return (DDI_SUCCESS);
}

/*
 * Page fault handling is done here. The prerequisite mapping setup
 * has been done in rsm_devmap with calls to ddi_devmem_setup or
 * ddi_umem_setup
 */
static int
rsmmap_access(devmap_cookie_t dhp, void *pvt, offset_t offset, size_t len,
    uint_t type, uint_t rw)
{
	int e;
	rsmseg_t *seg = (rsmseg_t *)pvt;
	DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_FUNC_ALL | RSM_DDI);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmmap_access enter\n"));

	rsmseglock_acquire(seg);

	ASSERT(seg->s_hdr.rsmrc_type == RSM_RESOURCE_IMPORT_SEGMENT);

	while (seg->s_state == RSM_STATE_MAP_QUIESCE) {
		if (cv_wait_sig(&seg->s_cv, &seg->s_lock) == 0) {
			DBG_PRINTF((category, RSM_DEBUG,
			    "rsmmap_access done: cv_wait INTR"));
			rsmseglock_release(seg);
			return (RSMERR_INTERRUPTED);
		}
	}

	ASSERT(seg->s_state == RSM_STATE_DISCONNECT ||
	    seg->s_state == RSM_STATE_ACTIVE);

	if (seg->s_state == RSM_STATE_DISCONNECT)
		seg->s_flags |= RSM_IMPORT_DUMMY;

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
	    "rsmmap_access: dhp = %x\n", dhp));

	rsmseglock_release(seg);

	if (e = devmap_load(dhp, offset, len, type, rw)) {
		DBG_PRINTF((category, RSM_ERR, "devmap_load failed\n"));
	}


	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmmap_access done\n"));

	return (e);
}

static int
rsmmap_dup(devmap_cookie_t dhp, void *oldpvt, devmap_cookie_t new_dhp,
	void **newpvt)
{
	rsmseg_t	*seg = (rsmseg_t *)oldpvt;
	rsmcookie_t	*p, *old;
	DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_FUNC_ALL | RSM_DDI);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmmap_dup enter\n"));

	/*
	 * Same as map, create an entry to hold cookie and add it to
	 * connect segment list. The oldpvt is a pointer to segment.
	 * Return segment pointer in newpvt.
	 */
	rsmseglock_acquire(seg);

	ASSERT(seg->s_hdr.rsmrc_type == RSM_RESOURCE_IMPORT_SEGMENT);

	/*
	 * Find old cookie
	 */
	for (old = seg->s_ckl; old != NULL; old = old->c_next) {
		if (old->c_dhp == dhp) {
			break;
		}
	}
	if (old == NULL) {
		DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
		    "rsmmap_dup done: EINVAL\n"));
		rsmseglock_release(seg);
		return (EINVAL);
	}

	p = kmem_alloc(sizeof (*p), KM_SLEEP);

	p->c_dhp = new_dhp;
	p->c_off = old->c_off;
	p->c_len = old->c_len;
	p->c_next = seg->s_ckl;
	seg->s_ckl = p;

	*newpvt = (void *)seg;

	rsmseglock_release(seg);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmmap_dup done\n"));

	return (DDI_SUCCESS);
}

static void
rsmmap_unmap(devmap_cookie_t dhp, void *pvtp, offset_t off, size_t len,
	devmap_cookie_t new_dhp1, void **pvtp1,
	devmap_cookie_t new_dhp2, void **pvtp2)
{
	/*
	 * Remove pvtp structure from segment list.
	 */
	rsmseg_t	*seg = (rsmseg_t *)pvtp;
	int freeflag;

	DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_FUNC_ALL | RSM_DDI);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmmap_unmap enter\n"));

	off = off; len = len;
	pvtp1 = pvtp1; pvtp2 = pvtp2;

	rsmseglock_acquire(seg);

	ASSERT(seg->s_hdr.rsmrc_type == RSM_RESOURCE_IMPORT_SEGMENT);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
	    "rsmmap_unmap: dhp = %x\n", dhp));
	/*
	 * We can go ahead and remove the dhps even if we are in
	 * the MAPPING state because the dhps being removed here
	 * belong to a different mmap and we are holding the segment
	 * lock.
	 */
	if (new_dhp1 == NULL && new_dhp2 == NULL) {
		/* find and remove dhp handle */
		rsmcookie_t *tmp, **back = &seg->s_ckl;

		while (*back != NULL) {
			tmp = *back;
			if (tmp->c_dhp == dhp) {
				*back = tmp->c_next;
				kmem_free(tmp, sizeof (*tmp));
				break;
			}
			back = &tmp->c_next;
		}
	} else {
		DBG_PRINTF((category, RSM_DEBUG_LVL2,
		    "rsmmap_unmap:parital unmap"
		    "new_dhp1 %lx, new_dhp2 %lx\n",
		    (size_t)new_dhp1, (size_t)new_dhp2));
	}

	/*
	 * rsmmap_unmap is called for each mapping cookie on the list.
	 * When the list becomes empty and we are not in the MAPPING
	 * state then unmap in the rsmpi driver.
	 */
	if ((seg->s_ckl == NULL) && (seg->s_state != RSM_STATE_MAPPING))
		(void) rsm_unmap(seg);

	if (seg->s_state == RSM_STATE_END && seg->s_ckl == NULL) {
		freeflag = 1;
	} else {
		freeflag = 0;
	}

	rsmseglock_release(seg);

	if (freeflag) {
		/* Free the segment structure */
		rsmseg_free(seg);
	}
	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsmmap_unmap done\n"));

}

static struct devmap_callback_ctl rsmmap_ops = {
	DEVMAP_OPS_REV,	/* devmap_ops version number	*/
	rsmmap_map,	/* devmap_ops map routine */
	rsmmap_access,	/* devmap_ops access routine */
	rsmmap_dup,		/* devmap_ops dup routine		*/
	rsmmap_unmap,	/* devmap_ops unmap routine */
};

static int
rsm_devmap(dev_t dev, devmap_cookie_t dhc, offset_t off, size_t len,
    size_t *maplen, uint_t model /*ARGSUSED*/)
{
	struct devmap_callback_ctl *callbackops = &rsmmap_ops;
	int		err;
	uint_t		maxprot;
	minor_t		rnum;
	rsmseg_t	*seg;
	off_t		dev_offset;
	size_t		cur_len;
	DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_FUNC_ALL | RSM_DDI);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_devmap enter\n"));

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
	    "rsm_devmap: off = %lx, len = %lx\n", off, len));
	rnum = getminor(dev);
	seg = (rsmseg_t *)rsmresource_lookup(rnum, RSM_NOLOCK);
	ASSERT(seg != NULL);

	if (seg->s_hdr.rsmrc_type == RSM_RESOURCE_BAR) {
		if ((off == barrier_offset) &&
		    (len == barrier_size)) {

			ASSERT(bar_va != NULL && bar_cookie != NULL);

			/*
			 * The offset argument in devmap_umem_setup represents
			 * the offset within the kernel memory defined by the
			 * cookie. We use this offset as barrier_offset.
			 */
			err = devmap_umem_setup(dhc, rsm_dip, NULL, bar_cookie,
			    barrier_offset, len, PROT_USER|PROT_READ,
			    DEVMAP_DEFAULTS, 0);

			if (err != 0) {
				DBG_PRINTF((category, RSM_ERR,
				    "rsm_devmap done: %d\n", err));
				return (RSMERR_MAP_FAILED);
			}
			DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
			    "rsm_devmap done: %d\n", err));

			*maplen = barrier_size;

			return (err);
		} else {
			DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
			    "rsm_devmap done: %d\n", err));
			return (RSMERR_MAP_FAILED);
		}
	}

	ASSERT(seg->s_hdr.rsmrc_type == RSM_RESOURCE_IMPORT_SEGMENT);
	ASSERT(seg->s_state == RSM_STATE_MAPPING);

	/*
	 * Make sure we still have permission for the map operation.
	 */
	maxprot = PROT_USER;
	if (seg->s_mode & RSM_PERM_READ) {
		maxprot |= PROT_READ;
	}

	if (seg->s_mode & RSM_PERM_WRITE) {
		maxprot |= PROT_WRITE;
	}

	/*
	 * For each devmap call, rsmmap_map is called. This maintains driver
	 * private information for the mapping. Thus, if there are multiple
	 * devmap calls there will be multiple rsmmap_map calls and for each
	 * call, the mapping information will be stored.
	 * In case of an error during the processing of the devmap call, error
	 * will be returned. This error return causes the caller of rsm_devmap
	 * to undo all the mappings by calling rsmmap_unmap for each one.
	 * rsmmap_unmap will free up the private information for the requested
	 * mapping.
	 */
	if (seg->s_node != my_nodeid) {
		rsm_mapinfo_t *p;

		p = rsm_get_mapinfo(seg, off, len, &dev_offset, &cur_len);
		if (p == NULL) {
			DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
			    "rsm_devmap: incorrect mapping info\n"));
			return (RSMERR_MAP_FAILED);
		}
		err = devmap_devmem_setup(dhc, p->dip,
		    callbackops, p->dev_register,
		    dev_offset, cur_len, maxprot,
		    DEVMAP_ALLOW_REMAP | DEVMAP_DEFAULTS, 0);

		DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
		    "rsm_devmap: dip=%lx,dreg=%lu,doff=%lx,"
		    "off=%lx,len=%lx\n",
		    p->dip, p->dev_register, dev_offset, off, cur_len));

		if (err != 0) {
			DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
			    "rsm_devmap: devmap_devmem_setup failed %d\n",
			    err));
			return (RSMERR_MAP_FAILED);
		}
		/* cur_len is always an integral multiple pagesize */
		ASSERT((cur_len & (PAGESIZE-1)) == 0);
		*maplen = cur_len;
		return (err);

	} else {
		err = devmap_umem_setup(dhc, rsm_dip, callbackops,
		    seg->s_cookie, off, len, maxprot,
		    DEVMAP_ALLOW_REMAP|DEVMAP_DEFAULTS, 0);
		if (err != 0) {
			DBG_PRINTF((category, RSM_DEBUG,
			    "rsm_devmap: devmap_umem_setup failed %d\n",
			    err));
			return (RSMERR_MAP_FAILED);
		}
		DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
		    "rsm_devmap: loopback done\n"));

		*maplen = ptob(btopr(len));

		return (err);
	}
}

/*
 * We can use the devmap framework for mapping device memory to user space by
 * specifying this routine in the rsm_cb_ops structure. The kernel mmap
 * processing calls this entry point and devmap_setup is called within this
 * function, which eventually calls rsm_devmap
 */
static int
rsm_segmap(dev_t dev, off_t off, struct as *as, caddr_t *addrp, off_t len,
    uint_t prot, uint_t maxprot, uint_t flags, struct cred *cred)
{
	int			error = 0;
	int			old_state;
	minor_t			rnum;
	rsmseg_t		*seg, *eseg;
	adapter_t		*adapter;
	rsm_import_share_t	*sharedp;
	DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_FUNC_ALL | RSM_DDI);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "rsm_segmap enter\n"));

	/*
	 * find segment
	 */
	rnum = getminor(dev);
	seg = (rsmseg_t *)rsmresource_lookup(rnum, RSM_LOCK);

	if (seg == NULL) {
		DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
		    "rsm_segmap done: invalid segment\n"));
		return (EINVAL);
	}

	/*
	 * the user is trying to map a resource that has not been
	 * defined yet. The library uses this to map in the
	 * barrier page.
	 */
	if (seg->s_hdr.rsmrc_type == RSM_RESOURCE_BAR) {
		rsmseglock_release(seg);

		/*
		 * The mapping for the barrier page is identified
		 * by the special offset barrier_offset
		 */

		if (off == (off_t)barrier_offset ||
		    len == (off_t)barrier_size) {
			if (bar_cookie == NULL || bar_va == NULL) {
				DBG_PRINTF((category, RSM_DEBUG,
				    "rsm_segmap: bar cookie/va is NULL\n"));
				return (EINVAL);
			}

			error = devmap_setup(dev, (offset_t)off, as, addrp,
			    (size_t)len, prot, maxprot, flags,  cred);

			DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
			    "rsm_segmap done: %d\n", error));
			return (error);
		} else {
			DBG_PRINTF((category, RSM_DEBUG,
			    "rsm_segmap: bad offset/length\n"));
			return (EINVAL);
		}
	}

	/* Make sure you can only map imported segments */
	if (seg->s_hdr.rsmrc_type != RSM_RESOURCE_IMPORT_SEGMENT) {
		rsmseglock_release(seg);
		DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
		    "rsm_segmap done: not an import segment\n"));
		return (EINVAL);
	}
	/* check means library is broken */
	ASSERT(seg->s_hdr.rsmrc_num == rnum);

	/* wait for the segment to become unquiesced */
	while (seg->s_state == RSM_STATE_CONN_QUIESCE) {
		if (cv_wait_sig(&seg->s_cv, &seg->s_lock) == 0) {
			rsmseglock_release(seg);
			DBG_PRINTF((category, RSM_DEBUG,
			    "rsm_segmap done: cv_wait INTR"));
			return (ENODEV);
		}
	}

	/* wait until segment leaves the mapping state */
	while (seg->s_state == RSM_STATE_MAPPING)
		cv_wait(&seg->s_cv, &seg->s_lock);

	/*
	 * we allow multiple maps of the same segment in the KA
	 * and it works because we do an rsmpi map of the whole
	 * segment during the first map and all the device mapping
	 * information needed in rsm_devmap is in the mapinfo list.
	 */
	if ((seg->s_state != RSM_STATE_CONNECT) &&
	    (seg->s_state != RSM_STATE_ACTIVE)) {
		rsmseglock_release(seg);
		DBG_PRINTF((category, RSM_DEBUG,
		    "rsm_segmap done: segment not connected\n"));
		return (ENODEV);
	}

	/*
	 * Make sure we are not mapping a larger segment than what's
	 * exported
	 */
	if ((size_t)off + ptob(btopr(len)) > seg->s_len) {
		rsmseglock_release(seg);
		DBG_PRINTF((category, RSM_DEBUG,
		    "rsm_segmap done: off+len>seg size\n"));
		return (ENXIO);
	}

	/*
	 * Make sure we still have permission for the map operation.
	 */
	maxprot = PROT_USER;
	if (seg->s_mode & RSM_PERM_READ) {
		maxprot |= PROT_READ;
	}

	if (seg->s_mode & RSM_PERM_WRITE) {
		maxprot |= PROT_WRITE;
	}

	if ((prot & maxprot) != prot) {
		/* No permission */
		rsmseglock_release(seg);
		DBG_PRINTF((category, RSM_DEBUG,
		    "rsm_segmap done: no permission\n"));
		return (EACCES);
	}

	old_state = seg->s_state;

	ASSERT(seg->s_share != NULL);

	rsmsharelock_acquire(seg);

	sharedp = seg->s_share;

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
	    "rsm_segmap:RSMSI_STATE=%d\n", sharedp->rsmsi_state));

	if ((sharedp->rsmsi_state != RSMSI_STATE_CONNECTED) &&
	    (sharedp->rsmsi_state != RSMSI_STATE_MAPPED)) {
		rsmsharelock_release(seg);
		rsmseglock_release(seg);
		DBG_PRINTF((category, RSM_DEBUG,
		    "rsm_segmap done:RSMSI_STATE %d invalid\n",
		    sharedp->rsmsi_state));
		return (ENODEV);
	}

	/*
	 * Do the map - since we want importers to share mappings
	 * we do the rsmpi map for the whole segment
	 */
	if (seg->s_node != my_nodeid) {
		uint_t dev_register;
		off_t dev_offset;
		dev_info_t *dip;
		size_t tmp_len;
		size_t total_length_mapped = 0;
		size_t length_to_map = seg->s_len;
		off_t tmp_off = 0;
		rsm_mapinfo_t *p;

		/*
		 * length_to_map = seg->s_len is always an integral
		 * multiple of PAGESIZE. Length mapped in each entry in mapinfo
		 * list is a multiple of PAGESIZE - RSMPI map ensures this
		 */

		adapter = seg->s_adapter;
		ASSERT(sharedp->rsmsi_state == RSMSI_STATE_CONNECTED ||
		    sharedp->rsmsi_state == RSMSI_STATE_MAPPED);

		if (sharedp->rsmsi_state == RSMSI_STATE_CONNECTED) {
			error = 0;
			/* map the whole segment */
			while (total_length_mapped < seg->s_len) {
				tmp_len = 0;

				error = adapter->rsmpi_ops->rsm_map(
				    seg->s_handle.in, tmp_off,
				    length_to_map, &tmp_len,
				    &dip, &dev_register, &dev_offset,
				    NULL, NULL);

				if (error != 0)
					break;

				/*
				 * Store the mapping info obtained from rsm_map
				 */
				p = kmem_alloc(sizeof (*p), KM_SLEEP);
				p->dev_register = dev_register;
				p->dev_offset = dev_offset;
				p->dip = dip;
				p->individual_len = tmp_len;
				p->start_offset = tmp_off;
				p->next = sharedp->rsmsi_mapinfo;
				sharedp->rsmsi_mapinfo = p;

				total_length_mapped += tmp_len;
				length_to_map -= tmp_len;
				tmp_off += tmp_len;
			}
			seg->s_mapinfo = sharedp->rsmsi_mapinfo;

			if (error != RSM_SUCCESS) {
				/* Check if this is the the first rsm_map */
				if (sharedp->rsmsi_mapinfo != NULL) {
					/*
					 * A single rsm_unmap undoes
					 * multiple rsm_maps.
					 */
					(void) seg->s_adapter->rsmpi_ops->
					    rsm_unmap(sharedp->rsmsi_handle);
					rsm_free_mapinfo(sharedp->
					    rsmsi_mapinfo);
				}
				sharedp->rsmsi_mapinfo = NULL;
				sharedp->rsmsi_state = RSMSI_STATE_CONNECTED;
				rsmsharelock_release(seg);
				rsmseglock_release(seg);
				DBG_PRINTF((category, RSM_DEBUG,
				    "rsm_segmap done: rsmpi map err %d\n",
				    error));
				ASSERT(error != RSMERR_BAD_LENGTH &&
				    error != RSMERR_BAD_MEM_ALIGNMENT &&
				    error != RSMERR_BAD_SEG_HNDL);
				if (error == RSMERR_UNSUPPORTED_OPERATION)
					return (ENOTSUP);
				else if (error == RSMERR_INSUFFICIENT_RESOURCES)
					return (EAGAIN);
				else if (error == RSMERR_CONN_ABORTED)
					return (ENODEV);
				else
					return (error);
			} else {
				sharedp->rsmsi_state = RSMSI_STATE_MAPPED;
			}
		} else {
			seg->s_mapinfo = sharedp->rsmsi_mapinfo;
		}

		sharedp->rsmsi_mapcnt++;

		rsmsharelock_release(seg);

		/* move to an intermediate mapping state */
		seg->s_state = RSM_STATE_MAPPING;
		rsmseglock_release(seg);

		error = devmap_setup(dev, (offset_t)off, as, addrp,
		    len, prot, maxprot, flags, cred);

		rsmseglock_acquire(seg);
		ASSERT(seg->s_state == RSM_STATE_MAPPING);

		if (error == DDI_SUCCESS) {
			seg->s_state = RSM_STATE_ACTIVE;
		} else {
			rsmsharelock_acquire(seg);

			ASSERT(sharedp->rsmsi_state == RSMSI_STATE_MAPPED);

			sharedp->rsmsi_mapcnt--;
			if (sharedp->rsmsi_mapcnt == 0) {
				/* unmap the shared RSMPI mapping */
				ASSERT(sharedp->rsmsi_handle != NULL);
				(void) adapter->rsmpi_ops->
				    rsm_unmap(sharedp->rsmsi_handle);
				rsm_free_mapinfo(sharedp->rsmsi_mapinfo);
				sharedp->rsmsi_mapinfo = NULL;
				sharedp->rsmsi_state = RSMSI_STATE_CONNECTED;
			}

			rsmsharelock_release(seg);
			seg->s_state = old_state;
			DBG_PRINTF((category, RSM_ERR,
			    "rsm: devmap_setup failed %d\n", error));
		}
		cv_broadcast(&seg->s_cv);
		rsmseglock_release(seg);
		DBG_PRINTF((category, RSM_DEBUG_LVL2, "rsm_segmap done: %d\n",
		    error));
		return (error);
	} else {
		/*
		 * For loopback, the export segment mapping cookie (s_cookie)
		 * is also used as the s_cookie value for its import segments
		 * during mapping.
		 * Note that reference counting for s_cookie of the export
		 * segment is not required due to the following:
		 * We never have a case of the export segment being destroyed,
		 * leaving the import segments with a stale value for the
		 * s_cookie field, since a force disconnect is done prior to a
		 * destroy of an export segment. The force disconnect causes
		 * the s_cookie value to be reset to NULL. Also for the
		 * rsm_rebind operation, we change the s_cookie value of the
		 * export segment as well as of all its local (loopback)
		 * importers.
		 */
		DBG_ADDCATEGORY(category, RSM_LOOPBACK);

		rsmsharelock_release(seg);
		/*
		 * In order to maintain the lock ordering between the export
		 * and import segment locks, we need to acquire the export
		 * segment lock first and only then acquire the import
		 * segment lock.
		 * The above is necessary to avoid any deadlock scenarios
		 * with rsm_rebind which also acquires both the export
		 * and import segment locks in the above mentioned order.
		 * Based on code inspection, there seem to be no other
		 * situations in which both the export and import segment
		 * locks are acquired either in the same or opposite order
		 * as mentioned above.
		 * Thus in order to conform to the above lock order, we
		 * need to change the state of the import segment to
		 * RSM_STATE_MAPPING, release the lock. Once this is done we
		 * can now safely acquire the export segment lock first
		 * followed by the import segment lock which is as per
		 * the lock order mentioned above.
		 */
		/* move to an intermediate mapping state */
		seg->s_state = RSM_STATE_MAPPING;
		rsmseglock_release(seg);

		eseg = rsmexport_lookup(seg->s_key);

		if (eseg == NULL) {
			rsmseglock_acquire(seg);
			/*
			 * Revert to old_state and signal any waiters
			 * The shared state is not changed
			 */

			seg->s_state = old_state;
			cv_broadcast(&seg->s_cv);
			rsmseglock_release(seg);
			DBG_PRINTF((category, RSM_DEBUG,
			    "rsm_segmap done: key %d not found\n", seg->s_key));
			return (ENODEV);
		}

		rsmsharelock_acquire(seg);
		ASSERT(sharedp->rsmsi_state == RSMSI_STATE_CONNECTED ||
		    sharedp->rsmsi_state == RSMSI_STATE_MAPPED);

		sharedp->rsmsi_mapcnt++;
		sharedp->rsmsi_state = RSMSI_STATE_MAPPED;
		rsmsharelock_release(seg);

		ASSERT(eseg->s_cookie != NULL);

		/*
		 * It is not required or necessary to acquire the import
		 * segment lock here to change the value of s_cookie since
		 * no one will touch the import segment as long as it is
		 * in the RSM_STATE_MAPPING state.
		 */
		seg->s_cookie = eseg->s_cookie;

		rsmseglock_release(eseg);

		error = devmap_setup(dev, (offset_t)off, as, addrp, (size_t)len,
		    prot, maxprot, flags, cred);

		rsmseglock_acquire(seg);
		ASSERT(seg->s_state == RSM_STATE_MAPPING);
		if (error == 0) {
			seg->s_state = RSM_STATE_ACTIVE;
		} else {
			rsmsharelock_acquire(seg);

			ASSERT(sharedp->rsmsi_state == RSMSI_STATE_MAPPED);

			sharedp->rsmsi_mapcnt--;
			if (sharedp->rsmsi_mapcnt == 0) {
				sharedp->rsmsi_mapinfo = NULL;
				sharedp->rsmsi_state = RSMSI_STATE_CONNECTED;
			}
			rsmsharelock_release(seg);
			seg->s_state = old_state;
			seg->s_cookie = NULL;
		}
		cv_broadcast(&seg->s_cv);
		rsmseglock_release(seg);
		DBG_PRINTF((category, RSM_DEBUG_LVL2,
		    "rsm_segmap done: %d\n", error));
		return (error);
	}
}

int
rsmka_null_seg_create(
    rsm_controller_handle_t argcp,
    rsm_memseg_export_handle_t *handle,
    size_t size,
    uint_t flags,
    rsm_memory_local_t *memory,
    rsm_resource_callback_t callback,
    rsm_resource_callback_arg_t callback_arg	/*ARGSUSED*/)
{
	return (RSM_SUCCESS);
}


int
rsmka_null_seg_destroy(
    rsm_memseg_export_handle_t argmemseg	/*ARGSUSED*/)
{
	return (RSM_SUCCESS);
}


int
rsmka_null_bind(
    rsm_memseg_export_handle_t argmemseg,
    off_t offset,
    rsm_memory_local_t *argmemory,
    rsm_resource_callback_t callback,
    rsm_resource_callback_arg_t callback_arg	/*ARGSUSED*/)
{
	return (RSM_SUCCESS);
}


int
rsmka_null_unbind(
    rsm_memseg_export_handle_t argmemseg,
    off_t offset,
    size_t length	/*ARGSUSED*/)
{
	return (DDI_SUCCESS);
}

int
rsmka_null_rebind(
    rsm_memseg_export_handle_t argmemseg,
    off_t offset,
    rsm_memory_local_t *memory,
    rsm_resource_callback_t callback,
    rsm_resource_callback_arg_t callback_arg	/*ARGSUSED*/)
{
	return (RSM_SUCCESS);
}

int
rsmka_null_publish(
    rsm_memseg_export_handle_t argmemseg,
    rsm_access_entry_t access_list[],
    uint_t access_list_length,
    rsm_memseg_id_t segment_id,
    rsm_resource_callback_t callback,
    rsm_resource_callback_arg_t callback_arg	/*ARGSUSED*/)
{
	return (RSM_SUCCESS);
}


int
rsmka_null_republish(
    rsm_memseg_export_handle_t memseg,
    rsm_access_entry_t access_list[],
    uint_t access_list_length,
    rsm_resource_callback_t callback,
    rsm_resource_callback_arg_t callback_arg	/*ARGSUSED*/)
{
	return (RSM_SUCCESS);
}

int
rsmka_null_unpublish(
    rsm_memseg_export_handle_t argmemseg	/*ARGSUSED*/)
{
	return (RSM_SUCCESS);
}


void
rsmka_init_loopback()
{
	rsm_ops_t	*ops = &null_rsmpi_ops;
	DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_FUNC_ALL | RSM_LOOPBACK);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
	    "rsmka_init_loopback enter\n"));

	/* initialize null ops vector */
	ops->rsm_seg_create = rsmka_null_seg_create;
	ops->rsm_seg_destroy = rsmka_null_seg_destroy;
	ops->rsm_bind = rsmka_null_bind;
	ops->rsm_unbind = rsmka_null_unbind;
	ops->rsm_rebind = rsmka_null_rebind;
	ops->rsm_publish = rsmka_null_publish;
	ops->rsm_unpublish = rsmka_null_unpublish;
	ops->rsm_republish = rsmka_null_republish;

	/* initialize attributes for loopback adapter */
	loopback_attr.attr_name = loopback_str;
	loopback_attr.attr_page_size = 0x8; /* 8K */

	/* initialize loopback adapter */
	loopback_adapter.rsm_attr = loopback_attr;
	loopback_adapter.rsmpi_ops = &null_rsmpi_ops;
	DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
	    "rsmka_init_loopback done\n"));
}

/* ************** DR functions ********************************** */
static void
rsm_quiesce_exp_seg(rsmresource_t *resp)
{
	int		recheck_state;
	rsmseg_t	*segp = (rsmseg_t *)resp;
	DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_FUNC_ALL);
	DBG_DEFINE_STR(function, "rsm_unquiesce_exp_seg");

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
	    "%s enter: key=%u\n", function, segp->s_key));

	rsmseglock_acquire(segp);
	do {
		recheck_state = 0;
		if ((segp->s_state == RSM_STATE_NEW_QUIESCED) ||
		    (segp->s_state == RSM_STATE_BIND_QUIESCED) ||
		    (segp->s_state == RSM_STATE_EXPORT_QUIESCING) ||
		    (segp->s_state == RSM_STATE_EXPORT_QUIESCED)) {
			rsmseglock_release(segp);
			DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
			    "%s done:state =%d\n", function,
			    segp->s_state));
			return;
		}

		if (segp->s_state == RSM_STATE_NEW) {
			segp->s_state = RSM_STATE_NEW_QUIESCED;
			rsmseglock_release(segp);
			DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
			    "%s done:state =%d\n", function,
			    segp->s_state));
			return;
		}

		if (segp->s_state == RSM_STATE_BIND) {
			/* unbind */
			(void) rsm_unbind_pages(segp);
			segp->s_state = RSM_STATE_BIND_QUIESCED;
			rsmseglock_release(segp);
			DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
			    "%s done:state =%d\n", function,
			    segp->s_state));
			return;
		}

		if (segp->s_state == RSM_STATE_EXPORT) {
			/*
			 * wait for putv/getv to complete if the segp is
			 * a local memory handle
			 */
			while ((segp->s_state == RSM_STATE_EXPORT) &&
			    (segp->s_rdmacnt != 0)) {
				cv_wait(&segp->s_cv, &segp->s_lock);
			}

			if (segp->s_state != RSM_STATE_EXPORT) {
				/*
				 * state changed need to see what it
				 * should be changed to.
				 */
				recheck_state = 1;
				continue;
			}

			segp->s_state = RSM_STATE_EXPORT_QUIESCING;
			rsmseglock_release(segp);
			/*
			 * send SUSPEND messages - currently it will be
			 * done at the end
			 */
			DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
			    "%s done:state =%d\n", function,
			    segp->s_state));
			return;
		}
	} while (recheck_state);

	rsmseglock_release(segp);
}

static void
rsm_unquiesce_exp_seg(rsmresource_t *resp)
{
	int			ret;
	rsmseg_t		*segp = (rsmseg_t *)resp;
	rsmapi_access_entry_t	*acl;
	rsm_access_entry_t	*rsmpi_acl;
	int			acl_len;
	int			create_flags = 0;
	struct buf		*xbuf;
	rsm_memory_local_t	mem;
	adapter_t		*adapter;
	dev_t			sdev = 0;
	rsm_resource_callback_t callback_flag;
	DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_FUNC_ALL);
	DBG_DEFINE_STR(function, "rsm_unquiesce_exp_seg");

	rsmseglock_acquire(segp);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
	    "%s enter: key=%u, state=%d\n", function, segp->s_key,
	    segp->s_state));

	if ((segp->s_state == RSM_STATE_NEW) ||
	    (segp->s_state == RSM_STATE_BIND) ||
	    (segp->s_state == RSM_STATE_EXPORT)) {
		rsmseglock_release(segp);
		DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "%s done:state=%d\n",
		    function, segp->s_state));
		return;
	}

	if (segp->s_state == RSM_STATE_NEW_QUIESCED) {
		segp->s_state = RSM_STATE_NEW;
		cv_broadcast(&segp->s_cv);
		rsmseglock_release(segp);
		DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "%s done:state=%d\n",
		    function, segp->s_state));
		return;
	}

	if (segp->s_state == RSM_STATE_BIND_QUIESCED) {
		/* bind the segment */
		ret = rsm_bind_pages(&segp->s_cookie, segp->s_region.r_vaddr,
		    segp->s_len, segp->s_proc);
		if (ret == RSM_SUCCESS) { /* bind successful */
			segp->s_state = RSM_STATE_BIND;
		} else { /* bind failed - resource unavailable */
			segp->s_state = RSM_STATE_NEW;
		}
		cv_broadcast(&segp->s_cv);
		rsmseglock_release(segp);
		DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
		    "%s done: bind_qscd bind = %d\n", function, ret));
		return;
	}

	while (segp->s_state == RSM_STATE_EXPORT_QUIESCING) {
		/* wait for the segment to move to EXPORT_QUIESCED state */
		cv_wait(&segp->s_cv, &segp->s_lock);
	}

	if (segp->s_state == RSM_STATE_EXPORT_QUIESCED) {
		/* bind the segment */
		ret = rsm_bind_pages(&segp->s_cookie, segp->s_region.r_vaddr,
		    segp->s_len, segp->s_proc);

		if (ret != RSM_SUCCESS) {
			/* bind failed - resource unavailable */
			acl_len = segp->s_acl_len;
			acl = segp->s_acl;
			rsmpi_acl = segp->s_acl_in;
			segp->s_acl_len = 0;
			segp->s_acl = NULL;
			segp->s_acl_in = NULL;
			rsmseglock_release(segp);

			rsmexport_rm(segp);
			rsmacl_free(acl, acl_len);
			rsmpiacl_free(rsmpi_acl, acl_len);

			rsmseglock_acquire(segp);
			segp->s_state = RSM_STATE_NEW;
			cv_broadcast(&segp->s_cv);
			rsmseglock_release(segp);
			DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
			    "%s done: exp_qscd bind failed = %d\n",
			    function, ret));
			return;
		}
		/*
		 * publish the segment
		 * if  successful
		 *   segp->s_state = RSM_STATE_EXPORT;
		 * else failed
		 *   segp->s_state = RSM_STATE_BIND;
		 */

		/* check whether it is a local_memory_handle */
		if (segp->s_acl != (rsmapi_access_entry_t *)NULL) {
			if ((segp->s_acl[0].ae_node == my_nodeid) &&
			    (segp->s_acl[0].ae_permission == 0)) {
				segp->s_state = RSM_STATE_EXPORT;
				cv_broadcast(&segp->s_cv);
				rsmseglock_release(segp);
				DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
				    "%s done:exp_qscd\n", function));
				return;
			}
		}
		xbuf = ddi_umem_iosetup(segp->s_cookie, 0, segp->s_len, B_WRITE,
		    sdev, 0, NULL, DDI_UMEM_SLEEP);
		ASSERT(xbuf != NULL);

		mem.ms_type = RSM_MEM_BUF;
		mem.ms_bp = xbuf;

		adapter = segp->s_adapter;

		if (segp->s_flags & RSMKA_ALLOW_UNBIND_REBIND) {
			create_flags = RSM_ALLOW_UNBIND_REBIND;
		}

		if (segp->s_flags & RSMKA_SET_RESOURCE_DONTWAIT) {
			callback_flag  = RSM_RESOURCE_DONTWAIT;
		} else {
			callback_flag  = RSM_RESOURCE_SLEEP;
		}

		ret = adapter->rsmpi_ops->rsm_seg_create(
		    adapter->rsmpi_handle, &segp->s_handle.out,
		    segp->s_len, create_flags, &mem,
		    callback_flag, NULL);

		if (ret != RSM_SUCCESS) {
			acl_len = segp->s_acl_len;
			acl = segp->s_acl;
			rsmpi_acl = segp->s_acl_in;
			segp->s_acl_len = 0;
			segp->s_acl = NULL;
			segp->s_acl_in = NULL;
			rsmseglock_release(segp);

			rsmexport_rm(segp);
			rsmacl_free(acl, acl_len);
			rsmpiacl_free(rsmpi_acl, acl_len);

			rsmseglock_acquire(segp);
			segp->s_state = RSM_STATE_BIND;
			cv_broadcast(&segp->s_cv);
			rsmseglock_release(segp);
			DBG_PRINTF((category, RSM_ERR,
			    "%s done: exp_qscd create failed = %d\n",
			    function, ret));
			return;
		}

		ret = adapter->rsmpi_ops->rsm_publish(
		    segp->s_handle.out, segp->s_acl_in, segp->s_acl_len,
		    segp->s_segid, RSM_RESOURCE_DONTWAIT, NULL);

		if (ret != RSM_SUCCESS) {
			acl_len = segp->s_acl_len;
			acl = segp->s_acl;
			rsmpi_acl = segp->s_acl_in;
			segp->s_acl_len = 0;
			segp->s_acl = NULL;
			segp->s_acl_in = NULL;
			adapter->rsmpi_ops->rsm_seg_destroy(segp->s_handle.out);
			rsmseglock_release(segp);

			rsmexport_rm(segp);
			rsmacl_free(acl, acl_len);
			rsmpiacl_free(rsmpi_acl, acl_len);

			rsmseglock_acquire(segp);
			segp->s_state = RSM_STATE_BIND;
			cv_broadcast(&segp->s_cv);
			rsmseglock_release(segp);
			DBG_PRINTF((category, RSM_ERR,
			    "%s done: exp_qscd publish failed = %d\n",
			    function, ret));
			return;
		}

		segp->s_state = RSM_STATE_EXPORT;
		cv_broadcast(&segp->s_cv);
		rsmseglock_release(segp);
		DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "%s done: exp_qscd\n",
		    function));
		return;
	}

	rsmseglock_release(segp);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "%s done\n", function));
}

static void
rsm_quiesce_imp_seg(rsmresource_t *resp)
{
	rsmseg_t	*segp = (rsmseg_t *)resp;
	DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_FUNC_ALL);
	DBG_DEFINE_STR(function, "rsm_quiesce_imp_seg");

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
	    "%s enter: key=%u\n", function, segp->s_key));

	rsmseglock_acquire(segp);
	segp->s_flags |= RSM_DR_INPROGRESS;

	while (segp->s_rdmacnt != 0) {
		/* wait for the RDMA to complete */
		cv_wait(&segp->s_cv, &segp->s_lock);
	}

	rsmseglock_release(segp);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "%s done\n", function));

}

static void
rsm_unquiesce_imp_seg(rsmresource_t *resp)
{
	rsmseg_t	*segp = (rsmseg_t *)resp;
	DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_FUNC_ALL);
	DBG_DEFINE_STR(function, "rsm_unquiesce_imp_seg");

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
	    "%s enter: key=%u\n", function, segp->s_key));

	rsmseglock_acquire(segp);

	segp->s_flags &= ~RSM_DR_INPROGRESS;
	/* wake up any waiting putv/getv ops */
	cv_broadcast(&segp->s_cv);

	rsmseglock_release(segp);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE, "%s done\n", function));


}

static void
rsm_process_exp_seg(rsmresource_t *resp, int event)
{
	if (event == RSM_DR_QUIESCE)
		rsm_quiesce_exp_seg(resp);
	else /* UNQUIESCE */
		rsm_unquiesce_exp_seg(resp);
}

static void
rsm_process_imp_seg(rsmresource_t *resp, int event)
{
	if (event == RSM_DR_QUIESCE)
		rsm_quiesce_imp_seg(resp);
	else /* UNQUIESCE */
		rsm_unquiesce_imp_seg(resp);
}

static void
rsm_dr_process_local_segments(int event)
{

	int i, j;
	rsmresource_blk_t	*blk;
	rsmresource_t		*p;
	DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_FUNC_ALL);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
	    "rsm_dr_process_local_segments enter\n"));

	/* iterate through the resource structure */

	rw_enter(&rsm_resource.rsmrc_lock, RW_READER);

	for (i = 0; i < rsm_resource.rsmrc_len; i++) {
		blk = rsm_resource.rsmrc_root[i];
		if (blk != NULL) {
			for (j = 0; j < RSMRC_BLKSZ; j++) {
				p = blk->rsmrcblk_blks[j];
				if ((p != NULL) && (p != RSMRC_RESERVED)) {
					/* valid resource */
					if (p->rsmrc_type ==
					    RSM_RESOURCE_EXPORT_SEGMENT)
						rsm_process_exp_seg(p, event);
					else if (p->rsmrc_type ==
					    RSM_RESOURCE_IMPORT_SEGMENT)
						rsm_process_imp_seg(p, event);
				}
			}
		}
	}

	rw_exit(&rsm_resource.rsmrc_lock);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
	    "rsm_dr_process_local_segments done\n"));
}

/* *************** DR callback functions ************ */
static void
rsm_dr_callback_post_add(void *arg, pgcnt_t delta /* ARGSUSED */)
{
	DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_FUNC_ALL);
	DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
	    "rsm_dr_callback_post_add is a no-op\n"));
	/* Noop */
}

static int
rsm_dr_callback_pre_del(void *arg, pgcnt_t delta /* ARGSUSED */)
{
	int	recheck_state = 0;
	DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_FUNC_ALL);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
	    "rsm_dr_callback_pre_del enter\n"));

	mutex_enter(&rsm_drv_data.drv_lock);

	do {
		recheck_state = 0;
		DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
		    "rsm_dr_callback_pre_del:state=%d\n",
		    rsm_drv_data.drv_state));

		switch (rsm_drv_data.drv_state) {
		case RSM_DRV_NEW:
			/*
			 * The state should usually never be RSM_DRV_NEW
			 * since in this state the callbacks have not yet
			 * been registered. So, ASSERT.
			 */
			ASSERT(0);
			return (0);
		case RSM_DRV_REG_PROCESSING:
			/*
			 * The driver is in the process of registering
			 * with the DR framework. So, wait till the
			 * registration process is complete.
			 */
			recheck_state = 1;
			cv_wait(&rsm_drv_data.drv_cv, &rsm_drv_data.drv_lock);
			break;
		case RSM_DRV_UNREG_PROCESSING:
			/*
			 * If the state is RSM_DRV_UNREG_PROCESSING, the
			 * module is in the process of detaching and
			 * unregistering the callbacks from the DR
			 * framework. So, simply return.
			 */
			mutex_exit(&rsm_drv_data.drv_lock);
			DBG_PRINTF((category, RSM_DEBUG,
			    "rsm_dr_callback_pre_del:"
			    "pre-del on NEW/UNREG\n"));
			return (0);
		case RSM_DRV_OK:
			rsm_drv_data.drv_state = RSM_DRV_PREDEL_STARTED;
			break;
		case RSM_DRV_PREDEL_STARTED:
			/* FALLTHRU */
		case RSM_DRV_PREDEL_COMPLETED:
			/* FALLTHRU */
		case RSM_DRV_POSTDEL_IN_PROGRESS:
			recheck_state = 1;
			cv_wait(&rsm_drv_data.drv_cv, &rsm_drv_data.drv_lock);
			break;
		case RSM_DRV_DR_IN_PROGRESS:
			rsm_drv_data.drv_memdel_cnt++;
			mutex_exit(&rsm_drv_data.drv_lock);
			DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
			    "rsm_dr_callback_pre_del done\n"));
			return (0);
			/* break; */
		default:
			ASSERT(0);
			break;
		}

	} while (recheck_state);

	rsm_drv_data.drv_memdel_cnt++;

	mutex_exit(&rsm_drv_data.drv_lock);

	/* Do all the quiescing stuff here */
	DBG_PRINTF((category, RSM_DEBUG,
	    "rsm_dr_callback_pre_del: quiesce things now\n"));

	rsm_dr_process_local_segments(RSM_DR_QUIESCE);

	/*
	 * now that all local segments have been quiesced lets inform
	 * the importers
	 */
	rsm_send_suspend();

	/*
	 * In response to the suspend message the remote node(s) will process
	 * the segments and send a suspend_complete message. Till all
	 * the nodes send the suspend_complete message we wait in the
	 * RSM_DRV_PREDEL_STARTED state. In the exporter_quiesce
	 * function we transition to the RSM_DRV_PREDEL_COMPLETED state.
	 */
	mutex_enter(&rsm_drv_data.drv_lock);

	while (rsm_drv_data.drv_state == RSM_DRV_PREDEL_STARTED) {
		cv_wait(&rsm_drv_data.drv_cv, &rsm_drv_data.drv_lock);
	}

	ASSERT(rsm_drv_data.drv_state == RSM_DRV_PREDEL_COMPLETED);

	rsm_drv_data.drv_state = RSM_DRV_DR_IN_PROGRESS;
	cv_broadcast(&rsm_drv_data.drv_cv);

	mutex_exit(&rsm_drv_data.drv_lock);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
	    "rsm_dr_callback_pre_del done\n"));

	return (0);
}

static void
rsm_dr_callback_post_del(void *arg, pgcnt_t delta, int cancelled /* ARGSUSED */)
{
	int	recheck_state = 0;
	DBG_DEFINE(category, RSM_KERNEL_AGENT | RSM_FUNC_ALL);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
	    "rsm_dr_callback_post_del enter\n"));

	mutex_enter(&rsm_drv_data.drv_lock);

	do {
		recheck_state = 0;
		DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
		    "rsm_dr_callback_post_del:state=%d\n",
		    rsm_drv_data.drv_state));

		switch (rsm_drv_data.drv_state) {
		case RSM_DRV_NEW:
			/*
			 * The driver state cannot not be RSM_DRV_NEW
			 * since in this state the callbacks have not
			 * yet been registered.
			 */
			ASSERT(0);
			return;
		case RSM_DRV_REG_PROCESSING:
			/*
			 * The driver is in the process of registering with
			 * the DR framework. Wait till the registration is
			 * complete.
			 */
			recheck_state = 1;
			cv_wait(&rsm_drv_data.drv_cv, &rsm_drv_data.drv_lock);
			break;
		case RSM_DRV_UNREG_PROCESSING:
			/*
			 * RSM_DRV_UNREG_PROCESSING state means the module
			 * is detaching and unregistering the callbacks
			 * from the DR framework. So simply return.
			 */
			/* FALLTHRU */
		case RSM_DRV_OK:
			/*
			 * RSM_DRV_OK means we missed the pre-del
			 * corresponding to this post-del coz we had not
			 * registered yet, so simply return.
			 */
			mutex_exit(&rsm_drv_data.drv_lock);
			DBG_PRINTF((category, RSM_DEBUG,
			    "rsm_dr_callback_post_del:"
			    "post-del on OK/UNREG\n"));
			return;
			/* break; */
		case RSM_DRV_PREDEL_STARTED:
			/* FALLTHRU */
		case RSM_DRV_PREDEL_COMPLETED:
			/* FALLTHRU */
		case RSM_DRV_POSTDEL_IN_PROGRESS:
			recheck_state = 1;
			cv_wait(&rsm_drv_data.drv_cv, &rsm_drv_data.drv_lock);
			break;
		case RSM_DRV_DR_IN_PROGRESS:
			rsm_drv_data.drv_memdel_cnt--;
			if (rsm_drv_data.drv_memdel_cnt > 0) {
				mutex_exit(&rsm_drv_data.drv_lock);
				DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
				    "rsm_dr_callback_post_del done:\n"));
				return;
			}
			rsm_drv_data.drv_state = RSM_DRV_POSTDEL_IN_PROGRESS;
			break;
		default:
			ASSERT(0);
			return;
			/* break; */
		}
	} while (recheck_state);

	mutex_exit(&rsm_drv_data.drv_lock);

	/* Do all the unquiescing stuff here */
	DBG_PRINTF((category, RSM_DEBUG,
	    "rsm_dr_callback_post_del: unquiesce things now\n"));

	rsm_dr_process_local_segments(RSM_DR_UNQUIESCE);

	/*
	 * now that all local segments have been unquiesced lets inform
	 * the importers
	 */
	rsm_send_resume();

	mutex_enter(&rsm_drv_data.drv_lock);

	rsm_drv_data.drv_state = RSM_DRV_OK;

	cv_broadcast(&rsm_drv_data.drv_cv);

	mutex_exit(&rsm_drv_data.drv_lock);

	DBG_PRINTF((category, RSM_DEBUG_VERBOSE,
	    "rsm_dr_callback_post_del done\n"));

	return;

}