common/io/dcopy.c

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

/*
 * dcopy.c
 *    dcopy misc module
 */

#include <sys/conf.h>
#include <sys/kmem.h>
#include <sys/ddi.h>
#include <sys/sunddi.h>
#include <sys/modctl.h>
#include <sys/sysmacros.h>
#include <sys/atomic.h>


#include <sys/dcopy.h>
#include <sys/dcopy_device.h>


/* Number of entries per channel to allocate */
uint_t dcopy_channel_size = 1024;


typedef struct dcopy_list_s {
	list_t			dl_list;
	kmutex_t		dl_mutex;
	uint_t			dl_cnt; /* num entries on list */
} dcopy_list_t;

/* device state for register/unregister */
struct dcopy_device_s {
	/* DMA device drivers private pointer */
	void			*dc_device_private;

	/* to track list of channels from this DMA device */
	dcopy_list_t		dc_devchan_list;
	list_node_t		dc_device_list_node;

	/*
	 * dc_removing_cnt track how many channels still have to be freed up
	 * before it's safe to allow the DMA device driver to detach.
	 */
	uint_t			dc_removing_cnt;
	dcopy_device_cb_t	*dc_cb;

	dcopy_device_info_t	dc_info;

};

typedef struct dcopy_stats_s {
	kstat_named_t	cs_bytes_xfer;
	kstat_named_t	cs_cmd_alloc;
	kstat_named_t	cs_cmd_post;
	kstat_named_t	cs_cmd_poll;
	kstat_named_t	cs_notify_poll;
	kstat_named_t	cs_notify_pending;
	kstat_named_t	cs_id;
	kstat_named_t	cs_capabilities;
} dcopy_stats_t;

/* DMA channel state */
struct dcopy_channel_s {
	/* DMA driver channel private pointer */
	void			*ch_channel_private;

	/* shortcut to device callbacks */
	dcopy_device_cb_t	*ch_cb;

	/*
	 * number of outstanding allocs for this channel. used to track when
	 * it's safe to free up this channel so the DMA device driver can
	 * detach.
	 */
	uint64_t		ch_ref_cnt;

	/* state for if channel needs to be removed when ch_ref_cnt gets to 0 */
	boolean_t		ch_removing;

	list_node_t		ch_devchan_list_node;
	list_node_t		ch_globalchan_list_node;

	/*
	 * per channel list of commands actively blocking waiting for
	 * completion.
	 */
	dcopy_list_t		ch_poll_list;

	/* pointer back to our device */
	struct dcopy_device_s	*ch_device;

	dcopy_query_channel_t	ch_info;

	kstat_t			*ch_kstat;
	dcopy_stats_t		ch_stat;
};

/*
 * If grabbing both device_list mutex & globalchan_list mutex,
 * Always grab globalchan_list mutex before device_list mutex
 */
typedef struct dcopy_state_s {
	dcopy_list_t		d_device_list;
	dcopy_list_t		d_globalchan_list;
} dcopy_state_t;
dcopy_state_t *dcopy_statep;


/* Module Driver Info */
static struct modlmisc dcopy_modlmisc = {
	&mod_miscops,
	"dcopy kernel module"
};

/* Module Linkage */
static struct modlinkage dcopy_modlinkage = {
	MODREV_1,
	&dcopy_modlmisc,
	NULL
};

static int dcopy_init();
static void dcopy_fini();

static int dcopy_list_init(dcopy_list_t *list, size_t node_size,
    offset_t link_offset);
static void dcopy_list_fini(dcopy_list_t *list);
static void dcopy_list_push(dcopy_list_t *list, void *list_node);
static void *dcopy_list_pop(dcopy_list_t *list);

static void dcopy_device_cleanup(dcopy_device_handle_t device,
    boolean_t do_callback);

static int dcopy_stats_init(dcopy_handle_t channel);
static void dcopy_stats_fini(dcopy_handle_t channel);


/*
 * _init()
 */
int
_init()
{
	int e;

	e = dcopy_init();
	if (e != 0) {
		return (e);
	}

	return (mod_install(&dcopy_modlinkage));
}


/*
 * _info()
 */
int
_info(struct modinfo *modinfop)
{
	return (mod_info(&dcopy_modlinkage, modinfop));
}


/*
 * _fini()
 */
int
_fini()
{
	int e;

	e = mod_remove(&dcopy_modlinkage);
	if (e != 0) {
		return (e);
	}

	dcopy_fini();

	return (e);
}

/*
 * dcopy_init()
 */
static int
dcopy_init()
{
	int e;


	dcopy_statep = kmem_zalloc(sizeof (*dcopy_statep), KM_SLEEP);

	/* Initialize the list we use to track device register/unregister */
	e = dcopy_list_init(&dcopy_statep->d_device_list,
	    sizeof (struct dcopy_device_s),
	    offsetof(struct dcopy_device_s, dc_device_list_node));
	if (e != DCOPY_SUCCESS) {
		goto dcopyinitfail_device;
	}

	/* Initialize the list we use to track all DMA channels */
	e = dcopy_list_init(&dcopy_statep->d_globalchan_list,
	    sizeof (struct dcopy_channel_s),
	    offsetof(struct dcopy_channel_s, ch_globalchan_list_node));
	if (e != DCOPY_SUCCESS) {
		goto dcopyinitfail_global;
	}

	return (0);

dcopyinitfail_global:
	dcopy_list_fini(&dcopy_statep->d_device_list);
dcopyinitfail_device:
	kmem_free(dcopy_statep, sizeof (*dcopy_statep));

	return (-1);
}


/*
 * dcopy_fini()
 */
static void
dcopy_fini()
{
	/*
	 * if mod_remove was successfull, we shouldn't have any
	 * devices/channels to worry about.
	 */
	ASSERT(list_head(&dcopy_statep->d_globalchan_list.dl_list) == NULL);
	ASSERT(list_head(&dcopy_statep->d_device_list.dl_list) == NULL);

	dcopy_list_fini(&dcopy_statep->d_globalchan_list);
	dcopy_list_fini(&dcopy_statep->d_device_list);
	kmem_free(dcopy_statep, sizeof (*dcopy_statep));
}


/* *** EXTERNAL INTERFACE *** */
/*
 * dcopy_query()
 */
void
dcopy_query(dcopy_query_t *query)
{
	query->dq_version = DCOPY_QUERY_V0;
	query->dq_num_channels = dcopy_statep->d_globalchan_list.dl_cnt;
}


/*
 * dcopy_alloc()
 */
/*ARGSUSED*/
int
dcopy_alloc(int flags, dcopy_handle_t *handle)
{
	dcopy_handle_t channel;
	dcopy_list_t *list;


	/*
	 * we don't use the dcopy_list_* code here because we need to due
	 * some non-standard stuff.
	 */

	list = &dcopy_statep->d_globalchan_list;

	/*
	 * if nothing is on the channel list, return DCOPY_NORESOURCES. This
	 * can happen if there aren't any DMA device registered.
	 */
	mutex_enter(&list->dl_mutex);
	channel = list_head(&list->dl_list);
	if (channel == NULL) {
		mutex_exit(&list->dl_mutex);
		return (DCOPY_NORESOURCES);
	}

	/*
	 * increment the reference count, and pop the channel off the head and
	 * push it on the tail. This ensures we rotate through the channels.
	 * DMA channels are shared.
	 */
	channel->ch_ref_cnt++;
	list_remove(&list->dl_list, channel);
	list_insert_tail(&list->dl_list, channel);
	mutex_exit(&list->dl_mutex);

	*handle = (dcopy_handle_t)channel;
	return (DCOPY_SUCCESS);
}


/*
 * dcopy_free()
 */
void
dcopy_free(dcopy_handle_t *channel)
{
	dcopy_device_handle_t device;
	dcopy_list_t *list;
	boolean_t cleanup = B_FALSE;


	ASSERT(*channel != NULL);

	/*
	 * we don't need to add the channel back to the list since we never
	 * removed it. decrement the reference count.
	 */
	list = &dcopy_statep->d_globalchan_list;
	mutex_enter(&list->dl_mutex);
	(*channel)->ch_ref_cnt--;

	/*
	 * if we need to remove this channel, and the reference count is down
	 * to 0, decrement the number of channels which still need to be
	 * removed on the device.
	 */
	if ((*channel)->ch_removing && ((*channel)->ch_ref_cnt == 0)) {
		device = (*channel)->ch_device;
		mutex_enter(&device->dc_devchan_list.dl_mutex);
		device->dc_removing_cnt--;
		if (device->dc_removing_cnt == 0) {
			cleanup = B_TRUE;
		}
		mutex_exit(&device->dc_devchan_list.dl_mutex);
	}
	mutex_exit(&list->dl_mutex);

	/*
	 * if there are no channels which still need to be removed, cleanup the
	 * device state and call back into the DMA device driver to tell them
	 * the device is free.
	 */
	if (cleanup) {
		dcopy_device_cleanup(device, B_TRUE);
	}

	*channel = NULL;
}


/*
 * dcopy_query_channel()
 */
void
dcopy_query_channel(dcopy_handle_t channel, dcopy_query_channel_t *query)
{
	*query = channel->ch_info;
}


/*
 * dcopy_cmd_alloc()
 */
int
dcopy_cmd_alloc(dcopy_handle_t handle, int flags, dcopy_cmd_t *cmd)
{
	dcopy_handle_t channel;
	dcopy_cmd_priv_t priv;
	int e;


	channel = handle;

	atomic_inc_64(&channel->ch_stat.cs_cmd_alloc.value.ui64);
	e = channel->ch_cb->cb_cmd_alloc(channel->ch_channel_private, flags,
	    cmd);
	if (e == DCOPY_SUCCESS) {
		priv = (*cmd)->dp_private;
		priv->pr_channel = channel;
		/*
		 * we won't initialize the blocking state until we actually
		 * need to block.
		 */
		priv->pr_block_init = B_FALSE;
	}

	return (e);
}


/*
 * dcopy_cmd_free()
 */
void
dcopy_cmd_free(dcopy_cmd_t *cmd)
{
	dcopy_handle_t channel;
	dcopy_cmd_priv_t priv;


	ASSERT(*cmd != NULL);

	priv = (*cmd)->dp_private;
	channel = priv->pr_channel;

	/* if we initialized the blocking state, clean it up too */
	if (priv->pr_block_init) {
		cv_destroy(&priv->pr_cv);
		mutex_destroy(&priv->pr_mutex);
	}

	channel->ch_cb->cb_cmd_free(channel->ch_channel_private, cmd);
}


/*
 * dcopy_cmd_post()
 */
int
dcopy_cmd_post(dcopy_cmd_t cmd)
{
	dcopy_handle_t channel;
	int e;


	channel = cmd->dp_private->pr_channel;

	atomic_inc_64(&channel->ch_stat.cs_cmd_post.value.ui64);
	if (cmd->dp_cmd == DCOPY_CMD_COPY) {
		atomic_add_64(&channel->ch_stat.cs_bytes_xfer.value.ui64,
		    cmd->dp.copy.cc_size);
	}
	e = channel->ch_cb->cb_cmd_post(channel->ch_channel_private, cmd);
	if (e != DCOPY_SUCCESS) {
		return (e);
	}

	return (DCOPY_SUCCESS);
}


/*
 * dcopy_cmd_poll()
 */
int
dcopy_cmd_poll(dcopy_cmd_t cmd, int flags)
{
	dcopy_handle_t channel;
	dcopy_cmd_priv_t priv;
	int e;


	priv = cmd->dp_private;
	channel = priv->pr_channel;

	/*
	 * if the caller is trying to block, they needed to post the
	 * command with DCOPY_CMD_INTR set.
	 */
	if ((flags & DCOPY_POLL_BLOCK) && !(cmd->dp_flags & DCOPY_CMD_INTR)) {
		return (DCOPY_FAILURE);
	}

	atomic_inc_64(&channel->ch_stat.cs_cmd_poll.value.ui64);

repoll:
	e = channel->ch_cb->cb_cmd_poll(channel->ch_channel_private, cmd);
	if (e == DCOPY_PENDING) {
		/*
		 * if the command is still active, and the blocking flag
		 * is set.
		 */
		if (flags & DCOPY_POLL_BLOCK) {

			/*
			 * if we haven't initialized the state, do it now. A
			 * command can be re-used, so it's possible it's
			 * already been initialized.
			 */
			if (!priv->pr_block_init) {
				priv->pr_block_init = B_TRUE;
				mutex_init(&priv->pr_mutex, NULL, MUTEX_DRIVER,
				    NULL);
				cv_init(&priv->pr_cv, NULL, CV_DRIVER, NULL);
				priv->pr_cmd = cmd;
			}

			/* push it on the list for blocking commands */
			priv->pr_wait = B_TRUE;
			dcopy_list_push(&channel->ch_poll_list, priv);

			mutex_enter(&priv->pr_mutex);
			/*
			 * it's possible we already cleared pr_wait before we
			 * grabbed the mutex.
			 */
			if (priv->pr_wait) {
				cv_wait(&priv->pr_cv, &priv->pr_mutex);
			}
			mutex_exit(&priv->pr_mutex);

			/*
			 * the command has completed, go back and poll so we
			 * get the status.
			 */
			goto repoll;
		}
	}

	return (e);
}

/* *** END OF EXTERNAL INTERFACE *** */

/*
 * dcopy_list_init()
 */
static int
dcopy_list_init(dcopy_list_t *list, size_t node_size, offset_t link_offset)
{
	mutex_init(&list->dl_mutex, NULL, MUTEX_DRIVER, NULL);
	list_create(&list->dl_list, node_size, link_offset);
	list->dl_cnt = 0;

	return (DCOPY_SUCCESS);
}


/*
 * dcopy_list_fini()
 */
static void
dcopy_list_fini(dcopy_list_t *list)
{
	list_destroy(&list->dl_list);
	mutex_destroy(&list->dl_mutex);
}


/*
 * dcopy_list_push()
 */
static void
dcopy_list_push(dcopy_list_t *list, void *list_node)
{
	mutex_enter(&list->dl_mutex);
	list_insert_tail(&list->dl_list, list_node);
	list->dl_cnt++;
	mutex_exit(&list->dl_mutex);
}


/*
 * dcopy_list_pop()
 */
static void *
dcopy_list_pop(dcopy_list_t *list)
{
	list_node_t *list_node;

	mutex_enter(&list->dl_mutex);
	list_node = list_head(&list->dl_list);
	if (list_node == NULL) {
		mutex_exit(&list->dl_mutex);
		return (list_node);
	}
	list->dl_cnt--;
	list_remove(&list->dl_list, list_node);
	mutex_exit(&list->dl_mutex);

	return (list_node);
}


/* *** DEVICE INTERFACE *** */
/*
 * dcopy_device_register()
 */
int
dcopy_device_register(void *device_private, dcopy_device_info_t *info,
    dcopy_device_handle_t *handle)
{
	struct dcopy_channel_s *channel;
	struct dcopy_device_s *device;
	int e;
	int i;


	/* initialize the per device state */
	device = kmem_zalloc(sizeof (*device), KM_SLEEP);
	device->dc_device_private = device_private;
	device->dc_info = *info;
	device->dc_removing_cnt = 0;
	device->dc_cb = info->di_cb;

	/*
	 * we have a per device channel list so we can remove a device in the
	 * future.
	 */
	e = dcopy_list_init(&device->dc_devchan_list,
	    sizeof (struct dcopy_channel_s),
	    offsetof(struct dcopy_channel_s, ch_devchan_list_node));
	if (e != DCOPY_SUCCESS) {
		goto registerfail_devchan;
	}

	/*
	 * allocate state for each channel, allocate the channel,  and then add
	 * the devices dma channels to the devices channel list.
	 */
	for (i = 0; i < info->di_num_dma; i++) {
		channel = kmem_zalloc(sizeof (*channel), KM_SLEEP);
		channel->ch_device = device;
		channel->ch_removing = B_FALSE;
		channel->ch_ref_cnt = 0;
		channel->ch_cb = info->di_cb;

		e = info->di_cb->cb_channel_alloc(device_private, channel,
		    DCOPY_SLEEP, dcopy_channel_size, &channel->ch_info,
		    &channel->ch_channel_private);
		if (e != DCOPY_SUCCESS) {
			kmem_free(channel, sizeof (*channel));
			goto registerfail_alloc;
		}

		e = dcopy_stats_init(channel);
		if (e != DCOPY_SUCCESS) {
			info->di_cb->cb_channel_free(
			    &channel->ch_channel_private);
			kmem_free(channel, sizeof (*channel));
			goto registerfail_alloc;
		}

		e = dcopy_list_init(&channel->ch_poll_list,
		    sizeof (struct dcopy_cmd_priv_s),
		    offsetof(struct dcopy_cmd_priv_s, pr_poll_list_node));
		if (e != DCOPY_SUCCESS) {
			dcopy_stats_fini(channel);
			info->di_cb->cb_channel_free(
			    &channel->ch_channel_private);
			kmem_free(channel, sizeof (*channel));
			goto registerfail_alloc;
		}

		dcopy_list_push(&device->dc_devchan_list, channel);
	}

	/* add the device to device list */
	dcopy_list_push(&dcopy_statep->d_device_list, device);

	/*
	 * add the device's dma channels to the global channel list (where
	 * dcopy_alloc's come from)
	 */
	mutex_enter(&dcopy_statep->d_globalchan_list.dl_mutex);
	mutex_enter(&dcopy_statep->d_device_list.dl_mutex);
	channel = list_head(&device->dc_devchan_list.dl_list);
	while (channel != NULL) {
		list_insert_tail(&dcopy_statep->d_globalchan_list.dl_list,
		    channel);
		dcopy_statep->d_globalchan_list.dl_cnt++;
		channel = list_next(&device->dc_devchan_list.dl_list, channel);
	}
	mutex_exit(&dcopy_statep->d_device_list.dl_mutex);
	mutex_exit(&dcopy_statep->d_globalchan_list.dl_mutex);

	*handle = device;

	/* last call-back into kernel for dcopy KAPI enabled */
	uioa_dcopy_enable();

	return (DCOPY_SUCCESS);

registerfail_alloc:
	channel = list_head(&device->dc_devchan_list.dl_list);
	while (channel != NULL) {
		/* remove from the list */
		channel = dcopy_list_pop(&device->dc_devchan_list);
		ASSERT(channel != NULL);

		dcopy_list_fini(&channel->ch_poll_list);
		dcopy_stats_fini(channel);
		info->di_cb->cb_channel_free(&channel->ch_channel_private);
		kmem_free(channel, sizeof (*channel));
	}

	dcopy_list_fini(&device->dc_devchan_list);
registerfail_devchan:
	kmem_free(device, sizeof (*device));

	return (DCOPY_FAILURE);
}


/*
 * dcopy_device_unregister()
 */
/*ARGSUSED*/
int
dcopy_device_unregister(dcopy_device_handle_t *handle)
{
	struct dcopy_channel_s *channel;
	dcopy_device_handle_t device;
	boolean_t device_busy;

	/* first call-back into kernel for dcopy KAPI disable */
	uioa_dcopy_disable();

	device = *handle;
	device_busy = B_FALSE;

	/*
	 * remove the devices dma channels from the global channel list (where
	 * dcopy_alloc's come from)
	 */
	mutex_enter(&dcopy_statep->d_globalchan_list.dl_mutex);
	mutex_enter(&device->dc_devchan_list.dl_mutex);
	channel = list_head(&device->dc_devchan_list.dl_list);
	while (channel != NULL) {
		/*
		 * if the channel has outstanding allocs, mark it as having
		 * to be removed and increment the number of channels which
		 * need to be removed in the device state too.
		 */
		if (channel->ch_ref_cnt != 0) {
			channel->ch_removing = B_TRUE;
			device_busy = B_TRUE;
			device->dc_removing_cnt++;
		}
		dcopy_statep->d_globalchan_list.dl_cnt--;
		list_remove(&dcopy_statep->d_globalchan_list.dl_list, channel);
		channel = list_next(&device->dc_devchan_list.dl_list, channel);
	}
	mutex_exit(&device->dc_devchan_list.dl_mutex);
	mutex_exit(&dcopy_statep->d_globalchan_list.dl_mutex);

	/*
	 * if there are channels which still need to be removed, we will clean
	 * up the device state after they are freed up.
	 */
	if (device_busy) {
		return (DCOPY_PENDING);
	}

	dcopy_device_cleanup(device, B_FALSE);

	*handle = NULL;
	return (DCOPY_SUCCESS);
}


/*
 * dcopy_device_cleanup()
 */
static void
dcopy_device_cleanup(dcopy_device_handle_t device, boolean_t do_callback)
{
	struct dcopy_channel_s *channel;

	/*
	 * remove all the channels in the device list, free them, and clean up
	 * the state.
	 */
	mutex_enter(&dcopy_statep->d_device_list.dl_mutex);
	channel = list_head(&device->dc_devchan_list.dl_list);
	while (channel != NULL) {
		device->dc_devchan_list.dl_cnt--;
		list_remove(&device->dc_devchan_list.dl_list, channel);
		dcopy_list_fini(&channel->ch_poll_list);
		dcopy_stats_fini(channel);
		channel->ch_cb->cb_channel_free(&channel->ch_channel_private);
		kmem_free(channel, sizeof (*channel));
		channel = list_head(&device->dc_devchan_list.dl_list);
	}

	/* remove it from the list of devices */
	list_remove(&dcopy_statep->d_device_list.dl_list, device);

	mutex_exit(&dcopy_statep->d_device_list.dl_mutex);

	/*
	 * notify the DMA device driver that the device is free to be
	 * detached.
	 */
	if (do_callback) {
		device->dc_cb->cb_unregister_complete(
		    device->dc_device_private, DCOPY_SUCCESS);
	}

	dcopy_list_fini(&device->dc_devchan_list);
	kmem_free(device, sizeof (*device));
}


/*
 * dcopy_device_channel_notify()
 */
/*ARGSUSED*/
void
dcopy_device_channel_notify(dcopy_handle_t handle, int status)
{
	struct dcopy_channel_s *channel;
	dcopy_list_t *poll_list;
	dcopy_cmd_priv_t priv;
	int e;


	ASSERT(status == DCOPY_COMPLETION);
	channel = handle;

	poll_list = &channel->ch_poll_list;

	/*
	 * when we get a completion notification from the device, go through
	 * all of the commands blocking on this channel and see if they have
	 * completed. Remove the command and wake up the block thread if they
	 * have. Once we hit a command which is still pending, we are done
	 * polling since commands in a channel complete in order.
	 */
	mutex_enter(&poll_list->dl_mutex);
	if (poll_list->dl_cnt != 0) {
		priv = list_head(&poll_list->dl_list);
		while (priv != NULL) {
			atomic_inc_64(&channel->
			    ch_stat.cs_notify_poll.value.ui64);
			e = channel->ch_cb->cb_cmd_poll(
			    channel->ch_channel_private,
			    priv->pr_cmd);
			if (e == DCOPY_PENDING) {
				atomic_inc_64(&channel->
				    ch_stat.cs_notify_pending.value.ui64);
				break;
			}

			poll_list->dl_cnt--;
			list_remove(&poll_list->dl_list, priv);

			mutex_enter(&priv->pr_mutex);
			priv->pr_wait = B_FALSE;
			cv_signal(&priv->pr_cv);
			mutex_exit(&priv->pr_mutex);

			priv = list_head(&poll_list->dl_list);
		}
	}

	mutex_exit(&poll_list->dl_mutex);
}


/*
 * dcopy_stats_init()
 */
static int
dcopy_stats_init(dcopy_handle_t channel)
{
#define	CHANSTRSIZE	20
	char chanstr[CHANSTRSIZE];
	dcopy_stats_t *stats;
	int instance;
	char *name;


	stats = &channel->ch_stat;
	name = (char *)ddi_driver_name(channel->ch_device->dc_info.di_dip);
	instance = ddi_get_instance(channel->ch_device->dc_info.di_dip);

	(void) snprintf(chanstr, CHANSTRSIZE, "channel%d",
	    (uint32_t)channel->ch_info.qc_chan_num);

	channel->ch_kstat = kstat_create(name, instance, chanstr, "misc",
	    KSTAT_TYPE_NAMED, sizeof (dcopy_stats_t) / sizeof (kstat_named_t),
	    KSTAT_FLAG_VIRTUAL);
	if (channel->ch_kstat == NULL) {
		return (DCOPY_FAILURE);
	}
	channel->ch_kstat->ks_data = stats;

	kstat_named_init(&stats->cs_bytes_xfer, "bytes_xfer",
	    KSTAT_DATA_UINT64);
	kstat_named_init(&stats->cs_cmd_alloc, "cmd_alloc",
	    KSTAT_DATA_UINT64);
	kstat_named_init(&stats->cs_cmd_post, "cmd_post",
	    KSTAT_DATA_UINT64);
	kstat_named_init(&stats->cs_cmd_poll, "cmd_poll",
	    KSTAT_DATA_UINT64);
	kstat_named_init(&stats->cs_notify_poll, "notify_poll",
	    KSTAT_DATA_UINT64);
	kstat_named_init(&stats->cs_notify_pending, "notify_pending",
	    KSTAT_DATA_UINT64);
	kstat_named_init(&stats->cs_id, "id",
	    KSTAT_DATA_UINT64);
	kstat_named_init(&stats->cs_capabilities, "capabilities",
	    KSTAT_DATA_UINT64);

	kstat_install(channel->ch_kstat);

	channel->ch_stat.cs_id.value.ui64 = channel->ch_info.qc_id;
	channel->ch_stat.cs_capabilities.value.ui64 =
	    channel->ch_info.qc_capabilities;

	return (DCOPY_SUCCESS);
}


/*
 * dcopy_stats_fini()
 */
static void
dcopy_stats_fini(dcopy_handle_t channel)
{
	kstat_delete(channel->ch_kstat);
}
/* *** END OF DEVICE INTERFACE *** */