usb/usba/hcdi.c

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License, Version 1.0 only
 * (the "License").  You may not use this file except in compliance
 * with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright 2005 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */
#pragma ident	"%Z%%M%	%I%	%E% SMI"

/*
 * USBA: Solaris USB Architecture support
 *
 * hcdi.c contains the code for client driver callbacks.  A host controller
 * driver registers/unregisters with usba through usba_hcdi_register/unregister.
 *
 * When the transfer has finished, the host controller driver will call into
 * usba with the result.  The call is usba_hcdi_cb().
 *
 * The callback queue is maintained in FIFO order.  usba_hcdi_cb
 * adds to the queue, and hcdi_cb_thread takes the callbacks off the queue
 * and executes them.
 */
#define	USBA_FRAMEWORK
#include <sys/usb/usba/usba_impl.h>
#include <sys/usb/usba/hcdi_impl.h>
#include <sys/kstat.h>
#include <sys/ddi_impldefs.h>

/* function prototypes, XXXX use hcdi_ prefix?	*/
static void usba_hcdi_create_stats(usba_hcdi_t *, int);
static void usba_hcdi_update_error_stats(usba_hcdi_t *, usb_cr_t);
static void usba_hcdi_destroy_stats(usba_hcdi_t *);

/* internal functions */
static uint_t hcdi_soft_intr(caddr_t arg1, caddr_t arg2);

static void hcdi_cb_thread(void *);
static void hcdi_shared_cb_thread(void *);
static void hcdi_do_cb(usba_pipe_handle_data_t *, usba_req_wrapper_t *,
							usba_hcdi_t *);
static void hcdi_autoclearing(usba_req_wrapper_t *);

/* private function from USBAI */
void	usba_pipe_clear(usb_pipe_handle_t);

/* for debug messages */
static uint_t	hcdi_errmask	= (uint_t)DPRINT_MASK_ALL;
static uint_t	hcdi_errlevel	= USB_LOG_L4;
static uint_t	hcdi_instance_debug = (uint_t)-1;

void
usba_hcdi_initialization()
{
}


void
usba_hcdi_destroy()
{
}


/*
 * store hcdi structure in the dip
 */
void
usba_hcdi_set_hcdi(dev_info_t *dip, usba_hcdi_t *hcdi)
{
	ddi_set_driver_private(dip, hcdi);
}


/*
 * retrieve hcdi structure from the dip
 */
usba_hcdi_t *
usba_hcdi_get_hcdi(dev_info_t *dip)
{
	return (ddi_get_driver_private(dip));
}

/*
 * Called by an	HCD to attach an instance of the driver
 *	make this instance known to USBA
 *	the HCD	should initialize usba_hcdi structure prior
 *	to calling this	interface
 */
int
usba_hcdi_register(usba_hcdi_register_args_t *args, uint_t flags)
{
	char		*datap;
	uint_t		soft_prip;
	usba_hcdi_t	*hcdi = kmem_zalloc(sizeof (usba_hcdi_t), KM_SLEEP);

	if (args->usba_hcdi_register_version != HCDI_REGISTER_VERS_0) {
		kmem_free(hcdi, sizeof (usba_hcdi_t));

		return (USB_FAILURE);
	}

	hcdi->hcdi_dip = args->usba_hcdi_register_dip;

	/*
	 * Create a log_handle
	 */
	hcdi->hcdi_log_handle = usb_alloc_log_hdl(hcdi->hcdi_dip, NULL,
			&hcdi_errlevel, &hcdi_errmask, &hcdi_instance_debug,
			0);

	USB_DPRINTF_L4(DPRINT_MASK_HCDI, hcdi->hcdi_log_handle,
	    "usba_hcdi_register: %s", ddi_node_name(hcdi->hcdi_dip));

	/*
	 * Initialize the mutex.  Use the iblock cookie passed in
	 * by the host controller driver.
	 */
	mutex_init(&hcdi->hcdi_mutex, NULL, MUTEX_DRIVER,
			args->usba_hcdi_register_iblock_cookie);

	/* add soft interrupt */
	if (ddi_intr_add_softint(hcdi->hcdi_dip, &hcdi->hcdi_softint_hdl,
	    DDI_INTR_SOFTPRI_MAX, hcdi_soft_intr, (caddr_t)hcdi) !=
	    DDI_SUCCESS) {
		USB_DPRINTF_L2(DPRINT_MASK_HCDI, hcdi->hcdi_log_handle,
		    "usba_hcd_register: add soft interrupt failed");
		mutex_destroy(&hcdi->hcdi_mutex);
		usb_free_log_hdl(hcdi->hcdi_log_handle);
		kmem_free(hcdi, sizeof (usba_hcdi_t));

		return (USB_FAILURE);
	}

	if (ddi_intr_get_softint_pri(hcdi->hcdi_softint_hdl, &soft_prip) !=
	    DDI_SUCCESS) {
		USB_DPRINTF_L2(DPRINT_MASK_HCDI, hcdi->hcdi_log_handle,
		    "usba_hcd_register: get soft interrupt priority failed");
		(void) ddi_intr_remove_softint(hcdi->hcdi_softint_hdl);
		mutex_destroy(&hcdi->hcdi_mutex);
		usb_free_log_hdl(hcdi->hcdi_log_handle);
		kmem_free(hcdi, sizeof (usba_hcdi_t));

		return (USB_FAILURE);
	}

	/*
	 * Priority and iblock_cookie are one and the same
	 * (However, retaining hcdi_soft_iblock_cookie for now
	 * assigning it w/ priority. In future all iblock_cookie
	 * could just go)
	 */
	hcdi->hcdi_soft_iblock_cookie =
	    (ddi_iblock_cookie_t)(uint64_t)soft_prip;

	usba_init_list(&hcdi->hcdi_cb_queue, NULL, NULL);

	hcdi->hcdi_dma_attr	= args->usba_hcdi_register_dma_attr;
	hcdi->hcdi_flags	= flags;
	hcdi->hcdi_ops		= args->usba_hcdi_register_ops;
	hcdi->hcdi_iblock_cookie = args->usba_hcdi_register_iblock_cookie;
	usba_hcdi_create_stats(hcdi, ddi_get_instance(hcdi->hcdi_dip));

	hcdi->hcdi_min_xfer	= hcdi->hcdi_dma_attr->dma_attr_minxfer;
	hcdi->hcdi_min_burst_size =
		(1<<(ddi_ffs(hcdi->hcdi_dma_attr->dma_attr_burstsizes)-1));
	hcdi->hcdi_max_burst_size =
		(1<<(ddi_fls(hcdi->hcdi_dma_attr->dma_attr_burstsizes)-1));

	usba_hcdi_set_hcdi(hcdi->hcdi_dip, hcdi);

	if (ddi_prop_lookup_string(DDI_DEV_T_ANY,
	    hcdi->hcdi_dip,
	    DDI_PROP_DONTPASS, "ugen-default-binding", &datap) ==
	    DDI_PROP_SUCCESS) {
		if (strcmp(datap, "device") == 0) {
			hcdi->hcdi_ugen_default_binding =
			    USBA_UGEN_DEVICE_BINDING;
		} else if (strcmp(datap, "interface") == 0) {
			hcdi->hcdi_ugen_default_binding =
			    USBA_UGEN_INTERFACE_BINDING;
		} else {
			USB_DPRINTF_L1(DPRINT_MASK_HCDI,
			    hcdi->hcdi_log_handle,
			    "illegal value (%s) for "
			    "ugen_default_binding property",
			    datap);
		}
		ddi_prop_free(datap);
	}

	return (USB_SUCCESS);
}


/*
 * Called by an	HCD to detach an instance of the driver
 */
/*ARGSUSED*/
void
usba_hcdi_unregister(dev_info_t *dip)
{
	usba_hcdi_t *hcdi = usba_hcdi_get_hcdi(dip);

	if (hcdi) {
		USB_DPRINTF_L4(DPRINT_MASK_HCDI, hcdi->hcdi_log_handle,
		    "usba_hcdi_unregister: %s", ddi_node_name(dip));

		usba_hcdi_set_hcdi(dip, NULL);

		mutex_destroy(&hcdi->hcdi_mutex);
		usba_hcdi_destroy_stats(hcdi);
		usb_free_log_hdl(hcdi->hcdi_log_handle);

		/* Destroy the soft interrupt */
		(void) ddi_intr_remove_softint(hcdi->hcdi_softint_hdl);
		kmem_free(hcdi, sizeof (usba_hcdi_t));
	}
}


/*
 * alloc usba_hcdi_ops structure
 *	called from the HCD attach routine
 */
usba_hcdi_ops_t *
usba_alloc_hcdi_ops()
{
	usba_hcdi_ops_t	*usba_hcdi_ops;

	usba_hcdi_ops = kmem_zalloc(sizeof (usba_hcdi_ops_t), KM_SLEEP);

	return (usba_hcdi_ops);
}


/*
 * dealloc usba_hcdi_ops structure
 */
void
usba_free_hcdi_ops(usba_hcdi_ops_t *hcdi_ops)
{
	if (hcdi_ops) {
		kmem_free(hcdi_ops, sizeof (usba_hcdi_ops_t));
	}
}


/*
 * Allocate the hotplug kstats structure
 */
void
usba_hcdi_create_stats(usba_hcdi_t *hcdi, int instance)
{
	char			kstatname[KSTAT_STRLEN];
	const char		*dname = ddi_driver_name(hcdi->hcdi_dip);
	hcdi_hotplug_stats_t	*hsp;
	hcdi_error_stats_t	*esp;

	if (HCDI_HOTPLUG_STATS(hcdi) == NULL) {
		(void) snprintf(kstatname, KSTAT_STRLEN, "%s%d,hotplug",
		    dname, instance);
		HCDI_HOTPLUG_STATS(hcdi) = kstat_create("usba", instance,
		    kstatname, "usb_hotplug", KSTAT_TYPE_NAMED,
		    sizeof (hcdi_hotplug_stats_t) / sizeof (kstat_named_t),
		    KSTAT_FLAG_PERSISTENT);

		if (HCDI_HOTPLUG_STATS(hcdi) == NULL) {

			return;
		}

		hsp = HCDI_HOTPLUG_STATS_DATA(hcdi);
		kstat_named_init(&hsp->hcdi_hotplug_total_success,
		    "Total Hotplug Successes", KSTAT_DATA_UINT64);
		kstat_named_init(&hsp->hcdi_hotplug_success,
		    "Hotplug Successes", KSTAT_DATA_UINT64);
		kstat_named_init(&hsp->hcdi_hotplug_total_failure,
		    "Hotplug Total Failures", KSTAT_DATA_UINT64);
		kstat_named_init(&hsp->hcdi_hotplug_failure,
		    "Hotplug Failures", KSTAT_DATA_UINT64);
		kstat_named_init(&hsp->hcdi_device_count,
		    "Device Count", KSTAT_DATA_UINT64);

		HCDI_HOTPLUG_STATS(hcdi)->ks_private = hcdi;
		HCDI_HOTPLUG_STATS(hcdi)->ks_update = nulldev;
		kstat_install(HCDI_HOTPLUG_STATS(hcdi));
	}

	if (HCDI_ERROR_STATS(hcdi) == NULL) {
		(void) snprintf(kstatname, KSTAT_STRLEN, "%s%d,error",
		    dname, instance);
		HCDI_ERROR_STATS(hcdi) = kstat_create("usba", instance,
		    kstatname, "usb_errors", KSTAT_TYPE_NAMED,
		    sizeof (hcdi_error_stats_t) / sizeof (kstat_named_t),
		    KSTAT_FLAG_PERSISTENT);

		if (HCDI_ERROR_STATS(hcdi) == NULL) {

			return;
		}

		esp = HCDI_ERROR_STATS_DATA(hcdi);
		kstat_named_init(&esp->cc_crc, "CRC Errors", KSTAT_DATA_UINT64);
		kstat_named_init(&esp->cc_bitstuffing,
		    "Bit Stuffing Violations", KSTAT_DATA_UINT64);
		kstat_named_init(&esp->cc_data_toggle_mm,
		    "Data Toggle PID Errors", KSTAT_DATA_UINT64);
		kstat_named_init(&esp->cc_stall,
		    "Endpoint Stalls", KSTAT_DATA_UINT64);
		kstat_named_init(&esp->cc_dev_not_resp,
		    "Device Not Responding", KSTAT_DATA_UINT64);
		kstat_named_init(&esp->cc_pid_checkfailure,
		    "PID Check Bit Errors", KSTAT_DATA_UINT64);
		kstat_named_init(&esp->cc_unexp_pid,
		    "Invalid PID Errors", KSTAT_DATA_UINT64);
		kstat_named_init(&esp->cc_data_overrun,
		    "Data Overruns", KSTAT_DATA_UINT64);
		kstat_named_init(&esp->cc_data_underrun,
		    "Data Underruns", KSTAT_DATA_UINT64);
		kstat_named_init(&esp->cc_buffer_overrun,
		    "Buffer Overruns", KSTAT_DATA_UINT64);
		kstat_named_init(&esp->cc_buffer_underrun,
		    "Buffer Underruns", KSTAT_DATA_UINT64);
		kstat_named_init(&esp->cc_timeout,
		    "Command Timed Out", KSTAT_DATA_UINT64);
		kstat_named_init(&esp->cc_not_accessed,
		    "Not Accessed By Hardware", KSTAT_DATA_UINT64);
		kstat_named_init(&esp->cc_unspecified_err,
		    "Unspecified Error", KSTAT_DATA_UINT64);
#ifdef	NOTYETNEEDED
		kstat_named_init(&esp->hcdi_usb_failure,
		    "USB Failure", KSTAT_DATA_UINT64);
		kstat_named_init(&esp->hcdi_usb_no_resources,
		    "No Resources", KSTAT_DATA_UINT64);
		kstat_named_init(&esp->hcdi_usb_no_bandwidth,
		    "No Bandwidth", KSTAT_DATA_UINT64);
		kstat_named_init(&esp->hcdi_usb_pipe_reserved,
		    "Pipe Reserved", KSTAT_DATA_UINT64);
		kstat_named_init(&esp->hcdi_usb_pipe_unshareable,
		    "Pipe Unshareable", KSTAT_DATA_UINT64);
		kstat_named_init(&esp->hcdi_usb_not_supported,
		    "Function Not Supported", KSTAT_DATA_UINT64);
		kstat_named_init(&esp->hcdi_usb_pipe_error,
		    "Pipe Error", KSTAT_DATA_UINT64);
		kstat_named_init(&esp->hcdi_usb_pipe_busy,
		    "Pipe Busy", KSTAT_DATA_UINT64);
#endif

		HCDI_ERROR_STATS(hcdi)->ks_private = hcdi;
		HCDI_ERROR_STATS(hcdi)->ks_update = nulldev;
		kstat_install(HCDI_ERROR_STATS(hcdi));
	}
}


/*
 * Do actual error stats
 */
void
usba_hcdi_update_error_stats(usba_hcdi_t *hcdi, usb_cr_t completion_reason)
{
	if (HCDI_ERROR_STATS(hcdi) == NULL) {

		return;
	}

	switch (completion_reason) {
	case USB_CR_OK:
		break;
	case USB_CR_CRC:
		HCDI_ERROR_STATS_DATA(hcdi)->cc_crc.value.ui64++;
		break;
	case USB_CR_BITSTUFFING:
		HCDI_ERROR_STATS_DATA(hcdi)->cc_bitstuffing.value.ui64++;
		break;
	case USB_CR_DATA_TOGGLE_MM:
		HCDI_ERROR_STATS_DATA(hcdi)->cc_data_toggle_mm.value.ui64++;
		break;
	case USB_CR_STALL:
		HCDI_ERROR_STATS_DATA(hcdi)->cc_stall.value.ui64++;
		break;
	case USB_CR_DEV_NOT_RESP:
		HCDI_ERROR_STATS_DATA(hcdi)->cc_dev_not_resp.value.ui64++;
		break;
	case USB_CR_PID_CHECKFAILURE:
		HCDI_ERROR_STATS_DATA(hcdi)->cc_pid_checkfailure.value.ui64++;
		break;
	case USB_CR_UNEXP_PID:
		HCDI_ERROR_STATS_DATA(hcdi)->cc_unexp_pid.value.ui64++;
		break;
	case USB_CR_DATA_OVERRUN:
		HCDI_ERROR_STATS_DATA(hcdi)->cc_data_overrun.value.ui64++;
		break;
	case USB_CR_DATA_UNDERRUN:
		HCDI_ERROR_STATS_DATA(hcdi)->cc_data_underrun.value.ui64++;
		break;
	case USB_CR_BUFFER_OVERRUN:
		HCDI_ERROR_STATS_DATA(hcdi)->cc_buffer_overrun.value.ui64++;
		break;
	case USB_CR_BUFFER_UNDERRUN:
		HCDI_ERROR_STATS_DATA(hcdi)->cc_buffer_underrun.value.ui64++;
		break;
	case USB_CR_TIMEOUT:
		HCDI_ERROR_STATS_DATA(hcdi)->cc_timeout.value.ui64++;
		break;
	case USB_CR_NOT_ACCESSED:
		HCDI_ERROR_STATS_DATA(hcdi)->cc_not_accessed.value.ui64++;
		break;
	case USB_CR_NO_RESOURCES:
		HCDI_ERROR_STATS_DATA(hcdi)->cc_no_resources.value.ui64++;
		break;
	case USB_CR_UNSPECIFIED_ERR:
		HCDI_ERROR_STATS_DATA(hcdi)->cc_unspecified_err.value.ui64++;
		break;
	case USB_CR_STOPPED_POLLING:
		HCDI_ERROR_STATS_DATA(hcdi)->cc_stopped_polling.value.ui64++;
		break;
	case USB_CR_PIPE_CLOSING:
		HCDI_ERROR_STATS_DATA(hcdi)->cc_pipe_closing.value.ui64++;
		break;
	case USB_CR_PIPE_RESET:
		HCDI_ERROR_STATS_DATA(hcdi)->cc_pipe_reset.value.ui64++;
		break;
	case USB_CR_NOT_SUPPORTED:
		HCDI_ERROR_STATS_DATA(hcdi)->cc_not_supported.value.ui64++;
		break;
	case USB_CR_FLUSHED:
		HCDI_ERROR_STATS_DATA(hcdi)->cc_flushed.value.ui64++;
		break;
	default:
		break;
	}
}


/*
 * Destroy the hotplug kstats structure
 */
static void
usba_hcdi_destroy_stats(usba_hcdi_t *hcdi)
{
	if (HCDI_HOTPLUG_STATS(hcdi)) {
		kstat_delete(HCDI_HOTPLUG_STATS(hcdi));
		HCDI_HOTPLUG_STATS(hcdi) = NULL;
	}

	if (HCDI_ERROR_STATS(hcdi)) {
		kstat_delete(HCDI_ERROR_STATS(hcdi));
		HCDI_ERROR_STATS(hcdi) = NULL;
	}
}


/*
 * HCD callback handling
 */
void
usba_hcdi_cb(usba_pipe_handle_data_t *ph_data,
	usb_opaque_t	req,
	usb_cr_t	completion_reason)
{

	usba_device_t		*usba_device = ph_data->p_usba_device;
	usba_hcdi_t		*hcdi =	usba_hcdi_get_hcdi(
					usba_device->usb_root_hub_dip);
	usba_req_wrapper_t	*req_wrp = USBA_REQ2WRP(req);
	usb_ep_descr_t		*eptd = &ph_data->p_ep;

	mutex_enter(&ph_data->p_mutex);

#ifdef DEBUG
	mutex_enter(&ph_data->p_ph_impl->usba_ph_mutex);

	USB_DPRINTF_L4(DPRINT_MASK_HCDI, hcdi->hcdi_log_handle,
	    "usba_hcdi_cb: "
	    "ph_data=0x%p req=0x%p state=%d ref=%d cnt=%d cr=%d",
	    ph_data, req, ph_data->p_ph_impl->usba_ph_state,
	    ph_data->p_ph_impl->usba_ph_ref_count, ph_data->p_req_count,
	    completion_reason);

	mutex_exit(&ph_data->p_ph_impl->usba_ph_mutex);
#endif

	/* Set the completion reason */
	switch (eptd->bmAttributes & USB_EP_ATTR_MASK) {
	case USB_EP_ATTR_CONTROL:
		((usb_ctrl_req_t *)req)->
		    ctrl_completion_reason = completion_reason;
		break;
	case USB_EP_ATTR_BULK:
		((usb_bulk_req_t *)req)->
		    bulk_completion_reason = completion_reason;
		break;
	case USB_EP_ATTR_INTR:
		((usb_intr_req_t *)req)->
		    intr_completion_reason = completion_reason;
		break;
	case USB_EP_ATTR_ISOCH:
		((usb_isoc_req_t *)req)->
		    isoc_completion_reason = completion_reason;
		break;
	}

	/*
	 * exception callbacks will still go thru a taskq thread
	 * but should occur after the soft interrupt callback
	 * By design of periodic pipes, polling will stop on any
	 * exception
	 */
	if ((ph_data->p_spec_flag & USBA_PH_FLAG_USE_SOFT_INTR) &&
	    (completion_reason == USB_CR_OK)) {
		ph_data->p_soft_intr++;
		mutex_exit(&ph_data->p_mutex);

		usba_add_to_list(&hcdi->hcdi_cb_queue, &req_wrp->wr_queue);

		if (ddi_intr_trigger_softint(hcdi->hcdi_softint_hdl, NULL) !=
		    DDI_SUCCESS)
			USB_DPRINTF_L2(DPRINT_MASK_HCDI, hcdi->hcdi_log_handle,
			    "usba_hcdi_cb: ddi_intr_trigger_softint  failed");

		return;
	}

	/*
	 * USBA_PH_FLAG_TQ_SHARE is for bulk and intr requests,
	 * USBA_PH_FLAG_USE_SOFT_INTR is only for isoch,
	 * so there are no conflicts.
	 */
	if (ph_data->p_spec_flag & USBA_PH_FLAG_TQ_SHARE) {
		int iface;

		mutex_exit(&ph_data->p_mutex);
		iface = usb_get_if_number(ph_data->p_dip);
		if (iface < 0) {
			/* we own the device, use the first taskq */
			iface = 0;
		}
		if (taskq_dispatch(usba_device->usb_shared_taskq[iface],
		    hcdi_shared_cb_thread, req_wrp, TQ_NOSLEEP) ==
		    NULL) {
			usba_req_exc_cb(req_wrp,
			    USB_CR_NO_RESOURCES, USB_CB_ASYNC_REQ_FAILED);
		}

		return;
	}

	/* Add the callback to the pipehandles callback list */
	usba_add_to_list(&ph_data->p_cb_queue, &req_wrp->wr_queue);

	/* only dispatch if there is no thread running */
	if (ph_data->p_thread_id == 0) {
		if (usba_async_ph_req(ph_data, hcdi_cb_thread,
		    ph_data, USB_FLAGS_NOSLEEP) != USB_SUCCESS) {
			USB_DPRINTF_L2(DPRINT_MASK_HCDI, hcdi->hcdi_log_handle,
			    "usba_hcdi_cb: taskq_dispatch failed");
			if (usba_rm_from_list(&ph_data->p_cb_queue,
			    &req_wrp->wr_queue) == USB_SUCCESS) {
				mutex_exit(&ph_data->p_mutex);
				usba_req_exc_cb(req_wrp,
				    USB_CR_NO_RESOURCES,
				    USB_CB_ASYNC_REQ_FAILED);

				return;
			}
		} else {
			ph_data->p_thread_id = (kthread_t *)1;
		}
	}
	mutex_exit(&ph_data->p_mutex);
}


/*
 * thread to perform the callbacks
 */
static void
hcdi_cb_thread(void *arg)
{
	usba_pipe_handle_data_t	*ph_data =
				(usba_pipe_handle_data_t *)arg;
	usba_ph_impl_t		*ph_impl = ph_data->p_ph_impl;
	usba_hcdi_t		*hcdi = usba_hcdi_get_hcdi(ph_data->
				p_usba_device->usb_root_hub_dip);
	usba_req_wrapper_t	*req_wrp;

	mutex_enter(&ph_data->p_mutex);
	ASSERT(ph_data->p_thread_id == (kthread_t *)1);
	ph_data->p_thread_id = curthread;

	/*
	 * hold the ph_data. we can't use usba_hold_ph_data() since
	 * it will return NULL if we are closing the pipe which would
	 * then leave all requests stuck in the cb_queue
	 */
	mutex_enter(&ph_impl->usba_ph_mutex);
	ph_impl->usba_ph_ref_count++;

	USB_DPRINTF_L4(DPRINT_MASK_HCDI, hcdi->hcdi_log_handle,
	    "hcdi_cb_thread: ph_data=0x%p ref=%d", ph_data,
	    ph_impl->usba_ph_ref_count);

	mutex_exit(&ph_impl->usba_ph_mutex);

	/*
	 * wait till soft interrupt callbacks are taken care of
	 */
	while (ph_data->p_soft_intr) {
		mutex_exit(&ph_data->p_mutex);
		delay(1);
		mutex_enter(&ph_data->p_mutex);
	}

	while ((req_wrp = (usba_req_wrapper_t *)
	    usba_rm_first_pvt_from_list(&ph_data->p_cb_queue)) != NULL) {
		hcdi_do_cb(ph_data, req_wrp, hcdi);
	}

	ph_data->p_thread_id = 0;
	mutex_exit(&ph_data->p_mutex);

	USB_DPRINTF_L4(DPRINT_MASK_HCDI, hcdi->hcdi_log_handle,
	    "hcdi_cb_thread done: ph_data=0x%p", ph_data);

	usba_release_ph_data(ph_impl);
}


static void
hcdi_do_cb(usba_pipe_handle_data_t *ph_data, usba_req_wrapper_t *req_wrp,
    usba_hcdi_t *hcdi)
{
	usb_cr_t		completion_reason;
	usb_req_attrs_t		attrs = req_wrp->wr_attrs;

	switch (req_wrp->wr_ph_data->p_ep.bmAttributes &
	    USB_EP_ATTR_MASK) {
	case USB_EP_ATTR_CONTROL:
		completion_reason =
		    USBA_WRP2CTRL_REQ(req_wrp)->ctrl_completion_reason;
		break;
	case USB_EP_ATTR_INTR:
		completion_reason =
		    USBA_WRP2INTR_REQ(req_wrp)->intr_completion_reason;
		break;
	case USB_EP_ATTR_BULK:
		completion_reason =
		    USBA_WRP2BULK_REQ(req_wrp)->bulk_completion_reason;
		break;
	case USB_EP_ATTR_ISOCH:
		completion_reason =
		    USBA_WRP2ISOC_REQ(req_wrp)->isoc_completion_reason;
		break;
	}
	req_wrp->wr_cr = completion_reason;

	USB_DPRINTF_L4(DPRINT_MASK_HCDI, hcdi->hcdi_log_handle,
	    "hcdi_do_cb: wrp=0x%p cr=0x%x", req_wrp, completion_reason);

	/*
	 * Normal callbacks:
	 */
	if (completion_reason == USB_CR_OK) {
		mutex_exit(&ph_data->p_mutex);
		usba_req_normal_cb(req_wrp);
		mutex_enter(&ph_data->p_mutex);
	} else {
		usb_pipe_state_t pipe_state;

		USB_DPRINTF_L4(DPRINT_MASK_HCDI, hcdi->hcdi_log_handle,
		    "exception callback handling: attrs=0x%x", attrs);

		/*
		 * In exception callback handling, if we were
		 * not able to clear stall, we need to modify
		 * pipe state. Also if auto-clearing is not set
		 * pipe state needs to be modified.
		 */
		pipe_state = usba_get_ph_state(ph_data);

		if (!USBA_PIPE_CLOSING(pipe_state)) {
			switch (completion_reason) {
			case USB_CR_STOPPED_POLLING:
				if (pipe_state ==
				    USB_PIPE_STATE_ACTIVE) {
					usba_pipe_new_state(ph_data,
					    USB_PIPE_STATE_IDLE);
				}
				break;
			case USB_CR_NOT_SUPPORTED:
				usba_pipe_new_state(ph_data,
				    USB_PIPE_STATE_IDLE);
				break;
			case USB_CR_PIPE_RESET:
			case USB_CR_FLUSHED:
				break;
			default:
				usba_pipe_new_state(ph_data,
				    USB_PIPE_STATE_ERROR);
				break;
			}
		}

		pipe_state = usba_get_ph_state(ph_data);

		mutex_exit(&ph_data->p_mutex);
		if (attrs & USB_ATTRS_PIPE_RESET) {
			if ((completion_reason != USB_CR_PIPE_RESET) &&
			    (pipe_state == USB_PIPE_STATE_ERROR)) {

				hcdi_autoclearing(req_wrp);
			}
		}

		usba_req_exc_cb(req_wrp, 0, 0);
		mutex_enter(&ph_data->p_mutex);
	}

	/* Update the hcdi error kstats */
	if (completion_reason) {
		mutex_enter(&hcdi->hcdi_mutex);
		usba_hcdi_update_error_stats(hcdi, completion_reason);
		mutex_exit(&hcdi->hcdi_mutex);
	}

	/*
	 * Once the callback is finished, release the pipe handle
	 * we start the next request first to avoid that the
	 * pipe gets closed while starting the next request
	 */
	mutex_exit(&ph_data->p_mutex);
	usba_start_next_req(ph_data);

	mutex_enter(&ph_data->p_mutex);
}


/*
 * thread to perform callbacks on the shared queue
 */
static void
hcdi_shared_cb_thread(void *arg)
{
	usba_req_wrapper_t *req_wrp = (usba_req_wrapper_t *)arg;
	usba_pipe_handle_data_t	*ph_data = req_wrp->wr_ph_data;
	usba_ph_impl_t		*ph_impl = ph_data->p_ph_impl;
	usba_hcdi_t		*hcdi = usba_hcdi_get_hcdi(ph_data->
				p_usba_device->usb_root_hub_dip);
	/*
	 * hold the ph_data. we can't use usba_hold_ph_data() since
	 * it will return NULL if we are closing the pipe which would
	 * then leave all requests stuck in the cb_queue
	 */
	mutex_enter(&ph_impl->usba_ph_mutex);
	ph_impl->usba_ph_ref_count++;

	USB_DPRINTF_L4(DPRINT_MASK_HCDI, hcdi->hcdi_log_handle,
	    "hcdi_shared_cb_thread: ph_data=0x%p ref=%d req=0x%p",
	    ph_data, ph_impl->usba_ph_ref_count, req_wrp);
	mutex_exit(&ph_impl->usba_ph_mutex);

	/* do the callback */
	mutex_enter(&ph_data->p_mutex);
	hcdi_do_cb(ph_data, req_wrp, hcdi);
	mutex_exit(&ph_data->p_mutex);

	USB_DPRINTF_L4(DPRINT_MASK_HCDI, hcdi->hcdi_log_handle,
	    "hcdi_cb_thread done: ph_data=0x%p", ph_data);

	usba_release_ph_data(ph_impl);
}


/*
 * soft interrupt handler
 */
/*ARGSUSED*/
static uint_t
hcdi_soft_intr(caddr_t arg1, caddr_t arg2)
{
	usba_hcdi_t		*hcdi = (usba_hcdi_t *)arg1;
	usba_req_wrapper_t	*req_wrp;
	int			count = 0;

	while ((req_wrp = (usba_req_wrapper_t *)
	    usba_rm_first_pvt_from_list(&hcdi->hcdi_cb_queue)) != NULL) {
		usba_pipe_handle_data_t *ph_data = req_wrp->wr_ph_data;
		usba_ph_impl_t		*ph_impl = ph_data->p_ph_impl;

		/* hold the pipe */
		mutex_enter(&ph_impl->usba_ph_mutex);
		ph_impl->usba_ph_ref_count++;
		mutex_exit(&ph_impl->usba_ph_mutex);

		/* do the callback */
		usba_req_normal_cb(req_wrp);

		/* decrement the soft interrupt count */
		mutex_enter(&ph_data->p_mutex);
		ph_data->p_soft_intr--;
		mutex_exit(&ph_data->p_mutex);

		/* release the pipe */
		mutex_enter(&ph_impl->usba_ph_mutex);
		ph_impl->usba_ph_ref_count--;
		mutex_exit(&ph_impl->usba_ph_mutex);

		count++;
	}

	return (count == 0 ? DDI_INTR_UNCLAIMED : DDI_INTR_CLAIMED);
}


/*
 * hcdi_autoclearing:
 *	This function is called under the taskq context. It
 *	resets the pipe, and clears the stall, if necessary
 */
static void
hcdi_autoclearing(usba_req_wrapper_t *req_wrp)
{
	usb_cr_t		cr = req_wrp->wr_cr;
	usb_pipe_handle_t	pipe_handle, def_pipe_handle;
	usb_cr_t		completion_reason;
	usb_cb_flags_t		cb_flags;
	int			rval;
	usba_device_t		*usba_device =
				    req_wrp->wr_ph_data->p_usba_device;
	usba_hcdi_t		*hcdi = usba_hcdi_get_hcdi(
				    usba_device->usb_root_hub_dip);
	usb_req_attrs_t		attrs = req_wrp->wr_attrs;

	USB_DPRINTF_L4(DPRINT_MASK_HCDI, hcdi->hcdi_log_handle,
	    "hcdi_autoclearing: wrp=0x%p", req_wrp);

	pipe_handle = usba_get_pipe_handle(req_wrp->wr_ph_data);
	def_pipe_handle = usba_get_dflt_pipe_handle(req_wrp->wr_ph_data->p_dip);

	/*
	 * first reset the pipe synchronously
	 */
	if ((attrs & USB_ATTRS_PIPE_RESET) == USB_ATTRS_PIPE_RESET) {
		usba_pipe_clear(pipe_handle);
		usba_req_set_cb_flags(req_wrp, USB_CB_RESET_PIPE);
	}

	ASSERT(def_pipe_handle);

	/* Do not clear if this request was a usb_get_status request */
	if ((pipe_handle == def_pipe_handle) &&
	    (USBA_WRP2CTRL_REQ(req_wrp)->ctrl_bRequest ==
	    USB_REQ_GET_STATUS)) {
		USB_DPRINTF_L2(DPRINT_MASK_USBAI, hcdi->hcdi_log_handle,
		    "hcdi_autoclearing: usb_get_status failed, no clearing");

	/* if default pipe and stall no auto clearing */
	} else if ((pipe_handle == def_pipe_handle) && (cr == USB_CR_STALL)) {
		USB_DPRINTF_L2(DPRINT_MASK_USBAI, hcdi->hcdi_log_handle,
		    "hcdi_autoclearing: default pipe stalled, no clearing");

		usba_req_set_cb_flags(req_wrp, USB_CB_PROTOCOL_STALL);

	/* else do auto clearing */
	} else if (((attrs & USB_ATTRS_AUTOCLEARING) ==
	    USB_ATTRS_AUTOCLEARING) && (cr == USB_CR_STALL)) {
		ushort_t status = 0;

		rval = usb_get_status(req_wrp->wr_dip, def_pipe_handle,
		    USB_DEV_REQ_DEV_TO_HOST | USB_DEV_REQ_RCPT_EP,
		    req_wrp->wr_ph_data->p_ep.bEndpointAddress,
		    &status, USB_FLAGS_SLEEP);
		if (rval != USB_SUCCESS) {
			USB_DPRINTF_L2(DPRINT_MASK_USBAI, hcdi->hcdi_log_handle,
			    "get status (STALL) failed: rval=%d", rval);

			usba_pipe_clear(def_pipe_handle);
		}

		if ((rval != USB_SUCCESS) ||
		    (status & USB_EP_HALT_STATUS)) {
			usba_req_set_cb_flags(req_wrp, USB_CB_FUNCTIONAL_STALL);

			if ((rval = usb_pipe_sync_ctrl_xfer(
			    req_wrp->wr_dip, def_pipe_handle,
			    USB_DEV_REQ_HOST_TO_DEV |
			    USB_DEV_REQ_RCPT_EP,
			    USB_REQ_CLEAR_FEATURE,
			    0,
			    req_wrp->wr_ph_data->p_ep.bEndpointAddress,
			    0,
			    NULL, 0,
			    &completion_reason,
			    &cb_flags, USB_FLAGS_SLEEP)) != USB_SUCCESS) {
				USB_DPRINTF_L2(DPRINT_MASK_USBAI,
				    hcdi->hcdi_log_handle,
				    "auto clearing (STALL) failed: "
				    "rval=%d, cr=0x%x cb=0x%x",
				    rval, completion_reason, cb_flags);

				usba_pipe_clear(def_pipe_handle);
			} else {
				usba_req_set_cb_flags(req_wrp,
						USB_CB_STALL_CLEARED);
			}
		} else {
			usba_req_set_cb_flags(req_wrp, USB_CB_PROTOCOL_STALL);
		}
	}
}


/*
 * usba_hcdi_get_req_private:
 *	This function is used to get the HCD private field
 *	maintained by USBA. HCD calls this function.
 *
 * Arguments:
 *	req		- pointer to usb_*_req_t
 *
 * Return Values:
 *	wr_hcd_private field from wrapper
 */
usb_opaque_t
usba_hcdi_get_req_private(usb_opaque_t req)
{
	usba_req_wrapper_t *wrp = USBA_REQ2WRP(req);

	return (wrp->wr_hcd_private);
}


/*
 * usba_hcdi_set_req_private:
 *	This function is used to set the HCD private field
 *	maintained by USBA. HCD calls this function.
 *
 * Arguments:
 *	req		- pointer to usb_*_req_t
 *	hcd_private	- wr_hcd_private field from wrapper
 */
void
usba_hcdi_set_req_private(usb_opaque_t req,
			usb_opaque_t	hcd_private)
{
	usba_req_wrapper_t *wrp = USBA_REQ2WRP(req);

	wrp->wr_hcd_private = hcd_private;
}


/* get data toggle information for this endpoint */
uchar_t
usba_hcdi_get_data_toggle(usba_device_t *usba_device, uint8_t ep_addr)
{
	uchar_t		toggle;
	usba_ph_impl_t	*ph_impl;
	int		ep_index;

	ep_index = usb_get_ep_index(ep_addr);
	mutex_enter(&usba_device->usb_mutex);
	ph_impl = &usba_device->usb_ph_list[ep_index];
	mutex_enter(&ph_impl->usba_ph_mutex);
	toggle = (uchar_t)(ph_impl->usba_ph_flags & USBA_PH_DATA_TOGGLE);
	mutex_exit(&ph_impl->usba_ph_mutex);
	mutex_exit(&usba_device->usb_mutex);

	return (toggle);
}


/* set data toggle information for this endpoint */
void
usba_hcdi_set_data_toggle(usba_device_t *usba_device, uint8_t ep_addr,
    uchar_t toggle)
{
	usba_ph_impl_t	*ph_impl;
	int		ep_index;

	ep_index = usb_get_ep_index(ep_addr);
	mutex_enter(&usba_device->usb_mutex);
	ph_impl = &usba_device->usb_ph_list[ep_index];
	mutex_enter(&ph_impl->usba_ph_mutex);
	ph_impl->usba_ph_flags &= ~USBA_PH_DATA_TOGGLE;
	ph_impl->usba_ph_flags |= (USBA_PH_DATA_TOGGLE & toggle);
	mutex_exit(&ph_impl->usba_ph_mutex);
	mutex_exit(&usba_device->usb_mutex);
}


/* get pipe_handle_impl ptr for this ep */
usba_pipe_handle_data_t *
usba_hcdi_get_ph_data(usba_device_t *usba_device, uint8_t ep_addr)
{
	return (usba_device->usb_ph_list[usb_get_ep_index(ep_addr)].
							usba_ph_data);
}