xref: /illumos-gate/usr/src/uts/common/io/usb/hcd/xhci/xhci_endpoint.c (revision 60afb9d1b449f489b493961c1c14893a7a74b287)

/*
 * This file and its contents are supplied under the terms of the
 * Common Development and Distribution License ("CDDL"), version 1.0.
 * You may only use this file in accordance with the terms of version
 * 1.0 of the CDDL.
 *
 * A full copy of the text of the CDDL should have accompanied this
 * source.  A copy of the CDDL is also available via the Internet at
 * http://www.illumos.org/license/CDDL.
 */

/*
 * Copyright (c) 2018, Joyent, Inc.
 * Copyright (c) 2019 by Western Digital Corporation
 * Copyright 2024 Oxide Computer Company
 */

/*
 * xHCI Endpoint Initialization and Management
 *
 * Please see the big theory statement in xhci.c for more information.
 */

#include <sys/usb/hcd/xhci/xhci.h>
#include <sys/sdt.h>

boolean_t
xhci_endpoint_is_periodic_in(xhci_endpoint_t *xep)
{
	usba_pipe_handle_data_t *ph;

	ASSERT(xep != NULL);
	ph = xep->xep_pipe;
	ASSERT(ph != NULL);

	return ((xep->xep_type == USB_EP_ATTR_INTR ||
	    xep->xep_type == USB_EP_ATTR_ISOCH) &&
	    (ph->p_ep.bEndpointAddress & USB_EP_DIR_MASK) == USB_EP_DIR_IN);
}

/*
 * Endpoints are a bit weirdly numbered. Endpoint zero is the default control
 * endpoint, so the direction doesn't matter. For all the others, they're
 * arranged as ep 1 out, ep 1 in, ep 2 out, ep 2 in. This is based on the layout
 * of the Device Context Structure in xHCI 1.1 / 6.2.1. Therefore to go from the
 * endpoint and direction, we know that endpoint n starts at 2n - 1.  e.g.
 * endpoint 1 starts at entry 1, endpoint 2 at entry 3, etc. Finally, the OUT
 * direction comes first, followed by the IN direction. So if we're getting the
 * endpoint for one of those, then we have to deal with that.
 */
uint_t
xhci_endpoint_pipe_to_epid(usba_pipe_handle_data_t *ph)
{
	int ep;

	ep = ph->p_ep.bEndpointAddress & USB_EP_NUM_MASK;
	if (ep == 0)
		return (ep);
	ep = ep * 2 - 1;
	if ((ph->p_ep.bEndpointAddress & USB_EP_DIR_MASK) == USB_EP_DIR_IN)
		ep++;

	VERIFY(ep < XHCI_NUM_ENDPOINTS);
	return (ep);
}

void
xhci_endpoint_timeout_cancel(xhci_t *xhcip, xhci_endpoint_t *xep)
{
	xep->xep_state |= XHCI_ENDPOINT_TEARDOWN;
	if (xep->xep_timeout != 0) {
		mutex_exit(&xhcip->xhci_lock);
		(void) untimeout(xep->xep_timeout);
		mutex_enter(&xhcip->xhci_lock);
		xep->xep_timeout = 0;
	}
}

void
xhci_endpoint_release(xhci_t *xhcip, xhci_endpoint_t *xep)
{
	VERIFY(MUTEX_HELD(&xhcip->xhci_lock));
	VERIFY3U(xep->xep_num, !=, XHCI_DEFAULT_ENDPOINT);
	VERIFY(list_is_empty(&xep->xep_transfers));

	VERIFY(xep->xep_pipe != NULL);
	xep->xep_pipe = NULL;

	VERIFY(xep->xep_state & XHCI_ENDPOINT_OPEN);
	xep->xep_state &= ~XHCI_ENDPOINT_OPEN;

	xhci_endpoint_timeout_cancel(xhcip, xep);
}

/*
 * The assumption is that someone calling this owns this endpoint / device and
 * that it's in a state where it's safe to zero out that information.
 */
void
xhci_endpoint_fini(xhci_device_t *xd, int endpoint)
{
	xhci_endpoint_t *xep = xd->xd_endpoints[endpoint];

	VERIFY(xep != NULL);
	xd->xd_endpoints[endpoint] = NULL;

	if (endpoint != XHCI_DEFAULT_ENDPOINT) {
		VERIFY(!(xep->xep_state & XHCI_ENDPOINT_OPEN));
	}

	xhci_ring_free(&xep->xep_ring);
	cv_destroy(&xep->xep_state_cv);
	list_destroy(&xep->xep_transfers);
	kmem_free(xep, sizeof (xhci_endpoint_t));
}

/*
 * Set up the default control endpoint input context. This needs to be done
 * before we address the device. Note, we separate out the default endpoint from
 * others, as we must set this up before we have a pipe handle.
 */
int
xhci_endpoint_setup_default_context(xhci_t *xhcip, xhci_device_t *xd,
    xhci_endpoint_t *xep)
{
	uint_t mps;
	xhci_endpoint_context_t *ectx;
	uint64_t deq;

	ectx = xd->xd_endin[xep->xep_num];
	VERIFY(ectx != NULL);

	/*
	 * We may or may not have a device descriptor. This should match the
	 * same initial sizes that are done in hubd_create_child().
	 *
	 * Note, since we don't necessarily have an endpoint descriptor yet to
	 * base this on we instead use the device's defaults if available. This
	 * is different from normal endpoints for which there's always a
	 * specific descriptor.
	 */
	switch (xd->xd_usbdev->usb_port_status) {
	case USBA_LOW_SPEED_DEV:
		if (xd->xd_usbdev->usb_dev_descr != NULL) {
			mps = xd->xd_usbdev->usb_dev_descr->bMaxPacketSize0;
		} else {
			mps = 8;
		}
		break;
	case USBA_FULL_SPEED_DEV:
	case USBA_HIGH_SPEED_DEV:
		if (xd->xd_usbdev->usb_dev_descr != NULL) {
			mps = xd->xd_usbdev->usb_dev_descr->bMaxPacketSize0;
		} else {
			mps = 64;
		}
		break;
	case USBA_SUPER_SPEED_DEV:
	default:
		if (xd->xd_usbdev->usb_dev_descr != NULL) {
			mps = xd->xd_usbdev->usb_dev_descr->bMaxPacketSize0;
			mps = 1 << mps;
		} else {
			mps = 512;
		}
		break;
	}

	bzero(ectx, sizeof (xhci_endpoint_context_t));
	ectx->xec_info = LE_32(0);
	ectx->xec_info2 = LE_32(XHCI_EPCTX_SET_CERR(3) |
	    XHCI_EPCTX_SET_EPTYPE(XHCI_EPCTX_TYPE_CTRL) |
	    XHCI_EPCTX_SET_MAXB(0) | XHCI_EPCTX_SET_MPS(mps));
	deq = xhci_dma_pa(&xep->xep_ring.xr_dma) + sizeof (xhci_trb_t) *
	    xep->xep_ring.xr_tail;
	ectx->xec_dequeue = LE_64(deq | xep->xep_ring.xr_cycle);
	ectx->xec_txinfo = LE_32(XHCI_EPCTX_MAX_ESIT_PAYLOAD(0) |
	    XHCI_EPCTX_AVG_TRB_LEN(XHCI_CONTEXT_DEF_CTRL_ATL));

	XHCI_DMA_SYNC(xd->xd_ictx, DDI_DMA_SYNC_FORDEV);
	if (xhci_check_dma_handle(xhcip, &xd->xd_ictx) != DDI_FM_OK) {
		xhci_error(xhcip, "failed to initialize default device input "
		    "context on slot %d and port %d for endpoint %u:  "
		    "encountered fatal FM error synchronizing input context "
		    "DMA memory", xd->xd_slot, xd->xd_port, xep->xep_num);
		xhci_fm_runtime_reset(xhcip);
		return (EIO);
	}

	return (0);
}

/*
 * Determine if we need to update the maximum packet size of the default
 * control endpoint. This may happen because we start with the default size
 * before we have a descriptor and then it may change. For example, with
 * full-speed devices that may have either an 8 or 64 byte maximum packet size.
 */
int
xhci_endpoint_update_default(xhci_t *xhcip, xhci_device_t *xd,
    xhci_endpoint_t *xep)
{
	int mps, desc, info, ret;
	ASSERT(xd->xd_usbdev != NULL);

	mps = XHCI_EPCTX_GET_MPS(xd->xd_endout[xep->xep_num]->xec_info2);
	desc = xd->xd_usbdev->usb_dev_descr->bMaxPacketSize0;
	if (xd->xd_usbdev->usb_port_status >= USBA_SUPER_SPEED_DEV) {
		desc = 1 << desc;
	}

	if (mps == desc)
		return (USB_SUCCESS);

	/*
	 * Update only the context for the default control endpoint.
	 */
	mutex_enter(&xd->xd_imtx);
	info = LE_32(xd->xd_endout[xep->xep_num]->xec_info2);
	info &= ~XHCI_EPCTX_SET_MPS(mps);
	info |= XHCI_EPCTX_SET_MPS(desc);
	xd->xd_endin[xep->xep_num]->xec_info2 = LE_32(info);
	xd->xd_input->xic_drop_flags = LE_32(0);
	xd->xd_input->xic_add_flags = LE_32(XHCI_INCTX_MASK_DCI(1));

	ret = xhci_command_evaluate_context(xhcip, xd);
	mutex_exit(&xd->xd_imtx);

	return (ret);
}

static uint_t
xhci_endpoint_epdesc_to_type(usb_ep_descr_t *ep)
{
	int type = ep->bmAttributes & USB_EP_ATTR_MASK;
	boolean_t in = (ep->bEndpointAddress & USB_EP_DIR_MASK) ==
	    USB_EP_DIR_IN;

	switch (type) {
	case USB_EP_ATTR_CONTROL:
		return (XHCI_EPCTX_TYPE_CTRL);
	case USB_EP_ATTR_ISOCH:
		if (in == B_TRUE)
			return (XHCI_EPCTX_TYPE_ISOCH_IN);
		return (XHCI_EPCTX_TYPE_ISOCH_OUT);
	case USB_EP_ATTR_BULK:
		if (in == B_TRUE)
			return (XHCI_EPCTX_TYPE_BULK_IN);
		return (XHCI_EPCTX_TYPE_BULK_OUT);
	case USB_EP_ATTR_INTR:
		if (in == B_TRUE)
			return (XHCI_EPCTX_TYPE_INTR_IN);
		return (XHCI_EPCTX_TYPE_INTR_OUT);
	default:
		panic("bad USB attribute type: %d", type);
	}

	/* LINTED: E_FUNC_NO_RET_VAL */
}

static uint_t
xhci_endpoint_determine_burst(xhci_device_t *xd, xhci_endpoint_t *xep)
{
	switch (xd->xd_usbdev->usb_port_status) {
	case USBA_LOW_SPEED_DEV:
	case USBA_FULL_SPEED_DEV:
		/*
		 * Per xHCI 1.1 / 6.2.3.4, burst is always zero for these
		 * devices.
		 */
		return (0);
	case USBA_HIGH_SPEED_DEV:
		if (xep->xep_type == USB_EP_ATTR_CONTROL ||
		    xep->xep_type == USB_EP_ATTR_BULK)
			return (0);
		return ((xep->xep_pipe->p_xep.uex_ep.wMaxPacketSize &
		    XHCI_CONTEXT_BURST_MASK) >> XHCI_CONTEXT_BURST_SHIFT);
	default:
		/*
		 * For these USB >= 3.0, this comes from the companion
		 * descriptor.
		 */
		ASSERT(xep->xep_pipe->p_xep.uex_flags & USB_EP_XFLAGS_SS_COMP);
		return (xep->xep_pipe->p_xep.uex_ep_ss.bMaxBurst);
	}
}

/*
 * Convert a linear mapping of values that are in in the range of 1-255 into a
 * 2^x value. Because we're supposed to round down for these calculations (see
 * the note in xHCI 1.1 / 6.2.3.6) we can do this simply with a fls() and
 * subtracting one.
 */
static uint_t
xhci_endpoint_linear_interval(usb_ep_descr_t *ep)
{
	int exp;
	int ival = ep->bInterval;
	if (ival < 1)
		ival = 1;
	if (ival > 255)
		ival = 255;
	exp = ddi_fls(ival) - 1;
	ASSERT(exp >= 0 && exp <= 7);
	return (exp);
}

/*
 * Convert the set of values that use a 2^(x-1) value for interval into a 2^x
 * range. Note the valid input range is 1-16, so we clamp values based on this.
 * See xHCI 1.1 / 6.2.3.6 for more information.
 */
static uint_t
xhci_endpoint_exponential_interval(usb_ep_descr_t *ep)
{
	int ival;

	ival = ep->bInterval;
	if (ival < 1)
		ival = 1;
	if (ival > 16)
		ival = 16;
	ival--;
	ASSERT(ival >= 0 && ival <= 15);
	return (ival);
}


/*
 * Determining the interval is unfortunately somewhat complicated as there are
 * many differnet forms that things can take. This is all summarized in a
 * somewhat helpful table, number 65, in xHCI 1.1 / 6.2.3.6. But here's
 * basically the six different cases we have to consider:
 *
 * Case 1: Non-High Speed Bulk and Control Endpoints
 *	Always return 0.
 *
 * Case 2: Super Speed and High Speed Isoch and Intr endpoints
 *	Convert from a 2^(x-1) range to a 2^x range.
 *
 * Case 3: Full Speed Isochronous Endpoints
 *	As case 2, but add 3 as its values are in frames and we need to convert
 *	to microframes. Adding three to the result is the same as multiplying
 *	the initial value by 8.
 *
 * Case 4: Full speed and Low Speed Interrupt Endpoints
 *	These have a 1-255 ms range that we need to convert to a 2^x * 128 us
 *	range. We use the linear conversion and then add 3 to account for the
 *	multiplying by 8 conversion from frames to microframes.
 *
 * Case 5: High Speed Interrupt and Bulk Output
 *	These are a bit of a weird case. The spec and other implementations make
 *	it seem that it's similar to case 4, but without the fixed addition as
 *	its interpreted differently due to NAKs.
 *
 * Case 6: Low Speed Isochronous Endpoints
 *	These are not actually defined; however, like other implementations we
 *	treat them like case 4.
 */
static uint_t
xhci_endpoint_interval(xhci_device_t *xd, usb_ep_descr_t *ep)
{
	int type = ep->bmAttributes & USB_EP_ATTR_MASK;
	int speed = xd->xd_usbdev->usb_port_status;

	/*
	 * Handle Cases 1 and 5 first.
	 */
	if (type == USB_EP_ATTR_CONTROL || type == USB_EP_ATTR_BULK) {
		if (speed != USBA_HIGH_SPEED_DEV)
			return (0);
		return (xhci_endpoint_linear_interval(ep));
	}

	/*
	 * Handle Isoch and Intr cases next.
	 */
	switch (speed) {
	case USBA_LOW_SPEED_DEV:
		/*
		 * Interrupt endpoints at low speed are the same as full speed,
		 * hence the fall through.
		 */
		if (type == USB_EP_ATTR_ISOCH) {
			return (xhci_endpoint_exponential_interval(ep) + 3);
		}
		/* FALLTHROUGH */
	case USBA_FULL_SPEED_DEV:
		return (xhci_endpoint_linear_interval(ep) + 3);
	case USBA_HIGH_SPEED_DEV:
	case USBA_SUPER_SPEED_DEV:
	default:
		/*
		 * Case 2. Treat any newer and faster speeds as Super Speed by
		 * default as USB 3.1 is effectively treated the same here.
		 */
		return (xhci_endpoint_exponential_interval(ep));
	}
}

/*
 * The way to calculate the Maximum ESIT is described in xHCI 1.1 / 4.14.2.
 * First off, this only applies to Interrupt and Isochronous descriptors. For
 * Super Speed and newer things, it comes out of a descriptor. Otherwise we
 * calculate it by doing 'Max Packet Size' * ('Max Burst' + 1).
 */
static uint_t
xhci_endpoint_max_esit(xhci_device_t *xd, xhci_endpoint_t *xep, uint_t mps,
    uint_t burst)
{
	if (xep->xep_type == USB_EP_ATTR_CONTROL ||
	    xep->xep_type == USB_EP_ATTR_BULK) {
		return (0);
	}

	/*
	 * Note that this will need to be updated for SuperSpeedPlus ISOC
	 * devices to pull from the secondary companion descriptor they use.
	 */
	if (xd->xd_usbdev->usb_port_status >= USBA_SUPER_SPEED_DEV) {
		usb_ep_xdescr_t *ep_xdesc = &xep->xep_pipe->p_xep;
		ASSERT(xep->xep_pipe->p_xep.uex_flags & USB_EP_XFLAGS_SS_COMP);
		return (ep_xdesc->uex_ep_ss.wBytesPerInterval);
	}

	return (mps * (burst + 1));
}

/*
 * We've been asked to calculate and tell the xHCI controller an average TRB
 * data length. This is talked about in an implementation note in xHCI 1.1 /
 * 4.14.1.1. So, the reality is that it's hard to actually calculate this, as
 * we're supposed to take into account all of the TRBs that we use on that ring.
 *
 * Surveying other xHCI drivers, they all agree on using the default of 8 for
 * control endpoints; however, from there things get a little more fluid. For
 * interrupt and isochronous endpoints, many device use the minimum of the max
 * packet size and the device's pagesize. For bulk endpoints some folks punt and
 * don't set anything and others try and set it to the pagesize. The xHCI
 * implementation note suggests a 3k size here initially. For now, we'll just
 * guess for bulk endpoints and use our page size as a determining factor for
 * this and use the BSD style for others. Note Linux here only sets this value
 * for control devices.
 */
static uint_t
xhci_endpoint_avg_trb(xhci_t *xhcip, usb_ep_descr_t *ep, int mps)
{
	int type = ep->bmAttributes & USB_EP_ATTR_MASK;

	switch (type) {
	case USB_EP_ATTR_ISOCH:
	case USB_EP_ATTR_INTR:
		return (MIN(xhcip->xhci_caps.xcap_pagesize, mps));
	case USB_EP_ATTR_CONTROL:
		return (XHCI_CONTEXT_DEF_CTRL_ATL);
	case USB_EP_ATTR_BULK:
		return (xhcip->xhci_caps.xcap_pagesize);
	default:
		panic("bad USB endpoint type: %d", type);
	}

	/* LINTED: E_FUNC_NO_RET_VAL */
}

/*
 * Set up the input context for this endpoint.  If this endpoint is already
 * open, just confirm that the current parameters and the originally programmed
 * parameters match.
 */
int
xhci_endpoint_setup_context(xhci_t *xhcip, xhci_device_t *xd,
    xhci_endpoint_t *xep)
{
	xhci_endpoint_params_t new_xepp;
	xhci_endpoint_context_t *ectx;
	uint64_t deq;

	/*
	 * Explicitly zero this entire struct to start so that we can compare
	 * it with bcmp().
	 */
	bzero(&new_xepp, sizeof (new_xepp));
	new_xepp.xepp_configured = B_TRUE;

	/*
	 * For a USB >=3.0 device we should always have its companion descriptor
	 * provided for us by USBA. If it's not here, complain loudly and fail.
	 */
	if (xd->xd_usbdev->usb_port_status >= USBA_SUPER_SPEED_DEV &&
	    (xep->xep_pipe->p_xep.uex_flags & USB_EP_XFLAGS_SS_COMP) == 0) {
		const char *prod, *mfg;

		prod = xd->xd_usbdev->usb_product_str;
		if (prod == NULL)
			prod = "Unknown Device";
		mfg = xd->xd_usbdev->usb_mfg_str;
		if (mfg == NULL)
			mfg = "Unknown Manufacturer";

		xhci_log(xhcip, "Encountered USB >=3.0 device without endpoint "
		    "companion descriptor. Ensure driver %s is properly using "
		    "usb_pipe_xopen() for device %s %s",
		    ddi_driver_name(xd->xd_usbdev->usb_dip), prod, mfg);
		return (EINVAL);
	}

	ectx = xd->xd_endin[xep->xep_num];
	VERIFY(ectx != NULL);
	VERIFY(xd->xd_usbdev->usb_dev_descr != NULL);
	VERIFY(xep->xep_pipe != NULL);

	new_xepp.xepp_mps =
	    xep->xep_pipe->p_ep.wMaxPacketSize & XHCI_CONTEXT_MPS_MASK;
	new_xepp.xepp_mult = XHCI_CONTEXT_DEF_MULT;
	new_xepp.xepp_cerr = XHCI_CONTEXT_DEF_CERR;

	switch (xep->xep_type) {
	case USB_EP_ATTR_ISOCH:
		/*
		 * When we have support for USB 3.1 SuperSpeedPlus devices,
		 * we'll need to make sure that we also check for its secondary
		 * endpoint companion descriptor here.
		 */
		/*
		 * Super Speed devices nominally have these xHCI super speed
		 * companion descriptors. We know that we're not properly
		 * grabbing them right now, so until we do, we should basically
		 * error about it.
		 */
		if (xd->xd_usbdev->usb_port_status >= USBA_SUPER_SPEED_DEV) {
			ASSERT(xep->xep_pipe->p_xep.uex_flags &
			    USB_EP_XFLAGS_SS_COMP);
			new_xepp.xepp_mult =
			    xep->xep_pipe->p_xep.uex_ep_ss.bmAttributes &
			    USB_EP_SS_COMP_ISOC_MULT_MASK;
		}

		new_xepp.xepp_mps &= XHCI_CONTEXT_MPS_MASK;
		new_xepp.xepp_cerr = XHCI_CONTEXT_ISOCH_CERR;
		break;
	default:
		/*
		 * No explicit changes needed for CONTROL, INTR, and BULK
		 * endpoints. They've been handled already and don't have any
		 * differences.
		 */
		break;
	}

	new_xepp.xepp_eptype = xhci_endpoint_epdesc_to_type(
	    &xep->xep_pipe->p_xep.uex_ep);
	new_xepp.xepp_burst = xhci_endpoint_determine_burst(xd, xep);
	new_xepp.xepp_ival = xhci_endpoint_interval(xd,
	    &xep->xep_pipe->p_xep.uex_ep);
	new_xepp.xepp_max_esit = xhci_endpoint_max_esit(xd, xep,
	    new_xepp.xepp_mps, new_xepp.xepp_burst);
	new_xepp.xepp_avgtrb = xhci_endpoint_avg_trb(xhcip,
	    &xep->xep_pipe->p_xep.uex_ep, new_xepp.xepp_mps);

	/*
	 * The multi field may be reserved as zero if the LEC feature flag is
	 * set. See the description of mult in xHCI 1.1 / 6.2.3.
	 */
	if (xhcip->xhci_caps.xcap_flags2 & XCAP2_LEC)
		new_xepp.xepp_mult = 0;

	if (xep->xep_params.xepp_configured) {
		/*
		 * The endpoint context has been configured already.  We are
		 * reopening the pipe, so just confirm that the parameters are
		 * the same.
		 */
		if (bcmp(&xep->xep_params, &new_xepp, sizeof (new_xepp)) == 0) {
			/*
			 * Everything matches up.
			 */
			return (0);
		}

		DTRACE_PROBE3(xhci__context__mismatch,
		    xhci_t *, xhcip,
		    xhci_endpoint_t *, xep,
		    xhci_endpoint_params_t *, &new_xepp);

		xhci_error(xhcip, "device input context on slot %d and "
		    "port %d for endpoint %u was already initialized but "
		    "with incompatible parameters",
		    xd->xd_slot, xd->xd_port, xep->xep_num);
		return (EINVAL);
	}

	bzero(ectx, sizeof (xhci_endpoint_context_t));

	ectx->xec_info = LE_32(XHCI_EPCTX_SET_MULT(new_xepp.xepp_mult) |
	    XHCI_EPCTX_SET_IVAL(new_xepp.xepp_ival));
	if (xhcip->xhci_caps.xcap_flags2 & XCAP2_LEC) {
		ectx->xec_info |=
		    LE_32(XHCI_EPCTX_SET_MAX_ESIT_HI(new_xepp.xepp_max_esit));
	}

	ectx->xec_info2 = LE_32(XHCI_EPCTX_SET_CERR(new_xepp.xepp_cerr) |
	    XHCI_EPCTX_SET_EPTYPE(new_xepp.xepp_eptype) |
	    XHCI_EPCTX_SET_MAXB(new_xepp.xepp_burst) |
	    XHCI_EPCTX_SET_MPS(new_xepp.xepp_mps));

	deq = xhci_dma_pa(&xep->xep_ring.xr_dma) + sizeof (xhci_trb_t) *
	    xep->xep_ring.xr_tail;
	ectx->xec_dequeue = LE_64(deq | xep->xep_ring.xr_cycle);

	ectx->xec_txinfo = LE_32(
	    XHCI_EPCTX_MAX_ESIT_PAYLOAD(new_xepp.xepp_max_esit) |
	    XHCI_EPCTX_AVG_TRB_LEN(new_xepp.xepp_avgtrb));

	XHCI_DMA_SYNC(xd->xd_ictx, DDI_DMA_SYNC_FORDEV);
	if (xhci_check_dma_handle(xhcip, &xd->xd_ictx) != DDI_FM_OK) {
		xhci_error(xhcip, "failed to initialize device input "
		    "context on slot %d and port %d for endpoint %u:  "
		    "encountered fatal FM error synchronizing input context "
		    "DMA memory", xd->xd_slot, xd->xd_port, xep->xep_num);
		xhci_fm_runtime_reset(xhcip);
		return (EIO);
	}

	bcopy(&new_xepp, &xep->xep_params, sizeof (new_xepp));
	VERIFY(xep->xep_params.xepp_configured);
	return (0);
}

/*
 * Initialize the endpoint and its input context for a given device. This is
 * called from two different contexts:
 *
 *   1. Initializing a device
 *   2. Opening a USB pipe
 *
 * In the second case, we need to worry about locking around the device. We
 * don't need to worry about the locking in the first case because the USBA
 * doesn't know about it yet.
 */
int
xhci_endpoint_init(xhci_t *xhcip, xhci_device_t *xd,
    usba_pipe_handle_data_t *ph)
{
	int ret;
	uint_t epid;
	xhci_endpoint_t *xep;

	if (ph == NULL) {
		epid = XHCI_DEFAULT_ENDPOINT;
	} else {
		ASSERT(MUTEX_HELD(&xhcip->xhci_lock));
		epid = xhci_endpoint_pipe_to_epid(ph);
	}
	VERIFY(xd->xd_endpoints[epid] == NULL);

	xep = kmem_zalloc(sizeof (xhci_endpoint_t), KM_SLEEP);
	list_create(&xep->xep_transfers, sizeof (xhci_transfer_t),
	    offsetof(xhci_transfer_t, xt_link));
	cv_init(&xep->xep_state_cv, NULL, CV_DRIVER, NULL);
	xep->xep_xd = xd;
	xep->xep_xhci = xhcip;
	xep->xep_num = epid;
	if (ph == NULL) {
		xep->xep_pipe = NULL;
		xep->xep_type = USB_EP_ATTR_CONTROL;
	} else {
		xep->xep_pipe = ph;
		xep->xep_type = ph->p_ep.bmAttributes & USB_EP_ATTR_MASK;
	}

	if ((ret = xhci_ring_alloc(xhcip, &xep->xep_ring)) != 0) {
		cv_destroy(&xep->xep_state_cv);
		list_destroy(&xep->xep_transfers);
		kmem_free(xep, sizeof (xhci_endpoint_t));
		return (ret);
	}

	if ((ret = xhci_ring_reset(xhcip, &xep->xep_ring)) != 0) {
		xhci_ring_free(&xep->xep_ring);
		cv_destroy(&xep->xep_state_cv);
		list_destroy(&xep->xep_transfers);
		kmem_free(xep, sizeof (xhci_endpoint_t));
		return (ret);
	}

	xd->xd_endpoints[epid] = xep;
	if (ph == NULL) {
		ret = xhci_endpoint_setup_default_context(xhcip, xd, xep);
	} else {
		ret = xhci_endpoint_setup_context(xhcip, xd, xep);
	}
	if (ret != 0) {
		xhci_endpoint_fini(xd, xep->xep_num);
		return (ret);
	}

	xep->xep_state |= XHCI_ENDPOINT_OPEN;
	return (0);
}

int
xhci_endpoint_reinit(xhci_t *xhcip, xhci_device_t *xd, xhci_endpoint_t *xep,
    usba_pipe_handle_data_t *ph)
{
	VERIFY(MUTEX_HELD(&xhcip->xhci_lock));
	VERIFY(ph != NULL);
	VERIFY3U(xhci_endpoint_pipe_to_epid(ph), ==, xep->xep_num);
	VERIFY3U(xep->xep_num, !=, XHCI_DEFAULT_ENDPOINT);

	if (xep->xep_type != (ph->p_ep.bmAttributes & USB_EP_ATTR_MASK)) {
		/*
		 * The endpoint type should not change unless the device has
		 * been torn down and recreated by the framework.
		 */
		return (EINVAL);
	}

	if (xep->xep_state & XHCI_ENDPOINT_OPEN) {
		return (EBUSY);
	}

	VERIFY(xep->xep_state & XHCI_ENDPOINT_TEARDOWN);
	xep->xep_state &= ~XHCI_ENDPOINT_TEARDOWN;

	VERIFY3U(xep->xep_timeout, ==, 0);
	VERIFY(list_is_empty(&xep->xep_transfers));

	VERIFY3P(xep->xep_pipe, ==, NULL);
	xep->xep_pipe = ph;

	/*
	 * Verify that the endpoint context parameters have not changed in a
	 * way that requires us to tell the controller about it.
	 */
	int ret;
	if ((ret = xhci_endpoint_setup_context(xhcip, xd, xep)) != 0) {
		xep->xep_pipe = NULL;
		xhci_endpoint_timeout_cancel(xhcip, xep);
		return (ret);
	}

	xep->xep_state |= XHCI_ENDPOINT_OPEN;
	return (0);
}

/*
 * Wait until any ongoing resets or time outs are completed.
 */
void
xhci_endpoint_serialize(xhci_t *xhcip, xhci_endpoint_t *xep)
{
	VERIFY(MUTEX_HELD(&xhcip->xhci_lock));

	while ((xep->xep_state & XHCI_ENDPOINT_SERIALIZE) != 0) {
		cv_wait(&xep->xep_state_cv, &xhcip->xhci_lock);
	}
}

/*
 * Attempt to quiesce an endpoint. Depending on the state of the endpoint, we
 * may need to simply stop it. Alternatively, we may need to explicitly reset
 * the endpoint. Once done, this endpoint should be stopped and can be
 * manipulated.
 */
int
xhci_endpoint_quiesce(xhci_t *xhcip, xhci_device_t *xd, xhci_endpoint_t *xep)
{
	int ret = USB_SUCCESS;
	xhci_endpoint_context_t *epctx = xd->xd_endout[xep->xep_num];

	ASSERT(MUTEX_HELD(&xhcip->xhci_lock));
	ASSERT(xep->xep_state & XHCI_ENDPOINT_QUIESCE);

	/*
	 * First attempt to stop the endpoint, unless it's halted. We don't
	 * really care what state it is in. Note that because other activity
	 * could be going on, the state may change on us; however, if it's
	 * running, it will always transition to a stopped state and none of the
	 * other valid states will allow transitions without us taking an active
	 * action.
	 */
	if (!(xep->xep_state & XHCI_ENDPOINT_HALTED)) {
		mutex_exit(&xhcip->xhci_lock);
		ret = xhci_command_stop_endpoint(xhcip, xd, xep);
		mutex_enter(&xhcip->xhci_lock);

		if (ret == USB_INVALID_CONTEXT) {
			XHCI_DMA_SYNC(xd->xd_octx, DDI_DMA_SYNC_FORKERNEL);
		}
	}

	/*
	 * Now, if we had the HALTED flag set or we failed to stop it due to a
	 * context error and we're in the HALTED state now, reset the end point.
	 */
	if ((xep->xep_state & XHCI_ENDPOINT_HALTED) ||
	    (ret == USB_INVALID_CONTEXT &&
	    XHCI_EPCTX_STATE(LE_32(epctx->xec_info)) == XHCI_EP_HALTED)) {
		mutex_exit(&xhcip->xhci_lock);
		ret = xhci_command_reset_endpoint(xhcip, xd, xep);
		mutex_enter(&xhcip->xhci_lock);
	}

	/*
	 * Ideally, one of the two commands should have worked; however, we
	 * could have had a context error due to being in the wrong state.
	 * Verify that we're either in the ERROR or STOPPED state and treat both
	 * as success. All callers are assumed to be doing this so they can
	 * change the dequeue pointer.
	 */
	if (ret != USB_SUCCESS && ret != USB_INVALID_CONTEXT) {
		return (ret);
	}

	if (ret == USB_INVALID_CONTEXT) {
		XHCI_DMA_SYNC(xd->xd_octx, DDI_DMA_SYNC_FORKERNEL);

		switch (XHCI_EPCTX_STATE(LE_32(epctx->xec_info))) {
		case XHCI_EP_STOPPED:
		case XHCI_EP_ERROR:
			/*
			 * This is where we wanted to go, so let's just take it.
			 */
			ret = USB_SUCCESS;
			break;
		case XHCI_EP_DISABLED:
		case XHCI_EP_RUNNING:
		case XHCI_EP_HALTED:
		default:
			/*
			 * If we're in any of these, something really weird has
			 * happened and it's not worth trying to recover at this
			 * point.
			 */
			xhci_error(xhcip, "!asked to stop endpoint %u on slot "
			    "%d and port %d: ended up in unexpected state %d",
			    xep->xep_num, xd->xd_slot, xd->xd_port,
			    XHCI_EPCTX_STATE(LE_32(epctx->xec_info)));
			return (ret);
		}
	}

	/*
	 * Now that we're successful, we can clear any possible halted state
	 * tracking that we might have had.
	 */
	if (ret == USB_SUCCESS) {
		xep->xep_state &= ~XHCI_ENDPOINT_HALTED;
	}

	return (ret);
}

int
xhci_endpoint_ring(xhci_t *xhcip, xhci_device_t *xd, xhci_endpoint_t *xep)
{
	/*
	 * The doorbell ID's are offset by one from the endpoint numbers that we
	 * keep.
	 */
	xhci_put32(xhcip, XHCI_R_DOOR, XHCI_DOORBELL(xd->xd_slot),
	    xep->xep_num + 1);
	if (xhci_check_regs_acc(xhcip) != DDI_FM_OK) {
		xhci_error(xhcip, "failed to ring doorbell for slot %d and "
		    "endpoint %u: encountered fatal FM register access error",
		    xd->xd_slot, xep->xep_num);
		xhci_fm_runtime_reset(xhcip);
		return (USB_HC_HARDWARE_ERROR);
	}

	DTRACE_PROBE3(xhci__doorbell__ring, xhci_t *, xhcip, uint32_t,
	    XHCI_DOORBELL(xd->xd_slot), uint32_t, xep->xep_num + 1);

	return (USB_SUCCESS);
}

static void
xhci_endpoint_tick(void *arg)
{
	int ret;
	xhci_transfer_t *xt;
	xhci_endpoint_t *xep = arg;
	xhci_device_t *xd = xep->xep_xd;
	xhci_t *xhcip = xep->xep_xhci;

	mutex_enter(&xhcip->xhci_lock);

	/*
	 * If we have the teardown flag set, then this is going away, don't try
	 * to do anything. Also, if somehow a periodic endpoint has something
	 * scheduled, just quit now and don't bother.
	 */
	if (xep->xep_state & (XHCI_ENDPOINT_TEARDOWN |
	    XHCI_ENDPOINT_PERIODIC)) {
		xep->xep_timeout = 0;
		mutex_exit(&xhcip->xhci_lock);
		return;
	}

	/*
	 * If something else has already kicked off, something potentially
	 * dangerous, just don't bother waiting for it and reschedule.
	 */
	if (xep->xep_state & XHCI_ENDPOINT_DONT_SCHEDULE) {
		xep->xep_timeout = timeout(xhci_endpoint_tick, xep,
		    drv_usectohz(XHCI_TICK_TIMEOUT_US));
		mutex_exit(&xhcip->xhci_lock);
		return;
	}

	/*
	 * At this point, we have an endpoint that we need to consider. See if
	 * there are any transfers on it, if none, we're done. If so, check if
	 * we have exceeded the timeout. If we have, then we have some work to
	 * do.
	 */
	xt = list_head(&xep->xep_transfers);
	if (xt == NULL) {
		xep->xep_timeout = 0;
		mutex_exit(&xhcip->xhci_lock);
		return;
	}

	if (xt->xt_timeout > 0) {
		xt->xt_timeout--;
		xep->xep_timeout = timeout(xhci_endpoint_tick, xep,
		    drv_usectohz(XHCI_TICK_TIMEOUT_US));
		mutex_exit(&xhcip->xhci_lock);
		return;
	}

	/*
	 * This item has timed out. We need to stop the ring and take action.
	 */
	xep->xep_state |= XHCI_ENDPOINT_TIMED_OUT | XHCI_ENDPOINT_QUIESCE;
	ret = xhci_endpoint_quiesce(xhcip, xd, xep);
	if (ret != USB_SUCCESS) {
		/*
		 * If we fail to quiesce during the timeout, then remove the
		 * state flags and hopefully we'll be able to the next time
		 * around or if a reset or polling stop comes in, maybe it can
		 * deal with it.
		 */
		xep->xep_state &= ~(XHCI_ENDPOINT_QUIESCE |
		    XHCI_ENDPOINT_TIMED_OUT);
		xep->xep_timeout = timeout(xhci_endpoint_tick, xep,
		    drv_usectohz(XHCI_TICK_TIMEOUT_US));
		mutex_exit(&xhcip->xhci_lock);
		cv_broadcast(&xep->xep_state_cv);
		xhci_error(xhcip, "failed to successfully quiesce timed out "
		    "endpoint %u of device on slot %d and port %d: device "
		    "remains timed out", xep->xep_num, xd->xd_slot,
		    xd->xd_port);
		return;
	}

	xhci_ring_skip_transfer(&xep->xep_ring, xt);
	(void) list_remove_head(&xep->xep_transfers);
	mutex_exit(&xhcip->xhci_lock);

	/*
	 * At this point, we try and set the ring's dequeue pointer. If this
	 * fails, we're left in an awkward state. We've already adjusted the
	 * ring and removed the transfer. All we can really do is go through and
	 * return the transfer and hope that they perhaps attempt to reset the
	 * ring and that will succeed at this point. Based on everything we've
	 * done to set things up, it'd be odd if this did fail.
	 */
	ret = xhci_command_set_tr_dequeue(xhcip, xd, xep);
	mutex_enter(&xhcip->xhci_lock);
	xep->xep_state &= ~XHCI_ENDPOINT_QUIESCE;
	if (ret == USB_SUCCESS) {
		xep->xep_state &= ~XHCI_ENDPOINT_TIMED_OUT;
	} else {
		xhci_error(xhcip, "failed to successfully set transfer ring "
		    "dequeue pointer of timed out endpoint %u of "
		    "device on slot %d and port %d: device remains timed out, "
		    "please use cfgadm to recover", xep->xep_num, xd->xd_slot,
		    xd->xd_port);
	}
	xep->xep_timeout = timeout(xhci_endpoint_tick, xep,
	    drv_usectohz(XHCI_TICK_TIMEOUT_US));
	mutex_exit(&xhcip->xhci_lock);
	cv_broadcast(&xep->xep_state_cv);

	/*
	 * Because we never time out periodic related activity, we will always
	 * have the request on the transfer.
	 */
	ASSERT(xt->xt_usba_req != NULL);
	usba_hcdi_cb(xep->xep_pipe, xt->xt_usba_req, USB_CR_TIMEOUT);
	xhci_transfer_free(xhcip, xt);
}

/*
 * We've been asked to schedule a series of frames onto the specified endpoint.
 * We need to make sure that there is enough room, at which point we can queue
 * it and then ring the door bell. Note that we queue in reverse order to make
 * sure that if the ring moves on, it won't see the correct cycle bit.
 */
int
xhci_endpoint_schedule(xhci_t *xhcip, xhci_device_t *xd, xhci_endpoint_t *xep,
    xhci_transfer_t *xt, boolean_t ring)
{
	int i;
	xhci_ring_t *rp = &xep->xep_ring;

	ASSERT(MUTEX_HELD(&xhcip->xhci_lock));
	ASSERT(xt->xt_ntrbs > 0);
	ASSERT(xt->xt_trbs != NULL);

	if ((xep->xep_state & XHCI_ENDPOINT_DONT_SCHEDULE) != 0)
		return (USB_FAILURE);

	if (xhci_ring_trb_space(rp, xt->xt_ntrbs) == B_FALSE)
		return (USB_NO_RESOURCES);

	for (i = xt->xt_ntrbs - 1; i > 0; i--) {
		xhci_ring_trb_fill(rp, i, &xt->xt_trbs[i], &xt->xt_trbs_pa[i],
		    B_TRUE);
	}
	xhci_ring_trb_fill(rp, 0U, &xt->xt_trbs[0], &xt->xt_trbs_pa[0],
	    B_FALSE);

	XHCI_DMA_SYNC(rp->xr_dma, DDI_DMA_SYNC_FORDEV);
	xhci_ring_trb_produce(rp, xt->xt_ntrbs);
	list_insert_tail(&xep->xep_transfers, xt);

	XHCI_DMA_SYNC(rp->xr_dma, DDI_DMA_SYNC_FORDEV);
	if (xhci_check_dma_handle(xhcip, &rp->xr_dma) != DDI_FM_OK) {
		xhci_error(xhcip, "failed to write out TRB for device on slot "
		    "%d, port %d, and endpoint %u: encountered fatal FM error "
		    "synchronizing ring DMA memory", xd->xd_slot, xd->xd_port,
		    xep->xep_num);
		xhci_fm_runtime_reset(xhcip);
		return (USB_HC_HARDWARE_ERROR);
	}

	if (xep->xep_timeout == 0 &&
	    !(xep->xep_state & XHCI_ENDPOINT_PERIODIC)) {
		xep->xep_timeout = timeout(xhci_endpoint_tick, xep,
		    drv_usectohz(XHCI_TICK_TIMEOUT_US));
	}

	xt->xt_sched_time = gethrtime();

	if (ring == B_FALSE)
		return (USB_SUCCESS);

	return (xhci_endpoint_ring(xhcip, xd, xep));
}

xhci_transfer_t *
xhci_endpoint_determine_transfer(xhci_t *xhcip, xhci_endpoint_t *xep,
    xhci_trb_t *trb, uint_t *offp)
{
	uint_t i;
	uint64_t addr;
	xhci_transfer_t *xt;

	ASSERT(xhcip != NULL);
	ASSERT(offp != NULL);
	ASSERT(xep != NULL);
	ASSERT(trb != NULL);
	ASSERT(MUTEX_HELD(&xhcip->xhci_lock));

	if ((xt = list_head(&xep->xep_transfers)) == NULL)
		return (NULL);

	addr = LE_64(trb->trb_addr);

	/*
	 * Check if this is the simple case of an event data. If it is, then all
	 * we need to do is look and see its data matches the address of the
	 * transfer.
	 */
	if (XHCI_TRB_GET_ED(LE_32(trb->trb_flags)) != 0) {
		if (LE_64(trb->trb_addr) != (uintptr_t)xt)
			return (NULL);

		*offp = xt->xt_ntrbs - 1;
		return (xt);
	}

	/*
	 * This represents an error that has occurred. We need to check two
	 * different things. The first is that the TRB PA maps to one of the
	 * TRBs in the transfer. Secondly, we need to make sure that it makes
	 * sense in the context of the ring and our notion of where the tail is.
	 */
	for (i = 0; i < xt->xt_ntrbs; i++) {
		if (xt->xt_trbs_pa[i] == addr)
			break;
	}

	if (i == xt->xt_ntrbs)
		return (NULL);

	if (xhci_ring_trb_valid_range(&xep->xep_ring, LE_64(trb->trb_addr),
	    xt->xt_ntrbs) == -1)
		return (NULL);

	*offp = i;
	return (xt);
}

static void
xhci_endpoint_reschedule_periodic(xhci_t *xhcip, xhci_device_t *xd,
    xhci_endpoint_t *xep, xhci_transfer_t *xt)
{
	int ret;
	xhci_pipe_t *xp = (xhci_pipe_t *)xep->xep_pipe->p_hcd_private;
	xhci_periodic_pipe_t *xpp = &xp->xp_periodic;

	ASSERT3U(xpp->xpp_tsize, >, 0);

	xt->xt_short = 0;
	xt->xt_cr = USB_CR_OK;

	mutex_enter(&xhcip->xhci_lock);

	/*
	 * If we don't have an active poll, then we shouldn't bother trying to
	 * reschedule it. This means that we're trying to stop or we ran out of
	 * memory.
	 */
	if (xpp->xpp_poll_state != XHCI_PERIODIC_POLL_ACTIVE) {
		mutex_exit(&xhcip->xhci_lock);
		return;
	}

	if (xep->xep_type == USB_EP_ATTR_ISOCH) {
		int i;
		for (i = 0; i < xt->xt_ntrbs; i++) {
			xt->xt_isoc[i].isoc_pkt_actual_length =
			    xt->xt_isoc[i].isoc_pkt_length;
			xt->xt_isoc[i].isoc_pkt_status = USB_CR_OK;
		}
	}

	/*
	 * In general, there should always be space on the ring for this. The
	 * only reason that rescheduling an existing transfer for a periodic
	 * endpoint wouldn't work is because we have a hardware error, at which
	 * point we're going to be going down hard anyways. We log and bump a
	 * stat here to make this case discoverable in case our assumptions our
	 * wrong.
	 */
	ret = xhci_endpoint_schedule(xhcip, xd, xep, xt, B_TRUE);
	if (ret != 0) {
		xhci_log(xhcip, "!failed to reschedule periodic endpoint %u "
		    "(type %u) on slot %d: %d\n", xep->xep_num, xep->xep_type,
		    xd->xd_slot, ret);
	}
	mutex_exit(&xhcip->xhci_lock);
}

/*
 * We're dealing with a message on a control endpoint. This may be a default
 * endpoint or otherwise. These usually come in groups of 3+ TRBs where you have
 * a setup stage, data stage (which may have one or more other TRBs) and then a
 * final status stage.
 *
 * We generally set ourselves up such that we get interrupted and notified only
 * on the status stage and for short transfers in the data stage. If we
 * encounter a short transfer in the data stage, then we need to go through and
 * check whether or not the short transfer is allowed. If it is, then there's
 * nothing to do. We'll update everything and call back the framework once we
 * get the status stage.
 */
static boolean_t
xhci_endpoint_control_callback(xhci_t *xhcip, xhci_device_t *xd,
    xhci_endpoint_t *xep, xhci_transfer_t *xt, uint_t off, xhci_trb_t *trb)
{
	int code;
	usb_ctrl_req_t *ucrp;
	xhci_transfer_t *rem;

	ASSERT(MUTEX_HELD(&xhcip->xhci_lock));

	code = XHCI_TRB_GET_CODE(LE_32(trb->trb_status));
	ucrp = (usb_ctrl_req_t *)xt->xt_usba_req;

	/*
	 * Now that we know what this TRB is for, was it for a data/normal stage
	 * or is it the status stage. We cheat by looking at the last entry. If
	 * it's a data stage, then we must have gotten a short write. We record
	 * this fact and whether we should consider the transfer fatal for the
	 * subsequent status stage.
	 */
	if (off != xt->xt_ntrbs - 1) {
		uint_t remain;
		usb_ctrl_req_t *ucrp = (usb_ctrl_req_t *)xt->xt_usba_req;

		/*
		 * This is a data stage TRB. The only reason we should have
		 * gotten something for this is beacuse it was short. Make sure
		 * it's okay before we continue.
		 */
		VERIFY3S(code, ==, XHCI_CODE_SHORT_XFER);
		if (!(ucrp->ctrl_attributes & USB_ATTRS_SHORT_XFER_OK)) {
			xt->xt_cr = USB_CR_DATA_UNDERRUN;
			mutex_exit(&xhcip->xhci_lock);
			return (B_TRUE);
		}

		/*
		 * The value in the resulting trb is how much data remained to
		 * be transferred. Normalize that against the original buffer
		 * size.
		 */
		remain = XHCI_TRB_REMAIN(LE_32(trb->trb_status));
		xt->xt_short = xt->xt_buffer.xdb_len - remain;
		mutex_exit(&xhcip->xhci_lock);
		return (B_TRUE);
	}

	/*
	 * Okay, this is a status stage trb that's in good health. We should
	 * finally go ahead, sync data and try and finally do the callback. If
	 * we have short data, then xt->xt_short will be non-zero.
	 */
	if (xt->xt_data_tohost == B_TRUE) {
		size_t len;
		if (xt->xt_short != 0) {
			len = xt->xt_short;
		} else {
			len = xt->xt_buffer.xdb_len;
		}

		if (xhci_transfer_sync(xhcip, xt, DDI_DMA_SYNC_FORCPU) !=
		    DDI_FM_OK) {
			xhci_error(xhcip, "failed to process control transfer "
			    "callback for endpoint %u of device on slot %d and "
			    "port %d: encountered fatal FM error synchronizing "
			    "DMA memory, resetting device", xep->xep_num,
			    xd->xd_slot, xd->xd_port);
			xhci_fm_runtime_reset(xhcip);
			mutex_exit(&xhcip->xhci_lock);
			return (B_FALSE);
		}

		xhci_transfer_copy(xt, ucrp->ctrl_data->b_rptr, len, B_TRUE);
		ucrp->ctrl_data->b_wptr += len;
	}

	/*
	 * Now we're done. We can go ahead and bump the ring. Free the transfer
	 * outside of the lock and call back into the framework.
	 */
	VERIFY(xhci_ring_trb_consumed(&xep->xep_ring, LE_64(trb->trb_addr)));
	rem = list_remove_head(&xep->xep_transfers);
	VERIFY3P(rem, ==, xt);
	mutex_exit(&xhcip->xhci_lock);

	usba_hcdi_cb(xep->xep_pipe, (usb_opaque_t)ucrp, xt->xt_cr);
	xhci_transfer_free(xhcip, xt);

	return (B_TRUE);
}

/*
 * Cons up a new usb request for the periodic data transfer if we can. If there
 * isn't one available, change the return code to NO_RESOURCES and stop polling
 * on this endpoint, thus using and consuming the original request.
 */
static usb_opaque_t
xhci_endpoint_dup_periodic(xhci_endpoint_t *xep, xhci_transfer_t *xt,
    usb_cr_t *cr)
{
	usb_opaque_t urp;

	xhci_pipe_t *xp = (xhci_pipe_t *)xep->xep_pipe->p_hcd_private;
	xhci_periodic_pipe_t *xpp = &xp->xp_periodic;

	if (XHCI_IS_ONESHOT_XFER(xt)) {
		/*
		 * Oneshot Interrupt IN transfers already have a USB request
		 * which we can just return:
		 */
		return (xt->xt_usba_req);
	}

	if (xep->xep_type == USB_EP_ATTR_INTR) {
		urp = (usb_opaque_t)usba_hcdi_dup_intr_req(xep->xep_pipe->p_dip,
		    (usb_intr_req_t *)xpp->xpp_usb_req, xpp->xpp_tsize, 0);
	} else {
		urp = (usb_opaque_t)usba_hcdi_dup_isoc_req(xep->xep_pipe->p_dip,
		    (usb_isoc_req_t *)xpp->xpp_usb_req, 0);
	}
	if (urp == NULL) {
		xpp->xpp_poll_state = XHCI_PERIODIC_POLL_NOMEM;
		urp = xpp->xpp_usb_req;
		xpp->xpp_usb_req = NULL;
		*cr = USB_CR_NO_RESOURCES;
	} else {
		mutex_enter(&xep->xep_pipe->p_mutex);
		xep->xep_pipe->p_req_count++;
		mutex_exit(&xep->xep_pipe->p_mutex);
	}

	return (urp);
}

xhci_device_t *
xhci_device_lookup_by_slot(xhci_t *xhcip, int slot)
{
	xhci_device_t *xd;

	ASSERT(MUTEX_HELD(&xhcip->xhci_lock));

	for (xd = list_head(&xhcip->xhci_usba.xa_devices); xd != NULL;
	    xd = list_next(&xhcip->xhci_usba.xa_devices, xd)) {
		if (xd->xd_slot == slot)
			return (xd);
	}

	return (NULL);
}

/*
 * Handle things which consist solely of normal tranfers, in other words, bulk
 * and interrupt transfers.
 */
static boolean_t
xhci_endpoint_norm_callback(xhci_t *xhcip, xhci_device_t *xd,
    xhci_endpoint_t *xep, xhci_transfer_t *xt, uint_t off, xhci_trb_t *trb)
{
	int code;
	usb_cr_t cr;
	xhci_transfer_t *rem;
	int attrs;
	mblk_t *mp;
	boolean_t periodic = B_FALSE;
	usb_opaque_t urp;

	ASSERT(MUTEX_HELD(&xhcip->xhci_lock));
	ASSERT(xep->xep_type == USB_EP_ATTR_BULK ||
	    xep->xep_type == USB_EP_ATTR_INTR);

	code = XHCI_TRB_GET_CODE(LE_32(trb->trb_status));

	if (code == XHCI_CODE_SHORT_XFER) {
		uint_t residue;
		residue = XHCI_TRB_REMAIN(LE_32(trb->trb_status));

		if (xep->xep_type == USB_EP_ATTR_BULK) {
			VERIFY3U(XHCI_TRB_GET_ED(LE_32(trb->trb_flags)), !=, 0);
			xt->xt_short = residue;
		} else {
			xt->xt_short = xt->xt_buffer.xdb_len - residue;
		}
	}

	/*
	 * If we have an interrupt from something that's not the last entry,
	 * that must mean we had a short transfer, so there's nothing more for
	 * us to do at the moment. We won't call back until everything's
	 * finished for the general transfer.
	 */
	if (off < xt->xt_ntrbs - 1) {
		mutex_exit(&xhcip->xhci_lock);
		return (B_TRUE);
	}

	urp = xt->xt_usba_req;
	if (xep->xep_type == USB_EP_ATTR_BULK) {
		usb_bulk_req_t *ubrp = (usb_bulk_req_t *)xt->xt_usba_req;
		attrs = ubrp->bulk_attributes;
		mp = ubrp->bulk_data;
	} else {
		usb_intr_req_t *uirp = (usb_intr_req_t *)xt->xt_usba_req;

		if (uirp == NULL) {
			periodic = B_TRUE;
			urp = xhci_endpoint_dup_periodic(xep, xt, &cr);
			uirp = (usb_intr_req_t *)urp;

			/*
			 * If we weren't able to duplicate the interrupt, then
			 * we can't put any data in it.
			 */
			if (cr == USB_CR_NO_RESOURCES)
				goto out;
		}

		attrs = uirp->intr_attributes;
		mp = uirp->intr_data;
	}

	if (xt->xt_data_tohost == B_TRUE) {
		size_t len;
		if (xt->xt_short != 0) {
			if (!(attrs & USB_ATTRS_SHORT_XFER_OK)) {
				cr = USB_CR_DATA_UNDERRUN;
				goto out;
			}
			len = xt->xt_short;
		} else {
			len = xt->xt_buffer.xdb_len;
		}

		if (xhci_transfer_sync(xhcip, xt, DDI_DMA_SYNC_FORCPU) !=
		    DDI_FM_OK) {
			xhci_error(xhcip, "failed to process normal transfer "
			    "callback for endpoint %u of device on slot %d and "
			    "port %d: encountered fatal FM error synchronizing "
			    "DMA memory, resetting device", xep->xep_num,
			    xd->xd_slot, xd->xd_port);
			xhci_fm_runtime_reset(xhcip);
			mutex_exit(&xhcip->xhci_lock);
			return (B_FALSE);
		}

		xhci_transfer_copy(xt, mp->b_rptr, len, B_TRUE);
		mp->b_wptr += len;
	}
	cr = USB_CR_OK;

out:
	/*
	 * Don't use the address from the TRB here. When we're dealing with
	 * event data that will be entirely wrong.
	 */
	VERIFY(xhci_ring_trb_consumed(&xep->xep_ring, xt->xt_trbs_pa[off]));
	rem = list_remove_head(&xep->xep_transfers);
	VERIFY3P(rem, ==, xt);
	mutex_exit(&xhcip->xhci_lock);

	usba_hcdi_cb(xep->xep_pipe, urp, cr);
	if (periodic == B_TRUE) {
		xhci_endpoint_reschedule_periodic(xhcip, xd, xep, xt);
	} else {
		xhci_transfer_free(xhcip, xt);
	}

	return (B_TRUE);
}

static boolean_t
xhci_endpoint_isoch_callback(xhci_t *xhcip, xhci_device_t *xd,
    xhci_endpoint_t *xep, xhci_transfer_t *xt, uint_t off, xhci_trb_t *trb)
{
	int code;
	usb_cr_t cr;
	xhci_transfer_t *rem;
	usb_isoc_pkt_descr_t *desc;
	usb_isoc_req_t *usrp;

	ASSERT(MUTEX_HELD(&xhcip->xhci_lock));
	ASSERT3S(xep->xep_type, ==, USB_EP_ATTR_ISOCH);

	code = XHCI_TRB_GET_CODE(LE_32(trb->trb_status));

	/*
	 * The descriptors that we copy the data from are set up to assume that
	 * everything was OK and we transferred all the requested data.
	 */
	desc = &xt->xt_isoc[off];
	if (code == XHCI_CODE_SHORT_XFER) {
		int residue = XHCI_TRB_REMAIN(LE_32(trb->trb_status));
		desc->isoc_pkt_actual_length -= residue;
	}

	/*
	 * We don't perform the callback until the very last TRB is returned
	 * here. If we have a TRB report on something else, that means that we
	 * had a short transfer.
	 */
	if (off < xt->xt_ntrbs - 1) {
		mutex_exit(&xhcip->xhci_lock);
		return (B_TRUE);
	}

	VERIFY(xhci_ring_trb_consumed(&xep->xep_ring, LE_64(trb->trb_addr)));
	rem = list_remove_head(&xep->xep_transfers);
	VERIFY3P(rem, ==, xt);
	mutex_exit(&xhcip->xhci_lock);

	cr = USB_CR_OK;

	if (xt->xt_data_tohost == B_TRUE) {
		usb_opaque_t urp;
		urp = xhci_endpoint_dup_periodic(xep, xt, &cr);
		usrp = (usb_isoc_req_t *)urp;

		if (cr == USB_CR_OK) {
			mblk_t *mp;
			size_t len;
			if (xhci_transfer_sync(xhcip, xt,
			    DDI_DMA_SYNC_FORCPU) != DDI_FM_OK) {
				xhci_error(xhcip, "failed to process "
				    "isochronous transfer callback for "
				    "endpoint %u of device on slot %d and port "
				    "%d: encountered fatal FM error "
				    "synchronizing DMA memory, resetting "
				    "device",
				    xep->xep_num, xd->xd_slot, xd->xd_port);
				xhci_fm_runtime_reset(xhcip);
				mutex_exit(&xhcip->xhci_lock);
				return (B_FALSE);
			}

			mp = usrp->isoc_data;
			len = xt->xt_buffer.xdb_len;
			xhci_transfer_copy(xt, mp->b_rptr, len, B_TRUE);
			mp->b_wptr += len;
		}
	} else {
		usrp = (usb_isoc_req_t *)xt->xt_usba_req;
	}

	if (cr == USB_CR_OK) {
		bcopy(xt->xt_isoc, usrp->isoc_pkt_descr,
		    sizeof (usb_isoc_pkt_descr_t) * usrp->isoc_pkts_count);
	}

	usba_hcdi_cb(xep->xep_pipe, (usb_opaque_t)usrp, cr);
	if (xt->xt_data_tohost == B_TRUE) {
		xhci_endpoint_reschedule_periodic(xhcip, xd, xep, xt);
	} else {
		xhci_transfer_free(xhcip, xt);
	}

	return (B_TRUE);
}

boolean_t
xhci_endpoint_transfer_callback(xhci_t *xhcip, xhci_trb_t *trb)
{
	boolean_t ret;
	int slot, endpoint, code;
	uint_t off;
	xhci_device_t *xd;
	xhci_endpoint_t *xep;
	xhci_transfer_t *xt;
	boolean_t transfer_done;

	endpoint = XHCI_TRB_GET_EP(LE_32(trb->trb_flags));
	slot = XHCI_TRB_GET_SLOT(LE_32(trb->trb_flags));
	code = XHCI_TRB_GET_CODE(LE_32(trb->trb_status));

	switch (code) {
	case XHCI_CODE_RING_UNDERRUN:
	case XHCI_CODE_RING_OVERRUN:
		/*
		 * If we have an ISOC overrun or underrun then there will be no
		 * valid data pointer in the TRB associated with it. Just drive
		 * on.
		 */
		return (B_TRUE);
	case XHCI_CODE_UNDEFINED:
		xhci_error(xhcip, "received transfer trb with undefined fatal "
		    "error: resetting device");
		xhci_fm_runtime_reset(xhcip);
		return (B_FALSE);
	case XHCI_CODE_XFER_STOPPED:
	case XHCI_CODE_XFER_STOPINV:
	case XHCI_CODE_XFER_STOPSHORT:
		/*
		 * This causes us to transition the endpoint to a stopped state.
		 * Each of these indicate a different possible state that we
		 * have to deal with. Effectively we're going to drop it and
		 * leave it up to the consumers to figure out what to do. For
		 * the moment, that's generally okay because stops are only used
		 * in cases where we're cleaning up outstanding reqs, etc.
		 *
		 * We do this before we check for the corresponding transfer as
		 * this will generally be generated by a command issued that's
		 * stopping the ring.
		 */
		return (B_TRUE);
	default:
		break;
	}

	mutex_enter(&xhcip->xhci_lock);
	xd = xhci_device_lookup_by_slot(xhcip, slot);
	if (xd == NULL) {
		xhci_error(xhcip, "received transfer trb with code %d for "
		    "unknown slot %d and endpoint %d: resetting device", code,
		    slot, endpoint);
		mutex_exit(&xhcip->xhci_lock);
		xhci_fm_runtime_reset(xhcip);
		return (B_FALSE);
	}

	/*
	 * Endpoint IDs are indexed based on their Device Context Index, which
	 * means that we need to subtract one to get the actual ID that we use.
	 */
	xep = xd->xd_endpoints[endpoint - 1];
	if (xep == NULL) {
		xhci_error(xhcip, "received transfer trb with code %d, slot "
		    "%d, and unknown endpoint %d: resetting device", code,
		    slot, endpoint);
		mutex_exit(&xhcip->xhci_lock);
		xhci_fm_runtime_reset(xhcip);
		return (B_FALSE);
	}

	/*
	 * The TRB that we recieved may be an event data TRB for a bulk
	 * endpoint, a normal or short completion for any other endpoint or an
	 * error. In all cases, we need to figure out what transfer this
	 * corresponds to. If this is an error, then we need to make sure that
	 * the generating ring has been cleaned up.
	 *
	 * TRBs should be delivered in order, based on the ring. If for some
	 * reason we find something that doesn't add up here, then we need to
	 * assume that something has gone horribly wrong in the system and issue
	 * a runtime reset. We issue the runtime reset rather than just trying
	 * to stop and flush the ring, because it's unclear if we could stop
	 * the ring in time.
	 */
	if ((xt = xhci_endpoint_determine_transfer(xhcip, xep, trb, &off)) ==
	    NULL) {
		xhci_error(xhcip, "received transfer trb with code %d, slot "
		    "%d, and endpoint %d, but does not match current transfer "
		    "for endpoint: resetting device", code, slot, endpoint);
		mutex_exit(&xhcip->xhci_lock);
		xhci_fm_runtime_reset(xhcip);
		return (B_FALSE);
	}

	transfer_done = B_FALSE;

	switch (code) {
	case XHCI_CODE_SUCCESS:
	case XHCI_CODE_SHORT_XFER:
		/* Handled by endpoint logic */
		break;
	case XHCI_CODE_STALL:
		/*
		 * This causes us to transition to the halted state;
		 * however, downstream clients are able to handle this just
		 * fine.
		 */
		xep->xep_state |= XHCI_ENDPOINT_HALTED;
		xt->xt_cr = USB_CR_STALL;
		transfer_done = B_TRUE;
		break;
	case XHCI_CODE_BABBLE:
		transfer_done = B_TRUE;
		xt->xt_cr = USB_CR_DATA_OVERRUN;
		xep->xep_state |= XHCI_ENDPOINT_HALTED;
		break;
	case XHCI_CODE_TXERR:
	case XHCI_CODE_SPLITERR:
		transfer_done = B_TRUE;
		xt->xt_cr = USB_CR_DEV_NOT_RESP;
		xep->xep_state |= XHCI_ENDPOINT_HALTED;
		break;
	case XHCI_CODE_BW_OVERRUN:
		transfer_done = B_TRUE;
		xt->xt_cr = USB_CR_DATA_OVERRUN;
		break;
	case XHCI_CODE_DATA_BUF:
		transfer_done = B_TRUE;
		if (xt->xt_data_tohost)
			xt->xt_cr = USB_CR_DATA_OVERRUN;
		else
			xt->xt_cr = USB_CR_DATA_UNDERRUN;
		break;
	default:
		/*
		 * Treat these as general unspecified errors that don't cause a
		 * stop of the ring. Even if it does, a subsequent timeout
		 * should occur which causes us to end up dropping a pipe reset
		 * or at least issuing a reset of the device as part of
		 * quiescing.
		 */
		transfer_done = B_TRUE;
		xt->xt_cr = USB_CR_HC_HARDWARE_ERR;
		break;
	}

	if (transfer_done == B_TRUE) {
		xhci_transfer_t *alt;

		alt = list_remove_head(&xep->xep_transfers);
		VERIFY3P(alt, ==, xt);
		mutex_exit(&xhcip->xhci_lock);
		if (xt->xt_usba_req == NULL) {
			usb_opaque_t urp;

			urp = xhci_endpoint_dup_periodic(xep, xt, &xt->xt_cr);
			usba_hcdi_cb(xep->xep_pipe, urp, xt->xt_cr);
		} else {
			usba_hcdi_cb(xep->xep_pipe,
			    (usb_opaque_t)xt->xt_usba_req, xt->xt_cr);
			xhci_transfer_free(xhcip, xt);
		}
		return (B_TRUE);
	}

	/*
	 * Process the transfer callback based on the type of endpoint. Each of
	 * these callback functions will end up calling back into USBA via
	 * usba_hcdi_cb() to return transfer information (whether successful or
	 * not). Because we can't hold any locks across a call to that function,
	 * all of these callbacks will drop the xhci_t`xhci_lock by the time
	 * they return. This is why there's no mutex_exit() call before we
	 * return.
	 */
	switch (xep->xep_type) {
	case USB_EP_ATTR_CONTROL:
		ret = xhci_endpoint_control_callback(xhcip, xd, xep, xt, off,
		    trb);
		break;
	case USB_EP_ATTR_BULK:
		ret = xhci_endpoint_norm_callback(xhcip, xd, xep, xt, off, trb);
		break;
	case USB_EP_ATTR_INTR:
		ret = xhci_endpoint_norm_callback(xhcip, xd, xep, xt, off,
		    trb);
		break;
	case USB_EP_ATTR_ISOCH:
		ret = xhci_endpoint_isoch_callback(xhcip, xd, xep, xt, off,
		    trb);
		break;
	default:
		panic("bad endpoint type: %u", xep->xep_type);
	}

	return (ret);
}