xref: /illumos-gate/usr/src/uts/common/io/usb/hcd/xhci/xhci_usba.c (revision 60afb9d1b449f489b493961c1c14893a7a74b287)
1 /*
2  * This file and its contents are supplied under the terms of the
3  * Common Development and Distribution License ("CDDL"), version 1.0.
4  * You may only use this file in accordance with the terms of version
5  * 1.0 of the CDDL.
6  *
7  * A full copy of the text of the CDDL should have accompanied this
8  * source.  A copy of the CDDL is also available via the Internet at
9  * http://www.illumos.org/license/CDDL.
10  */
11 
12 /*
13  * Copyright (c) 2018, Joyent, Inc.
14  * Copyright (c) 2019 by Western Digital Corporation
15  * Copyright 2024 Oxide Computer Company
16  */
17 
18 /*
19  * illumos USB framework endpoints and functions for xHCI.
20  *
21  * Please see the big theory statement in xhci.c for more information.
22  */
23 
24 #include <sys/usb/hcd/xhci/xhci.h>
25 #include <sys/sysmacros.h>
26 #include <sys/strsun.h>
27 #include <sys/strsubr.h>
28 
29 xhci_t *
xhci_hcdi_get_xhcip_from_dev(usba_device_t * ud)30 xhci_hcdi_get_xhcip_from_dev(usba_device_t *ud)
31 {
32 	dev_info_t *dip = ud->usb_root_hub_dip;
33 	xhci_t *xhcip = ddi_get_soft_state(xhci_soft_state,
34 	    ddi_get_instance(dip));
35 	VERIFY(xhcip != NULL);
36 	return (xhcip);
37 }
38 
39 static xhci_t *
xhci_hcdi_get_xhcip(usba_pipe_handle_data_t * ph)40 xhci_hcdi_get_xhcip(usba_pipe_handle_data_t *ph)
41 {
42 	return (xhci_hcdi_get_xhcip_from_dev(ph->p_usba_device));
43 }
44 
45 /*
46  * While the xHCI hardware is capable of supporting power management, we don't
47  * in the driver right now. Note, USBA doesn't seem to end up calling this entry
48  * point.
49  */
50 /* ARGSUSED */
51 static int
xhci_hcdi_pm_support(dev_info_t * dip)52 xhci_hcdi_pm_support(dev_info_t *dip)
53 {
54 	return (USB_FAILURE);
55 }
56 
57 static int
xhci_hcdi_pipe_open(usba_pipe_handle_data_t * ph,usb_flags_t usb_flags)58 xhci_hcdi_pipe_open(usba_pipe_handle_data_t *ph, usb_flags_t usb_flags)
59 {
60 	xhci_t *xhcip = xhci_hcdi_get_xhcip(ph);
61 	xhci_pipe_t *pipe;
62 	xhci_endpoint_t *xep = NULL;
63 	xhci_device_t *xd;
64 	int kmflags = usb_flags & USB_FLAGS_SLEEP ? KM_SLEEP : KM_NOSLEEP;
65 	int ret;
66 	uint_t epid;
67 
68 	mutex_enter(&xhcip->xhci_lock);
69 	if (xhcip->xhci_state & XHCI_S_ERROR) {
70 		mutex_exit(&xhcip->xhci_lock);
71 		return (USB_HC_HARDWARE_ERROR);
72 	}
73 	mutex_exit(&xhcip->xhci_lock);
74 
75 	/*
76 	 * If we're here, something must be trying to open an already-opened
77 	 * pipe which is bad news.
78 	 */
79 	if (ph->p_hcd_private != NULL) {
80 		return (USB_FAILURE);
81 	}
82 
83 	pipe = kmem_zalloc(sizeof (xhci_pipe_t), kmflags);
84 	if (pipe == NULL) {
85 		return (USB_NO_RESOURCES);
86 	}
87 	pipe->xp_opentime = gethrtime();
88 	pipe->xp_pipe = ph;
89 
90 	/*
91 	 * If this is the root hub, there's nothing special to do on open. Just
92 	 * go ahead and allow it to be opened. All we have to do is add this to
93 	 * the list of our tracking structures for open pipes.
94 	 */
95 	if (ph->p_usba_device->usb_addr == ROOT_HUB_ADDR) {
96 		xep = NULL;
97 		goto add;
98 	}
99 
100 	/*
101 	 * Now that we're here, we're being asked to open up an endpoint of some
102 	 * kind. Because we've already handled the case of the root hub,
103 	 * everything should have a device.
104 	 */
105 	epid = xhci_endpoint_pipe_to_epid(ph);
106 	xd = usba_hcdi_get_device_private(ph->p_usba_device);
107 	if (xd == NULL) {
108 		xhci_error(xhcip, "!encountered endpoint (%d) without device "
109 		    "during pipe open", epid);
110 		kmem_free(pipe, sizeof (xhci_pipe_t));
111 		return (USB_FAILURE);
112 	}
113 
114 	/*
115 	 * See if this endpoint exists or not, in general endpoints should not
116 	 * exist except for the default control endpoint, which we don't tear
117 	 * down until the device itself is cleaned up. Otherwise, a given pipe
118 	 * can only be open once.
119 	 */
120 	mutex_enter(&xhcip->xhci_lock);
121 	if (epid == XHCI_DEFAULT_ENDPOINT) {
122 		xep = xd->xd_endpoints[epid];
123 		VERIFY(xep != NULL);
124 		VERIFY(xep->xep_pipe == NULL);
125 		xep->xep_pipe = ph;
126 		mutex_exit(&xhcip->xhci_lock);
127 		ret = xhci_endpoint_update_default(xhcip, xd, xep);
128 		if (ret != USB_SUCCESS) {
129 			kmem_free(pipe, sizeof (xhci_pipe_t));
130 			return (ret);
131 		}
132 		goto add;
133 	}
134 
135 	/*
136 	 * If we're opening an endpoint other than the default control endpoint,
137 	 * then the device should have had a USB address assigned by the
138 	 * controller. Sanity check that before continuing.
139 	 */
140 	VERIFY(xd->xd_addressed == B_TRUE);
141 
142 	/*
143 	 * We may have already initialized the endpoint with a previous pipe
144 	 * open.
145 	 */
146 	if ((xep = xd->xd_endpoints[epid]) != NULL &&
147 	    (xep->xep_state & XHCI_ENDPOINT_OPEN)) {
148 		mutex_exit(&xhcip->xhci_lock);
149 
150 		kmem_free(pipe, sizeof (xhci_pipe_t));
151 		xhci_log(xhcip, "!asked to open endpoint %d on slot %d and "
152 		    "port %d, but endpoint already exists", epid, xd->xd_slot,
153 		    xd->xd_port);
154 		return (USB_FAILURE);
155 	}
156 
157 	if (xep != NULL) {
158 		/*
159 		 * The endpoint is already initialized but is not presently
160 		 * open so we can take it over here.
161 		 */
162 		if ((ret = xhci_endpoint_reinit(xhcip, xd, xep, ph) != 0)) {
163 			mutex_exit(&xhcip->xhci_lock);
164 
165 			kmem_free(pipe, sizeof (xhci_pipe_t));
166 			xhci_log(xhcip, "!asked to reopen endpoint %d on "
167 			    "slot %d and port %d, but reinit failed (%d)",
168 			    epid, xd->xd_slot, xd->xd_port, ret);
169 			return (ret);
170 		}
171 
172 		/*
173 		 * We need to ensure the endpoint is stopped before we try to
174 		 * reset the transfer ring.
175 		 */
176 		xep->xep_state |= XHCI_ENDPOINT_QUIESCE;
177 		if ((ret = xhci_endpoint_quiesce(xhcip, xd, xep)) !=
178 		    USB_SUCCESS) {
179 			/*
180 			 * If we could not quiesce the endpoint, release it so
181 			 * that another open can try again.
182 			 */
183 			xep->xep_state &= ~XHCI_ENDPOINT_QUIESCE;
184 			xhci_endpoint_release(xhcip, xep);
185 			mutex_exit(&xhcip->xhci_lock);
186 
187 			kmem_free(pipe, sizeof (xhci_pipe_t));
188 			xhci_log(xhcip, "!asked to reopen endpoint %d on "
189 			    "slot %d and port %d, but quiesce failed (%d)",
190 			    epid, xd->xd_slot, xd->xd_port, ret);
191 			return (ret);
192 		}
193 
194 		/*
195 		 * Reset the transfer ring dequeue pointer.  The initial
196 		 * Configure Endpoint command leaves the endpoint in the
197 		 * Running state (xHCI 1.2 / 4.6.6), so even though the ring is
198 		 * still empty we ring the doorbell to end up in the same state
199 		 * (Running but Inactive).
200 		 */
201 		mutex_exit(&xhcip->xhci_lock);
202 		if ((ret = xhci_command_set_tr_dequeue(xhcip, xd, xep)) != 0 ||
203 		    (ret = xhci_endpoint_ring(xhcip, xd, xep)) != 0) {
204 			mutex_enter(&xhcip->xhci_lock);
205 			xep->xep_state &= ~XHCI_ENDPOINT_QUIESCE;
206 			xhci_endpoint_release(xhcip, xep);
207 			mutex_exit(&xhcip->xhci_lock);
208 
209 			kmem_free(pipe, sizeof (xhci_pipe_t));
210 			xhci_log(xhcip, "!asked to open endpoint %d on "
211 			    "slot %d and port %d, but restart failed (%d)",
212 			    epid, xd->xd_slot, xd->xd_port, ret);
213 			return (USB_FAILURE);
214 		}
215 		mutex_enter(&xhcip->xhci_lock);
216 		xep->xep_state &= ~XHCI_ENDPOINT_QUIESCE;
217 		mutex_exit(&xhcip->xhci_lock);
218 
219 		goto add;
220 	}
221 
222 	/*
223 	 * Okay, at this point we need to go create and set up an endpoint from
224 	 * scratch.  Once we're done, we'll try to install it and make sure
225 	 * that it doesn't conflict with something else going on.
226 	 */
227 	ret = xhci_endpoint_init(xhcip, xd, ph);
228 	if (ret != 0) {
229 		mutex_exit(&xhcip->xhci_lock);
230 		kmem_free(pipe, sizeof (xhci_pipe_t));
231 		if (ret == EIO) {
232 			xhci_error(xhcip, "failed to initialize endpoint %d "
233 			    "on device slot %d and port %d: encountered fatal "
234 			    "FM error, resetting device", epid, xd->xd_slot,
235 			    xd->xd_port);
236 			xhci_fm_runtime_reset(xhcip);
237 		}
238 		return (USB_HC_HARDWARE_ERROR);
239 	}
240 	xep = xd->xd_endpoints[epid];
241 
242 	mutex_enter(&xd->xd_imtx);
243 	mutex_exit(&xhcip->xhci_lock);
244 
245 	/*
246 	 * Update the slot and input context for this endpoint. We make sure to
247 	 * always set the slot as having changed in the context field as the
248 	 * specification suggests we should and some hardware requires it.
249 	 */
250 	xd->xd_input->xic_drop_flags = LE_32(0);
251 	xd->xd_input->xic_add_flags = LE_32(XHCI_INCTX_MASK_DCI(0) |
252 	    XHCI_INCTX_MASK_DCI(epid + 1));
253 
254 	if (epid + 1 > XHCI_SCTX_GET_DCI(LE_32(xd->xd_slotin->xsc_info))) {
255 		uint32_t info;
256 
257 		info = xd->xd_slotin->xsc_info;
258 		info &= ~XHCI_SCTX_DCI_MASK;
259 		info |= XHCI_SCTX_SET_DCI(epid + 1);
260 		xd->xd_slotin->xsc_info = info;
261 	}
262 
263 	XHCI_DMA_SYNC(xd->xd_ictx, DDI_DMA_SYNC_FORDEV);
264 	if (xhci_check_dma_handle(xhcip, &xd->xd_ictx) != DDI_FM_OK) {
265 		mutex_exit(&xd->xd_imtx);
266 		xhci_endpoint_fini(xd, epid);
267 		kmem_free(pipe, sizeof (xhci_pipe_t));
268 		xhci_error(xhcip, "failed to open pipe on endpoint %d of "
269 		    "device with slot %d and port %d: encountered fatal FM "
270 		    "error syncing device input context, resetting device",
271 		    epid, xd->xd_slot, xd->xd_port);
272 		xhci_fm_runtime_reset(xhcip);
273 		return (USB_HC_HARDWARE_ERROR);
274 	}
275 
276 	if ((ret = xhci_command_configure_endpoint(xhcip, xd)) != USB_SUCCESS) {
277 		mutex_exit(&xd->xd_imtx);
278 		xhci_endpoint_fini(xd, epid);
279 		kmem_free(pipe, sizeof (xhci_pipe_t));
280 		return (ret);
281 	}
282 
283 	mutex_exit(&xd->xd_imtx);
284 add:
285 	pipe->xp_ep = xep;
286 	ph->p_hcd_private = (usb_opaque_t)pipe;
287 	mutex_enter(&xhcip->xhci_lock);
288 	list_insert_tail(&xhcip->xhci_usba.xa_pipes, pipe);
289 	mutex_exit(&xhcip->xhci_lock);
290 
291 	return (USB_SUCCESS);
292 }
293 
294 static void
xhci_hcdi_periodic_free(xhci_t * xhcip,xhci_pipe_t * xp)295 xhci_hcdi_periodic_free(xhci_t *xhcip, xhci_pipe_t *xp)
296 {
297 	int i;
298 	xhci_periodic_pipe_t *xpp = &xp->xp_periodic;
299 
300 	if (xpp->xpp_tsize == 0)
301 		return;
302 
303 	for (i = 0; i < xpp->xpp_ntransfers; i++) {
304 		if (xpp->xpp_transfers[i] == NULL)
305 			continue;
306 		xhci_transfer_free(xhcip, xpp->xpp_transfers[i]);
307 		xpp->xpp_transfers[i] = NULL;
308 	}
309 
310 	xpp->xpp_ntransfers = 0;
311 	xpp->xpp_tsize = 0;
312 }
313 
314 /*
315  * Iterate over all transfers and free everything on the pipe. Once done, update
316  * the ring to basically 'consume' everything. For periodic IN endpoints, we
317  * need to handle this somewhat differently and actually close the original
318  * request and not deallocate the related pieces as those exist for the lifetime
319  * of the endpoint and are constantly reused. While interrupt IN endpoints are
320  * mostly seen as periodic, oneshot transfers on those endpoints still bear a
321  * USB request that must be completed, and are a special case where we must
322  * still free the transfer.
323  */
324 static void
xhci_hcdi_pipe_flush(xhci_t * xhcip,xhci_endpoint_t * xep,int intr_code)325 xhci_hcdi_pipe_flush(xhci_t *xhcip, xhci_endpoint_t *xep, int intr_code)
326 {
327 	xhci_transfer_t *xt;
328 
329 	ASSERT(MUTEX_HELD(&xhcip->xhci_lock));
330 
331 	while ((xt = list_remove_head(&xep->xep_transfers)) != NULL) {
332 		if (xhci_endpoint_is_periodic_in(xep) == B_FALSE ||
333 		    XHCI_IS_ONESHOT_XFER(xt)) {
334 			usba_hcdi_cb(xep->xep_pipe, xt->xt_usba_req,
335 			    USB_CR_FLUSHED);
336 			xhci_transfer_free(xhcip, xt);
337 		}
338 	}
339 
340 	if (xhci_endpoint_is_periodic_in(xep) == B_TRUE) {
341 		xhci_pipe_t *xp = (xhci_pipe_t *)xep->xep_pipe->p_hcd_private;
342 		xhci_periodic_pipe_t *xpp = &xp->xp_periodic;
343 
344 		if (xpp->xpp_usb_req != NULL) {
345 			usba_hcdi_cb(xep->xep_pipe, xpp->xpp_usb_req,
346 			    intr_code);
347 			xpp->xpp_usb_req = NULL;
348 		}
349 	}
350 }
351 
352 /*
353  * We've been asked to terminate some set of regular I/O on an interrupt pipe.
354  * If this is for the root device, e.g. the xhci driver itself, then we remove
355  * our interrupt callback. Otherwise we stop the device for interrupt polling as
356  * follows:
357  *
358  * 1. Issue a stop endpoint command
359  * 2. Check to make sure that the endpoint stopped and reset it if needed.
360  * 3. Any thing that gets resolved can callback in the interim.
361  * 4. Ensure that nothing is scheduled on the ring
362  * 5. Skip the contents of the ring and set the TR dequeue pointer.
363  * 6. Return the original callback with a USB_CR_STOPPED_POLLING, NULL out the
364  *    callback in the process.
365  */
366 static int
xhci_hcdi_pipe_poll_fini(usba_pipe_handle_data_t * ph,boolean_t is_close)367 xhci_hcdi_pipe_poll_fini(usba_pipe_handle_data_t *ph, boolean_t is_close)
368 {
369 	int ret;
370 	uint_t epid;
371 	xhci_t *xhcip = xhci_hcdi_get_xhcip(ph);
372 	xhci_device_t *xd;
373 	xhci_endpoint_t *xep;
374 	xhci_pipe_t *xp;
375 	xhci_periodic_pipe_t *xpp;
376 	usb_opaque_t urp;
377 
378 	mutex_enter(&xhcip->xhci_lock);
379 	if (xhcip->xhci_state & XHCI_S_ERROR) {
380 		mutex_exit(&xhcip->xhci_lock);
381 		return (USB_HC_HARDWARE_ERROR);
382 	}
383 
384 	if (ph->p_usba_device->usb_addr == ROOT_HUB_ADDR) {
385 		xhci_root_hub_intr_root_disable(xhcip);
386 		ret = USB_SUCCESS;
387 		mutex_exit(&xhcip->xhci_lock);
388 		return (ret);
389 	}
390 
391 	xd = usba_hcdi_get_device_private(ph->p_usba_device);
392 	epid = xhci_endpoint_pipe_to_epid(ph);
393 	if (xd->xd_endpoints[epid] == NULL) {
394 		mutex_exit(&xhcip->xhci_lock);
395 		xhci_error(xhcip, "asked to stop intr polling on slot %d, "
396 		    "port %d, endpoint: %d, but no endpoint structure",
397 		    xd->xd_slot, xd->xd_port, epid);
398 		return (USB_FAILURE);
399 	}
400 	xep = xd->xd_endpoints[epid];
401 	xp = (xhci_pipe_t *)ph->p_hcd_private;
402 	if (xp == NULL) {
403 		mutex_exit(&xhcip->xhci_lock);
404 		xhci_error(xhcip, "asked to do finish polling on slot %d, "
405 		    "port %d, endpoint: %d, but no pipe structure",
406 		    xd->xd_slot, xd->xd_port, epid);
407 		return (USB_FAILURE);
408 	}
409 	xpp = &xp->xp_periodic;
410 
411 	/*
412 	 * Ensure that no other resets or time outs are going on right now.
413 	 */
414 	xhci_endpoint_serialize(xhcip, xep);
415 
416 	if (xpp->xpp_poll_state == XHCI_PERIODIC_POLL_IDLE) {
417 		mutex_exit(&xhcip->xhci_lock);
418 		return (USB_SUCCESS);
419 	}
420 
421 	if (xpp->xpp_poll_state == XHCI_PERIODIC_POLL_STOPPING) {
422 		mutex_exit(&xhcip->xhci_lock);
423 		return (USB_FAILURE);
424 	}
425 
426 	xpp->xpp_poll_state = XHCI_PERIODIC_POLL_STOPPING;
427 	xep->xep_state |= XHCI_ENDPOINT_QUIESCE;
428 	ret = xhci_endpoint_quiesce(xhcip, xd, xep);
429 	if (ret != USB_SUCCESS) {
430 		xhci_error(xhcip, "!failed to quiesce endpoint on slot %d, "
431 		    "port %d, endpoint: %d, failed with %d.",
432 		    xd->xd_slot, xd->xd_port, epid, ret);
433 		xep->xep_state &= ~XHCI_ENDPOINT_QUIESCE;
434 		cv_broadcast(&xep->xep_state_cv);
435 		mutex_exit(&xhcip->xhci_lock);
436 		return (ret);
437 	}
438 
439 	/*
440 	 * Okay, we've stopped this ring time to wrap it all up. Remove all the
441 	 * transfers, note they aren't freed like a pipe reset.
442 	 */
443 	while (list_is_empty(&xep->xep_transfers) == 0)
444 		(void) list_remove_head(&xep->xep_transfers);
445 	xhci_ring_skip(&xep->xep_ring);
446 	mutex_exit(&xhcip->xhci_lock);
447 
448 	if ((ret = xhci_command_set_tr_dequeue(xhcip, xd, xep)) !=
449 	    USB_SUCCESS) {
450 		xhci_error(xhcip, "!failed to reset endpoint ring on slot %d, "
451 		    "port %d, endpoint: %d, failed with %d.",
452 		    xd->xd_slot, xd->xd_port, epid, ret);
453 		mutex_enter(&xhcip->xhci_lock);
454 		xep->xep_state &= ~XHCI_ENDPOINT_QUIESCE;
455 		cv_broadcast(&xep->xep_state_cv);
456 		mutex_exit(&xhcip->xhci_lock);
457 		return (ret);
458 	}
459 
460 	mutex_enter(&xhcip->xhci_lock);
461 	urp = xpp->xpp_usb_req;
462 	xpp->xpp_usb_req = NULL;
463 	xpp->xpp_poll_state = XHCI_PERIODIC_POLL_IDLE;
464 	xep->xep_state &= ~XHCI_ENDPOINT_PERIODIC;
465 	mutex_exit(&xhcip->xhci_lock);
466 
467 	/*
468 	 * It's possible that with a persistent pipe, we may not actually have
469 	 * anything left to call back on, because we already had.
470 	 */
471 	if (urp != NULL) {
472 		usba_hcdi_cb(ph, urp, is_close == B_TRUE ?
473 		    USB_CR_PIPE_CLOSING : USB_CR_STOPPED_POLLING);
474 	}
475 
476 	/*
477 	 * Notify anything waiting for us that we're done quiescing this device.
478 	 */
479 	mutex_enter(&xhcip->xhci_lock);
480 	xep->xep_state &= ~XHCI_ENDPOINT_QUIESCE;
481 	cv_broadcast(&xep->xep_state_cv);
482 	mutex_exit(&xhcip->xhci_lock);
483 
484 	return (USB_SUCCESS);
485 
486 }
487 
488 /*
489  * Tear down everything that we did in open. After this, the consumer of this
490  * USB device is done.
491  */
492 static int
xhci_hcdi_pipe_close(usba_pipe_handle_data_t * ph,usb_flags_t usb_flags)493 xhci_hcdi_pipe_close(usba_pipe_handle_data_t *ph, usb_flags_t usb_flags)
494 {
495 	xhci_t *xhcip = xhci_hcdi_get_xhcip(ph);
496 	xhci_pipe_t *xp;
497 	xhci_device_t *xd;
498 	xhci_endpoint_t *xep;
499 	int ret;
500 	uint_t epid;
501 
502 	if ((ph->p_ep.bmAttributes & USB_EP_ATTR_MASK) == USB_EP_ATTR_INTR &&
503 	    xhcip->xhci_usba.xa_intr_cb_ph != NULL) {
504 		if ((ret = xhci_hcdi_pipe_poll_fini(ph, B_TRUE)) !=
505 		    USB_SUCCESS) {
506 			return (ret);
507 		}
508 	}
509 
510 	mutex_enter(&xhcip->xhci_lock);
511 
512 	xp = (xhci_pipe_t *)ph->p_hcd_private;
513 	VERIFY(xp != NULL);
514 
515 	/*
516 	 * The default endpoint is special. It is created and destroyed with the
517 	 * device. So like with open, closing it is just state tracking. The
518 	 * same is true for the root hub.
519 	 */
520 	if (ph->p_usba_device->usb_addr == ROOT_HUB_ADDR)
521 		goto remove;
522 
523 	xd = usba_hcdi_get_device_private(ph->p_usba_device);
524 	epid = xhci_endpoint_pipe_to_epid(ph);
525 	if (xd->xd_endpoints[epid] == NULL) {
526 		mutex_exit(&xhcip->xhci_lock);
527 		xhci_error(xhcip, "asked to do close pipe on slot %d, "
528 		    "port %d, endpoint: %d, but no endpoint structure",
529 		    xd->xd_slot, xd->xd_port, epid);
530 		return (USB_FAILURE);
531 	}
532 	xep = xd->xd_endpoints[epid];
533 
534 	if (xp->xp_ep != NULL && xp->xp_ep->xep_num == XHCI_DEFAULT_ENDPOINT) {
535 		xep->xep_pipe = NULL;
536 		goto remove;
537 	}
538 
539 	/*
540 	 * We clean up the endpoint by stopping it and cancelling any transfers
541 	 * that were in flight at the time.  The endpoint is not unconfigured
542 	 * until the device is torn down later.
543 	 */
544 	xhci_endpoint_timeout_cancel(xhcip, xep);
545 	xep->xep_state |= XHCI_ENDPOINT_QUIESCE;
546 	if ((ret = xhci_endpoint_quiesce(xhcip, xd, xep)) != USB_SUCCESS) {
547 		/*
548 		 * If we cannot stop the ring, it is not safe to proceed and we
549 		 * must keep the pipe open.
550 		 */
551 		xep->xep_state &=
552 		    ~(XHCI_ENDPOINT_TEARDOWN | XHCI_ENDPOINT_QUIESCE);
553 		cv_broadcast(&xep->xep_state_cv);
554 		mutex_exit(&xhcip->xhci_lock);
555 		xhci_error(xhcip, "asked to do close pipe on slot %d, "
556 		    "port %d, endpoint: %d, but quiesce failed %d",
557 		    xd->xd_slot, xd->xd_port, epid, ret);
558 		return (USB_FAILURE);
559 	}
560 
561 	/*
562 	 * Now that we've stopped the endpoint, see if we need to flush any
563 	 * transfers.
564 	 */
565 	xhci_hcdi_pipe_flush(xhcip, xep, USB_CR_PIPE_CLOSING);
566 	if ((ph->p_ep.bEndpointAddress & USB_EP_DIR_MASK) == USB_EP_DIR_IN) {
567 		xhci_hcdi_periodic_free(xhcip, xp);
568 	}
569 
570 	xhci_endpoint_release(xhcip, xep);
571 
572 remove:
573 	ph->p_hcd_private = NULL;
574 	list_remove(&xhcip->xhci_usba.xa_pipes, xp);
575 	kmem_free(xp, sizeof (xhci_pipe_t));
576 
577 	mutex_exit(&xhcip->xhci_lock);
578 
579 	return (USB_SUCCESS);
580 }
581 
582 /*
583  * We've been asked to reset a pipe aka an endpoint. This endpoint may be in an
584  * arbitrary state, it may be running or it may be halted. In this case, we go
585  * through and check whether or not we know it's been halted or not. If it has
586  * not, then we stop the endpoint.
587  *
588  * Once the endpoint has been stopped, walk all transfers and go ahead and
589  * basically return them as being flushed. Then finally set the dequeue point
590  * for this endpoint.
591  */
592 /* ARGSUSED */
593 static int
xhci_hcdi_pipe_reset(usba_pipe_handle_data_t * ph,usb_flags_t usb_flags)594 xhci_hcdi_pipe_reset(usba_pipe_handle_data_t *ph, usb_flags_t usb_flags)
595 {
596 	xhci_t *xhcip = xhci_hcdi_get_xhcip(ph);
597 	xhci_device_t *xd;
598 	xhci_endpoint_t *xep;
599 	uint_t epid;
600 	int ret;
601 
602 	mutex_enter(&xhcip->xhci_lock);
603 	if (xhcip->xhci_state & XHCI_S_ERROR) {
604 		mutex_exit(&xhcip->xhci_lock);
605 		return (USB_HC_HARDWARE_ERROR);
606 	}
607 
608 	if (ph->p_usba_device->usb_addr == ROOT_HUB_ADDR) {
609 		mutex_exit(&xhcip->xhci_lock);
610 		return (USB_NOT_SUPPORTED);
611 	}
612 
613 	xd = usba_hcdi_get_device_private(ph->p_usba_device);
614 	epid = xhci_endpoint_pipe_to_epid(ph);
615 	if (xd->xd_endpoints[epid] == NULL) {
616 		mutex_exit(&xhcip->xhci_lock);
617 		xhci_error(xhcip, "asked to do reset pipe on slot %d, "
618 		    "port %d, endpoint: %d, but no endpoint structure",
619 		    xd->xd_slot, xd->xd_port, epid);
620 		return (USB_FAILURE);
621 	}
622 
623 	xep = xd->xd_endpoints[epid];
624 
625 	/*
626 	 * Ensure that no other resets or time outs are going on right now.
627 	 */
628 	xhci_endpoint_serialize(xhcip, xep);
629 
630 	xep->xep_state |= XHCI_ENDPOINT_QUIESCE;
631 	ret = xhci_endpoint_quiesce(xhcip, xd, xep);
632 	if (ret != USB_SUCCESS) {
633 		/*
634 		 * We failed to quiesce for some reason, remove the flag and let
635 		 * someone else give it a shot.
636 		 */
637 		xhci_error(xhcip, "!failed to quiesce endpoint on slot %d, "
638 		    "port %d, endpoint: %d, failed with %d.",
639 		    xd->xd_slot, xd->xd_port, epid, ret);
640 		xep->xep_state &= ~XHCI_ENDPOINT_QUIESCE;
641 		cv_broadcast(&xep->xep_state_cv);
642 		mutex_exit(&xhcip->xhci_lock);
643 		return (ret);
644 	}
645 
646 	xhci_ring_skip(&xep->xep_ring);
647 
648 	mutex_exit(&xhcip->xhci_lock);
649 	if ((ret = xhci_command_set_tr_dequeue(xhcip, xd, xep)) !=
650 	    USB_SUCCESS) {
651 		xhci_error(xhcip, "!failed to reset endpoint ring on slot %d, "
652 		    "port %d, endpoint: %d, failed setting ring dequeue with "
653 		    "%d.", xd->xd_slot, xd->xd_port, epid, ret);
654 		mutex_enter(&xhcip->xhci_lock);
655 		xep->xep_state &= ~XHCI_ENDPOINT_QUIESCE;
656 		cv_broadcast(&xep->xep_state_cv);
657 		mutex_exit(&xhcip->xhci_lock);
658 		return (ret);
659 	}
660 
661 	mutex_enter(&xhcip->xhci_lock);
662 	xhci_hcdi_pipe_flush(xhcip, xep, USB_CR_PIPE_RESET);
663 
664 	/*
665 	 * We need to remove the periodic flag as part of resetting, as if this
666 	 * was used for periodic activity, it no longer is and therefore can now
667 	 * be used for such purposes.
668 	 *
669 	 * Notify anything waiting for us that we're done quiescing this device.
670 	 */
671 	xep->xep_state &= ~(XHCI_ENDPOINT_QUIESCE | XHCI_ENDPOINT_PERIODIC);
672 	cv_broadcast(&xep->xep_state_cv);
673 	mutex_exit(&xhcip->xhci_lock);
674 
675 	return (USB_SUCCESS);
676 }
677 
678 /*
679  * We're asked to reset or change the data toggle, which is used in a few cases.
680  * However, there doesn't seem to be a good way to do this in xHCI as the data
681  * toggle isn't exposed. It seems that dropping a reset endpoint would
682  * theoretically do this; however, that can only be used when in the HALTED
683  * state. As such, for now we just return.
684  */
685 /* ARGSUSED */
686 void
xhci_hcdi_pipe_reset_data_toggle(usba_pipe_handle_data_t * pipe_handle)687 xhci_hcdi_pipe_reset_data_toggle(usba_pipe_handle_data_t *pipe_handle)
688 {
689 }
690 
691 /*
692  * We need to convert the USB request to an 8-byte little endian value. If we
693  * didn't have to think about big endian systems, this would be fine.
694  * Unfortunately, with them, this is a bit confusing. The problem is that if you
695  * think of this as a struct layout, the order that we or things together
696  * represents their byte layout. e.g. ctrl_bRequest is at offset 1 in the SETUP
697  * STAGE trb. However, when it becomes a part of a 64-bit big endian number, if
698  * ends up at byte 7, where as it needs to be at one. Hence why we do a final
699  * LE_64 at the end of this, to convert this into the byte order that it's
700  * expected to be in.
701  */
702 static uint64_t
xhci_hcdi_ctrl_req_to_trb(usb_ctrl_req_t * ucrp)703 xhci_hcdi_ctrl_req_to_trb(usb_ctrl_req_t *ucrp)
704 {
705 	uint64_t ret = ucrp->ctrl_bmRequestType |
706 	    (ucrp->ctrl_bRequest << 8) |
707 	    ((uint64_t)LE_16(ucrp->ctrl_wValue) << 16) |
708 	    ((uint64_t)LE_16(ucrp->ctrl_wIndex) << 32) |
709 	    ((uint64_t)LE_16(ucrp->ctrl_wLength) << 48);
710 	return (LE_64(ret));
711 }
712 
713 /*
714  * USBA calls us in order to make a specific control type request to a device,
715  * potentially even the root hub. If the request is for the root hub, then we
716  * need to intercept this and cons up the requested data.
717  */
718 static int
xhci_hcdi_pipe_ctrl_xfer(usba_pipe_handle_data_t * ph,usb_ctrl_req_t * ucrp,usb_flags_t usb_flags)719 xhci_hcdi_pipe_ctrl_xfer(usba_pipe_handle_data_t *ph, usb_ctrl_req_t *ucrp,
720     usb_flags_t usb_flags)
721 {
722 	int ret, statusdir, trt;
723 	uint_t ep;
724 	xhci_device_t *xd;
725 	xhci_endpoint_t *xep;
726 	xhci_transfer_t *xt;
727 	boolean_t datain;
728 
729 	xhci_t *xhcip = xhci_hcdi_get_xhcip(ph);
730 
731 	mutex_enter(&xhcip->xhci_lock);
732 	if (xhcip->xhci_state & XHCI_S_ERROR) {
733 		mutex_exit(&xhcip->xhci_lock);
734 		return (USB_HC_HARDWARE_ERROR);
735 	}
736 
737 	if (ph->p_usba_device->usb_addr == ROOT_HUB_ADDR) {
738 		ret = xhci_root_hub_ctrl_req(xhcip, ph, ucrp);
739 		mutex_exit(&xhcip->xhci_lock);
740 		return (ret);
741 	}
742 
743 	/*
744 	 * Determine the device and endpoint.
745 	 */
746 	xd = usba_hcdi_get_device_private(ph->p_usba_device);
747 	ep = xhci_endpoint_pipe_to_epid(ph);
748 	if (xd->xd_endpoints[ep] == NULL) {
749 		mutex_exit(&xhcip->xhci_lock);
750 		xhci_error(xhcip, "asked to do control transfer on slot %d, "
751 		    "port %d, endpoint: %d, but no endpoint structure",
752 		    xd->xd_slot, xd->xd_port, ep);
753 		return (USB_FAILURE);
754 	}
755 	xep = xd->xd_endpoints[ep];
756 
757 	/*
758 	 * There are several types of requests that we have to handle in special
759 	 * ways in xHCI. If we have one of those requests, then we don't
760 	 * necessarily go through the normal path. These special cases are all
761 	 * documented in xHCI 1.1 / 4.5.4.
762 	 *
763 	 * Looking at that, you may ask why aren't SET_CONFIGURATION and SET_IF
764 	 * special cased here. This action is a little confusing by default. The
765 	 * xHCI specification requires that we may need to issue a configure
766 	 * endpoint command as part of this. However, the xHCI 1.1 / 4.5.4.2
767 	 * states that we don't actually need to if nothing in the endpoint
768 	 * configuration context has changed. Because nothing in it should have
769 	 * changed as part of this, we don't need to do anything and instead
770 	 * just can issue the request normally. We're also assuming in the
771 	 * USB_REQ_SET_IF case that if something's changing the interface, the
772 	 * non-default endpoint will have yet to be opened.
773 	 */
774 	if (ucrp->ctrl_bmRequestType == USB_DEV_REQ_HOST_TO_DEV &&
775 	    ucrp->ctrl_bRequest == USB_REQ_SET_ADDRESS) {
776 		/*
777 		 * As we've defined an explicit set-address endpoint, we should
778 		 * never call this function. If we get here, always fail.
779 		 */
780 		mutex_exit(&xhcip->xhci_lock);
781 		usba_hcdi_cb(ph, (usb_opaque_t)ucrp, USB_CR_NOT_SUPPORTED);
782 		return (USB_SUCCESS);
783 	}
784 
785 	mutex_exit(&xhcip->xhci_lock);
786 
787 	/*
788 	 * Allocate the transfer memory, etc.
789 	 */
790 	xt = xhci_transfer_alloc(xhcip, xep, ucrp->ctrl_wLength, 2, usb_flags);
791 	if (xt == NULL) {
792 		return (USB_NO_RESOURCES);
793 	}
794 	xt->xt_usba_req = (usb_opaque_t)ucrp;
795 	xt->xt_timeout = ucrp->ctrl_timeout;
796 	if (xt->xt_timeout == 0) {
797 		xt->xt_timeout = HCDI_DEFAULT_TIMEOUT;
798 	}
799 
800 	if (ucrp->ctrl_wLength > 0) {
801 		if ((ucrp->ctrl_bmRequestType & USB_DEV_REQ_DEV_TO_HOST) != 0) {
802 			trt = XHCI_TRB_TRT_IN;
803 			datain = B_TRUE;
804 			statusdir = 0;
805 		} else {
806 			trt = XHCI_TRB_TRT_OUT;
807 			datain = B_FALSE;
808 			statusdir = XHCI_TRB_DIR_IN;
809 
810 			xhci_transfer_copy(xt, ucrp->ctrl_data->b_rptr,
811 			    ucrp->ctrl_wLength, B_FALSE);
812 			if (xhci_transfer_sync(xhcip, xt,
813 			    DDI_DMA_SYNC_FORDEV) != DDI_FM_OK) {
814 				xhci_transfer_free(xhcip, xt);
815 				xhci_error(xhcip, "failed to synchronize ctrl "
816 				    "transfer DMA memory on endpoint %u of "
817 				    "device on slot %d and port %d: resetting "
818 				    "device", xep->xep_num, xd->xd_slot,
819 				    xd->xd_port);
820 				xhci_fm_runtime_reset(xhcip);
821 				return (USB_HC_HARDWARE_ERROR);
822 			}
823 		}
824 	} else {
825 		trt = 0;
826 		datain = B_FALSE;
827 		statusdir = XHCI_TRB_DIR_IN;
828 	}
829 
830 	/*
831 	 * We always fill in the required setup and status TRBs ourselves;
832 	 * however, to minimize our knowledge about how the data has been split
833 	 * across multiple DMA cookies in an SGL, we leave that to the transfer
834 	 * logic to fill in.
835 	 */
836 	xt->xt_trbs[0].trb_addr = xhci_hcdi_ctrl_req_to_trb(ucrp);
837 	xt->xt_trbs[0].trb_status = LE_32(XHCI_TRB_LEN(8) | XHCI_TRB_INTR(0));
838 	xt->xt_trbs[0].trb_flags = LE_32(trt | XHCI_TRB_IDT |
839 	    XHCI_TRB_TYPE_SETUP);
840 
841 	if (ucrp->ctrl_wLength > 0)
842 		xhci_transfer_trb_fill_data(xep, xt, 1, datain);
843 
844 	xt->xt_trbs[xt->xt_ntrbs - 1].trb_addr = 0;
845 	xt->xt_trbs[xt->xt_ntrbs - 1].trb_status = LE_32(XHCI_TRB_INTR(0));
846 	xt->xt_trbs[xt->xt_ntrbs - 1].trb_flags = LE_32(XHCI_TRB_TYPE_STATUS |
847 	    XHCI_TRB_IOC | statusdir);
848 
849 
850 	mutex_enter(&xhcip->xhci_lock);
851 
852 	/*
853 	 * Schedule the transfer, allocating resources in the process.
854 	 */
855 	if (xhci_endpoint_schedule(xhcip, xd, xep, xt, B_TRUE) != 0) {
856 		xhci_transfer_free(xhcip, xt);
857 		mutex_exit(&xhcip->xhci_lock);
858 		return (USB_NO_RESOURCES);
859 	}
860 
861 	mutex_exit(&xhcip->xhci_lock);
862 
863 	return (USB_SUCCESS);
864 }
865 
866 /*
867  * This request is trying to get the upper bound on the amount of data we're
868  * willing transfer in one go. Note that this amount can be broken down into
869  * multiple SGL entries, this interface doesn't particularly care about that.
870  */
871 /* ARGSUSED */
872 static int
xhci_hcdi_bulk_transfer_size(usba_device_t * ud,size_t * sizep)873 xhci_hcdi_bulk_transfer_size(usba_device_t *ud, size_t *sizep)
874 {
875 	if (sizep != NULL)
876 		*sizep = XHCI_MAX_TRANSFER;
877 	return (USB_SUCCESS);
878 }
879 
880 /*
881  * Perform a bulk transfer. This is a pretty straightforward action. We
882  * basically just allocate the appropriate transfer and try to schedule it,
883  * hoping there is enough space.
884  */
885 static int
xhci_hcdi_pipe_bulk_xfer(usba_pipe_handle_data_t * ph,usb_bulk_req_t * ubrp,usb_flags_t usb_flags)886 xhci_hcdi_pipe_bulk_xfer(usba_pipe_handle_data_t *ph, usb_bulk_req_t *ubrp,
887     usb_flags_t usb_flags)
888 {
889 	uint_t epid;
890 	xhci_device_t *xd;
891 	xhci_endpoint_t *xep;
892 	xhci_transfer_t *xt;
893 	boolean_t datain;
894 
895 	xhci_t *xhcip = xhci_hcdi_get_xhcip(ph);
896 
897 	mutex_enter(&xhcip->xhci_lock);
898 	if (xhcip->xhci_state & XHCI_S_ERROR) {
899 		mutex_exit(&xhcip->xhci_lock);
900 		return (USB_HC_HARDWARE_ERROR);
901 	}
902 
903 	if (ph->p_usba_device->usb_addr == ROOT_HUB_ADDR) {
904 		mutex_exit(&xhcip->xhci_lock);
905 		return (USB_NOT_SUPPORTED);
906 	}
907 
908 	xd = usba_hcdi_get_device_private(ph->p_usba_device);
909 	epid = xhci_endpoint_pipe_to_epid(ph);
910 	if (xd->xd_endpoints[epid] == NULL) {
911 		mutex_exit(&xhcip->xhci_lock);
912 		xhci_error(xhcip, "asked to do bulk transfer on slot %d, "
913 		    "port %d, endpoint: %d, but no endpoint structure",
914 		    xd->xd_slot, xd->xd_port, epid);
915 		return (USB_FAILURE);
916 	}
917 	xep = xd->xd_endpoints[epid];
918 	mutex_exit(&xhcip->xhci_lock);
919 
920 	if ((ph->p_ep.bEndpointAddress & USB_EP_DIR_MASK) == USB_EP_DIR_IN) {
921 		datain = B_TRUE;
922 	} else {
923 		datain = B_FALSE;
924 	}
925 
926 	xt = xhci_transfer_alloc(xhcip, xep, ubrp->bulk_len, 0, usb_flags);
927 	if (xt == NULL) {
928 		return (USB_NO_RESOURCES);
929 	}
930 	xt->xt_usba_req = (usb_opaque_t)ubrp;
931 	xt->xt_timeout = ubrp->bulk_timeout;
932 	if (xt->xt_timeout == 0) {
933 		xt->xt_timeout = HCDI_DEFAULT_TIMEOUT;
934 	}
935 
936 	if (ubrp->bulk_len > 0 && datain == B_FALSE) {
937 		xhci_transfer_copy(xt, ubrp->bulk_data->b_rptr, ubrp->bulk_len,
938 		    B_FALSE);
939 		if (xhci_transfer_sync(xhcip, xt, DDI_DMA_SYNC_FORDEV) !=
940 		    DDI_FM_OK) {
941 			xhci_transfer_free(xhcip, xt);
942 			xhci_error(xhcip, "failed to synchronize bulk "
943 			    "transfer DMA memory on endpoint %u of "
944 			    "device on slot %d and port %d: resetting "
945 			    "device", xep->xep_num, xd->xd_slot,
946 			    xd->xd_port);
947 			xhci_fm_runtime_reset(xhcip);
948 			return (USB_HC_HARDWARE_ERROR);
949 		}
950 	}
951 
952 	xhci_transfer_trb_fill_data(xep, xt, 0, datain);
953 	mutex_enter(&xhcip->xhci_lock);
954 	if (xhci_endpoint_schedule(xhcip, xd, xep, xt, B_TRUE) != 0) {
955 		xhci_transfer_free(xhcip, xt);
956 		mutex_exit(&xhcip->xhci_lock);
957 		return (USB_NO_RESOURCES);
958 	}
959 	mutex_exit(&xhcip->xhci_lock);
960 
961 	return (USB_SUCCESS);
962 }
963 
964 static void
xhci_hcdi_isoc_transfer_fill(xhci_device_t * xd,xhci_endpoint_t * xep,xhci_transfer_t * xt,usb_isoc_req_t * usrp)965 xhci_hcdi_isoc_transfer_fill(xhci_device_t *xd, xhci_endpoint_t *xep,
966     xhci_transfer_t *xt, usb_isoc_req_t *usrp)
967 {
968 	int i;
969 	uintptr_t buf;
970 
971 	buf = xt->xt_buffer.xdb_cookies[0].dmac_laddress;
972 	for (i = 0; i < usrp->isoc_pkts_count; i++) {
973 		int flags;
974 		uint_t tbc, tlbpc;
975 
976 		ushort_t len = usrp->isoc_pkt_descr[i].isoc_pkt_length;
977 		xhci_trb_t *trb = &xt->xt_trbs[i];
978 
979 		trb->trb_addr = LE_64(buf);
980 
981 		/*
982 		 * Because we know that a single frame can have all of its data
983 		 * in a single instance, we know that we don't need to do
984 		 * anything special here.
985 		 */
986 		trb->trb_status = LE_32(XHCI_TRB_LEN(len) | XHCI_TRB_TDREM(0) |
987 		    XHCI_TRB_INTR(0));
988 
989 		/*
990 		 * Always enable SIA to start the frame ASAP. We also always
991 		 * enable an interrupt on a short packet. If this is the last
992 		 * trb, then we will set IOC. Each TRB created here is really
993 		 * its own TD. However, we only set an interrupt on the last
994 		 * entry to better deal with scheduling.
995 		 */
996 		flags = XHCI_TRB_SIA | XHCI_TRB_ISP | XHCI_TRB_SET_FRAME(0);
997 		flags |= XHCI_TRB_TYPE_ISOCH;
998 
999 		if (i + 1 == usrp->isoc_pkts_count)
1000 			flags |= XHCI_TRB_IOC;
1001 
1002 		/*
1003 		 * Now we need to calculate the TBC and the TLBPC.
1004 		 */
1005 		xhci_transfer_calculate_isoc(xd, xep, len, &tbc, &tlbpc);
1006 		flags |= XHCI_TRB_SET_TBC(tbc);
1007 		flags |= XHCI_TRB_SET_TLBPC(tlbpc);
1008 
1009 		trb->trb_flags = LE_32(flags);
1010 		buf += len;
1011 
1012 		/*
1013 		 * Go through and copy the required data to our local copy of
1014 		 * the isoc descriptor. By default, we assume that all data will
1015 		 * be copied and the status set to OK. This mirrors the fact
1016 		 * that we won't get a notification unless there's been an
1017 		 * error or short packet transfer.
1018 		 */
1019 		xt->xt_isoc[i].isoc_pkt_length = len;
1020 		xt->xt_isoc[i].isoc_pkt_actual_length = len;
1021 		xt->xt_isoc[i].isoc_pkt_status = USB_CR_OK;
1022 	}
1023 }
1024 
1025 /*
1026  * Initialize periodic IN requests (both interrupt and isochronous)
1027  */
1028 static int
xhci_hcdi_periodic_init(xhci_t * xhcip,usba_pipe_handle_data_t * ph,usb_opaque_t usb_req,size_t len,int usb_flags)1029 xhci_hcdi_periodic_init(xhci_t *xhcip, usba_pipe_handle_data_t *ph,
1030     usb_opaque_t usb_req, size_t len, int usb_flags)
1031 {
1032 	int i, ret;
1033 	uint_t epid;
1034 	xhci_device_t *xd;
1035 	xhci_endpoint_t *xep;
1036 	xhci_pipe_t *xp;
1037 	xhci_periodic_pipe_t *xpp;
1038 
1039 	mutex_enter(&xhcip->xhci_lock);
1040 	if (xhcip->xhci_state & XHCI_S_ERROR) {
1041 		mutex_exit(&xhcip->xhci_lock);
1042 		return (USB_HC_HARDWARE_ERROR);
1043 	}
1044 
1045 	xd = usba_hcdi_get_device_private(ph->p_usba_device);
1046 	epid = xhci_endpoint_pipe_to_epid(ph);
1047 	if (xd->xd_endpoints[epid] == NULL) {
1048 		xhci_error(xhcip, "asked to do periodic transfer on slot %d, "
1049 		    "port %d, endpoint: %d, but no endpoint structure",
1050 		    xd->xd_slot, xd->xd_port, epid);
1051 		mutex_exit(&xhcip->xhci_lock);
1052 		return (USB_FAILURE);
1053 	}
1054 	xep = xd->xd_endpoints[epid];
1055 	xp = (xhci_pipe_t *)ph->p_hcd_private;
1056 	if (xp == NULL) {
1057 		xhci_error(xhcip, "asked to do periodic transfer on slot %d, "
1058 		    "port %d, endpoint: %d, but no pipe structure",
1059 		    xd->xd_slot, xd->xd_port, epid);
1060 		mutex_exit(&xhcip->xhci_lock);
1061 		return (USB_FAILURE);
1062 	}
1063 	xpp = &xp->xp_periodic;
1064 
1065 	/*
1066 	 * Only allow a single polling request at any given time.
1067 	 */
1068 	if (xpp->xpp_usb_req != NULL) {
1069 		mutex_exit(&xhcip->xhci_lock);
1070 		return (USB_BUSY);
1071 	}
1072 
1073 	/*
1074 	 * We keep allocations around in case we restart polling, which most
1075 	 * devices do (not really caring about a lost event). However, we don't
1076 	 * support a driver changing that size on us, which it probably won't.
1077 	 * If we stumble across driver that does, then this will need to become
1078 	 * a lot more complicated.
1079 	 */
1080 	if (xpp->xpp_tsize > 0 && xpp->xpp_tsize < len) {
1081 		mutex_exit(&xhcip->xhci_lock);
1082 		return (USB_INVALID_REQUEST);
1083 	}
1084 
1085 	if (xpp->xpp_tsize == 0) {
1086 		int ntrbs;
1087 		int ntransfers;
1088 
1089 		/*
1090 		 * What we allocate varies based on whether or not this is an
1091 		 * isochronous or interrupt IN periodic.
1092 		 */
1093 		if (xep->xep_type == USB_EP_ATTR_INTR) {
1094 			ntrbs = 0;
1095 			ntransfers = XHCI_INTR_IN_NTRANSFERS;
1096 		} else {
1097 			usb_isoc_req_t *usrp;
1098 			ASSERT(xep->xep_type == USB_EP_ATTR_ISOCH);
1099 
1100 			usrp = (usb_isoc_req_t *)usb_req;
1101 			ntrbs = usrp->isoc_pkts_count;
1102 			ntransfers = XHCI_ISOC_IN_NTRANSFERS;
1103 		}
1104 
1105 		xpp->xpp_tsize = len;
1106 		xpp->xpp_ntransfers = ntransfers;
1107 
1108 		for (i = 0; i < xpp->xpp_ntransfers; i++) {
1109 			xhci_transfer_t *xt = xhci_transfer_alloc(xhcip, xep,
1110 			    len, ntrbs, usb_flags);
1111 			if (xt == NULL) {
1112 				xhci_hcdi_periodic_free(xhcip, xp);
1113 				mutex_exit(&xhcip->xhci_lock);
1114 				return (USB_NO_RESOURCES);
1115 			}
1116 
1117 			if (xep->xep_type == USB_EP_ATTR_INTR) {
1118 				xhci_transfer_trb_fill_data(xep, xt, 0, B_TRUE);
1119 			} else {
1120 				usb_isoc_req_t *usrp;
1121 				usrp = (usb_isoc_req_t *)usb_req;
1122 				xhci_hcdi_isoc_transfer_fill(xd, xep, xt, usrp);
1123 				xt->xt_data_tohost = B_TRUE;
1124 			}
1125 			xpp->xpp_transfers[i] = xt;
1126 		}
1127 	}
1128 
1129 	/*
1130 	 * Mark the endpoint as periodic so we don't have timeouts at play.
1131 	 */
1132 	xep->xep_state |= XHCI_ENDPOINT_PERIODIC;
1133 
1134 	/*
1135 	 * Now that we've allocated everything, go ahead and schedule them and
1136 	 * kick off the ring.
1137 	 */
1138 	for (i = 0; i < xpp->xpp_ntransfers; i++) {
1139 		int ret;
1140 		ret = xhci_endpoint_schedule(xhcip, xd, xep,
1141 		    xpp->xpp_transfers[i], B_FALSE);
1142 		if (ret != 0) {
1143 			(void) xhci_ring_reset(xhcip, &xep->xep_ring);
1144 			xep->xep_state &= ~XHCI_ENDPOINT_PERIODIC;
1145 			mutex_exit(&xhcip->xhci_lock);
1146 			return (ret);
1147 		}
1148 	}
1149 
1150 	/*
1151 	 * Don't worry about freeing memory, it'll be done when the endpoint
1152 	 * closes and the whole system is reset.
1153 	 */
1154 	xpp->xpp_usb_req = usb_req;
1155 	xpp->xpp_poll_state = XHCI_PERIODIC_POLL_ACTIVE;
1156 
1157 	ret = xhci_endpoint_ring(xhcip, xd, xep);
1158 	mutex_exit(&xhcip->xhci_lock);
1159 	return (ret);
1160 }
1161 
1162 static int
xhci_hcdi_intr_oneshot(xhci_t * xhcip,usba_pipe_handle_data_t * ph,usb_intr_req_t * uirp,usb_flags_t usb_flags)1163 xhci_hcdi_intr_oneshot(xhci_t *xhcip, usba_pipe_handle_data_t *ph,
1164     usb_intr_req_t *uirp, usb_flags_t usb_flags)
1165 {
1166 	uint_t epid;
1167 	xhci_device_t *xd;
1168 	xhci_endpoint_t *xep;
1169 	xhci_transfer_t *xt;
1170 	boolean_t datain;
1171 	mblk_t *mp = NULL;
1172 
1173 	mutex_enter(&xhcip->xhci_lock);
1174 	if (xhcip->xhci_state & XHCI_S_ERROR) {
1175 		mutex_exit(&xhcip->xhci_lock);
1176 		return (USB_HC_HARDWARE_ERROR);
1177 	}
1178 
1179 	xd = usba_hcdi_get_device_private(ph->p_usba_device);
1180 	epid = xhci_endpoint_pipe_to_epid(ph);
1181 	if (xd->xd_endpoints[epid] == NULL) {
1182 		xhci_error(xhcip, "asked to do interrupt transfer on slot %d, "
1183 		    "port %d, endpoint: %d, but no endpoint structure",
1184 		    xd->xd_slot, xd->xd_port, epid);
1185 		mutex_exit(&xhcip->xhci_lock);
1186 		return (USB_FAILURE);
1187 	}
1188 	xep = xd->xd_endpoints[epid];
1189 
1190 	mutex_exit(&xhcip->xhci_lock);
1191 
1192 	if ((ph->p_ep.bEndpointAddress & USB_EP_DIR_MASK) == USB_EP_DIR_IN) {
1193 		datain = B_TRUE;
1194 	} else {
1195 		datain = B_FALSE;
1196 	}
1197 
1198 	xt = xhci_transfer_alloc(xhcip, xep, uirp->intr_len, 0, usb_flags);
1199 	if (xt == NULL) {
1200 		return (USB_NO_RESOURCES);
1201 	}
1202 
1203 	xt->xt_usba_req = (usb_opaque_t)uirp;
1204 	xt->xt_timeout = uirp->intr_timeout;
1205 	if (xt->xt_timeout == 0) {
1206 		xt->xt_timeout = HCDI_DEFAULT_TIMEOUT;
1207 	}
1208 
1209 	/*
1210 	 * Unlike other request types, USB Interrupt-IN requests aren't required
1211 	 * to have allocated the message block for data. If they haven't, we
1212 	 * take care of that now.
1213 	 */
1214 	if (uirp->intr_len > 0 && datain == B_TRUE && uirp->intr_data == NULL) {
1215 		if (usb_flags & USB_FLAGS_SLEEP) {
1216 			mp = allocb_wait(uirp->intr_len, BPRI_LO, STR_NOSIG,
1217 			    NULL);
1218 		} else {
1219 			mp = allocb(uirp->intr_len, 0);
1220 		}
1221 		if (mp == NULL) {
1222 			xhci_transfer_free(xhcip, xt);
1223 			mutex_exit(&xhcip->xhci_lock);
1224 			return (USB_NO_RESOURCES);
1225 		}
1226 		uirp->intr_data = mp;
1227 	}
1228 
1229 	if (uirp->intr_len > 0 && datain == B_FALSE) {
1230 		xhci_transfer_copy(xt, uirp->intr_data->b_rptr, uirp->intr_len,
1231 		    B_FALSE);
1232 		if (xhci_transfer_sync(xhcip, xt, DDI_DMA_SYNC_FORDEV) !=
1233 		    DDI_FM_OK) {
1234 			xhci_transfer_free(xhcip, xt);
1235 			xhci_error(xhcip, "failed to synchronize interrupt "
1236 			    "transfer DMA memory on endpoint %u of "
1237 			    "device on slot %d and port %d: resetting "
1238 			    "device", xep->xep_num, xd->xd_slot,
1239 			    xd->xd_port);
1240 			xhci_fm_runtime_reset(xhcip);
1241 			return (USB_HC_HARDWARE_ERROR);
1242 		}
1243 	}
1244 
1245 	xhci_transfer_trb_fill_data(xep, xt, 0, datain);
1246 	mutex_enter(&xhcip->xhci_lock);
1247 	if (xhci_endpoint_schedule(xhcip, xd, xep, xt, B_TRUE) != 0) {
1248 		if (mp != NULL) {
1249 			uirp->intr_data = NULL;
1250 			freemsg(mp);
1251 		}
1252 		xhci_transfer_free(xhcip, xt);
1253 		mutex_exit(&xhcip->xhci_lock);
1254 		return (USB_NO_RESOURCES);
1255 	}
1256 	mutex_exit(&xhcip->xhci_lock);
1257 
1258 	return (USB_SUCCESS);
1259 }
1260 
1261 /*
1262  * We've been asked to perform an interrupt transfer. When this is an interrupt
1263  * IN endpoint, that means that the hcd is being asked to start polling on the
1264  * endpoint. When the endpoint is the root hub, it effectively becomes synthetic
1265  * polling.
1266  *
1267  * When we have an interrupt out endpoint, then this is just a single simple
1268  * interrupt request that we send out and there isn't much special to do beyond
1269  * the normal activity.
1270  */
1271 static int
xhci_hcdi_pipe_intr_xfer(usba_pipe_handle_data_t * ph,usb_intr_req_t * uirp,usb_flags_t usb_flags)1272 xhci_hcdi_pipe_intr_xfer(usba_pipe_handle_data_t *ph, usb_intr_req_t *uirp,
1273     usb_flags_t usb_flags)
1274 {
1275 	int ret;
1276 	xhci_t *xhcip = xhci_hcdi_get_xhcip(ph);
1277 
1278 	if ((ph->p_ep.bEndpointAddress & USB_EP_DIR_MASK) == USB_EP_DIR_IN) {
1279 		if (ph->p_usba_device->usb_addr == ROOT_HUB_ADDR) {
1280 			ret = xhci_root_hub_intr_root_enable(xhcip, ph, uirp);
1281 		} else if (uirp->intr_attributes & USB_ATTRS_ONE_XFER) {
1282 			ret = xhci_hcdi_intr_oneshot(xhcip, ph, uirp,
1283 			    usb_flags);
1284 		} else {
1285 			ret = xhci_hcdi_periodic_init(xhcip, ph,
1286 			    (usb_opaque_t)uirp, uirp->intr_len, usb_flags);
1287 		}
1288 	} else {
1289 		if (ph->p_usba_device->usb_addr == ROOT_HUB_ADDR) {
1290 			return (USB_NOT_SUPPORTED);
1291 		}
1292 		ret = xhci_hcdi_intr_oneshot(xhcip, ph, uirp, usb_flags);
1293 	}
1294 
1295 	return (ret);
1296 }
1297 
1298 /* ARGSUSED */
1299 static int
xhci_hcdi_pipe_stop_intr_polling(usba_pipe_handle_data_t * ph,usb_flags_t usb_flags)1300 xhci_hcdi_pipe_stop_intr_polling(usba_pipe_handle_data_t *ph,
1301     usb_flags_t usb_flags)
1302 {
1303 	return (xhci_hcdi_pipe_poll_fini(ph, B_FALSE));
1304 }
1305 
1306 static int
xhci_hcdi_isoc_periodic(xhci_t * xhcip,usba_pipe_handle_data_t * ph,usb_isoc_req_t * usrp,usb_flags_t usb_flags)1307 xhci_hcdi_isoc_periodic(xhci_t *xhcip, usba_pipe_handle_data_t *ph,
1308     usb_isoc_req_t *usrp, usb_flags_t usb_flags)
1309 {
1310 	int i;
1311 	size_t count;
1312 
1313 	count = 0;
1314 	for (i = 0; i < usrp->isoc_pkts_count; i++) {
1315 		count += usrp->isoc_pkt_descr[i].isoc_pkt_length;
1316 	}
1317 
1318 	return (xhci_hcdi_periodic_init(xhcip, ph, (usb_opaque_t)usrp, count,
1319 	    usb_flags));
1320 }
1321 
1322 /*
1323  * This is used to create an isochronous request to send data out to the device.
1324  * This is a single one shot request, it is not something that we'll have to
1325  * repeat over and over.
1326  */
1327 static int
xhci_hcdi_isoc_oneshot(xhci_t * xhcip,usba_pipe_handle_data_t * ph,usb_isoc_req_t * usrp,usb_flags_t usb_flags)1328 xhci_hcdi_isoc_oneshot(xhci_t *xhcip, usba_pipe_handle_data_t *ph,
1329     usb_isoc_req_t *usrp, usb_flags_t usb_flags)
1330 {
1331 	int i, ret;
1332 	uint_t epid;
1333 	size_t count, mblen;
1334 	xhci_device_t *xd;
1335 	xhci_endpoint_t *xep;
1336 	xhci_transfer_t *xt;
1337 
1338 	count = 0;
1339 	for (i = 0; i < usrp->isoc_pkts_count; i++) {
1340 		count += usrp->isoc_pkt_descr[i].isoc_pkt_length;
1341 	}
1342 	mblen = MBLKL(usrp->isoc_data);
1343 
1344 	if (count != mblen) {
1345 		return (USB_INVALID_ARGS);
1346 	}
1347 
1348 	mutex_enter(&xhcip->xhci_lock);
1349 	if (xhcip->xhci_state & XHCI_S_ERROR) {
1350 		mutex_exit(&xhcip->xhci_lock);
1351 		return (USB_HC_HARDWARE_ERROR);
1352 	}
1353 
1354 	xd = usba_hcdi_get_device_private(ph->p_usba_device);
1355 	epid = xhci_endpoint_pipe_to_epid(ph);
1356 	if (xd->xd_endpoints[epid] == NULL) {
1357 		xhci_error(xhcip, "asked to do isochronous transfer on slot "
1358 		    "%d, port %d, endpoint: %d, but no endpoint structure",
1359 		    xd->xd_slot, xd->xd_port, epid);
1360 		mutex_exit(&xhcip->xhci_lock);
1361 		return (USB_FAILURE);
1362 	}
1363 	xep = xd->xd_endpoints[epid];
1364 	mutex_exit(&xhcip->xhci_lock);
1365 
1366 	xt = xhci_transfer_alloc(xhcip, xep, mblen, usrp->isoc_pkts_count,
1367 	    usb_flags);
1368 	if (xt == NULL) {
1369 		return (USB_NO_RESOURCES);
1370 	}
1371 	xt->xt_usba_req = (usb_opaque_t)usrp;
1372 
1373 	/*
1374 	 * USBA doesn't provide any real way for a timeout to be defined for an
1375 	 * isochronous event. However, since we technically aren't a periodic
1376 	 * endpoint, go ahead and always set the default timeout. It's better
1377 	 * than nothing.
1378 	 */
1379 	xt->xt_timeout = HCDI_DEFAULT_TIMEOUT;
1380 
1381 	xhci_transfer_copy(xt, usrp->isoc_data->b_rptr, mblen, B_FALSE);
1382 	if (xhci_transfer_sync(xhcip, xt, DDI_DMA_SYNC_FORDEV) != DDI_FM_OK) {
1383 		xhci_transfer_free(xhcip, xt);
1384 		xhci_error(xhcip, "failed to synchronize isochronous "
1385 		    "transfer DMA memory on endpoint %u of "
1386 		    "device on slot %d and port %d: resetting "
1387 		    "device", xep->xep_num, xd->xd_slot,
1388 		    xd->xd_port);
1389 		xhci_fm_runtime_reset(xhcip);
1390 		return (USB_HC_HARDWARE_ERROR);
1391 	}
1392 
1393 	/*
1394 	 * Fill in the ISOC data. Note, that we always use ASAP scheduling and
1395 	 * we don't support specifying the frame at this time, for better or
1396 	 * worse.
1397 	 */
1398 	xhci_hcdi_isoc_transfer_fill(xd, xep, xt, usrp);
1399 
1400 	mutex_enter(&xhcip->xhci_lock);
1401 	ret = xhci_endpoint_schedule(xhcip, xd, xep, xt, B_TRUE);
1402 	mutex_exit(&xhcip->xhci_lock);
1403 
1404 	return (ret);
1405 }
1406 
1407 static int
xhci_hcdi_pipe_isoc_xfer(usba_pipe_handle_data_t * ph,usb_isoc_req_t * usrp,usb_flags_t usb_flags)1408 xhci_hcdi_pipe_isoc_xfer(usba_pipe_handle_data_t *ph, usb_isoc_req_t *usrp,
1409     usb_flags_t usb_flags)
1410 {
1411 	int ret;
1412 	xhci_t *xhcip;
1413 
1414 	xhcip = xhci_hcdi_get_xhcip(ph);
1415 
1416 	/*
1417 	 * We don't support isochronous transactions on the root hub at all.
1418 	 * Always fail them if for some reason we end up here.
1419 	 */
1420 	if (ph->p_usba_device->usb_addr == ROOT_HUB_ADDR) {
1421 		return (USB_NOT_SUPPORTED);
1422 	}
1423 
1424 	/*
1425 	 * We do not support being asked to set the frame ID at this time. We
1426 	 * require that everything specify the attribute
1427 	 * USB_ATTRS_ISOC_XFER_ASAP.
1428 	 */
1429 	if (!(usrp->isoc_attributes & USB_ATTRS_ISOC_XFER_ASAP)) {
1430 		return (USB_NOT_SUPPORTED);
1431 	}
1432 
1433 	if ((ph->p_ep.bEndpointAddress & USB_EP_DIR_MASK) == USB_EP_DIR_IN) {
1434 		/*
1435 		 * Note, there is no such thing as a non-periodic isochronous
1436 		 * incoming transfer.
1437 		 */
1438 		ret = xhci_hcdi_isoc_periodic(xhcip, ph, usrp, usb_flags);
1439 	} else {
1440 		ret = xhci_hcdi_isoc_oneshot(xhcip, ph, usrp, usb_flags);
1441 	}
1442 
1443 	return (ret);
1444 }
1445 
1446 /* ARGSUSED */
1447 static int
xhci_hcdi_pipe_stop_isoc_polling(usba_pipe_handle_data_t * ph,usb_flags_t usb_flags)1448 xhci_hcdi_pipe_stop_isoc_polling(usba_pipe_handle_data_t *ph,
1449     usb_flags_t usb_flags)
1450 {
1451 	return (xhci_hcdi_pipe_poll_fini(ph, B_FALSE));
1452 }
1453 
1454 /*
1455  * This is asking us for the current frame number. The USBA expects this to
1456  * actually be a bit of a fiction, as it tries to maintain a frame number well
1457  * beyond what the hardware actually contains in its registers. Hardware
1458  * basically has a 14-bit counter, whereas we need to have a constant amount of
1459  * milliseconds.
1460  *
1461  * Today, no client drivers actually use this API and everyone specifies the
1462  * attribute to say that we should schedule things ASAP. So until we have some
1463  * real device that want this functionality, we're going to fail.
1464  */
1465 /* ARGSUSED */
1466 static int
xhci_hcdi_get_current_frame_number(usba_device_t * usba_device,usb_frame_number_t * frame_number)1467 xhci_hcdi_get_current_frame_number(usba_device_t *usba_device,
1468     usb_frame_number_t *frame_number)
1469 {
1470 	return (USB_FAILURE);
1471 }
1472 
1473 /*
1474  * See the comments around the XHCI_ISOC_MAX_TRB macro for more information.
1475  */
1476 /* ARGSUSED */
1477 static int
xhci_hcdi_get_max_isoc_pkts(usba_device_t * usba_device,uint_t * max_isoc_pkts_per_request)1478 xhci_hcdi_get_max_isoc_pkts(usba_device_t *usba_device,
1479     uint_t *max_isoc_pkts_per_request)
1480 {
1481 	*max_isoc_pkts_per_request = XHCI_ISOC_MAX_TRB;
1482 	return (USB_SUCCESS);
1483 }
1484 
1485 /*
1486  * VERSION 2 ops and helpers
1487  */
1488 
1489 static void
xhci_hcdi_device_free(xhci_device_t * xd)1490 xhci_hcdi_device_free(xhci_device_t *xd)
1491 {
1492 	xhci_dma_free(&xd->xd_ictx);
1493 	xhci_dma_free(&xd->xd_octx);
1494 	mutex_destroy(&xd->xd_imtx);
1495 	kmem_free(xd, sizeof (xhci_device_t));
1496 }
1497 
1498 /*
1499  * Calculate the device's route string. In USB 3.0 the route string is a 20-bit
1500  * number. Each four bits represent a port number attached to a deeper hub.
1501  * Particularly it represents the port on that current hub that you need to go
1502  * down to reach the next device. Bits 0-3 represent the first *external* hub.
1503  * So a device connected to a root hub has a route string of zero. Imagine the
1504  * following set of devices:
1505  *
1506  *               . port 2      . port 5
1507  *               .             .
1508  *  +----------+ .  +--------+ .  +-------+
1509  *  | root hub |-*->| hub 1  |-*->| hub 2 |
1510  *  +----------+    +--------+    +-------+
1511  *       * . port 12    * . port 8    * . port 1
1512  *       v              v             v
1513  *   +-------+      +-------+     +-------+
1514  *   | dev a |      | dev b |     | dev c |
1515  *   +-------+      +-------+     +-------+
1516  *
1517  * So, based on the above diagram, device a should have a route string of 0,
1518  * because it's directly connected to the root port. Device b would simply have
1519  * a route string of 8. This is because it travels through one non-root hub, hub
1520  * 1, and it does so on port 8. The root ports basically don't matter. Device c
1521  * would then have a route string of 0x15, as it's first traversing through hub
1522  * 1 on port 2 and then hub 2 on port 5.
1523  *
1524  * Finally, it's worth mentioning that because it's a four bit field, if for
1525  * some reason a device has more than 15 ports, we just treat the value as 15.
1526  *
1527  * Note, as part of this, we also grab what port on the root hub this whole
1528  * chain is on, as we're required to store that information in the slot context.
1529  */
1530 static void
xhci_hcdi_device_route(usba_device_t * ud,uint32_t * routep,uint32_t * root_port)1531 xhci_hcdi_device_route(usba_device_t *ud, uint32_t *routep, uint32_t *root_port)
1532 {
1533 	uint32_t route = 0;
1534 	usba_device_t *hub = ud->usb_parent_hub;
1535 	usba_device_t *port_dev = ud;
1536 
1537 	ASSERT(hub != NULL);
1538 
1539 	/*
1540 	 * Iterate over every hub, updating the route as we go. When we
1541 	 * encounter a hub without a parent, then we're at the root hub. At
1542 	 * which point, the port we want is on port_dev (the child of hub).
1543 	 */
1544 	while (hub->usb_parent_hub != NULL) {
1545 		uint32_t p;
1546 
1547 		p = port_dev->usb_port;
1548 		if (p > 15)
1549 			p = 15;
1550 		route <<= 4;
1551 		route |= p & 0xf;
1552 		port_dev = hub;
1553 		hub = hub->usb_parent_hub;
1554 	}
1555 
1556 	ASSERT(port_dev->usb_parent_hub == hub);
1557 	*root_port = port_dev->usb_port;
1558 	*routep = XHCI_ROUTE_MASK(route);
1559 }
1560 
1561 /*
1562  * If a low or full speed device is behind a high-speed device that is not a
1563  * root hub, then we must include the port and slot of that device. USBA already
1564  * stores this device in the usb_hs_hub_usba_dev member.
1565  */
1566 static uint32_t
xhci_hcdi_device_tt(usba_device_t * ud)1567 xhci_hcdi_device_tt(usba_device_t *ud)
1568 {
1569 	uint32_t ret;
1570 	xhci_device_t *xd;
1571 
1572 	if (ud->usb_port_status >= USBA_HIGH_SPEED_DEV)
1573 		return (0);
1574 
1575 	if (ud->usb_hs_hub_usba_dev == NULL)
1576 		return (0);
1577 
1578 	ASSERT(ud->usb_hs_hub_usba_dev != NULL);
1579 	ASSERT(ud->usb_hs_hub_usba_dev->usb_parent_hub != NULL);
1580 	xd = usba_hcdi_get_device_private(ud->usb_hs_hub_usba_dev);
1581 	ASSERT(xd != NULL);
1582 
1583 	ret = XHCI_SCTX_SET_TT_HUB_SID(xd->xd_slot);
1584 	ret |= XHCI_SCTX_SET_TT_PORT_NUM(ud->usb_hs_hub_usba_dev->usb_port);
1585 
1586 	return (ret);
1587 }
1588 
1589 /*
1590  * Initialize a new device. This allocates a device slot from the controller,
1591  * which tranfers it to our control.
1592  */
1593 static int
xhci_hcdi_device_init(usba_device_t * ud,usb_port_t port,void ** hcdpp)1594 xhci_hcdi_device_init(usba_device_t *ud, usb_port_t port, void **hcdpp)
1595 {
1596 	int ret, i;
1597 	xhci_device_t *xd;
1598 	ddi_device_acc_attr_t acc;
1599 	ddi_dma_attr_t attr;
1600 	xhci_t *xhcip = xhci_hcdi_get_xhcip_from_dev(ud);
1601 	size_t isize, osize, incr;
1602 	uint32_t route, rp, info, info2, tt;
1603 
1604 	xd = kmem_zalloc(sizeof (xhci_device_t), KM_SLEEP);
1605 	xd->xd_port = port;
1606 	xd->xd_usbdev = ud;
1607 	mutex_init(&xd->xd_imtx, NULL, MUTEX_DRIVER,
1608 	    (void *)(uintptr_t)xhcip->xhci_intr_pri);
1609 
1610 	/*
1611 	 * The size of the context structures is based upon the presence of the
1612 	 * context flag which determines whether we have a 32-byte or 64-byte
1613 	 * context. Note that the input context always has to account for the
1614 	 * entire size of the xhci_input_contex_t, which is 32-bytes by default.
1615 	 */
1616 	if (xhcip->xhci_caps.xcap_flags & XCAP_CSZ) {
1617 		incr = 64;
1618 		osize = XHCI_DEVICE_CONTEXT_64;
1619 		isize = XHCI_DEVICE_CONTEXT_64 + incr;
1620 	} else {
1621 		incr = 32;
1622 		osize = XHCI_DEVICE_CONTEXT_32;
1623 		isize = XHCI_DEVICE_CONTEXT_32 + incr;
1624 	}
1625 
1626 	xhci_dma_acc_attr(xhcip, &acc);
1627 	xhci_dma_dma_attr(xhcip, &attr);
1628 	if (xhci_dma_alloc(xhcip, &xd->xd_ictx, &attr, &acc, B_TRUE,
1629 	    isize, B_FALSE) == B_FALSE) {
1630 		xhci_hcdi_device_free(xd);
1631 		return (USB_NO_RESOURCES);
1632 	}
1633 
1634 	xd->xd_input = (xhci_input_context_t *)xd->xd_ictx.xdb_va;
1635 	xd->xd_slotin = (xhci_slot_context_t *)(xd->xd_ictx.xdb_va + incr);
1636 	for (i = 0; i < XHCI_NUM_ENDPOINTS; i++) {
1637 		xd->xd_endin[i] =
1638 		    (xhci_endpoint_context_t *)(xd->xd_ictx.xdb_va +
1639 		    (i + 2) * incr);
1640 	}
1641 
1642 	if (xhci_dma_alloc(xhcip, &xd->xd_octx, &attr, &acc, B_TRUE,
1643 	    osize, B_FALSE) == B_FALSE) {
1644 		xhci_hcdi_device_free(xd);
1645 		return (USB_NO_RESOURCES);
1646 	}
1647 	xd->xd_slotout = (xhci_slot_context_t *)xd->xd_octx.xdb_va;
1648 	for (i = 0; i < XHCI_NUM_ENDPOINTS; i++) {
1649 		xd->xd_endout[i] =
1650 		    (xhci_endpoint_context_t *)(xd->xd_octx.xdb_va +
1651 		    (i + 1) * incr);
1652 	}
1653 
1654 	ret = xhci_command_enable_slot(xhcip, &xd->xd_slot);
1655 	if (ret != USB_SUCCESS) {
1656 		xhci_hcdi_device_free(xd);
1657 		return (ret);
1658 	}
1659 
1660 	/*
1661 	 * These are the default slot context and the endpoint zero context that
1662 	 * we're enabling. See 4.3.3.
1663 	 */
1664 	xd->xd_input->xic_add_flags = LE_32(XHCI_INCTX_MASK_DCI(0) |
1665 	    XHCI_INCTX_MASK_DCI(1));
1666 
1667 	/*
1668 	 * Note, we never need to set the MTT bit as illumos never enables the
1669 	 * alternate MTT interface.
1670 	 */
1671 	xhci_hcdi_device_route(ud, &route, &rp);
1672 	info = XHCI_SCTX_SET_ROUTE(route) | XHCI_SCTX_SET_DCI(1);
1673 	switch (ud->usb_port_status) {
1674 	case USBA_LOW_SPEED_DEV:
1675 		info |= XHCI_SCTX_SET_SPEED(XHCI_SPEED_LOW);
1676 		break;
1677 	case USBA_HIGH_SPEED_DEV:
1678 		info |= XHCI_SCTX_SET_SPEED(XHCI_SPEED_HIGH);
1679 		break;
1680 	case USBA_FULL_SPEED_DEV:
1681 		info |= XHCI_SCTX_SET_SPEED(XHCI_SPEED_FULL);
1682 		break;
1683 	case USBA_SUPER_SPEED_DEV:
1684 	default:
1685 		info |= XHCI_SCTX_SET_SPEED(XHCI_SPEED_SUPER);
1686 		break;
1687 	}
1688 	info2 = XHCI_SCTX_SET_RHPORT(rp);
1689 	tt = XHCI_SCTX_SET_IRQ_TARGET(0);
1690 	tt |= xhci_hcdi_device_tt(ud);
1691 
1692 	xd->xd_slotin->xsc_info = LE_32(info);
1693 	xd->xd_slotin->xsc_info2 = LE_32(info2);
1694 	xd->xd_slotin->xsc_tt = LE_32(tt);
1695 
1696 	if ((ret = xhci_endpoint_init(xhcip, xd, NULL)) != 0) {
1697 		(void) xhci_command_disable_slot(xhcip, xd->xd_slot);
1698 		xhci_hcdi_device_free(xd);
1699 		return (USB_HC_HARDWARE_ERROR);
1700 	}
1701 
1702 	if (xhci_context_slot_output_init(xhcip, xd) != B_TRUE) {
1703 		(void) xhci_command_disable_slot(xhcip, xd->xd_slot);
1704 		xhci_endpoint_fini(xd, 0);
1705 		xhci_hcdi_device_free(xd);
1706 		return (USB_HC_HARDWARE_ERROR);
1707 	}
1708 
1709 	if ((ret = xhci_command_set_address(xhcip, xd, B_TRUE)) != 0) {
1710 		(void) xhci_command_disable_slot(xhcip, xd->xd_slot);
1711 		xhci_context_slot_output_fini(xhcip, xd);
1712 		xhci_endpoint_fini(xd, 0);
1713 		xhci_hcdi_device_free(xd);
1714 		return (ret);
1715 	}
1716 
1717 	mutex_enter(&xhcip->xhci_lock);
1718 	list_insert_tail(&xhcip->xhci_usba.xa_devices, xd);
1719 	mutex_exit(&xhcip->xhci_lock);
1720 
1721 	*hcdpp = xd;
1722 	return (ret);
1723 }
1724 
1725 /*
1726  * We're tearing down a device now.
1727  */
1728 static void
xhci_hcdi_device_fini(usba_device_t * ud,void * hcdp)1729 xhci_hcdi_device_fini(usba_device_t *ud, void *hcdp)
1730 {
1731 	int ret;
1732 	xhci_endpoint_t *xep;
1733 	xhci_device_t *xd;
1734 	xhci_t *xhcip;
1735 
1736 	/*
1737 	 * Right now, it's theoretically possible that USBA may try and call
1738 	 * us here even if we hadn't successfully finished the device_init()
1739 	 * endpoint. We should probably modify the USBA to make sure that this
1740 	 * can't happen.
1741 	 */
1742 	if (hcdp == NULL)
1743 		return;
1744 
1745 	xd = hcdp;
1746 	xhcip = xhci_hcdi_get_xhcip_from_dev(ud);
1747 
1748 	/*
1749 	 * Make sure we have no timeout running on the default endpoint still.
1750 	 */
1751 	xep = xd->xd_endpoints[XHCI_DEFAULT_ENDPOINT];
1752 	mutex_enter(&xhcip->xhci_lock);
1753 	xep->xep_state |= XHCI_ENDPOINT_TEARDOWN;
1754 	mutex_exit(&xhcip->xhci_lock);
1755 	(void) untimeout(xep->xep_timeout);
1756 
1757 	/*
1758 	 * Go ahead and disable the slot. There's no reason to do anything
1759 	 * special about the default endpoint as it will be disabled as a part
1760 	 * of the slot disabling. However, if this all fails, we'll leave this
1761 	 * sitting here in a failed state, eating up a device slot. It is
1762 	 * unlikely this will occur.
1763 	 */
1764 	ret = xhci_command_disable_slot(xhcip, xd->xd_slot);
1765 	if (ret != USB_SUCCESS) {
1766 		xhci_error(xhcip, "failed to disable slot %d: %d",
1767 		    xd->xd_slot, ret);
1768 		return;
1769 	}
1770 
1771 	xhci_context_slot_output_fini(xhcip, xd);
1772 
1773 	/*
1774 	 * Once the slot is disabled, we can free any endpoints that were
1775 	 * opened.
1776 	 */
1777 	for (uint_t n = 0; n < XHCI_NUM_ENDPOINTS; n++) {
1778 		if (xd->xd_endpoints[n] != NULL) {
1779 			xhci_endpoint_fini(xd, n);
1780 		}
1781 	}
1782 
1783 	mutex_enter(&xhcip->xhci_lock);
1784 	list_remove(&xhcip->xhci_usba.xa_devices, xd);
1785 	mutex_exit(&xhcip->xhci_lock);
1786 
1787 	xhci_hcdi_device_free(xd);
1788 }
1789 
1790 /*
1791  * Synchronously attempt to set the device address. For xHCI this involves it
1792  * deciding what address to use.
1793  */
1794 static int
xhci_hcdi_device_address(usba_device_t * ud)1795 xhci_hcdi_device_address(usba_device_t *ud)
1796 {
1797 	int ret;
1798 	xhci_t *xhcip = xhci_hcdi_get_xhcip_from_dev(ud);
1799 	xhci_device_t *xd = usba_hcdi_get_device_private(ud);
1800 	xhci_endpoint_t *xep;
1801 
1802 	mutex_enter(&xhcip->xhci_lock);
1803 
1804 	/*
1805 	 * This device may already be addressed from the perspective of the xhci
1806 	 * controller. For example, the device this represents may have been
1807 	 * unconfigured, which does not actually remove the slot or other
1808 	 * information, merely tears down all the active use of it and the child
1809 	 * driver. In such cases, if we're already addressed, just return
1810 	 * success. The actual USB address is a fiction for USBA anyways.
1811 	 */
1812 	if (xd->xd_addressed == B_TRUE) {
1813 		mutex_exit(&xhcip->xhci_lock);
1814 		return (USB_SUCCESS);
1815 	}
1816 
1817 	ASSERT(xd->xd_addressed == B_FALSE);
1818 	xd->xd_addressed = B_TRUE;
1819 	VERIFY3P(xd->xd_endpoints[XHCI_DEFAULT_ENDPOINT], !=, NULL);
1820 	xep = xd->xd_endpoints[XHCI_DEFAULT_ENDPOINT];
1821 	mutex_exit(&xhcip->xhci_lock);
1822 
1823 	if ((ret = xhci_endpoint_setup_default_context(xhcip, xd, xep)) != 0) {
1824 		ASSERT(ret == EIO);
1825 		return (USB_HC_HARDWARE_ERROR);
1826 	}
1827 
1828 	ret = xhci_command_set_address(xhcip, xd, B_FALSE);
1829 
1830 	if (ret != USB_SUCCESS) {
1831 		mutex_enter(&xhcip->xhci_lock);
1832 		xd->xd_addressed = B_FALSE;
1833 		mutex_exit(&xhcip->xhci_lock);
1834 	}
1835 
1836 	return (ret);
1837 }
1838 
1839 /*
1840  * This is called relatively early on in a hub's life time. At this point, it's
1841  * descriptors have all been pulled and the default control pipe is still open.
1842  * What we need to do is go through and update the slot context to indicate that
1843  * this is a hub, otherwise, the controller will never let us speak to
1844  * downstream ports.
1845  */
1846 static int
xhci_hcdi_hub_update(usba_device_t * ud,uint8_t nports,uint8_t tt)1847 xhci_hcdi_hub_update(usba_device_t *ud, uint8_t nports, uint8_t tt)
1848 {
1849 	int ret;
1850 	xhci_t *xhcip = xhci_hcdi_get_xhcip_from_dev(ud);
1851 	xhci_device_t *xd = usba_hcdi_get_device_private(ud);
1852 
1853 	if (xd == NULL)
1854 		return (USB_FAILURE);
1855 
1856 	if (ud->usb_hubdi == NULL) {
1857 		return (USB_FAILURE);
1858 	}
1859 
1860 	mutex_enter(&xd->xd_imtx);
1861 
1862 	/*
1863 	 * Note, that usba never sets the interface of a hub to Multi TT. Hence
1864 	 * why we're never setting the MTT bit in xsc_info.
1865 	 */
1866 	xd->xd_slotin->xsc_info |= LE_32(XHCI_SCTX_SET_HUB(1));
1867 	xd->xd_slotin->xsc_info2 |= LE_32(XHCI_SCTX_SET_NPORTS(nports));
1868 	if (ud->usb_port_status == USBA_HIGH_SPEED_DEV)
1869 		xd->xd_slotin->xsc_tt |= LE_32(XHCI_SCTX_SET_TT_THINK_TIME(tt));
1870 
1871 	/*
1872 	 * We're only updating the slot context, no endpoint contexts should be
1873 	 * touched.
1874 	 */
1875 	xd->xd_input->xic_drop_flags = LE_32(0);
1876 	xd->xd_input->xic_add_flags = LE_32(XHCI_INCTX_MASK_DCI(0));
1877 
1878 	ret = xhci_command_evaluate_context(xhcip, xd);
1879 	mutex_exit(&xd->xd_imtx);
1880 	return (ret);
1881 }
1882 
1883 void
xhci_hcd_fini(xhci_t * xhcip)1884 xhci_hcd_fini(xhci_t *xhcip)
1885 {
1886 	usba_hcdi_unregister(xhcip->xhci_dip);
1887 	usba_free_hcdi_ops(xhcip->xhci_usba.xa_ops);
1888 	list_destroy(&xhcip->xhci_usba.xa_pipes);
1889 	list_destroy(&xhcip->xhci_usba.xa_devices);
1890 }
1891 
1892 int
xhci_hcd_init(xhci_t * xhcip)1893 xhci_hcd_init(xhci_t *xhcip)
1894 {
1895 	usba_hcdi_register_args_t hreg;
1896 	usba_hcdi_ops_t *ops;
1897 
1898 	ops = usba_alloc_hcdi_ops();
1899 	VERIFY(ops != NULL);
1900 
1901 	ops->usba_hcdi_ops_version = HCDI_OPS_VERSION;
1902 	ops->usba_hcdi_dip = xhcip->xhci_dip;
1903 
1904 	ops->usba_hcdi_pm_support = xhci_hcdi_pm_support;
1905 	ops->usba_hcdi_pipe_open = xhci_hcdi_pipe_open;
1906 	ops->usba_hcdi_pipe_close = xhci_hcdi_pipe_close;
1907 	ops->usba_hcdi_pipe_reset = xhci_hcdi_pipe_reset;
1908 	ops->usba_hcdi_pipe_reset_data_toggle =
1909 	    xhci_hcdi_pipe_reset_data_toggle;
1910 	ops->usba_hcdi_pipe_ctrl_xfer = xhci_hcdi_pipe_ctrl_xfer;
1911 	ops->usba_hcdi_bulk_transfer_size = xhci_hcdi_bulk_transfer_size;
1912 	ops->usba_hcdi_pipe_bulk_xfer = xhci_hcdi_pipe_bulk_xfer;
1913 	ops->usba_hcdi_pipe_intr_xfer = xhci_hcdi_pipe_intr_xfer;
1914 	ops->usba_hcdi_pipe_stop_intr_polling =
1915 	    xhci_hcdi_pipe_stop_intr_polling;
1916 	ops->usba_hcdi_pipe_isoc_xfer = xhci_hcdi_pipe_isoc_xfer;
1917 	ops->usba_hcdi_pipe_stop_isoc_polling =
1918 	    xhci_hcdi_pipe_stop_isoc_polling;
1919 	ops->usba_hcdi_get_current_frame_number =
1920 	    xhci_hcdi_get_current_frame_number;
1921 	ops->usba_hcdi_get_max_isoc_pkts = xhci_hcdi_get_max_isoc_pkts;
1922 	ops->usba_hcdi_console_input_init = xhci_hcdi_console_input_init;
1923 	ops->usba_hcdi_console_input_fini = xhci_hcdi_console_input_fini;
1924 	ops->usba_hcdi_console_input_enter = xhci_hcdi_console_input_enter;
1925 	ops->usba_hcdi_console_read = xhci_hcdi_console_read;
1926 	ops->usba_hcdi_console_input_exit = xhci_hcdi_console_input_exit;
1927 
1928 	ops->usba_hcdi_console_output_init = xhci_hcdi_console_output_init;
1929 	ops->usba_hcdi_console_output_fini = xhci_hcdi_console_output_fini;
1930 	ops->usba_hcdi_console_output_enter = xhci_hcdi_console_output_enter;
1931 	ops->usba_hcdi_console_write = xhci_hcdi_console_write;
1932 	ops->usba_hcdi_console_output_exit = xhci_hcdi_console_output_exit;
1933 
1934 	ops->usba_hcdi_device_init = xhci_hcdi_device_init;
1935 	ops->usba_hcdi_device_fini = xhci_hcdi_device_fini;
1936 	ops->usba_hcdi_device_address = xhci_hcdi_device_address;
1937 	ops->usba_hcdi_hub_update = xhci_hcdi_hub_update;
1938 
1939 	hreg.usba_hcdi_register_version = HCDI_REGISTER_VERSION;
1940 	hreg.usba_hcdi_register_dip = xhcip->xhci_dip;
1941 	hreg.usba_hcdi_register_ops = ops;
1942 
1943 	/*
1944 	 * We're required to give xhci a set of DMA attributes that it may loan
1945 	 * out to other devices. Therefore we'll be conservative with what we
1946 	 * end up giving it.
1947 	 */
1948 	xhci_dma_dma_attr(xhcip, &xhcip->xhci_usba.xa_dma_attr);
1949 	hreg.usba_hcdi_register_dma_attr = &xhcip->xhci_usba.xa_dma_attr;
1950 
1951 	hreg.usba_hcdi_register_iblock_cookie =
1952 	    (ddi_iblock_cookie_t)(uintptr_t)xhcip->xhci_intr_pri;
1953 
1954 	if (usba_hcdi_register(&hreg, 0) != DDI_SUCCESS) {
1955 		usba_free_hcdi_ops(ops);
1956 		return (DDI_FAILURE);
1957 	}
1958 
1959 	xhcip->xhci_usba.xa_ops = ops;
1960 
1961 	list_create(&xhcip->xhci_usba.xa_devices, sizeof (xhci_device_t),
1962 	    offsetof(xhci_device_t, xd_link));
1963 	list_create(&xhcip->xhci_usba.xa_pipes, sizeof (xhci_pipe_t),
1964 	    offsetof(xhci_pipe_t, xp_link));
1965 
1966 
1967 	return (DDI_SUCCESS);
1968 }
1969