xref: /freebsd/sys/dev/usb/usb_dev.c (revision dc60165b73e4c4d829a2cb9fed5cce585e93d9a9)
1 /* $FreeBSD$ */
2 /*-
3  * Copyright (c) 2006-2008 Hans Petter Selasky. All rights reserved.
4  *
5  * Redistribution and use in source and binary forms, with or without
6  * modification, are permitted provided that the following conditions
7  * are met:
8  * 1. Redistributions of source code must retain the above copyright
9  *    notice, this list of conditions and the following disclaimer.
10  * 2. Redistributions in binary form must reproduce the above copyright
11  *    notice, this list of conditions and the following disclaimer in the
12  *    documentation and/or other materials provided with the distribution.
13  *
14  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24  * SUCH DAMAGE.
25  *
26  *
27  * usb2_dev.c - An abstraction layer for creating devices under /dev/...
28  */
29 
30 #include <dev/usb/usb.h>
31 #include <dev/usb/usb_ioctl.h>
32 #include <dev/usb/usb_mfunc.h>
33 #include <dev/usb/usb_error.h>
34 
35 #define	USB_DEBUG_VAR usb2_fifo_debug
36 
37 #include <dev/usb/usb_core.h>
38 #include <dev/usb/usb_mbuf.h>
39 #include <dev/usb/usb_dev.h>
40 #include <dev/usb/usb_process.h>
41 #include <dev/usb/usb_device.h>
42 #include <dev/usb/usb_debug.h>
43 #include <dev/usb/usb_busdma.h>
44 #include <dev/usb/usb_generic.h>
45 #include <dev/usb/usb_dynamic.h>
46 #include <dev/usb/usb_util.h>
47 
48 #include <dev/usb/usb_controller.h>
49 #include <dev/usb/usb_bus.h>
50 
51 #include <sys/filio.h>
52 #include <sys/ttycom.h>
53 #include <sys/syscallsubr.h>
54 
55 #include <machine/stdarg.h>
56 
57 #if USB_HAVE_UGEN
58 
59 #if USB_DEBUG
60 static int usb2_fifo_debug = 0;
61 
62 SYSCTL_NODE(_hw_usb2, OID_AUTO, dev, CTLFLAG_RW, 0, "USB device");
63 SYSCTL_INT(_hw_usb2_dev, OID_AUTO, debug, CTLFLAG_RW,
64     &usb2_fifo_debug, 0, "Debug Level");
65 #endif
66 
67 #if ((__FreeBSD_version >= 700001) || (__FreeBSD_version == 0) || \
68      ((__FreeBSD_version >= 600034) && (__FreeBSD_version < 700000)))
69 #define	USB_UCRED struct ucred *ucred,
70 #else
71 #define	USB_UCRED
72 #endif
73 
74 /* prototypes */
75 
76 static int	usb2_fifo_open(struct usb2_cdev_privdata *,
77 		    struct usb2_fifo *, int);
78 static void	usb2_fifo_close(struct usb2_fifo *, int);
79 static void	usb2_dev_init(void *);
80 static void	usb2_dev_init_post(void *);
81 static void	usb2_dev_uninit(void *);
82 static int	usb2_fifo_uiomove(struct usb2_fifo *, void *, int,
83 		    struct uio *);
84 static void	usb2_fifo_check_methods(struct usb2_fifo_methods *);
85 static struct	usb2_fifo *usb2_fifo_alloc(void);
86 static struct	usb2_pipe *usb2_dev_get_pipe(struct usb2_device *, uint8_t,
87 		    uint8_t);
88 static void	usb2_loc_fill(struct usb2_fs_privdata *,
89 		    struct usb2_cdev_privdata *);
90 static void	usb2_close(void *);
91 static usb2_error_t usb2_ref_device(struct usb2_cdev_privdata *, int);
92 static usb2_error_t usb2_uref_location(struct usb2_cdev_privdata *);
93 static void	usb2_unref_device(struct usb2_cdev_privdata *);
94 
95 static d_open_t usb2_open;
96 static d_ioctl_t usb2_ioctl;
97 static d_read_t usb2_read;
98 static d_write_t usb2_write;
99 static d_poll_t usb2_poll;
100 
101 static d_ioctl_t usb2_static_ioctl;
102 
103 static usb2_fifo_open_t usb2_fifo_dummy_open;
104 static usb2_fifo_close_t usb2_fifo_dummy_close;
105 static usb2_fifo_ioctl_t usb2_fifo_dummy_ioctl;
106 static usb2_fifo_cmd_t usb2_fifo_dummy_cmd;
107 
108 /* character device structure used for devices (/dev/ugenX.Y and /dev/uXXX) */
109 struct cdevsw usb2_devsw = {
110 	.d_version = D_VERSION,
111 	.d_open = usb2_open,
112 	.d_ioctl = usb2_ioctl,
113 	.d_name = "usbdev",
114 	.d_flags = D_TRACKCLOSE,
115 	.d_read = usb2_read,
116 	.d_write = usb2_write,
117 	.d_poll = usb2_poll
118 };
119 
120 static struct cdev* usb2_dev = NULL;
121 
122 /* character device structure used for /dev/usb */
123 struct cdevsw usb2_static_devsw = {
124 	.d_version = D_VERSION,
125 	.d_ioctl = usb2_static_ioctl,
126 	.d_name = "usb"
127 };
128 
129 static TAILQ_HEAD(, usb2_symlink) usb2_sym_head;
130 static struct sx usb2_sym_lock;
131 
132 struct mtx usb2_ref_lock;
133 
134 /*------------------------------------------------------------------------*
135  *	usb2_loc_fill
136  *
137  * This is used to fill out a usb2_cdev_privdata structure based on the
138  * device's address as contained in usb2_fs_privdata.
139  *------------------------------------------------------------------------*/
140 static void
141 usb2_loc_fill(struct usb2_fs_privdata* pd, struct usb2_cdev_privdata *cpd)
142 {
143 	cpd->bus_index = pd->bus_index;
144 	cpd->dev_index = pd->dev_index;
145 	cpd->ep_addr = pd->ep_addr;
146 	cpd->fifo_index = pd->fifo_index;
147 }
148 
149 /*------------------------------------------------------------------------*
150  *	usb2_ref_device
151  *
152  * This function is used to atomically refer an USB device by its
153  * device location. If this function returns success the USB device
154  * will not dissappear until the USB device is unreferenced.
155  *
156  * Return values:
157  *  0: Success, refcount incremented on the given USB device.
158  *  Else: Failure.
159  *------------------------------------------------------------------------*/
160 usb2_error_t
161 usb2_ref_device(struct usb2_cdev_privdata* cpd, int need_uref)
162 {
163 	struct usb2_fifo **ppf;
164 	struct usb2_fifo *f;
165 
166 	DPRINTFN(2, "usb2_ref_device, cpd=%p need uref=%d\n", cpd, need_uref);
167 
168 	mtx_lock(&usb2_ref_lock);
169 	cpd->bus = devclass_get_softc(usb2_devclass_ptr, cpd->bus_index);
170 	if (cpd->bus == NULL) {
171 		DPRINTFN(2, "no bus at %u\n", cpd->bus_index);
172 		goto error;
173 	}
174 	cpd->udev = cpd->bus->devices[cpd->dev_index];
175 	if (cpd->udev == NULL) {
176 		DPRINTFN(2, "no device at %u\n", cpd->dev_index);
177 		goto error;
178 	}
179 	if (cpd->udev->refcount == USB_DEV_REF_MAX) {
180 		DPRINTFN(2, "no dev ref\n");
181 		goto error;
182 	}
183 	/* check if we are doing an open */
184 	if (cpd->fflags == 0) {
185 		/* set defaults */
186 		cpd->txfifo = NULL;
187 		cpd->rxfifo = NULL;
188 		cpd->is_write = 0;
189 		cpd->is_read = 0;
190 		cpd->is_usbfs = 0;
191 	} else {
192 		/* initialise "is_usbfs" flag */
193 		cpd->is_usbfs = 0;
194 
195 		/* check for write */
196 		if (cpd->fflags & FWRITE) {
197 			ppf = cpd->udev->fifo;
198 			f = ppf[cpd->fifo_index + USB_FIFO_TX];
199 			cpd->txfifo = f;
200 			cpd->is_write = 1;	/* ref */
201 			if (f == NULL || f->refcount == USB_FIFO_REF_MAX)
202 				goto error;
203 			if (f->curr_cpd != cpd)
204 				goto error;
205 			/* check if USB-FS is active */
206 			if (f->fs_ep_max != 0) {
207 				cpd->is_usbfs = 1;
208 			}
209 		} else {
210 			cpd->txfifo = NULL;
211 			cpd->is_write = 0;	/* no ref */
212 		}
213 
214 		/* check for read */
215 		if (cpd->fflags & FREAD) {
216 			ppf = cpd->udev->fifo;
217 			f = ppf[cpd->fifo_index + USB_FIFO_RX];
218 			cpd->rxfifo = f;
219 			cpd->is_read = 1;	/* ref */
220 			if (f == NULL || f->refcount == USB_FIFO_REF_MAX)
221 				goto error;
222 			if (f->curr_cpd != cpd)
223 				goto error;
224 			/* check if USB-FS is active */
225 			if (f->fs_ep_max != 0) {
226 				cpd->is_usbfs = 1;
227 			}
228 		} else {
229 			cpd->rxfifo = NULL;
230 			cpd->is_read = 0;	/* no ref */
231 		}
232 	}
233 
234 	/* when everything is OK we increment the refcounts */
235 	if (cpd->is_write) {
236 		DPRINTFN(2, "ref write\n");
237 		cpd->txfifo->refcount++;
238 	}
239 	if (cpd->is_read) {
240 		DPRINTFN(2, "ref read\n");
241 		cpd->rxfifo->refcount++;
242 	}
243 	if (need_uref) {
244 		DPRINTFN(2, "ref udev - needed\n");
245 		cpd->udev->refcount++;
246 		cpd->is_uref = 1;
247 	}
248 	mtx_unlock(&usb2_ref_lock);
249 
250 	if (cpd->is_uref) {
251 		/*
252 		 * We are about to alter the bus-state. Apply the
253 		 * required locks.
254 		 */
255 		sx_xlock(cpd->udev->default_sx + 1);
256 		mtx_lock(&Giant);	/* XXX */
257 	}
258 	return (0);
259 
260 error:
261 	mtx_unlock(&usb2_ref_lock);
262 	DPRINTFN(2, "fail\n");
263 	return (USB_ERR_INVAL);
264 }
265 
266 /*------------------------------------------------------------------------*
267  *	usb2_uref_location
268  *
269  * This function is used to upgrade an USB reference to include the
270  * USB device reference on a USB location.
271  *
272  * Return values:
273  *  0: Success, refcount incremented on the given USB device.
274  *  Else: Failure.
275  *------------------------------------------------------------------------*/
276 static usb2_error_t
277 usb2_uref_location(struct usb2_cdev_privdata *cpd)
278 {
279 	/*
280 	 * Check if we already got an USB reference on this location:
281 	 */
282 	if (cpd->is_uref) {
283 		return (0);		/* success */
284 	}
285 	mtx_lock(&usb2_ref_lock);
286 	if (cpd->bus != devclass_get_softc(usb2_devclass_ptr, cpd->bus_index)) {
287 		DPRINTFN(2, "bus changed at %u\n", cpd->bus_index);
288 		goto error;
289 	}
290 	if (cpd->udev != cpd->bus->devices[cpd->dev_index]) {
291 		DPRINTFN(2, "device changed at %u\n", cpd->dev_index);
292 		goto error;
293 	}
294 	if (cpd->udev->refcount == USB_DEV_REF_MAX) {
295 		DPRINTFN(2, "no dev ref\n");
296 		goto error;
297 	}
298 	DPRINTFN(2, "ref udev\n");
299 	cpd->udev->refcount++;
300 	mtx_unlock(&usb2_ref_lock);
301 
302 	/* set "uref" */
303 	cpd->is_uref = 1;
304 
305 	/*
306 	 * We are about to alter the bus-state. Apply the
307 	 * required locks.
308 	 */
309 	sx_xlock(cpd->udev->default_sx + 1);
310 	mtx_lock(&Giant);		/* XXX */
311 	return (0);
312 
313 error:
314 	mtx_unlock(&usb2_ref_lock);
315 	DPRINTFN(2, "fail\n");
316 	return (USB_ERR_INVAL);
317 }
318 
319 /*------------------------------------------------------------------------*
320  *	usb2_unref_device
321  *
322  * This function will release the reference count by one unit for the
323  * given USB device.
324  *------------------------------------------------------------------------*/
325 void
326 usb2_unref_device(struct usb2_cdev_privdata *cpd)
327 {
328 	if (cpd->is_uref) {
329 		mtx_unlock(&Giant);	/* XXX */
330 		sx_unlock(cpd->udev->default_sx + 1);
331 	}
332 	mtx_lock(&usb2_ref_lock);
333 	if (cpd->is_read) {
334 		if (--(cpd->rxfifo->refcount) == 0) {
335 			usb2_cv_signal(&cpd->rxfifo->cv_drain);
336 		}
337 		cpd->is_read = 0;
338 	}
339 	if (cpd->is_write) {
340 		if (--(cpd->txfifo->refcount) == 0) {
341 			usb2_cv_signal(&cpd->txfifo->cv_drain);
342 		}
343 		cpd->is_write = 0;
344 	}
345 	if (cpd->is_uref) {
346 		if (--(cpd->udev->refcount) == 0) {
347 			usb2_cv_signal(cpd->udev->default_cv + 1);
348 		}
349 		cpd->is_uref = 0;
350 	}
351 	mtx_unlock(&usb2_ref_lock);
352 }
353 
354 static struct usb2_fifo *
355 usb2_fifo_alloc(void)
356 {
357 	struct usb2_fifo *f;
358 
359 	f = malloc(sizeof(*f), M_USBDEV, M_WAITOK | M_ZERO);
360 	if (f) {
361 		usb2_cv_init(&f->cv_io, "FIFO-IO");
362 		usb2_cv_init(&f->cv_drain, "FIFO-DRAIN");
363 		f->refcount = 1;
364 	}
365 	return (f);
366 }
367 
368 /*------------------------------------------------------------------------*
369  *	usb2_fifo_create
370  *------------------------------------------------------------------------*/
371 static int
372 usb2_fifo_create(struct usb2_cdev_privdata *cpd)
373 {
374 	struct usb2_device *udev = cpd->udev;
375 	struct usb2_fifo *f;
376 	struct usb2_pipe *pipe;
377 	uint8_t n;
378 	uint8_t is_tx;
379 	uint8_t is_rx;
380 	uint8_t no_null;
381 	uint8_t is_busy;
382 	int ep = cpd->ep_addr;
383 
384 	is_tx = (cpd->fflags & FWRITE) ? 1 : 0;
385 	is_rx = (cpd->fflags & FREAD) ? 1 : 0;
386 	no_null = 1;
387 	is_busy = 0;
388 
389 	/* Preallocated FIFO */
390 	if (ep < 0) {
391 		DPRINTFN(5, "Preallocated FIFO\n");
392 		if (is_tx) {
393 			f = udev->fifo[cpd->fifo_index + USB_FIFO_TX];
394 			if (f == NULL)
395 				return (EINVAL);
396 			cpd->txfifo = f;
397 		}
398 		if (is_rx) {
399 			f = udev->fifo[cpd->fifo_index + USB_FIFO_RX];
400 			if (f == NULL)
401 				return (EINVAL);
402 			cpd->rxfifo = f;
403 		}
404 		return (0);
405 	}
406 
407 	KASSERT(ep >= 0 && ep <= 15, ("endpoint %d out of range", ep));
408 
409 	/* search for a free FIFO slot */
410 	DPRINTFN(5, "Endpoint device, searching for 0x%02x\n", ep);
411 	for (n = 0;; n += 2) {
412 
413 		if (n == USB_FIFO_MAX) {
414 			if (no_null) {
415 				no_null = 0;
416 				n = 0;
417 			} else {
418 				/* end of FIFOs reached */
419 				DPRINTFN(5, "out of FIFOs\n");
420 				return (ENOMEM);
421 			}
422 		}
423 		/* Check for TX FIFO */
424 		if (is_tx) {
425 			f = udev->fifo[n + USB_FIFO_TX];
426 			if (f != NULL) {
427 				if (f->dev_ep_index != ep) {
428 					/* wrong endpoint index */
429 					continue;
430 				}
431 				if (f->curr_cpd != NULL) {
432 					/* FIFO is opened */
433 					is_busy = 1;
434 					continue;
435 				}
436 			} else if (no_null) {
437 				continue;
438 			}
439 		}
440 		/* Check for RX FIFO */
441 		if (is_rx) {
442 			f = udev->fifo[n + USB_FIFO_RX];
443 			if (f != NULL) {
444 				if (f->dev_ep_index != ep) {
445 					/* wrong endpoint index */
446 					continue;
447 				}
448 				if (f->curr_cpd != NULL) {
449 					/* FIFO is opened */
450 					is_busy = 1;
451 					continue;
452 				}
453 			} else if (no_null) {
454 				continue;
455 			}
456 		}
457 		break;
458 	}
459 
460 	if (no_null == 0) {
461 		if (ep >= (USB_EP_MAX / 2)) {
462 			/* we don't create any endpoints in this range */
463 			DPRINTFN(5, "ep out of range\n");
464 			return (is_busy ? EBUSY : EINVAL);
465 		}
466 	}
467 
468 	if ((ep != 0) && is_busy) {
469 		/*
470 		 * Only the default control endpoint is allowed to be
471 		 * opened multiple times!
472 		 */
473 		DPRINTFN(5, "busy\n");
474 		return (EBUSY);
475 	}
476 
477 	/* Check TX FIFO */
478 	if (is_tx &&
479 	    (udev->fifo[n + USB_FIFO_TX] == NULL)) {
480 		pipe = usb2_dev_get_pipe(udev, ep, USB_FIFO_TX);
481 		DPRINTFN(5, "dev_get_pipe(%d, 0x%x)\n", ep, USB_FIFO_TX);
482 		if (pipe == NULL) {
483 			DPRINTFN(5, "dev_get_pipe returned NULL\n");
484 			return (EINVAL);
485 		}
486 		f = usb2_fifo_alloc();
487 		if (f == NULL) {
488 			DPRINTFN(5, "could not alloc tx fifo\n");
489 			return (ENOMEM);
490 		}
491 		/* update some fields */
492 		f->fifo_index = n + USB_FIFO_TX;
493 		f->dev_ep_index = ep;
494 		f->priv_mtx = udev->default_mtx;
495 		f->priv_sc0 = pipe;
496 		f->methods = &usb2_ugen_methods;
497 		f->iface_index = pipe->iface_index;
498 		f->udev = udev;
499 		mtx_lock(&usb2_ref_lock);
500 		udev->fifo[n + USB_FIFO_TX] = f;
501 		mtx_unlock(&usb2_ref_lock);
502 	}
503 	/* Check RX FIFO */
504 	if (is_rx &&
505 	    (udev->fifo[n + USB_FIFO_RX] == NULL)) {
506 
507 		pipe = usb2_dev_get_pipe(udev, ep, USB_FIFO_RX);
508 		DPRINTFN(5, "dev_get_pipe(%d, 0x%x)\n", ep, USB_FIFO_RX);
509 		if (pipe == NULL) {
510 			DPRINTFN(5, "dev_get_pipe returned NULL\n");
511 			return (EINVAL);
512 		}
513 		f = usb2_fifo_alloc();
514 		if (f == NULL) {
515 			DPRINTFN(5, "could not alloc rx fifo\n");
516 			return (ENOMEM);
517 		}
518 		/* update some fields */
519 		f->fifo_index = n + USB_FIFO_RX;
520 		f->dev_ep_index = ep;
521 		f->priv_mtx = udev->default_mtx;
522 		f->priv_sc0 = pipe;
523 		f->methods = &usb2_ugen_methods;
524 		f->iface_index = pipe->iface_index;
525 		f->udev = udev;
526 		mtx_lock(&usb2_ref_lock);
527 		udev->fifo[n + USB_FIFO_RX] = f;
528 		mtx_unlock(&usb2_ref_lock);
529 	}
530 	if (is_tx) {
531 		cpd->txfifo = udev->fifo[n + USB_FIFO_TX];
532 	}
533 	if (is_rx) {
534 		cpd->rxfifo = udev->fifo[n + USB_FIFO_RX];
535 	}
536 	/* fill out fifo index */
537 	DPRINTFN(5, "fifo index = %d\n", n);
538 	cpd->fifo_index = n;
539 
540 	/* complete */
541 
542 	return (0);
543 }
544 
545 void
546 usb2_fifo_free(struct usb2_fifo *f)
547 {
548 	uint8_t n;
549 
550 	if (f == NULL) {
551 		/* be NULL safe */
552 		return;
553 	}
554 	/* destroy symlink devices, if any */
555 	for (n = 0; n != 2; n++) {
556 		if (f->symlink[n]) {
557 			usb2_free_symlink(f->symlink[n]);
558 			f->symlink[n] = NULL;
559 		}
560 	}
561 	mtx_lock(&usb2_ref_lock);
562 
563 	/* delink ourselves to stop calls from userland */
564 	if ((f->fifo_index < USB_FIFO_MAX) &&
565 	    (f->udev != NULL) &&
566 	    (f->udev->fifo[f->fifo_index] == f)) {
567 		f->udev->fifo[f->fifo_index] = NULL;
568 	} else {
569 		DPRINTFN(0, "USB FIFO %p has not been linked!\n", f);
570 	}
571 
572 	/* decrease refcount */
573 	f->refcount--;
574 	/* prevent any write flush */
575 	f->flag_iserror = 1;
576 	/* need to wait until all callers have exited */
577 	while (f->refcount != 0) {
578 		mtx_unlock(&usb2_ref_lock);	/* avoid LOR */
579 		mtx_lock(f->priv_mtx);
580 		/* get I/O thread out of any sleep state */
581 		if (f->flag_sleeping) {
582 			f->flag_sleeping = 0;
583 			usb2_cv_broadcast(&f->cv_io);
584 		}
585 		mtx_unlock(f->priv_mtx);
586 		mtx_lock(&usb2_ref_lock);
587 
588 		/* wait for sync */
589 		usb2_cv_wait(&f->cv_drain, &usb2_ref_lock);
590 	}
591 	mtx_unlock(&usb2_ref_lock);
592 
593 	/* take care of closing the device here, if any */
594 	usb2_fifo_close(f, 0);
595 
596 	usb2_cv_destroy(&f->cv_io);
597 	usb2_cv_destroy(&f->cv_drain);
598 
599 	free(f, M_USBDEV);
600 }
601 
602 static struct usb2_pipe *
603 usb2_dev_get_pipe(struct usb2_device *udev, uint8_t ep_index, uint8_t dir)
604 {
605 	struct usb2_pipe *pipe;
606 	uint8_t ep_dir;
607 
608 	if (ep_index == 0) {
609 		pipe = &udev->default_pipe;
610 	} else {
611 		if (dir == USB_FIFO_RX) {
612 			if (udev->flags.usb2_mode == USB_MODE_HOST) {
613 				ep_dir = UE_DIR_IN;
614 			} else {
615 				ep_dir = UE_DIR_OUT;
616 			}
617 		} else {
618 			if (udev->flags.usb2_mode == USB_MODE_HOST) {
619 				ep_dir = UE_DIR_OUT;
620 			} else {
621 				ep_dir = UE_DIR_IN;
622 			}
623 		}
624 		pipe = usb2_get_pipe_by_addr(udev, ep_index | ep_dir);
625 	}
626 
627 	if (pipe == NULL) {
628 		/* if the pipe does not exist then return */
629 		return (NULL);
630 	}
631 	if (pipe->edesc == NULL) {
632 		/* invalid pipe */
633 		return (NULL);
634 	}
635 	return (pipe);			/* success */
636 }
637 
638 /*------------------------------------------------------------------------*
639  *	usb2_fifo_open
640  *
641  * Returns:
642  * 0: Success
643  * Else: Failure
644  *------------------------------------------------------------------------*/
645 static int
646 usb2_fifo_open(struct usb2_cdev_privdata *cpd,
647     struct usb2_fifo *f, int fflags)
648 {
649 	int err;
650 
651 	if (f == NULL) {
652 		/* no FIFO there */
653 		DPRINTFN(2, "no FIFO\n");
654 		return (ENXIO);
655 	}
656 	/* remove FWRITE and FREAD flags */
657 	fflags &= ~(FWRITE | FREAD);
658 
659 	/* set correct file flags */
660 	if ((f->fifo_index & 1) == USB_FIFO_TX) {
661 		fflags |= FWRITE;
662 	} else {
663 		fflags |= FREAD;
664 	}
665 
666 	/* check if we are already opened */
667 	/* we don't need any locks when checking this variable */
668 	if (f->curr_cpd != NULL) {
669 		err = EBUSY;
670 		goto done;
671 	}
672 
673 	/* reset short flag before open */
674 	f->flag_short = 0;
675 
676 	/* call open method */
677 	err = (f->methods->f_open) (f, fflags);
678 	if (err) {
679 		goto done;
680 	}
681 	mtx_lock(f->priv_mtx);
682 
683 	/* reset sleep flag */
684 	f->flag_sleeping = 0;
685 
686 	/* reset error flag */
687 	f->flag_iserror = 0;
688 
689 	/* reset complete flag */
690 	f->flag_iscomplete = 0;
691 
692 	/* reset select flag */
693 	f->flag_isselect = 0;
694 
695 	/* reset flushing flag */
696 	f->flag_flushing = 0;
697 
698 	/* reset ASYNC proc flag */
699 	f->async_p = NULL;
700 
701 	mtx_lock(&usb2_ref_lock);
702 	/* flag the fifo as opened to prevent others */
703 	f->curr_cpd = cpd;
704 	mtx_unlock(&usb2_ref_lock);
705 
706 	/* reset queue */
707 	usb2_fifo_reset(f);
708 
709 	mtx_unlock(f->priv_mtx);
710 done:
711 	return (err);
712 }
713 
714 /*------------------------------------------------------------------------*
715  *	usb2_fifo_reset
716  *------------------------------------------------------------------------*/
717 void
718 usb2_fifo_reset(struct usb2_fifo *f)
719 {
720 	struct usb2_mbuf *m;
721 
722 	if (f == NULL) {
723 		return;
724 	}
725 	while (1) {
726 		USB_IF_DEQUEUE(&f->used_q, m);
727 		if (m) {
728 			USB_IF_ENQUEUE(&f->free_q, m);
729 		} else {
730 			break;
731 		}
732 	}
733 }
734 
735 /*------------------------------------------------------------------------*
736  *	usb2_fifo_close
737  *------------------------------------------------------------------------*/
738 static void
739 usb2_fifo_close(struct usb2_fifo *f, int fflags)
740 {
741 	int err;
742 
743 	/* check if we are not opened */
744 	if (f->curr_cpd == NULL) {
745 		/* nothing to do - already closed */
746 		return;
747 	}
748 	mtx_lock(f->priv_mtx);
749 
750 	/* clear current cdev private data pointer */
751 	f->curr_cpd = NULL;
752 
753 	/* check if we are selected */
754 	if (f->flag_isselect) {
755 		selwakeup(&f->selinfo);
756 		f->flag_isselect = 0;
757 	}
758 	/* check if a thread wants SIGIO */
759 	if (f->async_p != NULL) {
760 		PROC_LOCK(f->async_p);
761 		psignal(f->async_p, SIGIO);
762 		PROC_UNLOCK(f->async_p);
763 		f->async_p = NULL;
764 	}
765 	/* remove FWRITE and FREAD flags */
766 	fflags &= ~(FWRITE | FREAD);
767 
768 	/* flush written data, if any */
769 	if ((f->fifo_index & 1) == USB_FIFO_TX) {
770 
771 		if (!f->flag_iserror) {
772 
773 			/* set flushing flag */
774 			f->flag_flushing = 1;
775 
776 			/* start write transfer, if not already started */
777 			(f->methods->f_start_write) (f);
778 
779 			/* check if flushed already */
780 			while (f->flag_flushing &&
781 			    (!f->flag_iserror)) {
782 				/* wait until all data has been written */
783 				f->flag_sleeping = 1;
784 				err = usb2_cv_wait_sig(&f->cv_io, f->priv_mtx);
785 				if (err) {
786 					DPRINTF("signal received\n");
787 					break;
788 				}
789 			}
790 		}
791 		fflags |= FWRITE;
792 
793 		/* stop write transfer, if not already stopped */
794 		(f->methods->f_stop_write) (f);
795 	} else {
796 		fflags |= FREAD;
797 
798 		/* stop write transfer, if not already stopped */
799 		(f->methods->f_stop_read) (f);
800 	}
801 
802 	/* check if we are sleeping */
803 	if (f->flag_sleeping) {
804 		DPRINTFN(2, "Sleeping at close!\n");
805 	}
806 	mtx_unlock(f->priv_mtx);
807 
808 	/* call close method */
809 	(f->methods->f_close) (f, fflags);
810 
811 	DPRINTF("closed\n");
812 }
813 
814 /*------------------------------------------------------------------------*
815  *	usb2_open - cdev callback
816  *------------------------------------------------------------------------*/
817 static int
818 usb2_open(struct cdev *dev, int fflags, int devtype, struct thread *td)
819 {
820 	struct usb2_fs_privdata* pd = (struct usb2_fs_privdata*)dev->si_drv1;
821 	struct usb2_cdev_privdata *cpd;
822 	int err, ep;
823 
824 	DPRINTFN(2, "%s fflags=0x%08x\n", dev->si_name, fflags);
825 
826 	KASSERT(fflags & (FREAD|FWRITE), ("invalid open flags"));
827 	if (((fflags & FREAD) && !(pd->mode & FREAD)) ||
828 	    ((fflags & FWRITE) && !(pd->mode & FWRITE))) {
829 		DPRINTFN(2, "access mode not supported\n");
830 		return (EPERM);
831 	}
832 
833 	cpd = malloc(sizeof(*cpd), M_USBDEV, M_WAITOK | M_ZERO);
834 	ep = cpd->ep_addr = pd->ep_addr;
835 
836 	usb2_loc_fill(pd, cpd);
837 	err = usb2_ref_device(cpd, 1);
838 	if (err) {
839 		DPRINTFN(2, "cannot ref device\n");
840 		free(cpd, M_USBDEV);
841 		return (ENXIO);
842 	}
843 	cpd->fflags = fflags;	/* access mode for open lifetime */
844 
845 	/* create FIFOs, if any */
846 	err = usb2_fifo_create(cpd);
847 	/* check for error */
848 	if (err) {
849 		DPRINTFN(2, "cannot create fifo\n");
850 		usb2_unref_device(cpd);
851 		free(cpd, M_USBDEV);
852 		return (err);
853 	}
854 	if (fflags & FREAD) {
855 		err = usb2_fifo_open(cpd, cpd->rxfifo, fflags);
856 		if (err) {
857 			DPRINTFN(2, "read open failed\n");
858 			usb2_unref_device(cpd);
859 			free(cpd, M_USBDEV);
860 			return (err);
861 		}
862 	}
863 	if (fflags & FWRITE) {
864 		err = usb2_fifo_open(cpd, cpd->txfifo, fflags);
865 		if (err) {
866 			DPRINTFN(2, "write open failed\n");
867 			if (fflags & FREAD) {
868 				usb2_fifo_close(cpd->rxfifo, fflags);
869 			}
870 			usb2_unref_device(cpd);
871 			free(cpd, M_USBDEV);
872 			return (err);
873 		}
874 	}
875 	usb2_unref_device(cpd);
876 	devfs_set_cdevpriv(cpd, usb2_close);
877 
878 	return (0);
879 }
880 
881 /*------------------------------------------------------------------------*
882  *	usb2_close - cdev callback
883  *------------------------------------------------------------------------*/
884 static void
885 usb2_close(void *arg)
886 {
887 	struct usb2_cdev_privdata *cpd = arg;
888 	int err;
889 
890 	DPRINTFN(2, "cpd=%p\n", cpd);
891 
892 	err = usb2_ref_device(cpd, 1);
893 	if (err) {
894 		free(cpd, M_USBDEV);
895 		return;
896 	}
897 	if (cpd->fflags & FREAD) {
898 		usb2_fifo_close(cpd->rxfifo, cpd->fflags);
899 	}
900 	if (cpd->fflags & FWRITE) {
901 		usb2_fifo_close(cpd->txfifo, cpd->fflags);
902 	}
903 
904 	usb2_unref_device(cpd);
905 	free(cpd, M_USBDEV);
906 	return;
907 }
908 
909 static void
910 usb2_dev_init(void *arg)
911 {
912 	mtx_init(&usb2_ref_lock, "USB ref mutex", NULL, MTX_DEF);
913 	sx_init(&usb2_sym_lock, "USB sym mutex");
914 	TAILQ_INIT(&usb2_sym_head);
915 
916 	/* check the UGEN methods */
917 	usb2_fifo_check_methods(&usb2_ugen_methods);
918 }
919 
920 SYSINIT(usb2_dev_init, SI_SUB_KLD, SI_ORDER_FIRST, usb2_dev_init, NULL);
921 
922 static void
923 usb2_dev_init_post(void *arg)
924 {
925 	/*
926 	 * Create /dev/usb - this is needed for usbconfig(8), which
927 	 * needs a well-known device name to access.
928 	 */
929 	usb2_dev = make_dev(&usb2_static_devsw, 0, UID_ROOT, GID_OPERATOR,
930 	    0644, USB_DEVICE_NAME);
931 	if (usb2_dev == NULL) {
932 		DPRINTFN(0, "Could not create usb bus device!\n");
933 	}
934 }
935 
936 SYSINIT(usb2_dev_init_post, SI_SUB_KICK_SCHEDULER, SI_ORDER_FIRST, usb2_dev_init_post, NULL);
937 
938 static void
939 usb2_dev_uninit(void *arg)
940 {
941 	if (usb2_dev != NULL) {
942 		destroy_dev(usb2_dev);
943 		usb2_dev = NULL;
944 
945 	}
946 	mtx_destroy(&usb2_ref_lock);
947 	sx_destroy(&usb2_sym_lock);
948 }
949 
950 SYSUNINIT(usb2_dev_uninit, SI_SUB_KICK_SCHEDULER, SI_ORDER_ANY, usb2_dev_uninit, NULL);
951 
952 static int
953 usb2_ioctl_f_sub(struct usb2_fifo *f, u_long cmd, void *addr,
954     struct thread *td)
955 {
956 	int error = 0;
957 
958 	switch (cmd) {
959 	case FIODTYPE:
960 		*(int *)addr = 0;	/* character device */
961 		break;
962 
963 	case FIONBIO:
964 		/* handled by upper FS layer */
965 		break;
966 
967 	case FIOASYNC:
968 		if (*(int *)addr) {
969 			if (f->async_p != NULL) {
970 				error = EBUSY;
971 				break;
972 			}
973 			f->async_p = USB_TD_GET_PROC(td);
974 		} else {
975 			f->async_p = NULL;
976 		}
977 		break;
978 
979 		/* XXX this is not the most general solution */
980 	case TIOCSPGRP:
981 		if (f->async_p == NULL) {
982 			error = EINVAL;
983 			break;
984 		}
985 		if (*(int *)addr != USB_PROC_GET_GID(f->async_p)) {
986 			error = EPERM;
987 			break;
988 		}
989 		break;
990 	default:
991 		return (ENOIOCTL);
992 	}
993 	DPRINTFN(3, "cmd 0x%lx = %d\n", cmd, error);
994 	return (error);
995 }
996 
997 /*------------------------------------------------------------------------*
998  *	usb2_ioctl - cdev callback
999  *------------------------------------------------------------------------*/
1000 static int
1001 usb2_ioctl(struct cdev *dev, u_long cmd, caddr_t addr, int fflag, struct thread* td)
1002 {
1003 	struct usb2_cdev_privdata* cpd;
1004 	struct usb2_fifo *f;
1005 	int fflags;
1006 	int err;
1007 
1008 	DPRINTFN(2, "cmd=0x%lx\n", cmd);
1009 
1010 	err = devfs_get_cdevpriv((void **)&cpd);
1011 	if (err != 0)
1012 		return (err);
1013 
1014 	/*
1015 	 * Performance optimistaion: We try to check for IOCTL's that
1016 	 * don't need the USB reference first. Then we grab the USB
1017 	 * reference if we need it!
1018 	 */
1019 	err = usb2_ref_device(cpd, 0 /* no uref */ );
1020 	if (err) {
1021 		return (ENXIO);
1022 	}
1023 	fflags = cpd->fflags;
1024 
1025 	f = NULL;			/* set default value */
1026 	err = ENOIOCTL;			/* set default value */
1027 
1028 	if (fflags & FWRITE) {
1029 		f = cpd->txfifo;
1030 		err = usb2_ioctl_f_sub(f, cmd, addr, td);
1031 	}
1032 	if (fflags & FREAD) {
1033 		f = cpd->rxfifo;
1034 		err = usb2_ioctl_f_sub(f, cmd, addr, td);
1035 	}
1036 	KASSERT(f != NULL, ("fifo not found"));
1037 	if (err == ENOIOCTL) {
1038 		err = (f->methods->f_ioctl) (f, cmd, addr, fflags);
1039 		DPRINTFN(2, "f_ioctl cmd 0x%lx = %d\n", cmd, err);
1040 		if (err == ENOIOCTL) {
1041 			if (usb2_uref_location(cpd)) {
1042 				err = ENXIO;
1043 				goto done;
1044 			}
1045 			err = (f->methods->f_ioctl_post) (f, cmd, addr, fflags);
1046 			DPRINTFN(2, "f_ioctl_post cmd 0x%lx = %d\n", cmd, err);
1047 		}
1048 	}
1049 	if (err == ENOIOCTL) {
1050 		err = ENOTTY;
1051 	}
1052 done:
1053 	usb2_unref_device(cpd);
1054 	return (err);
1055 }
1056 
1057 /* ARGSUSED */
1058 static int
1059 usb2_poll(struct cdev* dev, int events, struct thread* td)
1060 {
1061 	struct usb2_cdev_privdata* cpd;
1062 	struct usb2_fifo *f;
1063 	struct usb2_mbuf *m;
1064 	int fflags, revents;
1065 
1066 	if (devfs_get_cdevpriv((void **)&cpd) != 0 ||
1067 	    usb2_ref_device(cpd, 0) != 0)
1068 		return (events &
1069 		    (POLLHUP|POLLIN|POLLRDNORM|POLLOUT|POLLWRNORM));
1070 
1071 	fflags = cpd->fflags;
1072 
1073 	/* Figure out who needs service */
1074 	revents = 0;
1075 	if ((events & (POLLOUT | POLLWRNORM)) &&
1076 	    (fflags & FWRITE)) {
1077 
1078 		f = cpd->txfifo;
1079 
1080 		mtx_lock(f->priv_mtx);
1081 
1082 		if (!cpd->is_usbfs) {
1083 			if (f->flag_iserror) {
1084 				/* we got an error */
1085 				m = (void *)1;
1086 			} else {
1087 				if (f->queue_data == NULL) {
1088 					/*
1089 					 * start write transfer, if not
1090 					 * already started
1091 					 */
1092 					(f->methods->f_start_write) (f);
1093 				}
1094 				/* check if any packets are available */
1095 				USB_IF_POLL(&f->free_q, m);
1096 			}
1097 		} else {
1098 			if (f->flag_iscomplete) {
1099 				m = (void *)1;
1100 			} else {
1101 				m = NULL;
1102 			}
1103 		}
1104 
1105 		if (m) {
1106 			revents |= events & (POLLOUT | POLLWRNORM);
1107 		} else {
1108 			f->flag_isselect = 1;
1109 			selrecord(td, &f->selinfo);
1110 		}
1111 
1112 		mtx_unlock(f->priv_mtx);
1113 	}
1114 	if ((events & (POLLIN | POLLRDNORM)) &&
1115 	    (fflags & FREAD)) {
1116 
1117 		f = cpd->rxfifo;
1118 
1119 		mtx_lock(f->priv_mtx);
1120 
1121 		if (!cpd->is_usbfs) {
1122 			if (f->flag_iserror) {
1123 				/* we have and error */
1124 				m = (void *)1;
1125 			} else {
1126 				if (f->queue_data == NULL) {
1127 					/*
1128 					 * start read transfer, if not
1129 					 * already started
1130 					 */
1131 					(f->methods->f_start_read) (f);
1132 				}
1133 				/* check if any packets are available */
1134 				USB_IF_POLL(&f->used_q, m);
1135 			}
1136 		} else {
1137 			if (f->flag_iscomplete) {
1138 				m = (void *)1;
1139 			} else {
1140 				m = NULL;
1141 			}
1142 		}
1143 
1144 		if (m) {
1145 			revents |= events & (POLLIN | POLLRDNORM);
1146 		} else {
1147 			f->flag_isselect = 1;
1148 			selrecord(td, &f->selinfo);
1149 
1150 			if (!cpd->is_usbfs) {
1151 				/* start reading data */
1152 				(f->methods->f_start_read) (f);
1153 			}
1154 		}
1155 
1156 		mtx_unlock(f->priv_mtx);
1157 	}
1158 	usb2_unref_device(cpd);
1159 	return (revents);
1160 }
1161 
1162 static int
1163 usb2_read(struct cdev *dev, struct uio *uio, int ioflag)
1164 {
1165 	struct usb2_cdev_privdata* cpd;
1166 	struct usb2_fifo *f;
1167 	struct usb2_mbuf *m;
1168 	int fflags;
1169 	int resid;
1170 	int io_len;
1171 	int err;
1172 	uint8_t tr_data = 0;
1173 
1174 	err = devfs_get_cdevpriv((void **)&cpd);
1175 	if (err != 0)
1176 		return (err);
1177 
1178 	err = usb2_ref_device(cpd, 0 /* no uref */ );
1179 	if (err) {
1180 		return (ENXIO);
1181 	}
1182 	fflags = cpd->fflags;
1183 
1184 	f = cpd->rxfifo;
1185 	if (f == NULL) {
1186 		/* should not happen */
1187 		return (EPERM);
1188 	}
1189 
1190 	resid = uio->uio_resid;
1191 
1192 	mtx_lock(f->priv_mtx);
1193 
1194 	/* check for permanent read error */
1195 	if (f->flag_iserror) {
1196 		err = EIO;
1197 		goto done;
1198 	}
1199 	/* check if USB-FS interface is active */
1200 	if (cpd->is_usbfs) {
1201 		/*
1202 		 * The queue is used for events that should be
1203 		 * retrieved using the "USB_FS_COMPLETE" ioctl.
1204 		 */
1205 		err = EINVAL;
1206 		goto done;
1207 	}
1208 	while (uio->uio_resid > 0) {
1209 
1210 		USB_IF_DEQUEUE(&f->used_q, m);
1211 
1212 		if (m == NULL) {
1213 
1214 			/* start read transfer, if not already started */
1215 
1216 			(f->methods->f_start_read) (f);
1217 
1218 			if (fflags & IO_NDELAY) {
1219 				if (tr_data) {
1220 					/* return length before error */
1221 					break;
1222 				}
1223 				err = EWOULDBLOCK;
1224 				break;
1225 			}
1226 			DPRINTF("sleeping\n");
1227 
1228 			err = usb2_fifo_wait(f);
1229 			if (err) {
1230 				break;
1231 			}
1232 			continue;
1233 		}
1234 		if (f->methods->f_filter_read) {
1235 			/*
1236 			 * Sometimes it is convenient to process data at the
1237 			 * expense of a userland process instead of a kernel
1238 			 * process.
1239 			 */
1240 			(f->methods->f_filter_read) (f, m);
1241 		}
1242 		tr_data = 1;
1243 
1244 		io_len = MIN(m->cur_data_len, uio->uio_resid);
1245 
1246 		DPRINTFN(2, "transfer %d bytes from %p\n",
1247 		    io_len, m->cur_data_ptr);
1248 
1249 		err = usb2_fifo_uiomove(f,
1250 		    m->cur_data_ptr, io_len, uio);
1251 
1252 		m->cur_data_len -= io_len;
1253 		m->cur_data_ptr += io_len;
1254 
1255 		if (m->cur_data_len == 0) {
1256 
1257 			uint8_t last_packet;
1258 
1259 			last_packet = m->last_packet;
1260 
1261 			USB_IF_ENQUEUE(&f->free_q, m);
1262 
1263 			if (last_packet) {
1264 				/* keep framing */
1265 				break;
1266 			}
1267 		} else {
1268 			USB_IF_PREPEND(&f->used_q, m);
1269 		}
1270 
1271 		if (err) {
1272 			break;
1273 		}
1274 	}
1275 done:
1276 	mtx_unlock(f->priv_mtx);
1277 
1278 	usb2_unref_device(cpd);
1279 
1280 	return (err);
1281 }
1282 
1283 static int
1284 usb2_write(struct cdev *dev, struct uio *uio, int ioflag)
1285 {
1286 	struct usb2_cdev_privdata* cpd;
1287 	struct usb2_fifo *f;
1288 	struct usb2_mbuf *m;
1289 	int fflags;
1290 	int resid;
1291 	int io_len;
1292 	int err;
1293 	uint8_t tr_data = 0;
1294 
1295 	DPRINTFN(2, "\n");
1296 
1297 	err = devfs_get_cdevpriv((void **)&cpd);
1298 	if (err != 0)
1299 		return (err);
1300 
1301 	err = usb2_ref_device(cpd, 0 /* no uref */ );
1302 	if (err) {
1303 		return (ENXIO);
1304 	}
1305 	fflags = cpd->fflags;
1306 
1307 	f = cpd->txfifo;
1308 	if (f == NULL) {
1309 		/* should not happen */
1310 		usb2_unref_device(cpd);
1311 		return (EPERM);
1312 	}
1313 	resid = uio->uio_resid;
1314 
1315 	mtx_lock(f->priv_mtx);
1316 
1317 	/* check for permanent write error */
1318 	if (f->flag_iserror) {
1319 		err = EIO;
1320 		goto done;
1321 	}
1322 	/* check if USB-FS interface is active */
1323 	if (cpd->is_usbfs) {
1324 		/*
1325 		 * The queue is used for events that should be
1326 		 * retrieved using the "USB_FS_COMPLETE" ioctl.
1327 		 */
1328 		err = EINVAL;
1329 		goto done;
1330 	}
1331 	if (f->queue_data == NULL) {
1332 		/* start write transfer, if not already started */
1333 		(f->methods->f_start_write) (f);
1334 	}
1335 	/* we allow writing zero length data */
1336 	do {
1337 		USB_IF_DEQUEUE(&f->free_q, m);
1338 
1339 		if (m == NULL) {
1340 
1341 			if (fflags & IO_NDELAY) {
1342 				if (tr_data) {
1343 					/* return length before error */
1344 					break;
1345 				}
1346 				err = EWOULDBLOCK;
1347 				break;
1348 			}
1349 			DPRINTF("sleeping\n");
1350 
1351 			err = usb2_fifo_wait(f);
1352 			if (err) {
1353 				break;
1354 			}
1355 			continue;
1356 		}
1357 		tr_data = 1;
1358 
1359 		USB_MBUF_RESET(m);
1360 
1361 		io_len = MIN(m->cur_data_len, uio->uio_resid);
1362 
1363 		m->cur_data_len = io_len;
1364 
1365 		DPRINTFN(2, "transfer %d bytes to %p\n",
1366 		    io_len, m->cur_data_ptr);
1367 
1368 		err = usb2_fifo_uiomove(f,
1369 		    m->cur_data_ptr, io_len, uio);
1370 
1371 		if (err) {
1372 			USB_IF_ENQUEUE(&f->free_q, m);
1373 			break;
1374 		}
1375 		if (f->methods->f_filter_write) {
1376 			/*
1377 			 * Sometimes it is convenient to process data at the
1378 			 * expense of a userland process instead of a kernel
1379 			 * process.
1380 			 */
1381 			(f->methods->f_filter_write) (f, m);
1382 		}
1383 		USB_IF_ENQUEUE(&f->used_q, m);
1384 
1385 		(f->methods->f_start_write) (f);
1386 
1387 	} while (uio->uio_resid > 0);
1388 done:
1389 	mtx_unlock(f->priv_mtx);
1390 
1391 	usb2_unref_device(cpd);
1392 
1393 	return (err);
1394 }
1395 
1396 int
1397 usb2_static_ioctl(struct cdev *dev, u_long cmd, caddr_t data, int fflag,
1398     struct thread *td)
1399 {
1400 	union {
1401 		struct usb2_read_dir *urd;
1402 		void* data;
1403 	} u;
1404 	int err = ENOTTY;
1405 
1406 	u.data = data;
1407 	switch (cmd) {
1408 		case USB_READ_DIR:
1409 			err = usb2_read_symlink(u.urd->urd_data,
1410 			    u.urd->urd_startentry, u.urd->urd_maxlen);
1411 			break;
1412 		case USB_DEV_QUIRK_GET:
1413 		case USB_QUIRK_NAME_GET:
1414 		case USB_DEV_QUIRK_ADD:
1415 		case USB_DEV_QUIRK_REMOVE:
1416 			err = usb2_quirk_ioctl_p(cmd, data, fflag, td);
1417 			break;
1418 		case USB_GET_TEMPLATE:
1419 			*(int *)data = usb2_template;
1420 			break;
1421 		case USB_SET_TEMPLATE:
1422 			err = priv_check(curthread, PRIV_DRIVER);
1423 			if (err)
1424 				break;
1425 			usb2_template = *(int *)data;
1426 			break;
1427 	}
1428 	return (err);
1429 }
1430 
1431 static int
1432 usb2_fifo_uiomove(struct usb2_fifo *f, void *cp,
1433     int n, struct uio *uio)
1434 {
1435 	int error;
1436 
1437 	mtx_unlock(f->priv_mtx);
1438 
1439 	/*
1440 	 * "uiomove()" can sleep so one needs to make a wrapper,
1441 	 * exiting the mutex and checking things:
1442 	 */
1443 	error = uiomove(cp, n, uio);
1444 
1445 	mtx_lock(f->priv_mtx);
1446 
1447 	return (error);
1448 }
1449 
1450 int
1451 usb2_fifo_wait(struct usb2_fifo *f)
1452 {
1453 	int err;
1454 
1455 	mtx_assert(f->priv_mtx, MA_OWNED);
1456 
1457 	if (f->flag_iserror) {
1458 		/* we are gone */
1459 		return (EIO);
1460 	}
1461 	f->flag_sleeping = 1;
1462 
1463 	err = usb2_cv_wait_sig(&f->cv_io, f->priv_mtx);
1464 
1465 	if (f->flag_iserror) {
1466 		/* we are gone */
1467 		err = EIO;
1468 	}
1469 	return (err);
1470 }
1471 
1472 void
1473 usb2_fifo_signal(struct usb2_fifo *f)
1474 {
1475 	if (f->flag_sleeping) {
1476 		f->flag_sleeping = 0;
1477 		usb2_cv_broadcast(&f->cv_io);
1478 	}
1479 }
1480 
1481 void
1482 usb2_fifo_wakeup(struct usb2_fifo *f)
1483 {
1484 	usb2_fifo_signal(f);
1485 
1486 	if (f->flag_isselect) {
1487 		selwakeup(&f->selinfo);
1488 		f->flag_isselect = 0;
1489 	}
1490 	if (f->async_p != NULL) {
1491 		PROC_LOCK(f->async_p);
1492 		psignal(f->async_p, SIGIO);
1493 		PROC_UNLOCK(f->async_p);
1494 	}
1495 }
1496 
1497 static int
1498 usb2_fifo_dummy_open(struct usb2_fifo *fifo, int fflags)
1499 {
1500 	return (0);
1501 }
1502 
1503 static void
1504 usb2_fifo_dummy_close(struct usb2_fifo *fifo, int fflags)
1505 {
1506 	return;
1507 }
1508 
1509 static int
1510 usb2_fifo_dummy_ioctl(struct usb2_fifo *fifo, u_long cmd, void *addr, int fflags)
1511 {
1512 	return (ENOIOCTL);
1513 }
1514 
1515 static void
1516 usb2_fifo_dummy_cmd(struct usb2_fifo *fifo)
1517 {
1518 	fifo->flag_flushing = 0;	/* not flushing */
1519 }
1520 
1521 static void
1522 usb2_fifo_check_methods(struct usb2_fifo_methods *pm)
1523 {
1524 	/* check that all callback functions are OK */
1525 
1526 	if (pm->f_open == NULL)
1527 		pm->f_open = &usb2_fifo_dummy_open;
1528 
1529 	if (pm->f_close == NULL)
1530 		pm->f_close = &usb2_fifo_dummy_close;
1531 
1532 	if (pm->f_ioctl == NULL)
1533 		pm->f_ioctl = &usb2_fifo_dummy_ioctl;
1534 
1535 	if (pm->f_ioctl_post == NULL)
1536 		pm->f_ioctl_post = &usb2_fifo_dummy_ioctl;
1537 
1538 	if (pm->f_start_read == NULL)
1539 		pm->f_start_read = &usb2_fifo_dummy_cmd;
1540 
1541 	if (pm->f_stop_read == NULL)
1542 		pm->f_stop_read = &usb2_fifo_dummy_cmd;
1543 
1544 	if (pm->f_start_write == NULL)
1545 		pm->f_start_write = &usb2_fifo_dummy_cmd;
1546 
1547 	if (pm->f_stop_write == NULL)
1548 		pm->f_stop_write = &usb2_fifo_dummy_cmd;
1549 }
1550 
1551 /*------------------------------------------------------------------------*
1552  *	usb2_fifo_attach
1553  *
1554  * The following function will create a duplex FIFO.
1555  *
1556  * Return values:
1557  * 0: Success.
1558  * Else: Failure.
1559  *------------------------------------------------------------------------*/
1560 int
1561 usb2_fifo_attach(struct usb2_device *udev, void *priv_sc,
1562     struct mtx *priv_mtx, struct usb2_fifo_methods *pm,
1563     struct usb2_fifo_sc *f_sc, uint16_t unit, uint16_t subunit,
1564     uint8_t iface_index, uid_t uid, gid_t gid, int mode)
1565 {
1566 	struct usb2_fifo *f_tx;
1567 	struct usb2_fifo *f_rx;
1568 	char devname[32];
1569 	uint8_t n;
1570 	struct usb2_fs_privdata* pd;
1571 
1572 	f_sc->fp[USB_FIFO_TX] = NULL;
1573 	f_sc->fp[USB_FIFO_RX] = NULL;
1574 
1575 	if (pm == NULL)
1576 		return (EINVAL);
1577 
1578 	/* check the methods */
1579 	usb2_fifo_check_methods(pm);
1580 
1581 	if (priv_mtx == NULL)
1582 		priv_mtx = &Giant;
1583 
1584 	/* search for a free FIFO slot */
1585 	for (n = 0;; n += 2) {
1586 
1587 		if (n == USB_FIFO_MAX) {
1588 			/* end of FIFOs reached */
1589 			return (ENOMEM);
1590 		}
1591 		/* Check for TX FIFO */
1592 		if (udev->fifo[n + USB_FIFO_TX] != NULL) {
1593 			continue;
1594 		}
1595 		/* Check for RX FIFO */
1596 		if (udev->fifo[n + USB_FIFO_RX] != NULL) {
1597 			continue;
1598 		}
1599 		break;
1600 	}
1601 
1602 	f_tx = usb2_fifo_alloc();
1603 	f_rx = usb2_fifo_alloc();
1604 
1605 	if ((f_tx == NULL) || (f_rx == NULL)) {
1606 		usb2_fifo_free(f_tx);
1607 		usb2_fifo_free(f_rx);
1608 		return (ENOMEM);
1609 	}
1610 	/* initialise FIFO structures */
1611 
1612 	f_tx->fifo_index = n + USB_FIFO_TX;
1613 	f_tx->dev_ep_index = -1;
1614 	f_tx->priv_mtx = priv_mtx;
1615 	f_tx->priv_sc0 = priv_sc;
1616 	f_tx->methods = pm;
1617 	f_tx->iface_index = iface_index;
1618 	f_tx->udev = udev;
1619 
1620 	f_rx->fifo_index = n + USB_FIFO_RX;
1621 	f_rx->dev_ep_index = -1;
1622 	f_rx->priv_mtx = priv_mtx;
1623 	f_rx->priv_sc0 = priv_sc;
1624 	f_rx->methods = pm;
1625 	f_rx->iface_index = iface_index;
1626 	f_rx->udev = udev;
1627 
1628 	f_sc->fp[USB_FIFO_TX] = f_tx;
1629 	f_sc->fp[USB_FIFO_RX] = f_rx;
1630 
1631 	mtx_lock(&usb2_ref_lock);
1632 	udev->fifo[f_tx->fifo_index] = f_tx;
1633 	udev->fifo[f_rx->fifo_index] = f_rx;
1634 	mtx_unlock(&usb2_ref_lock);
1635 
1636 	for (n = 0; n != 4; n++) {
1637 
1638 		if (pm->basename[n] == NULL) {
1639 			continue;
1640 		}
1641 		if (subunit == 0xFFFF) {
1642 			if (snprintf(devname, sizeof(devname),
1643 			    "%s%u%s", pm->basename[n],
1644 			    unit, pm->postfix[n] ?
1645 			    pm->postfix[n] : "")) {
1646 				/* ignore */
1647 			}
1648 		} else {
1649 			if (snprintf(devname, sizeof(devname),
1650 			    "%s%u.%u%s", pm->basename[n],
1651 			    unit, subunit, pm->postfix[n] ?
1652 			    pm->postfix[n] : "")) {
1653 				/* ignore */
1654 			}
1655 		}
1656 
1657 		/*
1658 		 * Distribute the symbolic links into two FIFO structures:
1659 		 */
1660 		if (n & 1) {
1661 			f_rx->symlink[n / 2] =
1662 			    usb2_alloc_symlink(devname);
1663 		} else {
1664 			f_tx->symlink[n / 2] =
1665 			    usb2_alloc_symlink(devname);
1666 		}
1667 
1668 		/*
1669 		 * Initialize device private data - this is used to find the
1670 		 * actual USB device itself.
1671 		 */
1672 		pd = malloc(sizeof(struct usb2_fs_privdata), M_USBDEV, M_WAITOK | M_ZERO);
1673 		pd->bus_index = device_get_unit(udev->bus->bdev);
1674 		pd->dev_index = udev->device_index;
1675 		pd->ep_addr = -1;	/* not an endpoint */
1676 		pd->fifo_index = f_tx->fifo_index & f_rx->fifo_index;
1677 		pd->mode = FREAD|FWRITE;
1678 
1679 		/* Now, create the device itself */
1680 		f_sc->dev = make_dev(&usb2_devsw, 0, uid, gid, mode,
1681 		    devname);
1682 		/* XXX setting si_drv1 and creating the device is not atomic! */
1683 		f_sc->dev->si_drv1 = pd;
1684 	}
1685 
1686 	DPRINTFN(2, "attached %p/%p\n", f_tx, f_rx);
1687 	return (0);
1688 }
1689 
1690 /*------------------------------------------------------------------------*
1691  *	usb2_fifo_alloc_buffer
1692  *
1693  * Return values:
1694  * 0: Success
1695  * Else failure
1696  *------------------------------------------------------------------------*/
1697 int
1698 usb2_fifo_alloc_buffer(struct usb2_fifo *f, usb2_size_t bufsize,
1699     uint16_t nbuf)
1700 {
1701 	usb2_fifo_free_buffer(f);
1702 
1703 	/* allocate an endpoint */
1704 	f->free_q.ifq_maxlen = nbuf;
1705 	f->used_q.ifq_maxlen = nbuf;
1706 
1707 	f->queue_data = usb2_alloc_mbufs(
1708 	    M_USBDEV, &f->free_q, bufsize, nbuf);
1709 
1710 	if ((f->queue_data == NULL) && bufsize && nbuf) {
1711 		return (ENOMEM);
1712 	}
1713 	return (0);			/* success */
1714 }
1715 
1716 /*------------------------------------------------------------------------*
1717  *	usb2_fifo_free_buffer
1718  *
1719  * This function will free the buffers associated with a FIFO. This
1720  * function can be called multiple times in a row.
1721  *------------------------------------------------------------------------*/
1722 void
1723 usb2_fifo_free_buffer(struct usb2_fifo *f)
1724 {
1725 	if (f->queue_data) {
1726 		/* free old buffer */
1727 		free(f->queue_data, M_USBDEV);
1728 		f->queue_data = NULL;
1729 	}
1730 	/* reset queues */
1731 
1732 	bzero(&f->free_q, sizeof(f->free_q));
1733 	bzero(&f->used_q, sizeof(f->used_q));
1734 }
1735 
1736 static void
1737 usb2_fifo_cleanup(void* ptr)
1738 {
1739 	free(ptr, M_USBDEV);
1740 }
1741 
1742 void
1743 usb2_fifo_detach(struct usb2_fifo_sc *f_sc)
1744 {
1745 	if (f_sc == NULL) {
1746 		return;
1747 	}
1748 	usb2_fifo_free(f_sc->fp[USB_FIFO_TX]);
1749 	usb2_fifo_free(f_sc->fp[USB_FIFO_RX]);
1750 
1751 	f_sc->fp[USB_FIFO_TX] = NULL;
1752 	f_sc->fp[USB_FIFO_RX] = NULL;
1753 
1754 	if (f_sc->dev != NULL) {
1755 		destroy_dev_sched_cb(f_sc->dev,
1756 		    usb2_fifo_cleanup, f_sc->dev->si_drv1);
1757 		f_sc->dev = NULL;
1758 	}
1759 
1760 	DPRINTFN(2, "detached %p\n", f_sc);
1761 }
1762 
1763 usb2_size_t
1764 usb2_fifo_put_bytes_max(struct usb2_fifo *f)
1765 {
1766 	struct usb2_mbuf *m;
1767 	usb2_size_t len;
1768 
1769 	USB_IF_POLL(&f->free_q, m);
1770 
1771 	if (m) {
1772 		len = m->max_data_len;
1773 	} else {
1774 		len = 0;
1775 	}
1776 	return (len);
1777 }
1778 
1779 /*------------------------------------------------------------------------*
1780  *	usb2_fifo_put_data
1781  *
1782  * what:
1783  *  0 - normal operation
1784  *  1 - set last packet flag to enforce framing
1785  *------------------------------------------------------------------------*/
1786 void
1787 usb2_fifo_put_data(struct usb2_fifo *f, struct usb2_page_cache *pc,
1788     usb2_frlength_t offset, usb2_frlength_t len, uint8_t what)
1789 {
1790 	struct usb2_mbuf *m;
1791 	usb2_frlength_t io_len;
1792 
1793 	while (len || (what == 1)) {
1794 
1795 		USB_IF_DEQUEUE(&f->free_q, m);
1796 
1797 		if (m) {
1798 			USB_MBUF_RESET(m);
1799 
1800 			io_len = MIN(len, m->cur_data_len);
1801 
1802 			usb2_copy_out(pc, offset, m->cur_data_ptr, io_len);
1803 
1804 			m->cur_data_len = io_len;
1805 			offset += io_len;
1806 			len -= io_len;
1807 
1808 			if ((len == 0) && (what == 1)) {
1809 				m->last_packet = 1;
1810 			}
1811 			USB_IF_ENQUEUE(&f->used_q, m);
1812 
1813 			usb2_fifo_wakeup(f);
1814 
1815 			if ((len == 0) || (what == 1)) {
1816 				break;
1817 			}
1818 		} else {
1819 			break;
1820 		}
1821 	}
1822 }
1823 
1824 void
1825 usb2_fifo_put_data_linear(struct usb2_fifo *f, void *ptr,
1826     usb2_size_t len, uint8_t what)
1827 {
1828 	struct usb2_mbuf *m;
1829 	usb2_size_t io_len;
1830 
1831 	while (len || (what == 1)) {
1832 
1833 		USB_IF_DEQUEUE(&f->free_q, m);
1834 
1835 		if (m) {
1836 			USB_MBUF_RESET(m);
1837 
1838 			io_len = MIN(len, m->cur_data_len);
1839 
1840 			bcopy(ptr, m->cur_data_ptr, io_len);
1841 
1842 			m->cur_data_len = io_len;
1843 			ptr = USB_ADD_BYTES(ptr, io_len);
1844 			len -= io_len;
1845 
1846 			if ((len == 0) && (what == 1)) {
1847 				m->last_packet = 1;
1848 			}
1849 			USB_IF_ENQUEUE(&f->used_q, m);
1850 
1851 			usb2_fifo_wakeup(f);
1852 
1853 			if ((len == 0) || (what == 1)) {
1854 				break;
1855 			}
1856 		} else {
1857 			break;
1858 		}
1859 	}
1860 }
1861 
1862 uint8_t
1863 usb2_fifo_put_data_buffer(struct usb2_fifo *f, void *ptr, usb2_size_t len)
1864 {
1865 	struct usb2_mbuf *m;
1866 
1867 	USB_IF_DEQUEUE(&f->free_q, m);
1868 
1869 	if (m) {
1870 		m->cur_data_len = len;
1871 		m->cur_data_ptr = ptr;
1872 		USB_IF_ENQUEUE(&f->used_q, m);
1873 		usb2_fifo_wakeup(f);
1874 		return (1);
1875 	}
1876 	return (0);
1877 }
1878 
1879 void
1880 usb2_fifo_put_data_error(struct usb2_fifo *f)
1881 {
1882 	f->flag_iserror = 1;
1883 	usb2_fifo_wakeup(f);
1884 }
1885 
1886 /*------------------------------------------------------------------------*
1887  *	usb2_fifo_get_data
1888  *
1889  * what:
1890  *  0 - normal operation
1891  *  1 - only get one "usb2_mbuf"
1892  *
1893  * returns:
1894  *  0 - no more data
1895  *  1 - data in buffer
1896  *------------------------------------------------------------------------*/
1897 uint8_t
1898 usb2_fifo_get_data(struct usb2_fifo *f, struct usb2_page_cache *pc,
1899     usb2_frlength_t offset, usb2_frlength_t len, usb2_frlength_t *actlen,
1900     uint8_t what)
1901 {
1902 	struct usb2_mbuf *m;
1903 	usb2_frlength_t io_len;
1904 	uint8_t tr_data = 0;
1905 
1906 	actlen[0] = 0;
1907 
1908 	while (1) {
1909 
1910 		USB_IF_DEQUEUE(&f->used_q, m);
1911 
1912 		if (m) {
1913 
1914 			tr_data = 1;
1915 
1916 			io_len = MIN(len, m->cur_data_len);
1917 
1918 			usb2_copy_in(pc, offset, m->cur_data_ptr, io_len);
1919 
1920 			len -= io_len;
1921 			offset += io_len;
1922 			actlen[0] += io_len;
1923 			m->cur_data_ptr += io_len;
1924 			m->cur_data_len -= io_len;
1925 
1926 			if ((m->cur_data_len == 0) || (what == 1)) {
1927 				USB_IF_ENQUEUE(&f->free_q, m);
1928 
1929 				usb2_fifo_wakeup(f);
1930 
1931 				if (what == 1) {
1932 					break;
1933 				}
1934 			} else {
1935 				USB_IF_PREPEND(&f->used_q, m);
1936 			}
1937 		} else {
1938 
1939 			if (tr_data) {
1940 				/* wait for data to be written out */
1941 				break;
1942 			}
1943 			if (f->flag_flushing) {
1944 				/* check if we should send a short packet */
1945 				if (f->flag_short != 0) {
1946 					f->flag_short = 0;
1947 					tr_data = 1;
1948 					break;
1949 				}
1950 				/* flushing complete */
1951 				f->flag_flushing = 0;
1952 				usb2_fifo_wakeup(f);
1953 			}
1954 			break;
1955 		}
1956 		if (len == 0) {
1957 			break;
1958 		}
1959 	}
1960 	return (tr_data);
1961 }
1962 
1963 uint8_t
1964 usb2_fifo_get_data_linear(struct usb2_fifo *f, void *ptr,
1965     usb2_size_t len, usb2_size_t *actlen, uint8_t what)
1966 {
1967 	struct usb2_mbuf *m;
1968 	usb2_size_t io_len;
1969 	uint8_t tr_data = 0;
1970 
1971 	actlen[0] = 0;
1972 
1973 	while (1) {
1974 
1975 		USB_IF_DEQUEUE(&f->used_q, m);
1976 
1977 		if (m) {
1978 
1979 			tr_data = 1;
1980 
1981 			io_len = MIN(len, m->cur_data_len);
1982 
1983 			bcopy(m->cur_data_ptr, ptr, io_len);
1984 
1985 			len -= io_len;
1986 			ptr = USB_ADD_BYTES(ptr, io_len);
1987 			actlen[0] += io_len;
1988 			m->cur_data_ptr += io_len;
1989 			m->cur_data_len -= io_len;
1990 
1991 			if ((m->cur_data_len == 0) || (what == 1)) {
1992 				USB_IF_ENQUEUE(&f->free_q, m);
1993 
1994 				usb2_fifo_wakeup(f);
1995 
1996 				if (what == 1) {
1997 					break;
1998 				}
1999 			} else {
2000 				USB_IF_PREPEND(&f->used_q, m);
2001 			}
2002 		} else {
2003 
2004 			if (tr_data) {
2005 				/* wait for data to be written out */
2006 				break;
2007 			}
2008 			if (f->flag_flushing) {
2009 				/* check if we should send a short packet */
2010 				if (f->flag_short != 0) {
2011 					f->flag_short = 0;
2012 					tr_data = 1;
2013 					break;
2014 				}
2015 				/* flushing complete */
2016 				f->flag_flushing = 0;
2017 				usb2_fifo_wakeup(f);
2018 			}
2019 			break;
2020 		}
2021 		if (len == 0) {
2022 			break;
2023 		}
2024 	}
2025 	return (tr_data);
2026 }
2027 
2028 uint8_t
2029 usb2_fifo_get_data_buffer(struct usb2_fifo *f, void **pptr, usb2_size_t *plen)
2030 {
2031 	struct usb2_mbuf *m;
2032 
2033 	USB_IF_POLL(&f->used_q, m);
2034 
2035 	if (m) {
2036 		*plen = m->cur_data_len;
2037 		*pptr = m->cur_data_ptr;
2038 
2039 		return (1);
2040 	}
2041 	return (0);
2042 }
2043 
2044 void
2045 usb2_fifo_get_data_error(struct usb2_fifo *f)
2046 {
2047 	f->flag_iserror = 1;
2048 	usb2_fifo_wakeup(f);
2049 }
2050 
2051 /*------------------------------------------------------------------------*
2052  *	usb2_alloc_symlink
2053  *
2054  * Return values:
2055  * NULL: Failure
2056  * Else: Pointer to symlink entry
2057  *------------------------------------------------------------------------*/
2058 struct usb2_symlink *
2059 usb2_alloc_symlink(const char *target)
2060 {
2061 	struct usb2_symlink *ps;
2062 
2063 	ps = malloc(sizeof(*ps), M_USBDEV, M_WAITOK);
2064 	if (ps == NULL) {
2065 		return (ps);
2066 	}
2067 	/* XXX no longer needed */
2068 	strlcpy(ps->src_path, target, sizeof(ps->src_path));
2069 	ps->src_len = strlen(ps->src_path);
2070 	strlcpy(ps->dst_path, target, sizeof(ps->dst_path));
2071 	ps->dst_len = strlen(ps->dst_path);
2072 
2073 	sx_xlock(&usb2_sym_lock);
2074 	TAILQ_INSERT_TAIL(&usb2_sym_head, ps, sym_entry);
2075 	sx_unlock(&usb2_sym_lock);
2076 	return (ps);
2077 }
2078 
2079 /*------------------------------------------------------------------------*
2080  *	usb2_free_symlink
2081  *------------------------------------------------------------------------*/
2082 void
2083 usb2_free_symlink(struct usb2_symlink *ps)
2084 {
2085 	if (ps == NULL) {
2086 		return;
2087 	}
2088 	sx_xlock(&usb2_sym_lock);
2089 	TAILQ_REMOVE(&usb2_sym_head, ps, sym_entry);
2090 	sx_unlock(&usb2_sym_lock);
2091 
2092 	free(ps, M_USBDEV);
2093 }
2094 
2095 /*------------------------------------------------------------------------*
2096  *	usb2_read_symlink
2097  *
2098  * Return value:
2099  * 0: Success
2100  * Else: Failure
2101  *------------------------------------------------------------------------*/
2102 int
2103 usb2_read_symlink(uint8_t *user_ptr, uint32_t startentry, uint32_t user_len)
2104 {
2105 	struct usb2_symlink *ps;
2106 	uint32_t temp;
2107 	uint32_t delta = 0;
2108 	uint8_t len;
2109 	int error = 0;
2110 
2111 	sx_xlock(&usb2_sym_lock);
2112 
2113 	TAILQ_FOREACH(ps, &usb2_sym_head, sym_entry) {
2114 
2115 		/*
2116 		 * Compute total length of source and destination symlink
2117 		 * strings pluss one length byte and two NUL bytes:
2118 		 */
2119 		temp = ps->src_len + ps->dst_len + 3;
2120 
2121 		if (temp > 255) {
2122 			/*
2123 			 * Skip entry because this length cannot fit
2124 			 * into one byte:
2125 			 */
2126 			continue;
2127 		}
2128 		if (startentry != 0) {
2129 			/* decrement read offset */
2130 			startentry--;
2131 			continue;
2132 		}
2133 		if (temp > user_len) {
2134 			/* out of buffer space */
2135 			break;
2136 		}
2137 		len = temp;
2138 
2139 		/* copy out total length */
2140 
2141 		error = copyout(&len,
2142 		    USB_ADD_BYTES(user_ptr, delta), 1);
2143 		if (error) {
2144 			break;
2145 		}
2146 		delta += 1;
2147 
2148 		/* copy out source string */
2149 
2150 		error = copyout(ps->src_path,
2151 		    USB_ADD_BYTES(user_ptr, delta), ps->src_len);
2152 		if (error) {
2153 			break;
2154 		}
2155 		len = 0;
2156 		delta += ps->src_len;
2157 		error = copyout(&len,
2158 		    USB_ADD_BYTES(user_ptr, delta), 1);
2159 		if (error) {
2160 			break;
2161 		}
2162 		delta += 1;
2163 
2164 		/* copy out destination string */
2165 
2166 		error = copyout(ps->dst_path,
2167 		    USB_ADD_BYTES(user_ptr, delta), ps->dst_len);
2168 		if (error) {
2169 			break;
2170 		}
2171 		len = 0;
2172 		delta += ps->dst_len;
2173 		error = copyout(&len,
2174 		    USB_ADD_BYTES(user_ptr, delta), 1);
2175 		if (error) {
2176 			break;
2177 		}
2178 		delta += 1;
2179 
2180 		user_len -= temp;
2181 	}
2182 
2183 	/* a zero length entry indicates the end */
2184 
2185 	if ((user_len != 0) && (error == 0)) {
2186 
2187 		len = 0;
2188 
2189 		error = copyout(&len,
2190 		    USB_ADD_BYTES(user_ptr, delta), 1);
2191 	}
2192 	sx_unlock(&usb2_sym_lock);
2193 	return (error);
2194 }
2195 
2196 void
2197 usb2_fifo_set_close_zlp(struct usb2_fifo *f, uint8_t onoff)
2198 {
2199 	if (f == NULL)
2200 		return;
2201 
2202 	/* send a Zero Length Packet, ZLP, before close */
2203 	f->flag_short = onoff;
2204 }
2205 #endif	/* USB_HAVE_UGEN */
2206