xref: /freebsd/sys/dev/usb/usb_dev.c (revision 0b3105a37d7adcadcb720112fed4dc4e8040be99)
1 /* $FreeBSD$ */
2 /*-
3  * Copyright (c) 2006-2008 Hans Petter Selasky. All rights reserved.
4  *
5  * Redistribution and use in source and binary forms, with or without
6  * modification, are permitted provided that the following conditions
7  * are met:
8  * 1. Redistributions of source code must retain the above copyright
9  *    notice, this list of conditions and the following disclaimer.
10  * 2. Redistributions in binary form must reproduce the above copyright
11  *    notice, this list of conditions and the following disclaimer in the
12  *    documentation and/or other materials provided with the distribution.
13  *
14  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24  * SUCH DAMAGE.
25  *
26  *
27  * usb_dev.c - An abstraction layer for creating devices under /dev/...
28  */
29 
30 #ifdef USB_GLOBAL_INCLUDE_FILE
31 #include USB_GLOBAL_INCLUDE_FILE
32 #else
33 #include <sys/stdint.h>
34 #include <sys/stddef.h>
35 #include <sys/param.h>
36 #include <sys/queue.h>
37 #include <sys/types.h>
38 #include <sys/systm.h>
39 #include <sys/kernel.h>
40 #include <sys/bus.h>
41 #include <sys/module.h>
42 #include <sys/lock.h>
43 #include <sys/mutex.h>
44 #include <sys/condvar.h>
45 #include <sys/sysctl.h>
46 #include <sys/sx.h>
47 #include <sys/unistd.h>
48 #include <sys/callout.h>
49 #include <sys/malloc.h>
50 #include <sys/priv.h>
51 #include <sys/vnode.h>
52 #include <sys/conf.h>
53 #include <sys/fcntl.h>
54 
55 #include <dev/usb/usb.h>
56 #include <dev/usb/usb_ioctl.h>
57 #include <dev/usb/usbdi.h>
58 #include <dev/usb/usbdi_util.h>
59 
60 #define	USB_DEBUG_VAR usb_fifo_debug
61 
62 #include <dev/usb/usb_core.h>
63 #include <dev/usb/usb_dev.h>
64 #include <dev/usb/usb_mbuf.h>
65 #include <dev/usb/usb_process.h>
66 #include <dev/usb/usb_device.h>
67 #include <dev/usb/usb_debug.h>
68 #include <dev/usb/usb_busdma.h>
69 #include <dev/usb/usb_generic.h>
70 #include <dev/usb/usb_dynamic.h>
71 #include <dev/usb/usb_util.h>
72 
73 #include <dev/usb/usb_controller.h>
74 #include <dev/usb/usb_bus.h>
75 
76 #include <sys/filio.h>
77 #include <sys/ttycom.h>
78 #include <sys/syscallsubr.h>
79 
80 #include <machine/stdarg.h>
81 #endif			/* USB_GLOBAL_INCLUDE_FILE */
82 
83 #if USB_HAVE_UGEN
84 
85 #ifdef USB_DEBUG
86 static int usb_fifo_debug = 0;
87 
88 static SYSCTL_NODE(_hw_usb, OID_AUTO, dev, CTLFLAG_RW, 0, "USB device");
89 SYSCTL_INT(_hw_usb_dev, OID_AUTO, debug, CTLFLAG_RWTUN,
90     &usb_fifo_debug, 0, "Debug Level");
91 #endif
92 
93 #if ((__FreeBSD_version >= 700001) || (__FreeBSD_version == 0) || \
94      ((__FreeBSD_version >= 600034) && (__FreeBSD_version < 700000)))
95 #define	USB_UCRED struct ucred *ucred,
96 #else
97 #define	USB_UCRED
98 #endif
99 
100 /* prototypes */
101 
102 static int	usb_fifo_open(struct usb_cdev_privdata *,
103 		    struct usb_fifo *, int);
104 static void	usb_fifo_close(struct usb_fifo *, int);
105 static void	usb_dev_init(void *);
106 static void	usb_dev_init_post(void *);
107 static void	usb_dev_uninit(void *);
108 static int	usb_fifo_uiomove(struct usb_fifo *, void *, int,
109 		    struct uio *);
110 static void	usb_fifo_check_methods(struct usb_fifo_methods *);
111 static struct	usb_fifo *usb_fifo_alloc(struct mtx *);
112 static struct	usb_endpoint *usb_dev_get_ep(struct usb_device *, uint8_t,
113 		    uint8_t);
114 static void	usb_loc_fill(struct usb_fs_privdata *,
115 		    struct usb_cdev_privdata *);
116 static void	usb_close(void *);
117 static usb_error_t usb_ref_device(struct usb_cdev_privdata *, struct usb_cdev_refdata *, int);
118 static usb_error_t usb_usb_ref_device(struct usb_cdev_privdata *, struct usb_cdev_refdata *);
119 static void	usb_unref_device(struct usb_cdev_privdata *, struct usb_cdev_refdata *);
120 
121 static d_open_t usb_open;
122 static d_ioctl_t usb_ioctl;
123 static d_read_t usb_read;
124 static d_write_t usb_write;
125 static d_poll_t usb_poll;
126 static d_kqfilter_t usb_kqfilter;
127 
128 static d_ioctl_t usb_static_ioctl;
129 
130 static usb_fifo_open_t usb_fifo_dummy_open;
131 static usb_fifo_close_t usb_fifo_dummy_close;
132 static usb_fifo_ioctl_t usb_fifo_dummy_ioctl;
133 static usb_fifo_cmd_t usb_fifo_dummy_cmd;
134 
135 /* character device structure used for devices (/dev/ugenX.Y and /dev/uXXX) */
136 struct cdevsw usb_devsw = {
137 	.d_version = D_VERSION,
138 	.d_open = usb_open,
139 	.d_ioctl = usb_ioctl,
140 	.d_name = "usbdev",
141 	.d_flags = D_TRACKCLOSE,
142 	.d_read = usb_read,
143 	.d_write = usb_write,
144 	.d_poll = usb_poll,
145 	.d_kqfilter = usb_kqfilter,
146 };
147 
148 static struct cdev* usb_dev = NULL;
149 
150 /* character device structure used for /dev/usb */
151 static struct cdevsw usb_static_devsw = {
152 	.d_version = D_VERSION,
153 	.d_ioctl = usb_static_ioctl,
154 	.d_name = "usb"
155 };
156 
157 static TAILQ_HEAD(, usb_symlink) usb_sym_head;
158 static struct sx usb_sym_lock;
159 
160 struct mtx usb_ref_lock;
161 
162 /*------------------------------------------------------------------------*
163  *	usb_loc_fill
164  *
165  * This is used to fill out a usb_cdev_privdata structure based on the
166  * device's address as contained in usb_fs_privdata.
167  *------------------------------------------------------------------------*/
168 static void
169 usb_loc_fill(struct usb_fs_privdata* pd, struct usb_cdev_privdata *cpd)
170 {
171 	cpd->bus_index = pd->bus_index;
172 	cpd->dev_index = pd->dev_index;
173 	cpd->ep_addr = pd->ep_addr;
174 	cpd->fifo_index = pd->fifo_index;
175 }
176 
177 /*------------------------------------------------------------------------*
178  *	usb_ref_device
179  *
180  * This function is used to atomically refer an USB device by its
181  * device location. If this function returns success the USB device
182  * will not dissappear until the USB device is unreferenced.
183  *
184  * Return values:
185  *  0: Success, refcount incremented on the given USB device.
186  *  Else: Failure.
187  *------------------------------------------------------------------------*/
188 static usb_error_t
189 usb_ref_device(struct usb_cdev_privdata *cpd,
190     struct usb_cdev_refdata *crd, int need_uref)
191 {
192 	struct usb_fifo **ppf;
193 	struct usb_fifo *f;
194 
195 	DPRINTFN(2, "cpd=%p need uref=%d\n", cpd, need_uref);
196 
197 	/* clear all refs */
198 	memset(crd, 0, sizeof(*crd));
199 
200 	mtx_lock(&usb_ref_lock);
201 	cpd->bus = devclass_get_softc(usb_devclass_ptr, cpd->bus_index);
202 	if (cpd->bus == NULL) {
203 		DPRINTFN(2, "no bus at %u\n", cpd->bus_index);
204 		goto error;
205 	}
206 	cpd->udev = cpd->bus->devices[cpd->dev_index];
207 	if (cpd->udev == NULL) {
208 		DPRINTFN(2, "no device at %u\n", cpd->dev_index);
209 		goto error;
210 	}
211 	if (cpd->udev->state == USB_STATE_DETACHED &&
212 	    (need_uref != 2)) {
213 		DPRINTFN(2, "device is detached\n");
214 		goto error;
215 	}
216 	if (need_uref) {
217 		DPRINTFN(2, "ref udev - needed\n");
218 
219 		if (cpd->udev->refcount == USB_DEV_REF_MAX) {
220 			DPRINTFN(2, "no dev ref\n");
221 			goto error;
222 		}
223 		cpd->udev->refcount++;
224 
225 		mtx_unlock(&usb_ref_lock);
226 
227 		/*
228 		 * We need to grab the enumeration SX-lock before
229 		 * grabbing the FIFO refs to avoid deadlock at detach!
230 		 */
231 		crd->do_unlock = usbd_enum_lock(cpd->udev);
232 
233 		mtx_lock(&usb_ref_lock);
234 
235 		/*
236 		 * Set "is_uref" after grabbing the default SX lock
237 		 */
238 		crd->is_uref = 1;
239 	}
240 
241 	/* check if we are doing an open */
242 	if (cpd->fflags == 0) {
243 		/* use zero defaults */
244 	} else {
245 		/* check for write */
246 		if (cpd->fflags & FWRITE) {
247 			ppf = cpd->udev->fifo;
248 			f = ppf[cpd->fifo_index + USB_FIFO_TX];
249 			crd->txfifo = f;
250 			crd->is_write = 1;	/* ref */
251 			if (f == NULL || f->refcount == USB_FIFO_REF_MAX)
252 				goto error;
253 			if (f->curr_cpd != cpd)
254 				goto error;
255 			/* check if USB-FS is active */
256 			if (f->fs_ep_max != 0) {
257 				crd->is_usbfs = 1;
258 			}
259 		}
260 
261 		/* check for read */
262 		if (cpd->fflags & FREAD) {
263 			ppf = cpd->udev->fifo;
264 			f = ppf[cpd->fifo_index + USB_FIFO_RX];
265 			crd->rxfifo = f;
266 			crd->is_read = 1;	/* ref */
267 			if (f == NULL || f->refcount == USB_FIFO_REF_MAX)
268 				goto error;
269 			if (f->curr_cpd != cpd)
270 				goto error;
271 			/* check if USB-FS is active */
272 			if (f->fs_ep_max != 0) {
273 				crd->is_usbfs = 1;
274 			}
275 		}
276 	}
277 
278 	/* when everything is OK we increment the refcounts */
279 	if (crd->is_write) {
280 		DPRINTFN(2, "ref write\n");
281 		crd->txfifo->refcount++;
282 	}
283 	if (crd->is_read) {
284 		DPRINTFN(2, "ref read\n");
285 		crd->rxfifo->refcount++;
286 	}
287 	mtx_unlock(&usb_ref_lock);
288 
289 	return (0);
290 
291 error:
292 	if (crd->do_unlock)
293 		usbd_enum_unlock(cpd->udev);
294 
295 	if (crd->is_uref) {
296 		if (--(cpd->udev->refcount) == 0)
297 			cv_broadcast(&cpd->udev->ref_cv);
298 	}
299 	mtx_unlock(&usb_ref_lock);
300 	DPRINTFN(2, "fail\n");
301 
302 	/* clear all refs */
303 	memset(crd, 0, sizeof(*crd));
304 
305 	return (USB_ERR_INVAL);
306 }
307 
308 /*------------------------------------------------------------------------*
309  *	usb_usb_ref_device
310  *
311  * This function is used to upgrade an USB reference to include the
312  * USB device reference on a USB location.
313  *
314  * Return values:
315  *  0: Success, refcount incremented on the given USB device.
316  *  Else: Failure.
317  *------------------------------------------------------------------------*/
318 static usb_error_t
319 usb_usb_ref_device(struct usb_cdev_privdata *cpd,
320     struct usb_cdev_refdata *crd)
321 {
322 	/*
323 	 * Check if we already got an USB reference on this location:
324 	 */
325 	if (crd->is_uref)
326 		return (0);		/* success */
327 
328 	/*
329 	 * To avoid deadlock at detach we need to drop the FIFO ref
330 	 * and re-acquire a new ref!
331 	 */
332 	usb_unref_device(cpd, crd);
333 
334 	return (usb_ref_device(cpd, crd, 1 /* need uref */));
335 }
336 
337 /*------------------------------------------------------------------------*
338  *	usb_unref_device
339  *
340  * This function will release the reference count by one unit for the
341  * given USB device.
342  *------------------------------------------------------------------------*/
343 static void
344 usb_unref_device(struct usb_cdev_privdata *cpd,
345     struct usb_cdev_refdata *crd)
346 {
347 
348 	DPRINTFN(2, "cpd=%p is_uref=%d\n", cpd, crd->is_uref);
349 
350 	if (crd->do_unlock)
351 		usbd_enum_unlock(cpd->udev);
352 
353 	mtx_lock(&usb_ref_lock);
354 	if (crd->is_read) {
355 		if (--(crd->rxfifo->refcount) == 0) {
356 			cv_signal(&crd->rxfifo->cv_drain);
357 		}
358 		crd->is_read = 0;
359 	}
360 	if (crd->is_write) {
361 		if (--(crd->txfifo->refcount) == 0) {
362 			cv_signal(&crd->txfifo->cv_drain);
363 		}
364 		crd->is_write = 0;
365 	}
366 	if (crd->is_uref) {
367 		crd->is_uref = 0;
368 		if (--(cpd->udev->refcount) == 0)
369 			cv_broadcast(&cpd->udev->ref_cv);
370 	}
371 	mtx_unlock(&usb_ref_lock);
372 }
373 
374 static struct usb_fifo *
375 usb_fifo_alloc(struct mtx *mtx)
376 {
377 	struct usb_fifo *f;
378 
379 	f = malloc(sizeof(*f), M_USBDEV, M_WAITOK | M_ZERO);
380 	if (f != NULL) {
381 		cv_init(&f->cv_io, "FIFO-IO");
382 		cv_init(&f->cv_drain, "FIFO-DRAIN");
383 		f->priv_mtx = mtx;
384 		f->refcount = 1;
385 		knlist_init_mtx(&f->selinfo.si_note, mtx);
386 	}
387 	return (f);
388 }
389 
390 /*------------------------------------------------------------------------*
391  *	usb_fifo_create
392  *------------------------------------------------------------------------*/
393 static int
394 usb_fifo_create(struct usb_cdev_privdata *cpd,
395     struct usb_cdev_refdata *crd)
396 {
397 	struct usb_device *udev = cpd->udev;
398 	struct usb_fifo *f;
399 	struct usb_endpoint *ep;
400 	uint8_t n;
401 	uint8_t is_tx;
402 	uint8_t is_rx;
403 	uint8_t no_null;
404 	uint8_t is_busy;
405 	int e = cpd->ep_addr;
406 
407 	is_tx = (cpd->fflags & FWRITE) ? 1 : 0;
408 	is_rx = (cpd->fflags & FREAD) ? 1 : 0;
409 	no_null = 1;
410 	is_busy = 0;
411 
412 	/* Preallocated FIFO */
413 	if (e < 0) {
414 		DPRINTFN(5, "Preallocated FIFO\n");
415 		if (is_tx) {
416 			f = udev->fifo[cpd->fifo_index + USB_FIFO_TX];
417 			if (f == NULL)
418 				return (EINVAL);
419 			crd->txfifo = f;
420 		}
421 		if (is_rx) {
422 			f = udev->fifo[cpd->fifo_index + USB_FIFO_RX];
423 			if (f == NULL)
424 				return (EINVAL);
425 			crd->rxfifo = f;
426 		}
427 		return (0);
428 	}
429 
430 	KASSERT(e >= 0 && e <= 15, ("endpoint %d out of range", e));
431 
432 	/* search for a free FIFO slot */
433 	DPRINTFN(5, "Endpoint device, searching for 0x%02x\n", e);
434 	for (n = 0;; n += 2) {
435 
436 		if (n == USB_FIFO_MAX) {
437 			if (no_null) {
438 				no_null = 0;
439 				n = 0;
440 			} else {
441 				/* end of FIFOs reached */
442 				DPRINTFN(5, "out of FIFOs\n");
443 				return (ENOMEM);
444 			}
445 		}
446 		/* Check for TX FIFO */
447 		if (is_tx) {
448 			f = udev->fifo[n + USB_FIFO_TX];
449 			if (f != NULL) {
450 				if (f->dev_ep_index != e) {
451 					/* wrong endpoint index */
452 					continue;
453 				}
454 				if (f->curr_cpd != NULL) {
455 					/* FIFO is opened */
456 					is_busy = 1;
457 					continue;
458 				}
459 			} else if (no_null) {
460 				continue;
461 			}
462 		}
463 		/* Check for RX FIFO */
464 		if (is_rx) {
465 			f = udev->fifo[n + USB_FIFO_RX];
466 			if (f != NULL) {
467 				if (f->dev_ep_index != e) {
468 					/* wrong endpoint index */
469 					continue;
470 				}
471 				if (f->curr_cpd != NULL) {
472 					/* FIFO is opened */
473 					is_busy = 1;
474 					continue;
475 				}
476 			} else if (no_null) {
477 				continue;
478 			}
479 		}
480 		break;
481 	}
482 
483 	if (no_null == 0) {
484 		if (e >= (USB_EP_MAX / 2)) {
485 			/* we don't create any endpoints in this range */
486 			DPRINTFN(5, "ep out of range\n");
487 			return (is_busy ? EBUSY : EINVAL);
488 		}
489 	}
490 
491 	if ((e != 0) && is_busy) {
492 		/*
493 		 * Only the default control endpoint is allowed to be
494 		 * opened multiple times!
495 		 */
496 		DPRINTFN(5, "busy\n");
497 		return (EBUSY);
498 	}
499 
500 	/* Check TX FIFO */
501 	if (is_tx &&
502 	    (udev->fifo[n + USB_FIFO_TX] == NULL)) {
503 		ep = usb_dev_get_ep(udev, e, USB_FIFO_TX);
504 		DPRINTFN(5, "dev_get_endpoint(%d, 0x%x)\n", e, USB_FIFO_TX);
505 		if (ep == NULL) {
506 			DPRINTFN(5, "dev_get_endpoint returned NULL\n");
507 			return (EINVAL);
508 		}
509 		f = usb_fifo_alloc(&udev->device_mtx);
510 		if (f == NULL) {
511 			DPRINTFN(5, "could not alloc tx fifo\n");
512 			return (ENOMEM);
513 		}
514 		/* update some fields */
515 		f->fifo_index = n + USB_FIFO_TX;
516 		f->dev_ep_index = e;
517 		f->priv_sc0 = ep;
518 		f->methods = &usb_ugen_methods;
519 		f->iface_index = ep->iface_index;
520 		f->udev = udev;
521 		mtx_lock(&usb_ref_lock);
522 		udev->fifo[n + USB_FIFO_TX] = f;
523 		mtx_unlock(&usb_ref_lock);
524 	}
525 	/* Check RX FIFO */
526 	if (is_rx &&
527 	    (udev->fifo[n + USB_FIFO_RX] == NULL)) {
528 
529 		ep = usb_dev_get_ep(udev, e, USB_FIFO_RX);
530 		DPRINTFN(5, "dev_get_endpoint(%d, 0x%x)\n", e, USB_FIFO_RX);
531 		if (ep == NULL) {
532 			DPRINTFN(5, "dev_get_endpoint returned NULL\n");
533 			return (EINVAL);
534 		}
535 		f = usb_fifo_alloc(&udev->device_mtx);
536 		if (f == NULL) {
537 			DPRINTFN(5, "could not alloc rx fifo\n");
538 			return (ENOMEM);
539 		}
540 		/* update some fields */
541 		f->fifo_index = n + USB_FIFO_RX;
542 		f->dev_ep_index = e;
543 		f->priv_sc0 = ep;
544 		f->methods = &usb_ugen_methods;
545 		f->iface_index = ep->iface_index;
546 		f->udev = udev;
547 		mtx_lock(&usb_ref_lock);
548 		udev->fifo[n + USB_FIFO_RX] = f;
549 		mtx_unlock(&usb_ref_lock);
550 	}
551 	if (is_tx) {
552 		crd->txfifo = udev->fifo[n + USB_FIFO_TX];
553 	}
554 	if (is_rx) {
555 		crd->rxfifo = udev->fifo[n + USB_FIFO_RX];
556 	}
557 	/* fill out fifo index */
558 	DPRINTFN(5, "fifo index = %d\n", n);
559 	cpd->fifo_index = n;
560 
561 	/* complete */
562 
563 	return (0);
564 }
565 
566 void
567 usb_fifo_free(struct usb_fifo *f)
568 {
569 	uint8_t n;
570 
571 	if (f == NULL) {
572 		/* be NULL safe */
573 		return;
574 	}
575 	/* destroy symlink devices, if any */
576 	for (n = 0; n != 2; n++) {
577 		if (f->symlink[n]) {
578 			usb_free_symlink(f->symlink[n]);
579 			f->symlink[n] = NULL;
580 		}
581 	}
582 	mtx_lock(&usb_ref_lock);
583 
584 	/* delink ourselves to stop calls from userland */
585 	if ((f->fifo_index < USB_FIFO_MAX) &&
586 	    (f->udev != NULL) &&
587 	    (f->udev->fifo[f->fifo_index] == f)) {
588 		f->udev->fifo[f->fifo_index] = NULL;
589 	} else {
590 		DPRINTFN(0, "USB FIFO %p has not been linked\n", f);
591 	}
592 
593 	/* decrease refcount */
594 	f->refcount--;
595 	/* need to wait until all callers have exited */
596 	while (f->refcount != 0) {
597 		mtx_unlock(&usb_ref_lock);	/* avoid LOR */
598 		mtx_lock(f->priv_mtx);
599 		/* prevent write flush, if any */
600 		f->flag_iserror = 1;
601 		/* get I/O thread out of any sleep state */
602 		if (f->flag_sleeping) {
603 			f->flag_sleeping = 0;
604 			cv_broadcast(&f->cv_io);
605 		}
606 		mtx_unlock(f->priv_mtx);
607 		mtx_lock(&usb_ref_lock);
608 
609 		/*
610 		 * Check if the "f->refcount" variable reached zero
611 		 * during the unlocked time before entering wait:
612 		 */
613 		if (f->refcount == 0)
614 			break;
615 
616 		/* wait for sync */
617 		cv_wait(&f->cv_drain, &usb_ref_lock);
618 	}
619 	mtx_unlock(&usb_ref_lock);
620 
621 	/* take care of closing the device here, if any */
622 	usb_fifo_close(f, 0);
623 
624 	cv_destroy(&f->cv_io);
625 	cv_destroy(&f->cv_drain);
626 
627 	knlist_clear(&f->selinfo.si_note, 0);
628 	seldrain(&f->selinfo);
629 	knlist_destroy(&f->selinfo.si_note);
630 
631 	free(f, M_USBDEV);
632 }
633 
634 static struct usb_endpoint *
635 usb_dev_get_ep(struct usb_device *udev, uint8_t ep_index, uint8_t dir)
636 {
637 	struct usb_endpoint *ep;
638 	uint8_t ep_dir;
639 
640 	if (ep_index == 0) {
641 		ep = &udev->ctrl_ep;
642 	} else {
643 		if (dir == USB_FIFO_RX) {
644 			if (udev->flags.usb_mode == USB_MODE_HOST) {
645 				ep_dir = UE_DIR_IN;
646 			} else {
647 				ep_dir = UE_DIR_OUT;
648 			}
649 		} else {
650 			if (udev->flags.usb_mode == USB_MODE_HOST) {
651 				ep_dir = UE_DIR_OUT;
652 			} else {
653 				ep_dir = UE_DIR_IN;
654 			}
655 		}
656 		ep = usbd_get_ep_by_addr(udev, ep_index | ep_dir);
657 	}
658 
659 	if (ep == NULL) {
660 		/* if the endpoint does not exist then return */
661 		return (NULL);
662 	}
663 	if (ep->edesc == NULL) {
664 		/* invalid endpoint */
665 		return (NULL);
666 	}
667 	return (ep);			/* success */
668 }
669 
670 /*------------------------------------------------------------------------*
671  *	usb_fifo_open
672  *
673  * Returns:
674  * 0: Success
675  * Else: Failure
676  *------------------------------------------------------------------------*/
677 static int
678 usb_fifo_open(struct usb_cdev_privdata *cpd,
679     struct usb_fifo *f, int fflags)
680 {
681 	int err;
682 
683 	if (f == NULL) {
684 		/* no FIFO there */
685 		DPRINTFN(2, "no FIFO\n");
686 		return (ENXIO);
687 	}
688 	/* remove FWRITE and FREAD flags */
689 	fflags &= ~(FWRITE | FREAD);
690 
691 	/* set correct file flags */
692 	if ((f->fifo_index & 1) == USB_FIFO_TX) {
693 		fflags |= FWRITE;
694 	} else {
695 		fflags |= FREAD;
696 	}
697 
698 	/* check if we are already opened */
699 	/* we don't need any locks when checking this variable */
700 	if (f->curr_cpd != NULL) {
701 		err = EBUSY;
702 		goto done;
703 	}
704 
705 	/* reset short flag before open */
706 	f->flag_short = 0;
707 
708 	/* call open method */
709 	err = (f->methods->f_open) (f, fflags);
710 	if (err) {
711 		goto done;
712 	}
713 	mtx_lock(f->priv_mtx);
714 
715 	/* reset sleep flag */
716 	f->flag_sleeping = 0;
717 
718 	/* reset error flag */
719 	f->flag_iserror = 0;
720 
721 	/* reset complete flag */
722 	f->flag_iscomplete = 0;
723 
724 	/* reset select flag */
725 	f->flag_isselect = 0;
726 
727 	/* reset flushing flag */
728 	f->flag_flushing = 0;
729 
730 	/* reset ASYNC proc flag */
731 	f->async_p = NULL;
732 
733 	mtx_lock(&usb_ref_lock);
734 	/* flag the fifo as opened to prevent others */
735 	f->curr_cpd = cpd;
736 	mtx_unlock(&usb_ref_lock);
737 
738 	/* reset queue */
739 	usb_fifo_reset(f);
740 
741 	mtx_unlock(f->priv_mtx);
742 done:
743 	return (err);
744 }
745 
746 /*------------------------------------------------------------------------*
747  *	usb_fifo_reset
748  *------------------------------------------------------------------------*/
749 void
750 usb_fifo_reset(struct usb_fifo *f)
751 {
752 	struct usb_mbuf *m;
753 
754 	if (f == NULL) {
755 		return;
756 	}
757 	while (1) {
758 		USB_IF_DEQUEUE(&f->used_q, m);
759 		if (m) {
760 			USB_IF_ENQUEUE(&f->free_q, m);
761 		} else {
762 			break;
763 		}
764 	}
765 	/* reset have fragment flag */
766 	f->flag_have_fragment = 0;
767 }
768 
769 /*------------------------------------------------------------------------*
770  *	usb_fifo_close
771  *------------------------------------------------------------------------*/
772 static void
773 usb_fifo_close(struct usb_fifo *f, int fflags)
774 {
775 	int err;
776 
777 	/* check if we are not opened */
778 	if (f->curr_cpd == NULL) {
779 		/* nothing to do - already closed */
780 		return;
781 	}
782 	mtx_lock(f->priv_mtx);
783 
784 	/* clear current cdev private data pointer */
785 	mtx_lock(&usb_ref_lock);
786 	f->curr_cpd = NULL;
787 	mtx_unlock(&usb_ref_lock);
788 
789 	/* check if we are watched by kevent */
790 	KNOTE_LOCKED(&f->selinfo.si_note, 0);
791 
792 	/* check if we are selected */
793 	if (f->flag_isselect) {
794 		selwakeup(&f->selinfo);
795 		f->flag_isselect = 0;
796 	}
797 	/* check if a thread wants SIGIO */
798 	if (f->async_p != NULL) {
799 		PROC_LOCK(f->async_p);
800 		kern_psignal(f->async_p, SIGIO);
801 		PROC_UNLOCK(f->async_p);
802 		f->async_p = NULL;
803 	}
804 	/* remove FWRITE and FREAD flags */
805 	fflags &= ~(FWRITE | FREAD);
806 
807 	/* flush written data, if any */
808 	if ((f->fifo_index & 1) == USB_FIFO_TX) {
809 
810 		if (!f->flag_iserror) {
811 
812 			/* set flushing flag */
813 			f->flag_flushing = 1;
814 
815 			/* get the last packet in */
816 			if (f->flag_have_fragment) {
817 				struct usb_mbuf *m;
818 				f->flag_have_fragment = 0;
819 				USB_IF_DEQUEUE(&f->free_q, m);
820 				if (m) {
821 					USB_IF_ENQUEUE(&f->used_q, m);
822 				}
823 			}
824 
825 			/* start write transfer, if not already started */
826 			(f->methods->f_start_write) (f);
827 
828 			/* check if flushed already */
829 			while (f->flag_flushing &&
830 			    (!f->flag_iserror)) {
831 				/* wait until all data has been written */
832 				f->flag_sleeping = 1;
833 				err = cv_timedwait_sig(&f->cv_io, f->priv_mtx,
834 				    USB_MS_TO_TICKS(USB_DEFAULT_TIMEOUT));
835 				if (err) {
836 					DPRINTF("signal received\n");
837 					break;
838 				}
839 			}
840 		}
841 		fflags |= FWRITE;
842 
843 		/* stop write transfer, if not already stopped */
844 		(f->methods->f_stop_write) (f);
845 	} else {
846 		fflags |= FREAD;
847 
848 		/* stop write transfer, if not already stopped */
849 		(f->methods->f_stop_read) (f);
850 	}
851 
852 	/* check if we are sleeping */
853 	if (f->flag_sleeping) {
854 		DPRINTFN(2, "Sleeping at close!\n");
855 	}
856 	mtx_unlock(f->priv_mtx);
857 
858 	/* call close method */
859 	(f->methods->f_close) (f, fflags);
860 
861 	DPRINTF("closed\n");
862 }
863 
864 /*------------------------------------------------------------------------*
865  *	usb_open - cdev callback
866  *------------------------------------------------------------------------*/
867 static int
868 usb_open(struct cdev *dev, int fflags, int devtype, struct thread *td)
869 {
870 	struct usb_fs_privdata* pd = (struct usb_fs_privdata*)dev->si_drv1;
871 	struct usb_cdev_refdata refs;
872 	struct usb_cdev_privdata *cpd;
873 	int err, ep;
874 
875 	DPRINTFN(2, "%s fflags=0x%08x\n", devtoname(dev), fflags);
876 
877 	KASSERT(fflags & (FREAD|FWRITE), ("invalid open flags"));
878 	if (((fflags & FREAD) && !(pd->mode & FREAD)) ||
879 	    ((fflags & FWRITE) && !(pd->mode & FWRITE))) {
880 		DPRINTFN(2, "access mode not supported\n");
881 		return (EPERM);
882 	}
883 
884 	cpd = malloc(sizeof(*cpd), M_USBDEV, M_WAITOK | M_ZERO);
885 	ep = cpd->ep_addr = pd->ep_addr;
886 
887 	usb_loc_fill(pd, cpd);
888 	err = usb_ref_device(cpd, &refs, 1);
889 	if (err) {
890 		DPRINTFN(2, "cannot ref device\n");
891 		free(cpd, M_USBDEV);
892 		return (ENXIO);
893 	}
894 	cpd->fflags = fflags;	/* access mode for open lifetime */
895 
896 	/* create FIFOs, if any */
897 	err = usb_fifo_create(cpd, &refs);
898 	/* check for error */
899 	if (err) {
900 		DPRINTFN(2, "cannot create fifo\n");
901 		usb_unref_device(cpd, &refs);
902 		free(cpd, M_USBDEV);
903 		return (err);
904 	}
905 	if (fflags & FREAD) {
906 		err = usb_fifo_open(cpd, refs.rxfifo, fflags);
907 		if (err) {
908 			DPRINTFN(2, "read open failed\n");
909 			usb_unref_device(cpd, &refs);
910 			free(cpd, M_USBDEV);
911 			return (err);
912 		}
913 	}
914 	if (fflags & FWRITE) {
915 		err = usb_fifo_open(cpd, refs.txfifo, fflags);
916 		if (err) {
917 			DPRINTFN(2, "write open failed\n");
918 			if (fflags & FREAD) {
919 				usb_fifo_close(refs.rxfifo, fflags);
920 			}
921 			usb_unref_device(cpd, &refs);
922 			free(cpd, M_USBDEV);
923 			return (err);
924 		}
925 	}
926 	usb_unref_device(cpd, &refs);
927 	devfs_set_cdevpriv(cpd, usb_close);
928 
929 	return (0);
930 }
931 
932 /*------------------------------------------------------------------------*
933  *	usb_close - cdev callback
934  *------------------------------------------------------------------------*/
935 static void
936 usb_close(void *arg)
937 {
938 	struct usb_cdev_refdata refs;
939 	struct usb_cdev_privdata *cpd = arg;
940 	int err;
941 
942 	DPRINTFN(2, "cpd=%p\n", cpd);
943 
944 	err = usb_ref_device(cpd, &refs,
945 	    2 /* uref and allow detached state */);
946 	if (err) {
947 		DPRINTFN(2, "Cannot grab USB reference when "
948 		    "closing USB file handle\n");
949 		goto done;
950 	}
951 	if (cpd->fflags & FREAD) {
952 		usb_fifo_close(refs.rxfifo, cpd->fflags);
953 	}
954 	if (cpd->fflags & FWRITE) {
955 		usb_fifo_close(refs.txfifo, cpd->fflags);
956 	}
957 	usb_unref_device(cpd, &refs);
958 done:
959 	free(cpd, M_USBDEV);
960 }
961 
962 static void
963 usb_dev_init(void *arg)
964 {
965 	mtx_init(&usb_ref_lock, "USB ref mutex", NULL, MTX_DEF);
966 	sx_init(&usb_sym_lock, "USB sym mutex");
967 	TAILQ_INIT(&usb_sym_head);
968 
969 	/* check the UGEN methods */
970 	usb_fifo_check_methods(&usb_ugen_methods);
971 }
972 
973 SYSINIT(usb_dev_init, SI_SUB_KLD, SI_ORDER_FIRST, usb_dev_init, NULL);
974 
975 static void
976 usb_dev_init_post(void *arg)
977 {
978 	/*
979 	 * Create /dev/usb - this is needed for usbconfig(8), which
980 	 * needs a well-known device name to access.
981 	 */
982 	usb_dev = make_dev(&usb_static_devsw, 0, UID_ROOT, GID_OPERATOR,
983 	    0644, USB_DEVICE_NAME);
984 	if (usb_dev == NULL) {
985 		DPRINTFN(0, "Could not create usb bus device\n");
986 	}
987 }
988 
989 SYSINIT(usb_dev_init_post, SI_SUB_KICK_SCHEDULER, SI_ORDER_FIRST, usb_dev_init_post, NULL);
990 
991 static void
992 usb_dev_uninit(void *arg)
993 {
994 	if (usb_dev != NULL) {
995 		destroy_dev(usb_dev);
996 		usb_dev = NULL;
997 	}
998 	mtx_destroy(&usb_ref_lock);
999 	sx_destroy(&usb_sym_lock);
1000 }
1001 
1002 SYSUNINIT(usb_dev_uninit, SI_SUB_KICK_SCHEDULER, SI_ORDER_ANY, usb_dev_uninit, NULL);
1003 
1004 static int
1005 usb_ioctl_f_sub(struct usb_fifo *f, u_long cmd, void *addr,
1006     struct thread *td)
1007 {
1008 	int error = 0;
1009 
1010 	switch (cmd) {
1011 	case FIODTYPE:
1012 		*(int *)addr = 0;	/* character device */
1013 		break;
1014 
1015 	case FIONBIO:
1016 		/* handled by upper FS layer */
1017 		break;
1018 
1019 	case FIOASYNC:
1020 		if (*(int *)addr) {
1021 			if (f->async_p != NULL) {
1022 				error = EBUSY;
1023 				break;
1024 			}
1025 			f->async_p = USB_TD_GET_PROC(td);
1026 		} else {
1027 			f->async_p = NULL;
1028 		}
1029 		break;
1030 
1031 		/* XXX this is not the most general solution */
1032 	case TIOCSPGRP:
1033 		if (f->async_p == NULL) {
1034 			error = EINVAL;
1035 			break;
1036 		}
1037 		if (*(int *)addr != USB_PROC_GET_GID(f->async_p)) {
1038 			error = EPERM;
1039 			break;
1040 		}
1041 		break;
1042 	default:
1043 		return (ENOIOCTL);
1044 	}
1045 	DPRINTFN(3, "cmd 0x%lx = %d\n", cmd, error);
1046 	return (error);
1047 }
1048 
1049 /*------------------------------------------------------------------------*
1050  *	usb_ioctl - cdev callback
1051  *------------------------------------------------------------------------*/
1052 static int
1053 usb_ioctl(struct cdev *dev, u_long cmd, caddr_t addr, int fflag, struct thread* td)
1054 {
1055 	struct usb_cdev_refdata refs;
1056 	struct usb_cdev_privdata* cpd;
1057 	struct usb_fifo *f;
1058 	int fflags;
1059 	int err;
1060 
1061 	DPRINTFN(2, "cmd=0x%lx\n", cmd);
1062 
1063 	err = devfs_get_cdevpriv((void **)&cpd);
1064 	if (err != 0)
1065 		return (err);
1066 
1067 	/*
1068 	 * Performance optimisation: We try to check for IOCTL's that
1069 	 * don't need the USB reference first. Then we grab the USB
1070 	 * reference if we need it!
1071 	 */
1072 	err = usb_ref_device(cpd, &refs, 0 /* no uref */ );
1073 	if (err)
1074 		return (ENXIO);
1075 
1076 	fflags = cpd->fflags;
1077 
1078 	f = NULL;			/* set default value */
1079 	err = ENOIOCTL;			/* set default value */
1080 
1081 	if (fflags & FWRITE) {
1082 		f = refs.txfifo;
1083 		err = usb_ioctl_f_sub(f, cmd, addr, td);
1084 	}
1085 	if (fflags & FREAD) {
1086 		f = refs.rxfifo;
1087 		err = usb_ioctl_f_sub(f, cmd, addr, td);
1088 	}
1089 	KASSERT(f != NULL, ("fifo not found"));
1090 	if (err != ENOIOCTL)
1091 		goto done;
1092 
1093 	err = (f->methods->f_ioctl) (f, cmd, addr, fflags);
1094 
1095 	DPRINTFN(2, "f_ioctl cmd 0x%lx = %d\n", cmd, err);
1096 
1097 	if (err != ENOIOCTL)
1098 		goto done;
1099 
1100 	if (usb_usb_ref_device(cpd, &refs)) {
1101 		/* we lost the reference */
1102 		return (ENXIO);
1103 	}
1104 
1105 	err = (f->methods->f_ioctl_post) (f, cmd, addr, fflags);
1106 
1107 	DPRINTFN(2, "f_ioctl_post cmd 0x%lx = %d\n", cmd, err);
1108 
1109 	if (err == ENOIOCTL)
1110 		err = ENOTTY;
1111 
1112 	if (err)
1113 		goto done;
1114 
1115 	/* Wait for re-enumeration, if any */
1116 
1117 	while (f->udev->re_enumerate_wait != USB_RE_ENUM_DONE) {
1118 
1119 		usb_unref_device(cpd, &refs);
1120 
1121 		usb_pause_mtx(NULL, hz / 128);
1122 
1123 		while (usb_ref_device(cpd, &refs, 1 /* need uref */)) {
1124 			if (usb_ref_device(cpd, &refs, 0)) {
1125 				/* device no longer exists */
1126 				return (ENXIO);
1127 			}
1128 			usb_unref_device(cpd, &refs);
1129 			usb_pause_mtx(NULL, hz / 128);
1130 		}
1131 	}
1132 
1133 done:
1134 	usb_unref_device(cpd, &refs);
1135 	return (err);
1136 }
1137 
1138 static void
1139 usb_filter_detach(struct knote *kn)
1140 {
1141 	struct usb_fifo *f = kn->kn_hook;
1142 	knlist_remove(&f->selinfo.si_note, kn, 0);
1143 }
1144 
1145 static int
1146 usb_filter_write(struct knote *kn, long hint)
1147 {
1148 	struct usb_cdev_privdata* cpd;
1149 	struct usb_fifo *f;
1150 	struct usb_mbuf *m;
1151 
1152 	DPRINTFN(2, "\n");
1153 
1154 	f = kn->kn_hook;
1155 
1156 	mtx_assert(f->priv_mtx, MA_OWNED);
1157 
1158 	cpd = f->curr_cpd;
1159 	if (cpd == NULL) {
1160 		m = (void *)1;
1161 	} else if (f->fs_ep_max == 0) {
1162 		if (f->flag_iserror) {
1163 			/* we got an error */
1164 			m = (void *)1;
1165 		} else {
1166 			if (f->queue_data == NULL) {
1167 				/*
1168 				 * start write transfer, if not
1169 				 * already started
1170 				 */
1171 				(f->methods->f_start_write) (f);
1172 			}
1173 			/* check if any packets are available */
1174 			USB_IF_POLL(&f->free_q, m);
1175 		}
1176 	} else {
1177 		if (f->flag_iscomplete) {
1178 			m = (void *)1;
1179 		} else {
1180 			m = NULL;
1181 		}
1182 	}
1183 	return (m ? 1 : 0);
1184 }
1185 
1186 static int
1187 usb_filter_read(struct knote *kn, long hint)
1188 {
1189 	struct usb_cdev_privdata* cpd;
1190 	struct usb_fifo *f;
1191 	struct usb_mbuf *m;
1192 
1193 	DPRINTFN(2, "\n");
1194 
1195 	f = kn->kn_hook;
1196 
1197 	mtx_assert(f->priv_mtx, MA_OWNED);
1198 
1199 	cpd = f->curr_cpd;
1200 	if (cpd == NULL) {
1201 		m = (void *)1;
1202 	} else if (f->fs_ep_max == 0) {
1203 		if (f->flag_iserror) {
1204 			/* we have an error */
1205 			m = (void *)1;
1206 		} else {
1207 			if (f->queue_data == NULL) {
1208 				/*
1209 				 * start read transfer, if not
1210 				 * already started
1211 				 */
1212 				(f->methods->f_start_read) (f);
1213 			}
1214 			/* check if any packets are available */
1215 			USB_IF_POLL(&f->used_q, m);
1216 
1217 			/* start reading data, if any */
1218 			if (m == NULL)
1219 				(f->methods->f_start_read) (f);
1220 		}
1221 	} else {
1222 		if (f->flag_iscomplete) {
1223 			m = (void *)1;
1224 		} else {
1225 			m = NULL;
1226 		}
1227 	}
1228 	return (m ? 1 : 0);
1229 }
1230 
1231 static struct filterops usb_filtops_write = {
1232 	.f_isfd = 1,
1233 	.f_detach = usb_filter_detach,
1234 	.f_event = usb_filter_write,
1235 };
1236 
1237 static struct filterops usb_filtops_read = {
1238 	.f_isfd = 1,
1239 	.f_detach = usb_filter_detach,
1240 	.f_event = usb_filter_read,
1241 };
1242 
1243 
1244 /* ARGSUSED */
1245 static int
1246 usb_kqfilter(struct cdev* dev, struct knote *kn)
1247 {
1248 	struct usb_cdev_refdata refs;
1249 	struct usb_cdev_privdata* cpd;
1250 	struct usb_fifo *f;
1251 	int fflags;
1252 	int err = EINVAL;
1253 
1254 	DPRINTFN(2, "\n");
1255 
1256 	if (devfs_get_cdevpriv((void **)&cpd) != 0 ||
1257 	    usb_ref_device(cpd, &refs, 0) != 0)
1258 		return (ENXIO);
1259 
1260 	fflags = cpd->fflags;
1261 
1262 	/* Figure out who needs service */
1263 	switch (kn->kn_filter) {
1264 	case EVFILT_WRITE:
1265 		if (fflags & FWRITE) {
1266 			f = refs.txfifo;
1267 			kn->kn_fop = &usb_filtops_write;
1268 			err = 0;
1269 		}
1270 		break;
1271 	case EVFILT_READ:
1272 		if (fflags & FREAD) {
1273 			f = refs.rxfifo;
1274 			kn->kn_fop = &usb_filtops_read;
1275 			err = 0;
1276 		}
1277 		break;
1278 	default:
1279 		err = EOPNOTSUPP;
1280 		break;
1281 	}
1282 
1283 	if (err == 0) {
1284 		kn->kn_hook = f;
1285 		mtx_lock(f->priv_mtx);
1286 		knlist_add(&f->selinfo.si_note, kn, 1);
1287 		mtx_unlock(f->priv_mtx);
1288 	}
1289 
1290 	usb_unref_device(cpd, &refs);
1291 	return (err);
1292 }
1293 
1294 /* ARGSUSED */
1295 static int
1296 usb_poll(struct cdev* dev, int events, struct thread* td)
1297 {
1298 	struct usb_cdev_refdata refs;
1299 	struct usb_cdev_privdata* cpd;
1300 	struct usb_fifo *f;
1301 	struct usb_mbuf *m;
1302 	int fflags, revents;
1303 
1304 	if (devfs_get_cdevpriv((void **)&cpd) != 0 ||
1305 	    usb_ref_device(cpd, &refs, 0) != 0)
1306 		return (events &
1307 		    (POLLHUP|POLLIN|POLLRDNORM|POLLOUT|POLLWRNORM));
1308 
1309 	fflags = cpd->fflags;
1310 
1311 	/* Figure out who needs service */
1312 	revents = 0;
1313 	if ((events & (POLLOUT | POLLWRNORM)) &&
1314 	    (fflags & FWRITE)) {
1315 
1316 		f = refs.txfifo;
1317 
1318 		mtx_lock(f->priv_mtx);
1319 
1320 		if (!refs.is_usbfs) {
1321 			if (f->flag_iserror) {
1322 				/* we got an error */
1323 				m = (void *)1;
1324 			} else {
1325 				if (f->queue_data == NULL) {
1326 					/*
1327 					 * start write transfer, if not
1328 					 * already started
1329 					 */
1330 					(f->methods->f_start_write) (f);
1331 				}
1332 				/* check if any packets are available */
1333 				USB_IF_POLL(&f->free_q, m);
1334 			}
1335 		} else {
1336 			if (f->flag_iscomplete) {
1337 				m = (void *)1;
1338 			} else {
1339 				m = NULL;
1340 			}
1341 		}
1342 
1343 		if (m) {
1344 			revents |= events & (POLLOUT | POLLWRNORM);
1345 		} else {
1346 			f->flag_isselect = 1;
1347 			selrecord(td, &f->selinfo);
1348 		}
1349 
1350 		mtx_unlock(f->priv_mtx);
1351 	}
1352 	if ((events & (POLLIN | POLLRDNORM)) &&
1353 	    (fflags & FREAD)) {
1354 
1355 		f = refs.rxfifo;
1356 
1357 		mtx_lock(f->priv_mtx);
1358 
1359 		if (!refs.is_usbfs) {
1360 			if (f->flag_iserror) {
1361 				/* we have an error */
1362 				m = (void *)1;
1363 			} else {
1364 				if (f->queue_data == NULL) {
1365 					/*
1366 					 * start read transfer, if not
1367 					 * already started
1368 					 */
1369 					(f->methods->f_start_read) (f);
1370 				}
1371 				/* check if any packets are available */
1372 				USB_IF_POLL(&f->used_q, m);
1373 			}
1374 		} else {
1375 			if (f->flag_iscomplete) {
1376 				m = (void *)1;
1377 			} else {
1378 				m = NULL;
1379 			}
1380 		}
1381 
1382 		if (m) {
1383 			revents |= events & (POLLIN | POLLRDNORM);
1384 		} else {
1385 			f->flag_isselect = 1;
1386 			selrecord(td, &f->selinfo);
1387 
1388 			if (!refs.is_usbfs) {
1389 				/* start reading data */
1390 				(f->methods->f_start_read) (f);
1391 			}
1392 		}
1393 
1394 		mtx_unlock(f->priv_mtx);
1395 	}
1396 	usb_unref_device(cpd, &refs);
1397 	return (revents);
1398 }
1399 
1400 static int
1401 usb_read(struct cdev *dev, struct uio *uio, int ioflag)
1402 {
1403 	struct usb_cdev_refdata refs;
1404 	struct usb_cdev_privdata* cpd;
1405 	struct usb_fifo *f;
1406 	struct usb_mbuf *m;
1407 	int fflags;
1408 	int resid;
1409 	int io_len;
1410 	int err;
1411 	uint8_t tr_data = 0;
1412 
1413 	err = devfs_get_cdevpriv((void **)&cpd);
1414 	if (err != 0)
1415 		return (err);
1416 
1417 	err = usb_ref_device(cpd, &refs, 0 /* no uref */ );
1418 	if (err)
1419 		return (ENXIO);
1420 
1421 	fflags = cpd->fflags;
1422 
1423 	f = refs.rxfifo;
1424 	if (f == NULL) {
1425 		/* should not happen */
1426 		usb_unref_device(cpd, &refs);
1427 		return (EPERM);
1428 	}
1429 
1430 	resid = uio->uio_resid;
1431 
1432 	mtx_lock(f->priv_mtx);
1433 
1434 	/* check for permanent read error */
1435 	if (f->flag_iserror) {
1436 		err = EIO;
1437 		goto done;
1438 	}
1439 	/* check if USB-FS interface is active */
1440 	if (refs.is_usbfs) {
1441 		/*
1442 		 * The queue is used for events that should be
1443 		 * retrieved using the "USB_FS_COMPLETE" ioctl.
1444 		 */
1445 		err = EINVAL;
1446 		goto done;
1447 	}
1448 	while (uio->uio_resid > 0) {
1449 
1450 		USB_IF_DEQUEUE(&f->used_q, m);
1451 
1452 		if (m == NULL) {
1453 
1454 			/* start read transfer, if not already started */
1455 
1456 			(f->methods->f_start_read) (f);
1457 
1458 			if (ioflag & IO_NDELAY) {
1459 				if (tr_data) {
1460 					/* return length before error */
1461 					break;
1462 				}
1463 				err = EWOULDBLOCK;
1464 				break;
1465 			}
1466 			DPRINTF("sleeping\n");
1467 
1468 			err = usb_fifo_wait(f);
1469 			if (err) {
1470 				break;
1471 			}
1472 			continue;
1473 		}
1474 		if (f->methods->f_filter_read) {
1475 			/*
1476 			 * Sometimes it is convenient to process data at the
1477 			 * expense of a userland process instead of a kernel
1478 			 * process.
1479 			 */
1480 			(f->methods->f_filter_read) (f, m);
1481 		}
1482 		tr_data = 1;
1483 
1484 		io_len = MIN(m->cur_data_len, uio->uio_resid);
1485 
1486 		DPRINTFN(2, "transfer %d bytes from %p\n",
1487 		    io_len, m->cur_data_ptr);
1488 
1489 		err = usb_fifo_uiomove(f,
1490 		    m->cur_data_ptr, io_len, uio);
1491 
1492 		m->cur_data_len -= io_len;
1493 		m->cur_data_ptr += io_len;
1494 
1495 		if (m->cur_data_len == 0) {
1496 
1497 			uint8_t last_packet;
1498 
1499 			last_packet = m->last_packet;
1500 
1501 			USB_IF_ENQUEUE(&f->free_q, m);
1502 
1503 			if (last_packet) {
1504 				/* keep framing */
1505 				break;
1506 			}
1507 		} else {
1508 			USB_IF_PREPEND(&f->used_q, m);
1509 		}
1510 
1511 		if (err) {
1512 			break;
1513 		}
1514 	}
1515 done:
1516 	mtx_unlock(f->priv_mtx);
1517 
1518 	usb_unref_device(cpd, &refs);
1519 
1520 	return (err);
1521 }
1522 
1523 static int
1524 usb_write(struct cdev *dev, struct uio *uio, int ioflag)
1525 {
1526 	struct usb_cdev_refdata refs;
1527 	struct usb_cdev_privdata* cpd;
1528 	struct usb_fifo *f;
1529 	struct usb_mbuf *m;
1530 	uint8_t *pdata;
1531 	int fflags;
1532 	int resid;
1533 	int io_len;
1534 	int err;
1535 	uint8_t tr_data = 0;
1536 
1537 	DPRINTFN(2, "\n");
1538 
1539 	err = devfs_get_cdevpriv((void **)&cpd);
1540 	if (err != 0)
1541 		return (err);
1542 
1543 	err = usb_ref_device(cpd, &refs, 0 /* no uref */ );
1544 	if (err)
1545 		return (ENXIO);
1546 
1547 	fflags = cpd->fflags;
1548 
1549 	f = refs.txfifo;
1550 	if (f == NULL) {
1551 		/* should not happen */
1552 		usb_unref_device(cpd, &refs);
1553 		return (EPERM);
1554 	}
1555 	resid = uio->uio_resid;
1556 
1557 	mtx_lock(f->priv_mtx);
1558 
1559 	/* check for permanent write error */
1560 	if (f->flag_iserror) {
1561 		err = EIO;
1562 		goto done;
1563 	}
1564 	/* check if USB-FS interface is active */
1565 	if (refs.is_usbfs) {
1566 		/*
1567 		 * The queue is used for events that should be
1568 		 * retrieved using the "USB_FS_COMPLETE" ioctl.
1569 		 */
1570 		err = EINVAL;
1571 		goto done;
1572 	}
1573 	if (f->queue_data == NULL) {
1574 		/* start write transfer, if not already started */
1575 		(f->methods->f_start_write) (f);
1576 	}
1577 	/* we allow writing zero length data */
1578 	do {
1579 		USB_IF_DEQUEUE(&f->free_q, m);
1580 
1581 		if (m == NULL) {
1582 
1583 			if (ioflag & IO_NDELAY) {
1584 				if (tr_data) {
1585 					/* return length before error */
1586 					break;
1587 				}
1588 				err = EWOULDBLOCK;
1589 				break;
1590 			}
1591 			DPRINTF("sleeping\n");
1592 
1593 			err = usb_fifo_wait(f);
1594 			if (err) {
1595 				break;
1596 			}
1597 			continue;
1598 		}
1599 		tr_data = 1;
1600 
1601 		if (f->flag_have_fragment == 0) {
1602 			USB_MBUF_RESET(m);
1603 			io_len = m->cur_data_len;
1604 			pdata = m->cur_data_ptr;
1605 			if (io_len > uio->uio_resid)
1606 				io_len = uio->uio_resid;
1607 			m->cur_data_len = io_len;
1608 		} else {
1609 			io_len = m->max_data_len - m->cur_data_len;
1610 			pdata = m->cur_data_ptr + m->cur_data_len;
1611 			if (io_len > uio->uio_resid)
1612 				io_len = uio->uio_resid;
1613 			m->cur_data_len += io_len;
1614 		}
1615 
1616 		DPRINTFN(2, "transfer %d bytes to %p\n",
1617 		    io_len, pdata);
1618 
1619 		err = usb_fifo_uiomove(f, pdata, io_len, uio);
1620 
1621 		if (err) {
1622 			f->flag_have_fragment = 0;
1623 			USB_IF_ENQUEUE(&f->free_q, m);
1624 			break;
1625 		}
1626 
1627 		/* check if the buffer is ready to be transmitted */
1628 
1629 		if ((f->flag_write_defrag == 0) ||
1630 		    (m->cur_data_len == m->max_data_len)) {
1631 			f->flag_have_fragment = 0;
1632 
1633 			/*
1634 			 * Check for write filter:
1635 			 *
1636 			 * Sometimes it is convenient to process data
1637 			 * at the expense of a userland process
1638 			 * instead of a kernel process.
1639 			 */
1640 			if (f->methods->f_filter_write) {
1641 				(f->methods->f_filter_write) (f, m);
1642 			}
1643 
1644 			/* Put USB mbuf in the used queue */
1645 			USB_IF_ENQUEUE(&f->used_q, m);
1646 
1647 			/* Start writing data, if not already started */
1648 			(f->methods->f_start_write) (f);
1649 		} else {
1650 			/* Wait for more data or close */
1651 			f->flag_have_fragment = 1;
1652 			USB_IF_PREPEND(&f->free_q, m);
1653 		}
1654 
1655 	} while (uio->uio_resid > 0);
1656 done:
1657 	mtx_unlock(f->priv_mtx);
1658 
1659 	usb_unref_device(cpd, &refs);
1660 
1661 	return (err);
1662 }
1663 
1664 int
1665 usb_static_ioctl(struct cdev *dev, u_long cmd, caddr_t data, int fflag,
1666     struct thread *td)
1667 {
1668 	union {
1669 		struct usb_read_dir *urd;
1670 		void* data;
1671 	} u;
1672 	int err;
1673 
1674 	u.data = data;
1675 	switch (cmd) {
1676 		case USB_READ_DIR:
1677 			err = usb_read_symlink(u.urd->urd_data,
1678 			    u.urd->urd_startentry, u.urd->urd_maxlen);
1679 			break;
1680 		case USB_DEV_QUIRK_GET:
1681 		case USB_QUIRK_NAME_GET:
1682 		case USB_DEV_QUIRK_ADD:
1683 		case USB_DEV_QUIRK_REMOVE:
1684 			err = usb_quirk_ioctl_p(cmd, data, fflag, td);
1685 			break;
1686 		case USB_GET_TEMPLATE:
1687 			*(int *)data = usb_template;
1688 			err = 0;
1689 			break;
1690 		case USB_SET_TEMPLATE:
1691 			err = priv_check(curthread, PRIV_DRIVER);
1692 			if (err)
1693 				break;
1694 			usb_template = *(int *)data;
1695 			break;
1696 		default:
1697 			err = ENOTTY;
1698 			break;
1699 	}
1700 	return (err);
1701 }
1702 
1703 static int
1704 usb_fifo_uiomove(struct usb_fifo *f, void *cp,
1705     int n, struct uio *uio)
1706 {
1707 	int error;
1708 
1709 	mtx_unlock(f->priv_mtx);
1710 
1711 	/*
1712 	 * "uiomove()" can sleep so one needs to make a wrapper,
1713 	 * exiting the mutex and checking things:
1714 	 */
1715 	error = uiomove(cp, n, uio);
1716 
1717 	mtx_lock(f->priv_mtx);
1718 
1719 	return (error);
1720 }
1721 
1722 int
1723 usb_fifo_wait(struct usb_fifo *f)
1724 {
1725 	int err;
1726 
1727 	mtx_assert(f->priv_mtx, MA_OWNED);
1728 
1729 	if (f->flag_iserror) {
1730 		/* we are gone */
1731 		return (EIO);
1732 	}
1733 	f->flag_sleeping = 1;
1734 
1735 	err = cv_wait_sig(&f->cv_io, f->priv_mtx);
1736 
1737 	if (f->flag_iserror) {
1738 		/* we are gone */
1739 		err = EIO;
1740 	}
1741 	return (err);
1742 }
1743 
1744 void
1745 usb_fifo_signal(struct usb_fifo *f)
1746 {
1747 	if (f->flag_sleeping) {
1748 		f->flag_sleeping = 0;
1749 		cv_broadcast(&f->cv_io);
1750 	}
1751 }
1752 
1753 void
1754 usb_fifo_wakeup(struct usb_fifo *f)
1755 {
1756 	usb_fifo_signal(f);
1757 
1758 	KNOTE_LOCKED(&f->selinfo.si_note, 0);
1759 
1760 	if (f->flag_isselect) {
1761 		selwakeup(&f->selinfo);
1762 		f->flag_isselect = 0;
1763 	}
1764 	if (f->async_p != NULL) {
1765 		PROC_LOCK(f->async_p);
1766 		kern_psignal(f->async_p, SIGIO);
1767 		PROC_UNLOCK(f->async_p);
1768 	}
1769 }
1770 
1771 static int
1772 usb_fifo_dummy_open(struct usb_fifo *fifo, int fflags)
1773 {
1774 	return (0);
1775 }
1776 
1777 static void
1778 usb_fifo_dummy_close(struct usb_fifo *fifo, int fflags)
1779 {
1780 	return;
1781 }
1782 
1783 static int
1784 usb_fifo_dummy_ioctl(struct usb_fifo *fifo, u_long cmd, void *addr, int fflags)
1785 {
1786 	return (ENOIOCTL);
1787 }
1788 
1789 static void
1790 usb_fifo_dummy_cmd(struct usb_fifo *fifo)
1791 {
1792 	fifo->flag_flushing = 0;	/* not flushing */
1793 }
1794 
1795 static void
1796 usb_fifo_check_methods(struct usb_fifo_methods *pm)
1797 {
1798 	/* check that all callback functions are OK */
1799 
1800 	if (pm->f_open == NULL)
1801 		pm->f_open = &usb_fifo_dummy_open;
1802 
1803 	if (pm->f_close == NULL)
1804 		pm->f_close = &usb_fifo_dummy_close;
1805 
1806 	if (pm->f_ioctl == NULL)
1807 		pm->f_ioctl = &usb_fifo_dummy_ioctl;
1808 
1809 	if (pm->f_ioctl_post == NULL)
1810 		pm->f_ioctl_post = &usb_fifo_dummy_ioctl;
1811 
1812 	if (pm->f_start_read == NULL)
1813 		pm->f_start_read = &usb_fifo_dummy_cmd;
1814 
1815 	if (pm->f_stop_read == NULL)
1816 		pm->f_stop_read = &usb_fifo_dummy_cmd;
1817 
1818 	if (pm->f_start_write == NULL)
1819 		pm->f_start_write = &usb_fifo_dummy_cmd;
1820 
1821 	if (pm->f_stop_write == NULL)
1822 		pm->f_stop_write = &usb_fifo_dummy_cmd;
1823 }
1824 
1825 /*------------------------------------------------------------------------*
1826  *	usb_fifo_attach
1827  *
1828  * The following function will create a duplex FIFO.
1829  *
1830  * Return values:
1831  * 0: Success.
1832  * Else: Failure.
1833  *------------------------------------------------------------------------*/
1834 int
1835 usb_fifo_attach(struct usb_device *udev, void *priv_sc,
1836     struct mtx *priv_mtx, struct usb_fifo_methods *pm,
1837     struct usb_fifo_sc *f_sc, uint16_t unit, int16_t subunit,
1838     uint8_t iface_index, uid_t uid, gid_t gid, int mode)
1839 {
1840 	struct usb_fifo *f_tx;
1841 	struct usb_fifo *f_rx;
1842 	char devname[32];
1843 	uint8_t n;
1844 
1845 	f_sc->fp[USB_FIFO_TX] = NULL;
1846 	f_sc->fp[USB_FIFO_RX] = NULL;
1847 
1848 	if (pm == NULL)
1849 		return (EINVAL);
1850 
1851 	/* check the methods */
1852 	usb_fifo_check_methods(pm);
1853 
1854 	if (priv_mtx == NULL)
1855 		priv_mtx = &Giant;
1856 
1857 	/* search for a free FIFO slot */
1858 	for (n = 0;; n += 2) {
1859 
1860 		if (n == USB_FIFO_MAX) {
1861 			/* end of FIFOs reached */
1862 			return (ENOMEM);
1863 		}
1864 		/* Check for TX FIFO */
1865 		if (udev->fifo[n + USB_FIFO_TX] != NULL) {
1866 			continue;
1867 		}
1868 		/* Check for RX FIFO */
1869 		if (udev->fifo[n + USB_FIFO_RX] != NULL) {
1870 			continue;
1871 		}
1872 		break;
1873 	}
1874 
1875 	f_tx = usb_fifo_alloc(priv_mtx);
1876 	f_rx = usb_fifo_alloc(priv_mtx);
1877 
1878 	if ((f_tx == NULL) || (f_rx == NULL)) {
1879 		usb_fifo_free(f_tx);
1880 		usb_fifo_free(f_rx);
1881 		return (ENOMEM);
1882 	}
1883 	/* initialise FIFO structures */
1884 
1885 	f_tx->fifo_index = n + USB_FIFO_TX;
1886 	f_tx->dev_ep_index = -1;
1887 	f_tx->priv_sc0 = priv_sc;
1888 	f_tx->methods = pm;
1889 	f_tx->iface_index = iface_index;
1890 	f_tx->udev = udev;
1891 
1892 	f_rx->fifo_index = n + USB_FIFO_RX;
1893 	f_rx->dev_ep_index = -1;
1894 	f_rx->priv_sc0 = priv_sc;
1895 	f_rx->methods = pm;
1896 	f_rx->iface_index = iface_index;
1897 	f_rx->udev = udev;
1898 
1899 	f_sc->fp[USB_FIFO_TX] = f_tx;
1900 	f_sc->fp[USB_FIFO_RX] = f_rx;
1901 
1902 	mtx_lock(&usb_ref_lock);
1903 	udev->fifo[f_tx->fifo_index] = f_tx;
1904 	udev->fifo[f_rx->fifo_index] = f_rx;
1905 	mtx_unlock(&usb_ref_lock);
1906 
1907 	for (n = 0; n != 4; n++) {
1908 
1909 		if (pm->basename[n] == NULL) {
1910 			continue;
1911 		}
1912 		if (subunit < 0) {
1913 			if (snprintf(devname, sizeof(devname),
1914 			    "%s%u%s", pm->basename[n],
1915 			    unit, pm->postfix[n] ?
1916 			    pm->postfix[n] : "")) {
1917 				/* ignore */
1918 			}
1919 		} else {
1920 			if (snprintf(devname, sizeof(devname),
1921 			    "%s%u.%d%s", pm->basename[n],
1922 			    unit, subunit, pm->postfix[n] ?
1923 			    pm->postfix[n] : "")) {
1924 				/* ignore */
1925 			}
1926 		}
1927 
1928 		/*
1929 		 * Distribute the symbolic links into two FIFO structures:
1930 		 */
1931 		if (n & 1) {
1932 			f_rx->symlink[n / 2] =
1933 			    usb_alloc_symlink(devname);
1934 		} else {
1935 			f_tx->symlink[n / 2] =
1936 			    usb_alloc_symlink(devname);
1937 		}
1938 
1939 		/* Create the device */
1940 		f_sc->dev = usb_make_dev(udev, devname, -1,
1941 		    f_tx->fifo_index & f_rx->fifo_index,
1942 		    FREAD|FWRITE, uid, gid, mode);
1943 	}
1944 
1945 	DPRINTFN(2, "attached %p/%p\n", f_tx, f_rx);
1946 	return (0);
1947 }
1948 
1949 /*------------------------------------------------------------------------*
1950  *	usb_fifo_alloc_buffer
1951  *
1952  * Return values:
1953  * 0: Success
1954  * Else failure
1955  *------------------------------------------------------------------------*/
1956 int
1957 usb_fifo_alloc_buffer(struct usb_fifo *f, usb_size_t bufsize,
1958     uint16_t nbuf)
1959 {
1960 	usb_fifo_free_buffer(f);
1961 
1962 	/* allocate an endpoint */
1963 	f->free_q.ifq_maxlen = nbuf;
1964 	f->used_q.ifq_maxlen = nbuf;
1965 
1966 	f->queue_data = usb_alloc_mbufs(
1967 	    M_USBDEV, &f->free_q, bufsize, nbuf);
1968 
1969 	if ((f->queue_data == NULL) && bufsize && nbuf) {
1970 		return (ENOMEM);
1971 	}
1972 	return (0);			/* success */
1973 }
1974 
1975 /*------------------------------------------------------------------------*
1976  *	usb_fifo_free_buffer
1977  *
1978  * This function will free the buffers associated with a FIFO. This
1979  * function can be called multiple times in a row.
1980  *------------------------------------------------------------------------*/
1981 void
1982 usb_fifo_free_buffer(struct usb_fifo *f)
1983 {
1984 	if (f->queue_data) {
1985 		/* free old buffer */
1986 		free(f->queue_data, M_USBDEV);
1987 		f->queue_data = NULL;
1988 	}
1989 	/* reset queues */
1990 
1991 	memset(&f->free_q, 0, sizeof(f->free_q));
1992 	memset(&f->used_q, 0, sizeof(f->used_q));
1993 }
1994 
1995 void
1996 usb_fifo_detach(struct usb_fifo_sc *f_sc)
1997 {
1998 	if (f_sc == NULL) {
1999 		return;
2000 	}
2001 	usb_fifo_free(f_sc->fp[USB_FIFO_TX]);
2002 	usb_fifo_free(f_sc->fp[USB_FIFO_RX]);
2003 
2004 	f_sc->fp[USB_FIFO_TX] = NULL;
2005 	f_sc->fp[USB_FIFO_RX] = NULL;
2006 
2007 	usb_destroy_dev(f_sc->dev);
2008 
2009 	f_sc->dev = NULL;
2010 
2011 	DPRINTFN(2, "detached %p\n", f_sc);
2012 }
2013 
2014 usb_size_t
2015 usb_fifo_put_bytes_max(struct usb_fifo *f)
2016 {
2017 	struct usb_mbuf *m;
2018 	usb_size_t len;
2019 
2020 	USB_IF_POLL(&f->free_q, m);
2021 
2022 	if (m) {
2023 		len = m->max_data_len;
2024 	} else {
2025 		len = 0;
2026 	}
2027 	return (len);
2028 }
2029 
2030 /*------------------------------------------------------------------------*
2031  *	usb_fifo_put_data
2032  *
2033  * what:
2034  *  0 - normal operation
2035  *  1 - set last packet flag to enforce framing
2036  *------------------------------------------------------------------------*/
2037 void
2038 usb_fifo_put_data(struct usb_fifo *f, struct usb_page_cache *pc,
2039     usb_frlength_t offset, usb_frlength_t len, uint8_t what)
2040 {
2041 	struct usb_mbuf *m;
2042 	usb_frlength_t io_len;
2043 
2044 	while (len || (what == 1)) {
2045 
2046 		USB_IF_DEQUEUE(&f->free_q, m);
2047 
2048 		if (m) {
2049 			USB_MBUF_RESET(m);
2050 
2051 			io_len = MIN(len, m->cur_data_len);
2052 
2053 			usbd_copy_out(pc, offset, m->cur_data_ptr, io_len);
2054 
2055 			m->cur_data_len = io_len;
2056 			offset += io_len;
2057 			len -= io_len;
2058 
2059 			if ((len == 0) && (what == 1)) {
2060 				m->last_packet = 1;
2061 			}
2062 			USB_IF_ENQUEUE(&f->used_q, m);
2063 
2064 			usb_fifo_wakeup(f);
2065 
2066 			if ((len == 0) || (what == 1)) {
2067 				break;
2068 			}
2069 		} else {
2070 			break;
2071 		}
2072 	}
2073 }
2074 
2075 void
2076 usb_fifo_put_data_linear(struct usb_fifo *f, void *ptr,
2077     usb_size_t len, uint8_t what)
2078 {
2079 	struct usb_mbuf *m;
2080 	usb_size_t io_len;
2081 
2082 	while (len || (what == 1)) {
2083 
2084 		USB_IF_DEQUEUE(&f->free_q, m);
2085 
2086 		if (m) {
2087 			USB_MBUF_RESET(m);
2088 
2089 			io_len = MIN(len, m->cur_data_len);
2090 
2091 			memcpy(m->cur_data_ptr, ptr, io_len);
2092 
2093 			m->cur_data_len = io_len;
2094 			ptr = USB_ADD_BYTES(ptr, io_len);
2095 			len -= io_len;
2096 
2097 			if ((len == 0) && (what == 1)) {
2098 				m->last_packet = 1;
2099 			}
2100 			USB_IF_ENQUEUE(&f->used_q, m);
2101 
2102 			usb_fifo_wakeup(f);
2103 
2104 			if ((len == 0) || (what == 1)) {
2105 				break;
2106 			}
2107 		} else {
2108 			break;
2109 		}
2110 	}
2111 }
2112 
2113 uint8_t
2114 usb_fifo_put_data_buffer(struct usb_fifo *f, void *ptr, usb_size_t len)
2115 {
2116 	struct usb_mbuf *m;
2117 
2118 	USB_IF_DEQUEUE(&f->free_q, m);
2119 
2120 	if (m) {
2121 		m->cur_data_len = len;
2122 		m->cur_data_ptr = ptr;
2123 		USB_IF_ENQUEUE(&f->used_q, m);
2124 		usb_fifo_wakeup(f);
2125 		return (1);
2126 	}
2127 	return (0);
2128 }
2129 
2130 void
2131 usb_fifo_put_data_error(struct usb_fifo *f)
2132 {
2133 	f->flag_iserror = 1;
2134 	usb_fifo_wakeup(f);
2135 }
2136 
2137 /*------------------------------------------------------------------------*
2138  *	usb_fifo_get_data
2139  *
2140  * what:
2141  *  0 - normal operation
2142  *  1 - only get one "usb_mbuf"
2143  *
2144  * returns:
2145  *  0 - no more data
2146  *  1 - data in buffer
2147  *------------------------------------------------------------------------*/
2148 uint8_t
2149 usb_fifo_get_data(struct usb_fifo *f, struct usb_page_cache *pc,
2150     usb_frlength_t offset, usb_frlength_t len, usb_frlength_t *actlen,
2151     uint8_t what)
2152 {
2153 	struct usb_mbuf *m;
2154 	usb_frlength_t io_len;
2155 	uint8_t tr_data = 0;
2156 
2157 	actlen[0] = 0;
2158 
2159 	while (1) {
2160 
2161 		USB_IF_DEQUEUE(&f->used_q, m);
2162 
2163 		if (m) {
2164 
2165 			tr_data = 1;
2166 
2167 			io_len = MIN(len, m->cur_data_len);
2168 
2169 			usbd_copy_in(pc, offset, m->cur_data_ptr, io_len);
2170 
2171 			len -= io_len;
2172 			offset += io_len;
2173 			actlen[0] += io_len;
2174 			m->cur_data_ptr += io_len;
2175 			m->cur_data_len -= io_len;
2176 
2177 			if ((m->cur_data_len == 0) || (what == 1)) {
2178 				USB_IF_ENQUEUE(&f->free_q, m);
2179 
2180 				usb_fifo_wakeup(f);
2181 
2182 				if (what == 1) {
2183 					break;
2184 				}
2185 			} else {
2186 				USB_IF_PREPEND(&f->used_q, m);
2187 			}
2188 		} else {
2189 
2190 			if (tr_data) {
2191 				/* wait for data to be written out */
2192 				break;
2193 			}
2194 			if (f->flag_flushing) {
2195 				/* check if we should send a short packet */
2196 				if (f->flag_short != 0) {
2197 					f->flag_short = 0;
2198 					tr_data = 1;
2199 					break;
2200 				}
2201 				/* flushing complete */
2202 				f->flag_flushing = 0;
2203 				usb_fifo_wakeup(f);
2204 			}
2205 			break;
2206 		}
2207 		if (len == 0) {
2208 			break;
2209 		}
2210 	}
2211 	return (tr_data);
2212 }
2213 
2214 uint8_t
2215 usb_fifo_get_data_linear(struct usb_fifo *f, void *ptr,
2216     usb_size_t len, usb_size_t *actlen, uint8_t what)
2217 {
2218 	struct usb_mbuf *m;
2219 	usb_size_t io_len;
2220 	uint8_t tr_data = 0;
2221 
2222 	actlen[0] = 0;
2223 
2224 	while (1) {
2225 
2226 		USB_IF_DEQUEUE(&f->used_q, m);
2227 
2228 		if (m) {
2229 
2230 			tr_data = 1;
2231 
2232 			io_len = MIN(len, m->cur_data_len);
2233 
2234 			memcpy(ptr, m->cur_data_ptr, io_len);
2235 
2236 			len -= io_len;
2237 			ptr = USB_ADD_BYTES(ptr, io_len);
2238 			actlen[0] += io_len;
2239 			m->cur_data_ptr += io_len;
2240 			m->cur_data_len -= io_len;
2241 
2242 			if ((m->cur_data_len == 0) || (what == 1)) {
2243 				USB_IF_ENQUEUE(&f->free_q, m);
2244 
2245 				usb_fifo_wakeup(f);
2246 
2247 				if (what == 1) {
2248 					break;
2249 				}
2250 			} else {
2251 				USB_IF_PREPEND(&f->used_q, m);
2252 			}
2253 		} else {
2254 
2255 			if (tr_data) {
2256 				/* wait for data to be written out */
2257 				break;
2258 			}
2259 			if (f->flag_flushing) {
2260 				/* check if we should send a short packet */
2261 				if (f->flag_short != 0) {
2262 					f->flag_short = 0;
2263 					tr_data = 1;
2264 					break;
2265 				}
2266 				/* flushing complete */
2267 				f->flag_flushing = 0;
2268 				usb_fifo_wakeup(f);
2269 			}
2270 			break;
2271 		}
2272 		if (len == 0) {
2273 			break;
2274 		}
2275 	}
2276 	return (tr_data);
2277 }
2278 
2279 uint8_t
2280 usb_fifo_get_data_buffer(struct usb_fifo *f, void **pptr, usb_size_t *plen)
2281 {
2282 	struct usb_mbuf *m;
2283 
2284 	USB_IF_POLL(&f->used_q, m);
2285 
2286 	if (m) {
2287 		*plen = m->cur_data_len;
2288 		*pptr = m->cur_data_ptr;
2289 
2290 		return (1);
2291 	}
2292 	return (0);
2293 }
2294 
2295 void
2296 usb_fifo_get_data_error(struct usb_fifo *f)
2297 {
2298 	f->flag_iserror = 1;
2299 	usb_fifo_wakeup(f);
2300 }
2301 
2302 /*------------------------------------------------------------------------*
2303  *	usb_alloc_symlink
2304  *
2305  * Return values:
2306  * NULL: Failure
2307  * Else: Pointer to symlink entry
2308  *------------------------------------------------------------------------*/
2309 struct usb_symlink *
2310 usb_alloc_symlink(const char *target)
2311 {
2312 	struct usb_symlink *ps;
2313 
2314 	ps = malloc(sizeof(*ps), M_USBDEV, M_WAITOK);
2315 	if (ps == NULL) {
2316 		return (ps);
2317 	}
2318 	/* XXX no longer needed */
2319 	strlcpy(ps->src_path, target, sizeof(ps->src_path));
2320 	ps->src_len = strlen(ps->src_path);
2321 	strlcpy(ps->dst_path, target, sizeof(ps->dst_path));
2322 	ps->dst_len = strlen(ps->dst_path);
2323 
2324 	sx_xlock(&usb_sym_lock);
2325 	TAILQ_INSERT_TAIL(&usb_sym_head, ps, sym_entry);
2326 	sx_unlock(&usb_sym_lock);
2327 	return (ps);
2328 }
2329 
2330 /*------------------------------------------------------------------------*
2331  *	usb_free_symlink
2332  *------------------------------------------------------------------------*/
2333 void
2334 usb_free_symlink(struct usb_symlink *ps)
2335 {
2336 	if (ps == NULL) {
2337 		return;
2338 	}
2339 	sx_xlock(&usb_sym_lock);
2340 	TAILQ_REMOVE(&usb_sym_head, ps, sym_entry);
2341 	sx_unlock(&usb_sym_lock);
2342 
2343 	free(ps, M_USBDEV);
2344 }
2345 
2346 /*------------------------------------------------------------------------*
2347  *	usb_read_symlink
2348  *
2349  * Return value:
2350  * 0: Success
2351  * Else: Failure
2352  *------------------------------------------------------------------------*/
2353 int
2354 usb_read_symlink(uint8_t *user_ptr, uint32_t startentry, uint32_t user_len)
2355 {
2356 	struct usb_symlink *ps;
2357 	uint32_t temp;
2358 	uint32_t delta = 0;
2359 	uint8_t len;
2360 	int error = 0;
2361 
2362 	sx_xlock(&usb_sym_lock);
2363 
2364 	TAILQ_FOREACH(ps, &usb_sym_head, sym_entry) {
2365 
2366 		/*
2367 		 * Compute total length of source and destination symlink
2368 		 * strings pluss one length byte and two NUL bytes:
2369 		 */
2370 		temp = ps->src_len + ps->dst_len + 3;
2371 
2372 		if (temp > 255) {
2373 			/*
2374 			 * Skip entry because this length cannot fit
2375 			 * into one byte:
2376 			 */
2377 			continue;
2378 		}
2379 		if (startentry != 0) {
2380 			/* decrement read offset */
2381 			startentry--;
2382 			continue;
2383 		}
2384 		if (temp > user_len) {
2385 			/* out of buffer space */
2386 			break;
2387 		}
2388 		len = temp;
2389 
2390 		/* copy out total length */
2391 
2392 		error = copyout(&len,
2393 		    USB_ADD_BYTES(user_ptr, delta), 1);
2394 		if (error) {
2395 			break;
2396 		}
2397 		delta += 1;
2398 
2399 		/* copy out source string */
2400 
2401 		error = copyout(ps->src_path,
2402 		    USB_ADD_BYTES(user_ptr, delta), ps->src_len);
2403 		if (error) {
2404 			break;
2405 		}
2406 		len = 0;
2407 		delta += ps->src_len;
2408 		error = copyout(&len,
2409 		    USB_ADD_BYTES(user_ptr, delta), 1);
2410 		if (error) {
2411 			break;
2412 		}
2413 		delta += 1;
2414 
2415 		/* copy out destination string */
2416 
2417 		error = copyout(ps->dst_path,
2418 		    USB_ADD_BYTES(user_ptr, delta), ps->dst_len);
2419 		if (error) {
2420 			break;
2421 		}
2422 		len = 0;
2423 		delta += ps->dst_len;
2424 		error = copyout(&len,
2425 		    USB_ADD_BYTES(user_ptr, delta), 1);
2426 		if (error) {
2427 			break;
2428 		}
2429 		delta += 1;
2430 
2431 		user_len -= temp;
2432 	}
2433 
2434 	/* a zero length entry indicates the end */
2435 
2436 	if ((user_len != 0) && (error == 0)) {
2437 
2438 		len = 0;
2439 
2440 		error = copyout(&len,
2441 		    USB_ADD_BYTES(user_ptr, delta), 1);
2442 	}
2443 	sx_unlock(&usb_sym_lock);
2444 	return (error);
2445 }
2446 
2447 void
2448 usb_fifo_set_close_zlp(struct usb_fifo *f, uint8_t onoff)
2449 {
2450 	if (f == NULL)
2451 		return;
2452 
2453 	/* send a Zero Length Packet, ZLP, before close */
2454 	f->flag_short = onoff;
2455 }
2456 
2457 void
2458 usb_fifo_set_write_defrag(struct usb_fifo *f, uint8_t onoff)
2459 {
2460 	if (f == NULL)
2461 		return;
2462 
2463 	/* defrag written data */
2464 	f->flag_write_defrag = onoff;
2465 	/* reset defrag state */
2466 	f->flag_have_fragment = 0;
2467 }
2468 
2469 void *
2470 usb_fifo_softc(struct usb_fifo *f)
2471 {
2472 	return (f->priv_sc0);
2473 }
2474 #endif	/* USB_HAVE_UGEN */
2475