xref: /freebsd/sys/dev/usb/usb_dev.c (revision da477bcdc0c335171bb0ed3813f570026de6df85)
1 /* $FreeBSD$ */
2 /*-
3  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
4  *
5  * Copyright (c) 2006-2008 Hans Petter Selasky. All rights reserved.
6  *
7  * Redistribution and use in source and binary forms, with or without
8  * modification, are permitted provided that the following conditions
9  * are met:
10  * 1. Redistributions of source code must retain the above copyright
11  *    notice, this list of conditions and the following disclaimer.
12  * 2. Redistributions in binary form must reproduce the above copyright
13  *    notice, this list of conditions and the following disclaimer in the
14  *    documentation and/or other materials provided with the distribution.
15  *
16  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
17  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
20  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26  * SUCH DAMAGE.
27  *
28  *
29  * usb_dev.c - An abstraction layer for creating devices under /dev/...
30  */
31 
32 #ifdef USB_GLOBAL_INCLUDE_FILE
33 #include USB_GLOBAL_INCLUDE_FILE
34 #else
35 #include <sys/stdint.h>
36 #include <sys/stddef.h>
37 #include <sys/param.h>
38 #include <sys/queue.h>
39 #include <sys/types.h>
40 #include <sys/systm.h>
41 #include <sys/kernel.h>
42 #include <sys/bus.h>
43 #include <sys/module.h>
44 #include <sys/lock.h>
45 #include <sys/mutex.h>
46 #include <sys/condvar.h>
47 #include <sys/sysctl.h>
48 #include <sys/sx.h>
49 #include <sys/unistd.h>
50 #include <sys/callout.h>
51 #include <sys/malloc.h>
52 #include <sys/priv.h>
53 #include <sys/vnode.h>
54 #include <sys/conf.h>
55 #include <sys/fcntl.h>
56 
57 #include <dev/usb/usb.h>
58 #include <dev/usb/usb_ioctl.h>
59 #include <dev/usb/usbdi.h>
60 #include <dev/usb/usbdi_util.h>
61 
62 #define	USB_DEBUG_VAR usb_fifo_debug
63 
64 #include <dev/usb/usb_core.h>
65 #include <dev/usb/usb_dev.h>
66 #include <dev/usb/usb_mbuf.h>
67 #include <dev/usb/usb_process.h>
68 #include <dev/usb/usb_device.h>
69 #include <dev/usb/usb_debug.h>
70 #include <dev/usb/usb_busdma.h>
71 #include <dev/usb/usb_generic.h>
72 #include <dev/usb/usb_dynamic.h>
73 #include <dev/usb/usb_util.h>
74 
75 #include <dev/usb/usb_controller.h>
76 #include <dev/usb/usb_bus.h>
77 
78 #include <sys/filio.h>
79 #include <sys/ttycom.h>
80 #include <sys/syscallsubr.h>
81 
82 #include <machine/stdarg.h>
83 #endif			/* USB_GLOBAL_INCLUDE_FILE */
84 
85 #if USB_HAVE_UGEN
86 
87 #ifdef USB_DEBUG
88 static int usb_fifo_debug = 0;
89 
90 static SYSCTL_NODE(_hw_usb, OID_AUTO, dev, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
91     "USB device");
92 SYSCTL_INT(_hw_usb_dev, OID_AUTO, debug, CTLFLAG_RWTUN,
93     &usb_fifo_debug, 0, "Debug Level");
94 #endif
95 
96 #if ((__FreeBSD_version >= 700001) || (__FreeBSD_version == 0) || \
97      ((__FreeBSD_version >= 600034) && (__FreeBSD_version < 700000)))
98 #define	USB_UCRED struct ucred *ucred,
99 #else
100 #define	USB_UCRED
101 #endif
102 
103 /* prototypes */
104 
105 static int	usb_fifo_open(struct usb_cdev_privdata *,
106 		    struct usb_fifo *, int);
107 static void	usb_fifo_close(struct usb_fifo *, int);
108 static void	usb_dev_init(void *);
109 static void	usb_dev_init_post(void *);
110 static void	usb_dev_uninit(void *);
111 static int	usb_fifo_uiomove(struct usb_fifo *, void *, int,
112 		    struct uio *);
113 static void	usb_fifo_check_methods(struct usb_fifo_methods *);
114 static struct	usb_fifo *usb_fifo_alloc(struct mtx *);
115 static struct	usb_endpoint *usb_dev_get_ep(struct usb_device *, uint8_t,
116 		    uint8_t);
117 static void	usb_loc_fill(struct usb_fs_privdata *,
118 		    struct usb_cdev_privdata *);
119 static void	usb_close(void *);
120 static usb_error_t usb_ref_device(struct usb_cdev_privdata *, struct usb_cdev_refdata *, int);
121 static usb_error_t usb_usb_ref_device(struct usb_cdev_privdata *, struct usb_cdev_refdata *);
122 static void	usb_unref_device(struct usb_cdev_privdata *, struct usb_cdev_refdata *);
123 
124 static d_open_t usb_open;
125 static d_ioctl_t usb_ioctl;
126 static d_read_t usb_read;
127 static d_write_t usb_write;
128 static d_poll_t usb_poll;
129 static d_kqfilter_t usb_kqfilter;
130 
131 static d_ioctl_t usb_static_ioctl;
132 
133 static usb_fifo_open_t usb_fifo_dummy_open;
134 static usb_fifo_close_t usb_fifo_dummy_close;
135 static usb_fifo_ioctl_t usb_fifo_dummy_ioctl;
136 static usb_fifo_cmd_t usb_fifo_dummy_cmd;
137 
138 /* character device structure used for devices (/dev/ugenX.Y and /dev/uXXX) */
139 struct cdevsw usb_devsw = {
140 	.d_version = D_VERSION,
141 	.d_open = usb_open,
142 	.d_ioctl = usb_ioctl,
143 	.d_name = "usbdev",
144 	.d_flags = D_TRACKCLOSE,
145 	.d_read = usb_read,
146 	.d_write = usb_write,
147 	.d_poll = usb_poll,
148 	.d_kqfilter = usb_kqfilter,
149 };
150 
151 static struct cdev* usb_dev = NULL;
152 
153 /* character device structure used for /dev/usb */
154 static struct cdevsw usb_static_devsw = {
155 	.d_version = D_VERSION,
156 	.d_ioctl = usb_static_ioctl,
157 	.d_name = "usb"
158 };
159 
160 static TAILQ_HEAD(, usb_symlink) usb_sym_head;
161 static struct sx usb_sym_lock;
162 
163 struct mtx usb_ref_lock;
164 
165 /*------------------------------------------------------------------------*
166  *	usb_loc_fill
167  *
168  * This is used to fill out a usb_cdev_privdata structure based on the
169  * device's address as contained in usb_fs_privdata.
170  *------------------------------------------------------------------------*/
171 static void
172 usb_loc_fill(struct usb_fs_privdata* pd, struct usb_cdev_privdata *cpd)
173 {
174 	cpd->bus_index = pd->bus_index;
175 	cpd->dev_index = pd->dev_index;
176 	cpd->ep_addr = pd->ep_addr;
177 	cpd->fifo_index = pd->fifo_index;
178 }
179 
180 /*------------------------------------------------------------------------*
181  *	usb_ref_device
182  *
183  * This function is used to atomically refer an USB device by its
184  * device location. If this function returns success the USB device
185  * will not disappear until the USB device is unreferenced.
186  *
187  * Return values:
188  *  0: Success, refcount incremented on the given USB device.
189  *  Else: Failure.
190  *------------------------------------------------------------------------*/
191 static usb_error_t
192 usb_ref_device(struct usb_cdev_privdata *cpd,
193     struct usb_cdev_refdata *crd, int need_uref)
194 {
195 	struct usb_fifo **ppf;
196 	struct usb_fifo *f;
197 
198 	DPRINTFN(2, "cpd=%p need uref=%d\n", cpd, need_uref);
199 
200 	/* clear all refs */
201 	memset(crd, 0, sizeof(*crd));
202 
203 	mtx_lock(&usb_ref_lock);
204 	cpd->bus = devclass_get_softc(usb_devclass_ptr, cpd->bus_index);
205 	if (cpd->bus == NULL) {
206 		DPRINTFN(2, "no bus at %u\n", cpd->bus_index);
207 		goto error;
208 	}
209 	cpd->udev = cpd->bus->devices[cpd->dev_index];
210 	if (cpd->udev == NULL) {
211 		DPRINTFN(2, "no device at %u\n", cpd->dev_index);
212 		goto error;
213 	}
214 	if (cpd->udev->state == USB_STATE_DETACHED &&
215 	    (need_uref != 2)) {
216 		DPRINTFN(2, "device is detached\n");
217 		goto error;
218 	}
219 	if (need_uref) {
220 		DPRINTFN(2, "ref udev - needed\n");
221 
222 		if (cpd->udev->refcount == USB_DEV_REF_MAX) {
223 			DPRINTFN(2, "no dev ref\n");
224 			goto error;
225 		}
226 		cpd->udev->refcount++;
227 
228 		mtx_unlock(&usb_ref_lock);
229 
230 		/*
231 		 * We need to grab the enumeration SX-lock before
232 		 * grabbing the FIFO refs to avoid deadlock at detach!
233 		 */
234 		crd->do_unlock = usbd_enum_lock_sig(cpd->udev);
235 
236 		mtx_lock(&usb_ref_lock);
237 
238 		/*
239 		 * Set "is_uref" after grabbing the default SX lock
240 		 */
241 		crd->is_uref = 1;
242 
243 		/* check for signal */
244 		if (crd->do_unlock > 1) {
245 			crd->do_unlock = 0;
246 			goto error;
247 		}
248 	}
249 
250 	/* check if we are doing an open */
251 	if (cpd->fflags == 0) {
252 		/* use zero defaults */
253 	} else {
254 		/* check for write */
255 		if (cpd->fflags & FWRITE) {
256 			ppf = cpd->udev->fifo;
257 			f = ppf[cpd->fifo_index + USB_FIFO_TX];
258 			crd->txfifo = f;
259 			crd->is_write = 1;	/* ref */
260 			if (f == NULL || f->refcount == USB_FIFO_REF_MAX)
261 				goto error;
262 			if (f->curr_cpd != cpd)
263 				goto error;
264 			/* check if USB-FS is active */
265 			if (f->fs_ep_max != 0) {
266 				crd->is_usbfs = 1;
267 			}
268 		}
269 
270 		/* check for read */
271 		if (cpd->fflags & FREAD) {
272 			ppf = cpd->udev->fifo;
273 			f = ppf[cpd->fifo_index + USB_FIFO_RX];
274 			crd->rxfifo = f;
275 			crd->is_read = 1;	/* ref */
276 			if (f == NULL || f->refcount == USB_FIFO_REF_MAX)
277 				goto error;
278 			if (f->curr_cpd != cpd)
279 				goto error;
280 			/* check if USB-FS is active */
281 			if (f->fs_ep_max != 0) {
282 				crd->is_usbfs = 1;
283 			}
284 		}
285 	}
286 
287 	/* when everything is OK we increment the refcounts */
288 	if (crd->is_write) {
289 		DPRINTFN(2, "ref write\n");
290 		crd->txfifo->refcount++;
291 	}
292 	if (crd->is_read) {
293 		DPRINTFN(2, "ref read\n");
294 		crd->rxfifo->refcount++;
295 	}
296 	mtx_unlock(&usb_ref_lock);
297 
298 	return (0);
299 
300 error:
301 	if (crd->do_unlock)
302 		usbd_enum_unlock(cpd->udev);
303 
304 	if (crd->is_uref) {
305 		if (--(cpd->udev->refcount) == 0)
306 			cv_broadcast(&cpd->udev->ref_cv);
307 	}
308 	mtx_unlock(&usb_ref_lock);
309 	DPRINTFN(2, "fail\n");
310 
311 	/* clear all refs */
312 	memset(crd, 0, sizeof(*crd));
313 
314 	return (USB_ERR_INVAL);
315 }
316 
317 /*------------------------------------------------------------------------*
318  *	usb_usb_ref_device
319  *
320  * This function is used to upgrade an USB reference to include the
321  * USB device reference on a USB location.
322  *
323  * Return values:
324  *  0: Success, refcount incremented on the given USB device.
325  *  Else: Failure.
326  *------------------------------------------------------------------------*/
327 static usb_error_t
328 usb_usb_ref_device(struct usb_cdev_privdata *cpd,
329     struct usb_cdev_refdata *crd)
330 {
331 	/*
332 	 * Check if we already got an USB reference on this location:
333 	 */
334 	if (crd->is_uref)
335 		return (0);		/* success */
336 
337 	/*
338 	 * To avoid deadlock at detach we need to drop the FIFO ref
339 	 * and re-acquire a new ref!
340 	 */
341 	usb_unref_device(cpd, crd);
342 
343 	return (usb_ref_device(cpd, crd, 1 /* need uref */));
344 }
345 
346 /*------------------------------------------------------------------------*
347  *	usb_unref_device
348  *
349  * This function will release the reference count by one unit for the
350  * given USB device.
351  *------------------------------------------------------------------------*/
352 static void
353 usb_unref_device(struct usb_cdev_privdata *cpd,
354     struct usb_cdev_refdata *crd)
355 {
356 
357 	DPRINTFN(2, "cpd=%p is_uref=%d\n", cpd, crd->is_uref);
358 
359 	if (crd->do_unlock)
360 		usbd_enum_unlock(cpd->udev);
361 
362 	mtx_lock(&usb_ref_lock);
363 	if (crd->is_read) {
364 		if (--(crd->rxfifo->refcount) == 0) {
365 			cv_signal(&crd->rxfifo->cv_drain);
366 		}
367 		crd->is_read = 0;
368 	}
369 	if (crd->is_write) {
370 		if (--(crd->txfifo->refcount) == 0) {
371 			cv_signal(&crd->txfifo->cv_drain);
372 		}
373 		crd->is_write = 0;
374 	}
375 	if (crd->is_uref) {
376 		crd->is_uref = 0;
377 		if (--(cpd->udev->refcount) == 0)
378 			cv_broadcast(&cpd->udev->ref_cv);
379 	}
380 	mtx_unlock(&usb_ref_lock);
381 }
382 
383 static struct usb_fifo *
384 usb_fifo_alloc(struct mtx *mtx)
385 {
386 	struct usb_fifo *f;
387 
388 	f = malloc(sizeof(*f), M_USBDEV, M_WAITOK | M_ZERO);
389 	cv_init(&f->cv_io, "FIFO-IO");
390 	cv_init(&f->cv_drain, "FIFO-DRAIN");
391 	f->priv_mtx = mtx;
392 	f->refcount = 1;
393 	knlist_init_mtx(&f->selinfo.si_note, mtx);
394 	return (f);
395 }
396 
397 /*------------------------------------------------------------------------*
398  *	usb_fifo_create
399  *------------------------------------------------------------------------*/
400 static int
401 usb_fifo_create(struct usb_cdev_privdata *cpd,
402     struct usb_cdev_refdata *crd)
403 {
404 	struct usb_device *udev = cpd->udev;
405 	struct usb_fifo *f;
406 	struct usb_endpoint *ep;
407 	uint8_t n;
408 	uint8_t is_tx;
409 	uint8_t is_rx;
410 	uint8_t no_null;
411 	uint8_t is_busy;
412 	int e = cpd->ep_addr;
413 
414 	is_tx = (cpd->fflags & FWRITE) ? 1 : 0;
415 	is_rx = (cpd->fflags & FREAD) ? 1 : 0;
416 	no_null = 1;
417 	is_busy = 0;
418 
419 	/* Preallocated FIFO */
420 	if (e < 0) {
421 		DPRINTFN(5, "Preallocated FIFO\n");
422 		if (is_tx) {
423 			f = udev->fifo[cpd->fifo_index + USB_FIFO_TX];
424 			if (f == NULL)
425 				return (EINVAL);
426 			crd->txfifo = f;
427 		}
428 		if (is_rx) {
429 			f = udev->fifo[cpd->fifo_index + USB_FIFO_RX];
430 			if (f == NULL)
431 				return (EINVAL);
432 			crd->rxfifo = f;
433 		}
434 		return (0);
435 	}
436 
437 	KASSERT(e >= 0 && e <= 15, ("endpoint %d out of range", e));
438 
439 	/* search for a free FIFO slot */
440 	DPRINTFN(5, "Endpoint device, searching for 0x%02x\n", e);
441 	for (n = 0;; n += 2) {
442 
443 		if (n == USB_FIFO_MAX) {
444 			if (no_null) {
445 				no_null = 0;
446 				n = 0;
447 			} else {
448 				/* end of FIFOs reached */
449 				DPRINTFN(5, "out of FIFOs\n");
450 				return (ENOMEM);
451 			}
452 		}
453 		/* Check for TX FIFO */
454 		if (is_tx) {
455 			f = udev->fifo[n + USB_FIFO_TX];
456 			if (f != NULL) {
457 				if (f->dev_ep_index != e) {
458 					/* wrong endpoint index */
459 					continue;
460 				}
461 				if (f->curr_cpd != NULL) {
462 					/* FIFO is opened */
463 					is_busy = 1;
464 					continue;
465 				}
466 			} else if (no_null) {
467 				continue;
468 			}
469 		}
470 		/* Check for RX FIFO */
471 		if (is_rx) {
472 			f = udev->fifo[n + USB_FIFO_RX];
473 			if (f != NULL) {
474 				if (f->dev_ep_index != e) {
475 					/* wrong endpoint index */
476 					continue;
477 				}
478 				if (f->curr_cpd != NULL) {
479 					/* FIFO is opened */
480 					is_busy = 1;
481 					continue;
482 				}
483 			} else if (no_null) {
484 				continue;
485 			}
486 		}
487 		break;
488 	}
489 
490 	if (no_null == 0) {
491 		if (e >= (USB_EP_MAX / 2)) {
492 			/* we don't create any endpoints in this range */
493 			DPRINTFN(5, "ep out of range\n");
494 			return (is_busy ? EBUSY : EINVAL);
495 		}
496 	}
497 
498 	if ((e != 0) && is_busy) {
499 		/*
500 		 * Only the default control endpoint is allowed to be
501 		 * opened multiple times!
502 		 */
503 		DPRINTFN(5, "busy\n");
504 		return (EBUSY);
505 	}
506 
507 	/* Check TX FIFO */
508 	if (is_tx &&
509 	    (udev->fifo[n + USB_FIFO_TX] == NULL)) {
510 		ep = usb_dev_get_ep(udev, e, USB_FIFO_TX);
511 		DPRINTFN(5, "dev_get_endpoint(%d, 0x%x)\n", e, USB_FIFO_TX);
512 		if (ep == NULL) {
513 			DPRINTFN(5, "dev_get_endpoint returned NULL\n");
514 			return (EINVAL);
515 		}
516 		f = usb_fifo_alloc(&udev->device_mtx);
517 		if (f == NULL) {
518 			DPRINTFN(5, "could not alloc tx fifo\n");
519 			return (ENOMEM);
520 		}
521 		/* update some fields */
522 		f->fifo_index = n + USB_FIFO_TX;
523 		f->dev_ep_index = e;
524 		f->priv_sc0 = ep;
525 		f->methods = &usb_ugen_methods;
526 		f->iface_index = ep->iface_index;
527 		f->udev = udev;
528 		mtx_lock(&usb_ref_lock);
529 		udev->fifo[n + USB_FIFO_TX] = f;
530 		mtx_unlock(&usb_ref_lock);
531 	}
532 	/* Check RX FIFO */
533 	if (is_rx &&
534 	    (udev->fifo[n + USB_FIFO_RX] == NULL)) {
535 
536 		ep = usb_dev_get_ep(udev, e, USB_FIFO_RX);
537 		DPRINTFN(5, "dev_get_endpoint(%d, 0x%x)\n", e, USB_FIFO_RX);
538 		if (ep == NULL) {
539 			DPRINTFN(5, "dev_get_endpoint returned NULL\n");
540 			return (EINVAL);
541 		}
542 		f = usb_fifo_alloc(&udev->device_mtx);
543 		if (f == NULL) {
544 			DPRINTFN(5, "could not alloc rx fifo\n");
545 			return (ENOMEM);
546 		}
547 		/* update some fields */
548 		f->fifo_index = n + USB_FIFO_RX;
549 		f->dev_ep_index = e;
550 		f->priv_sc0 = ep;
551 		f->methods = &usb_ugen_methods;
552 		f->iface_index = ep->iface_index;
553 		f->udev = udev;
554 		mtx_lock(&usb_ref_lock);
555 		udev->fifo[n + USB_FIFO_RX] = f;
556 		mtx_unlock(&usb_ref_lock);
557 	}
558 	if (is_tx) {
559 		crd->txfifo = udev->fifo[n + USB_FIFO_TX];
560 	}
561 	if (is_rx) {
562 		crd->rxfifo = udev->fifo[n + USB_FIFO_RX];
563 	}
564 	/* fill out fifo index */
565 	DPRINTFN(5, "fifo index = %d\n", n);
566 	cpd->fifo_index = n;
567 
568 	/* complete */
569 
570 	return (0);
571 }
572 
573 void
574 usb_fifo_free(struct usb_fifo *f)
575 {
576 	uint8_t n;
577 
578 	if (f == NULL) {
579 		/* be NULL safe */
580 		return;
581 	}
582 	/* destroy symlink devices, if any */
583 	for (n = 0; n != 2; n++) {
584 		if (f->symlink[n]) {
585 			usb_free_symlink(f->symlink[n]);
586 			f->symlink[n] = NULL;
587 		}
588 	}
589 	mtx_lock(&usb_ref_lock);
590 
591 	/* delink ourselves to stop calls from userland */
592 	if ((f->fifo_index < USB_FIFO_MAX) &&
593 	    (f->udev != NULL) &&
594 	    (f->udev->fifo[f->fifo_index] == f)) {
595 		f->udev->fifo[f->fifo_index] = NULL;
596 	} else {
597 		DPRINTFN(0, "USB FIFO %p has not been linked\n", f);
598 	}
599 
600 	/* decrease refcount */
601 	f->refcount--;
602 	/* need to wait until all callers have exited */
603 	while (f->refcount != 0) {
604 		mtx_unlock(&usb_ref_lock);	/* avoid LOR */
605 		mtx_lock(f->priv_mtx);
606 		/* prevent write flush, if any */
607 		f->flag_iserror = 1;
608 		/* get I/O thread out of any sleep state */
609 		if (f->flag_sleeping) {
610 			f->flag_sleeping = 0;
611 			cv_broadcast(&f->cv_io);
612 		}
613 		mtx_unlock(f->priv_mtx);
614 		mtx_lock(&usb_ref_lock);
615 
616 		/*
617 		 * Check if the "f->refcount" variable reached zero
618 		 * during the unlocked time before entering wait:
619 		 */
620 		if (f->refcount == 0)
621 			break;
622 
623 		/* wait for sync */
624 		cv_wait(&f->cv_drain, &usb_ref_lock);
625 	}
626 	mtx_unlock(&usb_ref_lock);
627 
628 	/* take care of closing the device here, if any */
629 	usb_fifo_close(f, 0);
630 
631 	cv_destroy(&f->cv_io);
632 	cv_destroy(&f->cv_drain);
633 
634 	knlist_clear(&f->selinfo.si_note, 0);
635 	seldrain(&f->selinfo);
636 	knlist_destroy(&f->selinfo.si_note);
637 
638 	free(f, M_USBDEV);
639 }
640 
641 static struct usb_endpoint *
642 usb_dev_get_ep(struct usb_device *udev, uint8_t ep_index, uint8_t dir)
643 {
644 	struct usb_endpoint *ep;
645 	uint8_t ep_dir;
646 
647 	if (ep_index == 0) {
648 		ep = &udev->ctrl_ep;
649 	} else {
650 		if (dir == USB_FIFO_RX) {
651 			if (udev->flags.usb_mode == USB_MODE_HOST) {
652 				ep_dir = UE_DIR_IN;
653 			} else {
654 				ep_dir = UE_DIR_OUT;
655 			}
656 		} else {
657 			if (udev->flags.usb_mode == USB_MODE_HOST) {
658 				ep_dir = UE_DIR_OUT;
659 			} else {
660 				ep_dir = UE_DIR_IN;
661 			}
662 		}
663 		ep = usbd_get_ep_by_addr(udev, ep_index | ep_dir);
664 	}
665 
666 	if (ep == NULL) {
667 		/* if the endpoint does not exist then return */
668 		return (NULL);
669 	}
670 	if (ep->edesc == NULL) {
671 		/* invalid endpoint */
672 		return (NULL);
673 	}
674 	return (ep);			/* success */
675 }
676 
677 /*------------------------------------------------------------------------*
678  *	usb_fifo_open
679  *
680  * Returns:
681  * 0: Success
682  * Else: Failure
683  *------------------------------------------------------------------------*/
684 static int
685 usb_fifo_open(struct usb_cdev_privdata *cpd,
686     struct usb_fifo *f, int fflags)
687 {
688 	int err;
689 
690 	if (f == NULL) {
691 		/* no FIFO there */
692 		DPRINTFN(2, "no FIFO\n");
693 		return (ENXIO);
694 	}
695 	/* remove FWRITE and FREAD flags */
696 	fflags &= ~(FWRITE | FREAD);
697 
698 	/* set correct file flags */
699 	if ((f->fifo_index & 1) == USB_FIFO_TX) {
700 		fflags |= FWRITE;
701 	} else {
702 		fflags |= FREAD;
703 	}
704 
705 	/* check if we are already opened */
706 	/* we don't need any locks when checking this variable */
707 	if (f->curr_cpd != NULL) {
708 		err = EBUSY;
709 		goto done;
710 	}
711 
712 	/* reset short flag before open */
713 	f->flag_short = 0;
714 
715 	/* call open method */
716 	err = (f->methods->f_open) (f, fflags);
717 	if (err) {
718 		goto done;
719 	}
720 	mtx_lock(f->priv_mtx);
721 
722 	/* reset sleep flag */
723 	f->flag_sleeping = 0;
724 
725 	/* reset error flag */
726 	f->flag_iserror = 0;
727 
728 	/* reset complete flag */
729 	f->flag_iscomplete = 0;
730 
731 	/* reset select flag */
732 	f->flag_isselect = 0;
733 
734 	/* reset flushing flag */
735 	f->flag_flushing = 0;
736 
737 	/* reset ASYNC proc flag */
738 	f->async_p = NULL;
739 
740 	mtx_lock(&usb_ref_lock);
741 	/* flag the fifo as opened to prevent others */
742 	f->curr_cpd = cpd;
743 	mtx_unlock(&usb_ref_lock);
744 
745 	/* reset queue */
746 	usb_fifo_reset(f);
747 
748 	mtx_unlock(f->priv_mtx);
749 done:
750 	return (err);
751 }
752 
753 /*------------------------------------------------------------------------*
754  *	usb_fifo_reset
755  *------------------------------------------------------------------------*/
756 void
757 usb_fifo_reset(struct usb_fifo *f)
758 {
759 	struct usb_mbuf *m;
760 
761 	if (f == NULL) {
762 		return;
763 	}
764 	while (1) {
765 		USB_IF_DEQUEUE(&f->used_q, m);
766 		if (m) {
767 			USB_IF_ENQUEUE(&f->free_q, m);
768 		} else {
769 			break;
770 		}
771 	}
772 	/* reset have fragment flag */
773 	f->flag_have_fragment = 0;
774 }
775 
776 /*------------------------------------------------------------------------*
777  *	usb_fifo_close
778  *------------------------------------------------------------------------*/
779 static void
780 usb_fifo_close(struct usb_fifo *f, int fflags)
781 {
782 	int err;
783 
784 	/* check if we are not opened */
785 	if (f->curr_cpd == NULL) {
786 		/* nothing to do - already closed */
787 		return;
788 	}
789 	mtx_lock(f->priv_mtx);
790 
791 	/* clear current cdev private data pointer */
792 	mtx_lock(&usb_ref_lock);
793 	f->curr_cpd = NULL;
794 	mtx_unlock(&usb_ref_lock);
795 
796 	/* check if we are watched by kevent */
797 	KNOTE_LOCKED(&f->selinfo.si_note, 0);
798 
799 	/* check if we are selected */
800 	if (f->flag_isselect) {
801 		selwakeup(&f->selinfo);
802 		f->flag_isselect = 0;
803 	}
804 	/* check if a thread wants SIGIO */
805 	if (f->async_p != NULL) {
806 		PROC_LOCK(f->async_p);
807 		kern_psignal(f->async_p, SIGIO);
808 		PROC_UNLOCK(f->async_p);
809 		f->async_p = NULL;
810 	}
811 	/* remove FWRITE and FREAD flags */
812 	fflags &= ~(FWRITE | FREAD);
813 
814 	/* flush written data, if any */
815 	if ((f->fifo_index & 1) == USB_FIFO_TX) {
816 
817 		if (!f->flag_iserror) {
818 
819 			/* set flushing flag */
820 			f->flag_flushing = 1;
821 
822 			/* get the last packet in */
823 			if (f->flag_have_fragment) {
824 				struct usb_mbuf *m;
825 				f->flag_have_fragment = 0;
826 				USB_IF_DEQUEUE(&f->free_q, m);
827 				if (m) {
828 					USB_IF_ENQUEUE(&f->used_q, m);
829 				}
830 			}
831 
832 			/* start write transfer, if not already started */
833 			(f->methods->f_start_write) (f);
834 
835 			/* check if flushed already */
836 			while (f->flag_flushing &&
837 			    (!f->flag_iserror)) {
838 				/* wait until all data has been written */
839 				f->flag_sleeping = 1;
840 				err = cv_timedwait_sig(&f->cv_io, f->priv_mtx,
841 				    USB_MS_TO_TICKS(USB_DEFAULT_TIMEOUT));
842 				if (err) {
843 					DPRINTF("signal received\n");
844 					break;
845 				}
846 			}
847 		}
848 		fflags |= FWRITE;
849 
850 		/* stop write transfer, if not already stopped */
851 		(f->methods->f_stop_write) (f);
852 	} else {
853 		fflags |= FREAD;
854 
855 		/* stop write transfer, if not already stopped */
856 		(f->methods->f_stop_read) (f);
857 	}
858 
859 	/* check if we are sleeping */
860 	if (f->flag_sleeping) {
861 		DPRINTFN(2, "Sleeping at close!\n");
862 	}
863 	mtx_unlock(f->priv_mtx);
864 
865 	/* call close method */
866 	(f->methods->f_close) (f, fflags);
867 
868 	DPRINTF("closed\n");
869 }
870 
871 /*------------------------------------------------------------------------*
872  *	usb_open - cdev callback
873  *------------------------------------------------------------------------*/
874 static int
875 usb_open(struct cdev *dev, int fflags, int devtype, struct thread *td)
876 {
877 	struct usb_fs_privdata* pd = (struct usb_fs_privdata*)dev->si_drv1;
878 	struct usb_cdev_refdata refs;
879 	struct usb_cdev_privdata *cpd;
880 	int err;
881 
882 	DPRINTFN(2, "%s fflags=0x%08x\n", devtoname(dev), fflags);
883 
884 	KASSERT(fflags & (FREAD|FWRITE), ("invalid open flags"));
885 	if (((fflags & FREAD) && !(pd->mode & FREAD)) ||
886 	    ((fflags & FWRITE) && !(pd->mode & FWRITE))) {
887 		DPRINTFN(2, "access mode not supported\n");
888 		return (EPERM);
889 	}
890 
891 	cpd = malloc(sizeof(*cpd), M_USBDEV, M_WAITOK | M_ZERO);
892 
893 	usb_loc_fill(pd, cpd);
894 	err = usb_ref_device(cpd, &refs, 1);
895 	if (err) {
896 		DPRINTFN(2, "cannot ref device\n");
897 		free(cpd, M_USBDEV);
898 		return (ENXIO);
899 	}
900 	cpd->fflags = fflags;	/* access mode for open lifetime */
901 
902 	/* create FIFOs, if any */
903 	err = usb_fifo_create(cpd, &refs);
904 	/* check for error */
905 	if (err) {
906 		DPRINTFN(2, "cannot create fifo\n");
907 		usb_unref_device(cpd, &refs);
908 		free(cpd, M_USBDEV);
909 		return (err);
910 	}
911 	if (fflags & FREAD) {
912 		err = usb_fifo_open(cpd, refs.rxfifo, fflags);
913 		if (err) {
914 			DPRINTFN(2, "read open failed\n");
915 			usb_unref_device(cpd, &refs);
916 			free(cpd, M_USBDEV);
917 			return (err);
918 		}
919 	}
920 	if (fflags & FWRITE) {
921 		err = usb_fifo_open(cpd, refs.txfifo, fflags);
922 		if (err) {
923 			DPRINTFN(2, "write open failed\n");
924 			if (fflags & FREAD) {
925 				usb_fifo_close(refs.rxfifo, fflags);
926 			}
927 			usb_unref_device(cpd, &refs);
928 			free(cpd, M_USBDEV);
929 			return (err);
930 		}
931 	}
932 	usb_unref_device(cpd, &refs);
933 	devfs_set_cdevpriv(cpd, usb_close);
934 
935 	return (0);
936 }
937 
938 /*------------------------------------------------------------------------*
939  *	usb_close - cdev callback
940  *------------------------------------------------------------------------*/
941 static void
942 usb_close(void *arg)
943 {
944 	struct usb_cdev_refdata refs;
945 	struct usb_cdev_privdata *cpd = arg;
946 	int err;
947 
948 	DPRINTFN(2, "cpd=%p\n", cpd);
949 
950 	err = usb_ref_device(cpd, &refs,
951 	    2 /* uref and allow detached state */);
952 	if (err) {
953 		DPRINTFN(2, "Cannot grab USB reference when "
954 		    "closing USB file handle\n");
955 		goto done;
956 	}
957 	if (cpd->fflags & FREAD) {
958 		usb_fifo_close(refs.rxfifo, cpd->fflags);
959 	}
960 	if (cpd->fflags & FWRITE) {
961 		usb_fifo_close(refs.txfifo, cpd->fflags);
962 	}
963 	usb_unref_device(cpd, &refs);
964 done:
965 	free(cpd, M_USBDEV);
966 }
967 
968 static void
969 usb_dev_init(void *arg)
970 {
971 	mtx_init(&usb_ref_lock, "USB ref mutex", NULL, MTX_DEF);
972 	sx_init(&usb_sym_lock, "USB sym mutex");
973 	TAILQ_INIT(&usb_sym_head);
974 
975 	/* check the UGEN methods */
976 	usb_fifo_check_methods(&usb_ugen_methods);
977 }
978 
979 SYSINIT(usb_dev_init, SI_SUB_KLD, SI_ORDER_FIRST, usb_dev_init, NULL);
980 
981 static void
982 usb_dev_init_post(void *arg)
983 {
984 	/*
985 	 * Create /dev/usb - this is needed for usbconfig(8), which
986 	 * needs a well-known device name to access.
987 	 */
988 	usb_dev = make_dev(&usb_static_devsw, 0, UID_ROOT, GID_OPERATOR,
989 	    0644, USB_DEVICE_NAME);
990 	if (usb_dev == NULL) {
991 		DPRINTFN(0, "Could not create usb bus device\n");
992 	}
993 }
994 
995 SYSINIT(usb_dev_init_post, SI_SUB_KICK_SCHEDULER, SI_ORDER_FIRST, usb_dev_init_post, NULL);
996 
997 static void
998 usb_dev_uninit(void *arg)
999 {
1000 	if (usb_dev != NULL) {
1001 		destroy_dev(usb_dev);
1002 		usb_dev = NULL;
1003 	}
1004 	mtx_destroy(&usb_ref_lock);
1005 	sx_destroy(&usb_sym_lock);
1006 }
1007 
1008 SYSUNINIT(usb_dev_uninit, SI_SUB_KICK_SCHEDULER, SI_ORDER_ANY, usb_dev_uninit, NULL);
1009 
1010 static int
1011 usb_ioctl_f_sub(struct usb_fifo *f, u_long cmd, void *addr,
1012     struct thread *td)
1013 {
1014 	int error = 0;
1015 
1016 	switch (cmd) {
1017 	case FIODTYPE:
1018 		*(int *)addr = 0;	/* character device */
1019 		break;
1020 
1021 	case FIONBIO:
1022 		/* handled by upper FS layer */
1023 		break;
1024 
1025 	case FIOASYNC:
1026 		if (*(int *)addr) {
1027 			if (f->async_p != NULL) {
1028 				error = EBUSY;
1029 				break;
1030 			}
1031 			f->async_p = USB_TD_GET_PROC(td);
1032 		} else {
1033 			f->async_p = NULL;
1034 		}
1035 		break;
1036 
1037 		/* XXX this is not the most general solution */
1038 	case TIOCSPGRP:
1039 		if (f->async_p == NULL) {
1040 			error = EINVAL;
1041 			break;
1042 		}
1043 		if (*(int *)addr != USB_PROC_GET_GID(f->async_p)) {
1044 			error = EPERM;
1045 			break;
1046 		}
1047 		break;
1048 	default:
1049 		return (ENOIOCTL);
1050 	}
1051 	DPRINTFN(3, "cmd 0x%lx = %d\n", cmd, error);
1052 	return (error);
1053 }
1054 
1055 /*------------------------------------------------------------------------*
1056  *	usb_ioctl - cdev callback
1057  *------------------------------------------------------------------------*/
1058 static int
1059 usb_ioctl(struct cdev *dev, u_long cmd, caddr_t addr, int fflag, struct thread* td)
1060 {
1061 	struct usb_cdev_refdata refs;
1062 	struct usb_cdev_privdata* cpd;
1063 	struct usb_fifo *f;
1064 	int fflags;
1065 	int err;
1066 
1067 	DPRINTFN(2, "cmd=0x%lx\n", cmd);
1068 
1069 	err = devfs_get_cdevpriv((void **)&cpd);
1070 	if (err != 0)
1071 		return (err);
1072 
1073 	/*
1074 	 * Performance optimisation: We try to check for IOCTL's that
1075 	 * don't need the USB reference first. Then we grab the USB
1076 	 * reference if we need it!
1077 	 */
1078 	err = usb_ref_device(cpd, &refs, 0 /* no uref */ );
1079 	if (err)
1080 		return (ENXIO);
1081 
1082 	fflags = cpd->fflags;
1083 
1084 	f = NULL;			/* set default value */
1085 	err = ENOIOCTL;			/* set default value */
1086 
1087 	if (fflags & FWRITE) {
1088 		f = refs.txfifo;
1089 		err = usb_ioctl_f_sub(f, cmd, addr, td);
1090 	}
1091 	if (fflags & FREAD) {
1092 		f = refs.rxfifo;
1093 		err = usb_ioctl_f_sub(f, cmd, addr, td);
1094 	}
1095 	KASSERT(f != NULL, ("fifo not found"));
1096 	if (err != ENOIOCTL)
1097 		goto done;
1098 
1099 	err = (f->methods->f_ioctl) (f, cmd, addr, fflags);
1100 
1101 	DPRINTFN(2, "f_ioctl cmd 0x%lx = %d\n", cmd, err);
1102 
1103 	if (err != ENOIOCTL)
1104 		goto done;
1105 
1106 	if (usb_usb_ref_device(cpd, &refs)) {
1107 		/* we lost the reference */
1108 		return (ENXIO);
1109 	}
1110 
1111 	err = (f->methods->f_ioctl_post) (f, cmd, addr, fflags);
1112 
1113 	DPRINTFN(2, "f_ioctl_post cmd 0x%lx = %d\n", cmd, err);
1114 
1115 	if (err == ENOIOCTL)
1116 		err = ENOTTY;
1117 
1118 	if (err)
1119 		goto done;
1120 
1121 	/* Wait for re-enumeration, if any */
1122 
1123 	while (f->udev->re_enumerate_wait != USB_RE_ENUM_DONE) {
1124 
1125 		usb_unref_device(cpd, &refs);
1126 
1127 		usb_pause_mtx(NULL, hz / 128);
1128 
1129 		while (usb_ref_device(cpd, &refs, 1 /* need uref */)) {
1130 			if (usb_ref_device(cpd, &refs, 0)) {
1131 				/* device no longer exists */
1132 				return (ENXIO);
1133 			}
1134 			usb_unref_device(cpd, &refs);
1135 			usb_pause_mtx(NULL, hz / 128);
1136 		}
1137 	}
1138 
1139 done:
1140 	usb_unref_device(cpd, &refs);
1141 	return (err);
1142 }
1143 
1144 static void
1145 usb_filter_detach(struct knote *kn)
1146 {
1147 	struct usb_fifo *f = kn->kn_hook;
1148 	knlist_remove(&f->selinfo.si_note, kn, 0);
1149 }
1150 
1151 static int
1152 usb_filter_write(struct knote *kn, long hint)
1153 {
1154 	struct usb_cdev_privdata* cpd;
1155 	struct usb_fifo *f;
1156 	struct usb_mbuf *m;
1157 
1158 	DPRINTFN(2, "\n");
1159 
1160 	f = kn->kn_hook;
1161 
1162 	USB_MTX_ASSERT(f->priv_mtx, MA_OWNED);
1163 
1164 	cpd = f->curr_cpd;
1165 	if (cpd == NULL) {
1166 		m = (void *)1;
1167 	} else if (f->fs_ep_max == 0) {
1168 		if (f->flag_iserror) {
1169 			/* we got an error */
1170 			m = (void *)1;
1171 		} else {
1172 			if (f->queue_data == NULL) {
1173 				/*
1174 				 * start write transfer, if not
1175 				 * already started
1176 				 */
1177 				(f->methods->f_start_write) (f);
1178 			}
1179 			/* check if any packets are available */
1180 			USB_IF_POLL(&f->free_q, m);
1181 		}
1182 	} else {
1183 		if (f->flag_iscomplete) {
1184 			m = (void *)1;
1185 		} else {
1186 			m = NULL;
1187 		}
1188 	}
1189 	return (m ? 1 : 0);
1190 }
1191 
1192 static int
1193 usb_filter_read(struct knote *kn, long hint)
1194 {
1195 	struct usb_cdev_privdata* cpd;
1196 	struct usb_fifo *f;
1197 	struct usb_mbuf *m;
1198 
1199 	DPRINTFN(2, "\n");
1200 
1201 	f = kn->kn_hook;
1202 
1203 	USB_MTX_ASSERT(f->priv_mtx, MA_OWNED);
1204 
1205 	cpd = f->curr_cpd;
1206 	if (cpd == NULL) {
1207 		m = (void *)1;
1208 	} else if (f->fs_ep_max == 0) {
1209 		if (f->flag_iserror) {
1210 			/* we have an error */
1211 			m = (void *)1;
1212 		} else {
1213 			if (f->queue_data == NULL) {
1214 				/*
1215 				 * start read transfer, if not
1216 				 * already started
1217 				 */
1218 				(f->methods->f_start_read) (f);
1219 			}
1220 			/* check if any packets are available */
1221 			USB_IF_POLL(&f->used_q, m);
1222 
1223 			/* start reading data, if any */
1224 			if (m == NULL)
1225 				(f->methods->f_start_read) (f);
1226 		}
1227 	} else {
1228 		if (f->flag_iscomplete) {
1229 			m = (void *)1;
1230 		} else {
1231 			m = NULL;
1232 		}
1233 	}
1234 	return (m ? 1 : 0);
1235 }
1236 
1237 static struct filterops usb_filtops_write = {
1238 	.f_isfd = 1,
1239 	.f_detach = usb_filter_detach,
1240 	.f_event = usb_filter_write,
1241 };
1242 
1243 static struct filterops usb_filtops_read = {
1244 	.f_isfd = 1,
1245 	.f_detach = usb_filter_detach,
1246 	.f_event = usb_filter_read,
1247 };
1248 
1249 
1250 /* ARGSUSED */
1251 static int
1252 usb_kqfilter(struct cdev* dev, struct knote *kn)
1253 {
1254 	struct usb_cdev_refdata refs;
1255 	struct usb_cdev_privdata* cpd;
1256 	struct usb_fifo *f;
1257 	int fflags;
1258 	int err = EINVAL;
1259 
1260 	DPRINTFN(2, "\n");
1261 
1262 	if (devfs_get_cdevpriv((void **)&cpd) != 0 ||
1263 	    usb_ref_device(cpd, &refs, 0) != 0)
1264 		return (ENXIO);
1265 
1266 	fflags = cpd->fflags;
1267 
1268 	/* Figure out who needs service */
1269 	switch (kn->kn_filter) {
1270 	case EVFILT_WRITE:
1271 		if (fflags & FWRITE) {
1272 			f = refs.txfifo;
1273 			kn->kn_fop = &usb_filtops_write;
1274 			err = 0;
1275 		}
1276 		break;
1277 	case EVFILT_READ:
1278 		if (fflags & FREAD) {
1279 			f = refs.rxfifo;
1280 			kn->kn_fop = &usb_filtops_read;
1281 			err = 0;
1282 		}
1283 		break;
1284 	default:
1285 		err = EOPNOTSUPP;
1286 		break;
1287 	}
1288 
1289 	if (err == 0) {
1290 		kn->kn_hook = f;
1291 		mtx_lock(f->priv_mtx);
1292 		knlist_add(&f->selinfo.si_note, kn, 1);
1293 		mtx_unlock(f->priv_mtx);
1294 	}
1295 
1296 	usb_unref_device(cpd, &refs);
1297 	return (err);
1298 }
1299 
1300 /* ARGSUSED */
1301 static int
1302 usb_poll(struct cdev* dev, int events, struct thread* td)
1303 {
1304 	struct usb_cdev_refdata refs;
1305 	struct usb_cdev_privdata* cpd;
1306 	struct usb_fifo *f;
1307 	struct usb_mbuf *m;
1308 	int fflags, revents;
1309 
1310 	if (devfs_get_cdevpriv((void **)&cpd) != 0 ||
1311 	    usb_ref_device(cpd, &refs, 0) != 0)
1312 		return (events &
1313 		    (POLLHUP|POLLIN|POLLRDNORM|POLLOUT|POLLWRNORM));
1314 
1315 	fflags = cpd->fflags;
1316 
1317 	/* Figure out who needs service */
1318 	revents = 0;
1319 	if ((events & (POLLOUT | POLLWRNORM)) &&
1320 	    (fflags & FWRITE)) {
1321 
1322 		f = refs.txfifo;
1323 
1324 		mtx_lock(f->priv_mtx);
1325 
1326 		if (!refs.is_usbfs) {
1327 			if (f->flag_iserror) {
1328 				/* we got an error */
1329 				m = (void *)1;
1330 			} else {
1331 				if (f->queue_data == NULL) {
1332 					/*
1333 					 * start write transfer, if not
1334 					 * already started
1335 					 */
1336 					(f->methods->f_start_write) (f);
1337 				}
1338 				/* check if any packets are available */
1339 				USB_IF_POLL(&f->free_q, m);
1340 			}
1341 		} else {
1342 			if (f->flag_iscomplete) {
1343 				m = (void *)1;
1344 			} else {
1345 				m = NULL;
1346 			}
1347 		}
1348 
1349 		if (m) {
1350 			revents |= events & (POLLOUT | POLLWRNORM);
1351 		} else {
1352 			f->flag_isselect = 1;
1353 			selrecord(td, &f->selinfo);
1354 		}
1355 
1356 		mtx_unlock(f->priv_mtx);
1357 	}
1358 	if ((events & (POLLIN | POLLRDNORM)) &&
1359 	    (fflags & FREAD)) {
1360 
1361 		f = refs.rxfifo;
1362 
1363 		mtx_lock(f->priv_mtx);
1364 
1365 		if (!refs.is_usbfs) {
1366 			if (f->flag_iserror) {
1367 				/* we have an error */
1368 				m = (void *)1;
1369 			} else {
1370 				if (f->queue_data == NULL) {
1371 					/*
1372 					 * start read transfer, if not
1373 					 * already started
1374 					 */
1375 					(f->methods->f_start_read) (f);
1376 				}
1377 				/* check if any packets are available */
1378 				USB_IF_POLL(&f->used_q, m);
1379 			}
1380 		} else {
1381 			if (f->flag_iscomplete) {
1382 				m = (void *)1;
1383 			} else {
1384 				m = NULL;
1385 			}
1386 		}
1387 
1388 		if (m) {
1389 			revents |= events & (POLLIN | POLLRDNORM);
1390 		} else {
1391 			f->flag_isselect = 1;
1392 			selrecord(td, &f->selinfo);
1393 
1394 			if (!refs.is_usbfs) {
1395 				/* start reading data */
1396 				(f->methods->f_start_read) (f);
1397 			}
1398 		}
1399 
1400 		mtx_unlock(f->priv_mtx);
1401 	}
1402 	usb_unref_device(cpd, &refs);
1403 	return (revents);
1404 }
1405 
1406 static int
1407 usb_read(struct cdev *dev, struct uio *uio, int ioflag)
1408 {
1409 	struct usb_cdev_refdata refs;
1410 	struct usb_cdev_privdata* cpd;
1411 	struct usb_fifo *f;
1412 	struct usb_mbuf *m;
1413 	int io_len;
1414 	int err;
1415 	uint8_t tr_data = 0;
1416 
1417 	err = devfs_get_cdevpriv((void **)&cpd);
1418 	if (err != 0)
1419 		return (err);
1420 
1421 	err = usb_ref_device(cpd, &refs, 0 /* no uref */ );
1422 	if (err)
1423 		return (ENXIO);
1424 
1425 	f = refs.rxfifo;
1426 	if (f == NULL) {
1427 		/* should not happen */
1428 		usb_unref_device(cpd, &refs);
1429 		return (EPERM);
1430 	}
1431 
1432 	mtx_lock(f->priv_mtx);
1433 
1434 	/* check for permanent read error */
1435 	if (f->flag_iserror) {
1436 		err = EIO;
1437 		goto done;
1438 	}
1439 	/* check if USB-FS interface is active */
1440 	if (refs.is_usbfs) {
1441 		/*
1442 		 * The queue is used for events that should be
1443 		 * retrieved using the "USB_FS_COMPLETE" ioctl.
1444 		 */
1445 		err = EINVAL;
1446 		goto done;
1447 	}
1448 	while (uio->uio_resid > 0) {
1449 
1450 		USB_IF_DEQUEUE(&f->used_q, m);
1451 
1452 		if (m == NULL) {
1453 
1454 			/* start read transfer, if not already started */
1455 
1456 			(f->methods->f_start_read) (f);
1457 
1458 			if (ioflag & IO_NDELAY) {
1459 				if (tr_data) {
1460 					/* return length before error */
1461 					break;
1462 				}
1463 				err = EWOULDBLOCK;
1464 				break;
1465 			}
1466 			DPRINTF("sleeping\n");
1467 
1468 			err = usb_fifo_wait(f);
1469 			if (err) {
1470 				break;
1471 			}
1472 			continue;
1473 		}
1474 		if (f->methods->f_filter_read) {
1475 			/*
1476 			 * Sometimes it is convenient to process data at the
1477 			 * expense of a userland process instead of a kernel
1478 			 * process.
1479 			 */
1480 			(f->methods->f_filter_read) (f, m);
1481 		}
1482 		tr_data = 1;
1483 
1484 		io_len = MIN(m->cur_data_len, uio->uio_resid);
1485 
1486 		DPRINTFN(2, "transfer %d bytes from %p\n",
1487 		    io_len, m->cur_data_ptr);
1488 
1489 		err = usb_fifo_uiomove(f,
1490 		    m->cur_data_ptr, io_len, uio);
1491 
1492 		m->cur_data_len -= io_len;
1493 		m->cur_data_ptr += io_len;
1494 
1495 		if (m->cur_data_len == 0) {
1496 
1497 			uint8_t last_packet;
1498 
1499 			last_packet = m->last_packet;
1500 
1501 			USB_IF_ENQUEUE(&f->free_q, m);
1502 
1503 			if (last_packet) {
1504 				/* keep framing */
1505 				break;
1506 			}
1507 		} else {
1508 			USB_IF_PREPEND(&f->used_q, m);
1509 		}
1510 
1511 		if (err) {
1512 			break;
1513 		}
1514 	}
1515 done:
1516 	mtx_unlock(f->priv_mtx);
1517 
1518 	usb_unref_device(cpd, &refs);
1519 
1520 	return (err);
1521 }
1522 
1523 static int
1524 usb_write(struct cdev *dev, struct uio *uio, int ioflag)
1525 {
1526 	struct usb_cdev_refdata refs;
1527 	struct usb_cdev_privdata* cpd;
1528 	struct usb_fifo *f;
1529 	struct usb_mbuf *m;
1530 	uint8_t *pdata;
1531 	int io_len;
1532 	int err;
1533 	uint8_t tr_data = 0;
1534 
1535 	DPRINTFN(2, "\n");
1536 
1537 	err = devfs_get_cdevpriv((void **)&cpd);
1538 	if (err != 0)
1539 		return (err);
1540 
1541 	err = usb_ref_device(cpd, &refs, 0 /* no uref */ );
1542 	if (err)
1543 		return (ENXIO);
1544 
1545 	f = refs.txfifo;
1546 	if (f == NULL) {
1547 		/* should not happen */
1548 		usb_unref_device(cpd, &refs);
1549 		return (EPERM);
1550 	}
1551 
1552 	mtx_lock(f->priv_mtx);
1553 
1554 	/* check for permanent write error */
1555 	if (f->flag_iserror) {
1556 		err = EIO;
1557 		goto done;
1558 	}
1559 	/* check if USB-FS interface is active */
1560 	if (refs.is_usbfs) {
1561 		/*
1562 		 * The queue is used for events that should be
1563 		 * retrieved using the "USB_FS_COMPLETE" ioctl.
1564 		 */
1565 		err = EINVAL;
1566 		goto done;
1567 	}
1568 	if (f->queue_data == NULL) {
1569 		/* start write transfer, if not already started */
1570 		(f->methods->f_start_write) (f);
1571 	}
1572 	/* we allow writing zero length data */
1573 	do {
1574 		USB_IF_DEQUEUE(&f->free_q, m);
1575 
1576 		if (m == NULL) {
1577 
1578 			if (ioflag & IO_NDELAY) {
1579 				if (tr_data) {
1580 					/* return length before error */
1581 					break;
1582 				}
1583 				err = EWOULDBLOCK;
1584 				break;
1585 			}
1586 			DPRINTF("sleeping\n");
1587 
1588 			err = usb_fifo_wait(f);
1589 			if (err) {
1590 				break;
1591 			}
1592 			continue;
1593 		}
1594 		tr_data = 1;
1595 
1596 		if (f->flag_have_fragment == 0) {
1597 			USB_MBUF_RESET(m);
1598 			io_len = m->cur_data_len;
1599 			pdata = m->cur_data_ptr;
1600 			if (io_len > uio->uio_resid)
1601 				io_len = uio->uio_resid;
1602 			m->cur_data_len = io_len;
1603 		} else {
1604 			io_len = m->max_data_len - m->cur_data_len;
1605 			pdata = m->cur_data_ptr + m->cur_data_len;
1606 			if (io_len > uio->uio_resid)
1607 				io_len = uio->uio_resid;
1608 			m->cur_data_len += io_len;
1609 		}
1610 
1611 		DPRINTFN(2, "transfer %d bytes to %p\n",
1612 		    io_len, pdata);
1613 
1614 		err = usb_fifo_uiomove(f, pdata, io_len, uio);
1615 
1616 		if (err) {
1617 			f->flag_have_fragment = 0;
1618 			USB_IF_ENQUEUE(&f->free_q, m);
1619 			break;
1620 		}
1621 
1622 		/* check if the buffer is ready to be transmitted */
1623 
1624 		if ((f->flag_write_defrag == 0) ||
1625 		    (m->cur_data_len == m->max_data_len)) {
1626 			f->flag_have_fragment = 0;
1627 
1628 			/*
1629 			 * Check for write filter:
1630 			 *
1631 			 * Sometimes it is convenient to process data
1632 			 * at the expense of a userland process
1633 			 * instead of a kernel process.
1634 			 */
1635 			if (f->methods->f_filter_write) {
1636 				(f->methods->f_filter_write) (f, m);
1637 			}
1638 
1639 			/* Put USB mbuf in the used queue */
1640 			USB_IF_ENQUEUE(&f->used_q, m);
1641 
1642 			/* Start writing data, if not already started */
1643 			(f->methods->f_start_write) (f);
1644 		} else {
1645 			/* Wait for more data or close */
1646 			f->flag_have_fragment = 1;
1647 			USB_IF_PREPEND(&f->free_q, m);
1648 		}
1649 
1650 	} while (uio->uio_resid > 0);
1651 done:
1652 	mtx_unlock(f->priv_mtx);
1653 
1654 	usb_unref_device(cpd, &refs);
1655 
1656 	return (err);
1657 }
1658 
1659 int
1660 usb_static_ioctl(struct cdev *dev, u_long cmd, caddr_t data, int fflag,
1661     struct thread *td)
1662 {
1663 	union {
1664 		struct usb_read_dir *urd;
1665 		void* data;
1666 	} u;
1667 	int err;
1668 
1669 	u.data = data;
1670 	switch (cmd) {
1671 		case USB_READ_DIR:
1672 			err = usb_read_symlink(u.urd->urd_data,
1673 			    u.urd->urd_startentry, u.urd->urd_maxlen);
1674 			break;
1675 		case USB_DEV_QUIRK_GET:
1676 		case USB_QUIRK_NAME_GET:
1677 		case USB_DEV_QUIRK_ADD:
1678 		case USB_DEV_QUIRK_REMOVE:
1679 			err = usb_quirk_ioctl_p(cmd, data, fflag, td);
1680 			break;
1681 		case USB_GET_TEMPLATE:
1682 			*(int *)data = usb_template;
1683 			err = 0;
1684 			break;
1685 		case USB_SET_TEMPLATE:
1686 			err = priv_check(curthread, PRIV_DRIVER);
1687 			if (err)
1688 				break;
1689 			usb_template = *(int *)data;
1690 			break;
1691 		default:
1692 			err = ENOTTY;
1693 			break;
1694 	}
1695 	return (err);
1696 }
1697 
1698 static int
1699 usb_fifo_uiomove(struct usb_fifo *f, void *cp,
1700     int n, struct uio *uio)
1701 {
1702 	int error;
1703 
1704 	mtx_unlock(f->priv_mtx);
1705 
1706 	/*
1707 	 * "uiomove()" can sleep so one needs to make a wrapper,
1708 	 * exiting the mutex and checking things:
1709 	 */
1710 	error = uiomove(cp, n, uio);
1711 
1712 	mtx_lock(f->priv_mtx);
1713 
1714 	return (error);
1715 }
1716 
1717 int
1718 usb_fifo_wait(struct usb_fifo *f)
1719 {
1720 	int err;
1721 
1722 	USB_MTX_ASSERT(f->priv_mtx, MA_OWNED);
1723 
1724 	if (f->flag_iserror) {
1725 		/* we are gone */
1726 		return (EIO);
1727 	}
1728 	f->flag_sleeping = 1;
1729 
1730 	err = cv_wait_sig(&f->cv_io, f->priv_mtx);
1731 
1732 	if (f->flag_iserror) {
1733 		/* we are gone */
1734 		err = EIO;
1735 	}
1736 	return (err);
1737 }
1738 
1739 void
1740 usb_fifo_signal(struct usb_fifo *f)
1741 {
1742 	if (f->flag_sleeping) {
1743 		f->flag_sleeping = 0;
1744 		cv_broadcast(&f->cv_io);
1745 	}
1746 }
1747 
1748 void
1749 usb_fifo_wakeup(struct usb_fifo *f)
1750 {
1751 	usb_fifo_signal(f);
1752 
1753 	KNOTE_LOCKED(&f->selinfo.si_note, 0);
1754 
1755 	if (f->flag_isselect) {
1756 		selwakeup(&f->selinfo);
1757 		f->flag_isselect = 0;
1758 	}
1759 	if (f->async_p != NULL) {
1760 		PROC_LOCK(f->async_p);
1761 		kern_psignal(f->async_p, SIGIO);
1762 		PROC_UNLOCK(f->async_p);
1763 	}
1764 }
1765 
1766 static int
1767 usb_fifo_dummy_open(struct usb_fifo *fifo, int fflags)
1768 {
1769 	return (0);
1770 }
1771 
1772 static void
1773 usb_fifo_dummy_close(struct usb_fifo *fifo, int fflags)
1774 {
1775 	return;
1776 }
1777 
1778 static int
1779 usb_fifo_dummy_ioctl(struct usb_fifo *fifo, u_long cmd, void *addr, int fflags)
1780 {
1781 	return (ENOIOCTL);
1782 }
1783 
1784 static void
1785 usb_fifo_dummy_cmd(struct usb_fifo *fifo)
1786 {
1787 	fifo->flag_flushing = 0;	/* not flushing */
1788 }
1789 
1790 static void
1791 usb_fifo_check_methods(struct usb_fifo_methods *pm)
1792 {
1793 	/* check that all callback functions are OK */
1794 
1795 	if (pm->f_open == NULL)
1796 		pm->f_open = &usb_fifo_dummy_open;
1797 
1798 	if (pm->f_close == NULL)
1799 		pm->f_close = &usb_fifo_dummy_close;
1800 
1801 	if (pm->f_ioctl == NULL)
1802 		pm->f_ioctl = &usb_fifo_dummy_ioctl;
1803 
1804 	if (pm->f_ioctl_post == NULL)
1805 		pm->f_ioctl_post = &usb_fifo_dummy_ioctl;
1806 
1807 	if (pm->f_start_read == NULL)
1808 		pm->f_start_read = &usb_fifo_dummy_cmd;
1809 
1810 	if (pm->f_stop_read == NULL)
1811 		pm->f_stop_read = &usb_fifo_dummy_cmd;
1812 
1813 	if (pm->f_start_write == NULL)
1814 		pm->f_start_write = &usb_fifo_dummy_cmd;
1815 
1816 	if (pm->f_stop_write == NULL)
1817 		pm->f_stop_write = &usb_fifo_dummy_cmd;
1818 }
1819 
1820 /*------------------------------------------------------------------------*
1821  *	usb_fifo_attach
1822  *
1823  * The following function will create a duplex FIFO.
1824  *
1825  * Return values:
1826  * 0: Success.
1827  * Else: Failure.
1828  *------------------------------------------------------------------------*/
1829 int
1830 usb_fifo_attach(struct usb_device *udev, void *priv_sc,
1831     struct mtx *priv_mtx, struct usb_fifo_methods *pm,
1832     struct usb_fifo_sc *f_sc, uint16_t unit, int16_t subunit,
1833     uint8_t iface_index, uid_t uid, gid_t gid, int mode)
1834 {
1835 	struct usb_fifo *f_tx;
1836 	struct usb_fifo *f_rx;
1837 	char devname[32];
1838 	uint8_t n;
1839 
1840 	f_sc->fp[USB_FIFO_TX] = NULL;
1841 	f_sc->fp[USB_FIFO_RX] = NULL;
1842 
1843 	if (pm == NULL)
1844 		return (EINVAL);
1845 
1846 	/* check the methods */
1847 	usb_fifo_check_methods(pm);
1848 
1849 	if (priv_mtx == NULL)
1850 		priv_mtx = &Giant;
1851 
1852 	/* search for a free FIFO slot */
1853 	for (n = 0;; n += 2) {
1854 
1855 		if (n == USB_FIFO_MAX) {
1856 			/* end of FIFOs reached */
1857 			return (ENOMEM);
1858 		}
1859 		/* Check for TX FIFO */
1860 		if (udev->fifo[n + USB_FIFO_TX] != NULL) {
1861 			continue;
1862 		}
1863 		/* Check for RX FIFO */
1864 		if (udev->fifo[n + USB_FIFO_RX] != NULL) {
1865 			continue;
1866 		}
1867 		break;
1868 	}
1869 
1870 	f_tx = usb_fifo_alloc(priv_mtx);
1871 	f_rx = usb_fifo_alloc(priv_mtx);
1872 
1873 	if ((f_tx == NULL) || (f_rx == NULL)) {
1874 		usb_fifo_free(f_tx);
1875 		usb_fifo_free(f_rx);
1876 		return (ENOMEM);
1877 	}
1878 	/* initialise FIFO structures */
1879 
1880 	f_tx->fifo_index = n + USB_FIFO_TX;
1881 	f_tx->dev_ep_index = -1;
1882 	f_tx->priv_sc0 = priv_sc;
1883 	f_tx->methods = pm;
1884 	f_tx->iface_index = iface_index;
1885 	f_tx->udev = udev;
1886 
1887 	f_rx->fifo_index = n + USB_FIFO_RX;
1888 	f_rx->dev_ep_index = -1;
1889 	f_rx->priv_sc0 = priv_sc;
1890 	f_rx->methods = pm;
1891 	f_rx->iface_index = iface_index;
1892 	f_rx->udev = udev;
1893 
1894 	f_sc->fp[USB_FIFO_TX] = f_tx;
1895 	f_sc->fp[USB_FIFO_RX] = f_rx;
1896 
1897 	mtx_lock(&usb_ref_lock);
1898 	udev->fifo[f_tx->fifo_index] = f_tx;
1899 	udev->fifo[f_rx->fifo_index] = f_rx;
1900 	mtx_unlock(&usb_ref_lock);
1901 
1902 	for (n = 0; n != 4; n++) {
1903 
1904 		if (pm->basename[n] == NULL) {
1905 			continue;
1906 		}
1907 		if (subunit < 0) {
1908 			if (snprintf(devname, sizeof(devname),
1909 			    "%s%u%s", pm->basename[n],
1910 			    unit, pm->postfix[n] ?
1911 			    pm->postfix[n] : "")) {
1912 				/* ignore */
1913 			}
1914 		} else {
1915 			if (snprintf(devname, sizeof(devname),
1916 			    "%s%u.%d%s", pm->basename[n],
1917 			    unit, subunit, pm->postfix[n] ?
1918 			    pm->postfix[n] : "")) {
1919 				/* ignore */
1920 			}
1921 		}
1922 
1923 		/*
1924 		 * Distribute the symbolic links into two FIFO structures:
1925 		 */
1926 		if (n & 1) {
1927 			f_rx->symlink[n / 2] =
1928 			    usb_alloc_symlink(devname);
1929 		} else {
1930 			f_tx->symlink[n / 2] =
1931 			    usb_alloc_symlink(devname);
1932 		}
1933 
1934 		/* Create the device */
1935 		f_sc->dev = usb_make_dev(udev, devname, -1,
1936 		    f_tx->fifo_index & f_rx->fifo_index,
1937 		    FREAD|FWRITE, uid, gid, mode);
1938 	}
1939 
1940 	DPRINTFN(2, "attached %p/%p\n", f_tx, f_rx);
1941 	return (0);
1942 }
1943 
1944 /*------------------------------------------------------------------------*
1945  *	usb_fifo_alloc_buffer
1946  *
1947  * Return values:
1948  * 0: Success
1949  * Else failure
1950  *------------------------------------------------------------------------*/
1951 int
1952 usb_fifo_alloc_buffer(struct usb_fifo *f, usb_size_t bufsize,
1953     uint16_t nbuf)
1954 {
1955 	usb_fifo_free_buffer(f);
1956 
1957 	/* allocate an endpoint */
1958 	f->free_q.ifq_maxlen = nbuf;
1959 	f->used_q.ifq_maxlen = nbuf;
1960 
1961 	f->queue_data = usb_alloc_mbufs(
1962 	    M_USBDEV, &f->free_q, bufsize, nbuf);
1963 
1964 	if ((f->queue_data == NULL) && bufsize && nbuf) {
1965 		return (ENOMEM);
1966 	}
1967 	return (0);			/* success */
1968 }
1969 
1970 /*------------------------------------------------------------------------*
1971  *	usb_fifo_free_buffer
1972  *
1973  * This function will free the buffers associated with a FIFO. This
1974  * function can be called multiple times in a row.
1975  *------------------------------------------------------------------------*/
1976 void
1977 usb_fifo_free_buffer(struct usb_fifo *f)
1978 {
1979 	if (f->queue_data) {
1980 		/* free old buffer */
1981 		free(f->queue_data, M_USBDEV);
1982 		f->queue_data = NULL;
1983 	}
1984 	/* reset queues */
1985 
1986 	memset(&f->free_q, 0, sizeof(f->free_q));
1987 	memset(&f->used_q, 0, sizeof(f->used_q));
1988 }
1989 
1990 void
1991 usb_fifo_detach(struct usb_fifo_sc *f_sc)
1992 {
1993 	if (f_sc == NULL) {
1994 		return;
1995 	}
1996 	usb_fifo_free(f_sc->fp[USB_FIFO_TX]);
1997 	usb_fifo_free(f_sc->fp[USB_FIFO_RX]);
1998 
1999 	f_sc->fp[USB_FIFO_TX] = NULL;
2000 	f_sc->fp[USB_FIFO_RX] = NULL;
2001 
2002 	usb_destroy_dev(f_sc->dev);
2003 
2004 	f_sc->dev = NULL;
2005 
2006 	DPRINTFN(2, "detached %p\n", f_sc);
2007 }
2008 
2009 usb_size_t
2010 usb_fifo_put_bytes_max(struct usb_fifo *f)
2011 {
2012 	struct usb_mbuf *m;
2013 	usb_size_t len;
2014 
2015 	USB_IF_POLL(&f->free_q, m);
2016 
2017 	if (m) {
2018 		len = m->max_data_len;
2019 	} else {
2020 		len = 0;
2021 	}
2022 	return (len);
2023 }
2024 
2025 /*------------------------------------------------------------------------*
2026  *	usb_fifo_put_data
2027  *
2028  * what:
2029  *  0 - normal operation
2030  *  1 - set last packet flag to enforce framing
2031  *------------------------------------------------------------------------*/
2032 void
2033 usb_fifo_put_data(struct usb_fifo *f, struct usb_page_cache *pc,
2034     usb_frlength_t offset, usb_frlength_t len, uint8_t what)
2035 {
2036 	struct usb_mbuf *m;
2037 	usb_frlength_t io_len;
2038 
2039 	while (len || (what == 1)) {
2040 
2041 		USB_IF_DEQUEUE(&f->free_q, m);
2042 
2043 		if (m) {
2044 			USB_MBUF_RESET(m);
2045 
2046 			io_len = MIN(len, m->cur_data_len);
2047 
2048 			usbd_copy_out(pc, offset, m->cur_data_ptr, io_len);
2049 
2050 			m->cur_data_len = io_len;
2051 			offset += io_len;
2052 			len -= io_len;
2053 
2054 			if ((len == 0) && (what == 1)) {
2055 				m->last_packet = 1;
2056 			}
2057 			USB_IF_ENQUEUE(&f->used_q, m);
2058 
2059 			usb_fifo_wakeup(f);
2060 
2061 			if ((len == 0) || (what == 1)) {
2062 				break;
2063 			}
2064 		} else {
2065 			break;
2066 		}
2067 	}
2068 }
2069 
2070 void
2071 usb_fifo_put_data_linear(struct usb_fifo *f, void *ptr,
2072     usb_size_t len, uint8_t what)
2073 {
2074 	struct usb_mbuf *m;
2075 	usb_size_t io_len;
2076 
2077 	while (len || (what == 1)) {
2078 
2079 		USB_IF_DEQUEUE(&f->free_q, m);
2080 
2081 		if (m) {
2082 			USB_MBUF_RESET(m);
2083 
2084 			io_len = MIN(len, m->cur_data_len);
2085 
2086 			memcpy(m->cur_data_ptr, ptr, io_len);
2087 
2088 			m->cur_data_len = io_len;
2089 			ptr = USB_ADD_BYTES(ptr, io_len);
2090 			len -= io_len;
2091 
2092 			if ((len == 0) && (what == 1)) {
2093 				m->last_packet = 1;
2094 			}
2095 			USB_IF_ENQUEUE(&f->used_q, m);
2096 
2097 			usb_fifo_wakeup(f);
2098 
2099 			if ((len == 0) || (what == 1)) {
2100 				break;
2101 			}
2102 		} else {
2103 			break;
2104 		}
2105 	}
2106 }
2107 
2108 uint8_t
2109 usb_fifo_put_data_buffer(struct usb_fifo *f, void *ptr, usb_size_t len)
2110 {
2111 	struct usb_mbuf *m;
2112 
2113 	USB_IF_DEQUEUE(&f->free_q, m);
2114 
2115 	if (m) {
2116 		m->cur_data_len = len;
2117 		m->cur_data_ptr = ptr;
2118 		USB_IF_ENQUEUE(&f->used_q, m);
2119 		usb_fifo_wakeup(f);
2120 		return (1);
2121 	}
2122 	return (0);
2123 }
2124 
2125 void
2126 usb_fifo_put_data_error(struct usb_fifo *f)
2127 {
2128 	f->flag_iserror = 1;
2129 	usb_fifo_wakeup(f);
2130 }
2131 
2132 /*------------------------------------------------------------------------*
2133  *	usb_fifo_get_data
2134  *
2135  * what:
2136  *  0 - normal operation
2137  *  1 - only get one "usb_mbuf"
2138  *
2139  * returns:
2140  *  0 - no more data
2141  *  1 - data in buffer
2142  *------------------------------------------------------------------------*/
2143 uint8_t
2144 usb_fifo_get_data(struct usb_fifo *f, struct usb_page_cache *pc,
2145     usb_frlength_t offset, usb_frlength_t len, usb_frlength_t *actlen,
2146     uint8_t what)
2147 {
2148 	struct usb_mbuf *m;
2149 	usb_frlength_t io_len;
2150 	uint8_t tr_data = 0;
2151 
2152 	actlen[0] = 0;
2153 
2154 	while (1) {
2155 
2156 		USB_IF_DEQUEUE(&f->used_q, m);
2157 
2158 		if (m) {
2159 
2160 			tr_data = 1;
2161 
2162 			io_len = MIN(len, m->cur_data_len);
2163 
2164 			usbd_copy_in(pc, offset, m->cur_data_ptr, io_len);
2165 
2166 			len -= io_len;
2167 			offset += io_len;
2168 			actlen[0] += io_len;
2169 			m->cur_data_ptr += io_len;
2170 			m->cur_data_len -= io_len;
2171 
2172 			if ((m->cur_data_len == 0) || (what == 1)) {
2173 				USB_IF_ENQUEUE(&f->free_q, m);
2174 
2175 				usb_fifo_wakeup(f);
2176 
2177 				if (what == 1) {
2178 					break;
2179 				}
2180 			} else {
2181 				USB_IF_PREPEND(&f->used_q, m);
2182 			}
2183 		} else {
2184 
2185 			if (tr_data) {
2186 				/* wait for data to be written out */
2187 				break;
2188 			}
2189 			if (f->flag_flushing) {
2190 				/* check if we should send a short packet */
2191 				if (f->flag_short != 0) {
2192 					f->flag_short = 0;
2193 					tr_data = 1;
2194 					break;
2195 				}
2196 				/* flushing complete */
2197 				f->flag_flushing = 0;
2198 				usb_fifo_wakeup(f);
2199 			}
2200 			break;
2201 		}
2202 		if (len == 0) {
2203 			break;
2204 		}
2205 	}
2206 	return (tr_data);
2207 }
2208 
2209 uint8_t
2210 usb_fifo_get_data_linear(struct usb_fifo *f, void *ptr,
2211     usb_size_t len, usb_size_t *actlen, uint8_t what)
2212 {
2213 	struct usb_mbuf *m;
2214 	usb_size_t io_len;
2215 	uint8_t tr_data = 0;
2216 
2217 	actlen[0] = 0;
2218 
2219 	while (1) {
2220 
2221 		USB_IF_DEQUEUE(&f->used_q, m);
2222 
2223 		if (m) {
2224 
2225 			tr_data = 1;
2226 
2227 			io_len = MIN(len, m->cur_data_len);
2228 
2229 			memcpy(ptr, m->cur_data_ptr, io_len);
2230 
2231 			len -= io_len;
2232 			ptr = USB_ADD_BYTES(ptr, io_len);
2233 			actlen[0] += io_len;
2234 			m->cur_data_ptr += io_len;
2235 			m->cur_data_len -= io_len;
2236 
2237 			if ((m->cur_data_len == 0) || (what == 1)) {
2238 				USB_IF_ENQUEUE(&f->free_q, m);
2239 
2240 				usb_fifo_wakeup(f);
2241 
2242 				if (what == 1) {
2243 					break;
2244 				}
2245 			} else {
2246 				USB_IF_PREPEND(&f->used_q, m);
2247 			}
2248 		} else {
2249 
2250 			if (tr_data) {
2251 				/* wait for data to be written out */
2252 				break;
2253 			}
2254 			if (f->flag_flushing) {
2255 				/* check if we should send a short packet */
2256 				if (f->flag_short != 0) {
2257 					f->flag_short = 0;
2258 					tr_data = 1;
2259 					break;
2260 				}
2261 				/* flushing complete */
2262 				f->flag_flushing = 0;
2263 				usb_fifo_wakeup(f);
2264 			}
2265 			break;
2266 		}
2267 		if (len == 0) {
2268 			break;
2269 		}
2270 	}
2271 	return (tr_data);
2272 }
2273 
2274 uint8_t
2275 usb_fifo_get_data_buffer(struct usb_fifo *f, void **pptr, usb_size_t *plen)
2276 {
2277 	struct usb_mbuf *m;
2278 
2279 	USB_IF_POLL(&f->used_q, m);
2280 
2281 	if (m) {
2282 		*plen = m->cur_data_len;
2283 		*pptr = m->cur_data_ptr;
2284 
2285 		return (1);
2286 	}
2287 	return (0);
2288 }
2289 
2290 void
2291 usb_fifo_get_data_error(struct usb_fifo *f)
2292 {
2293 	f->flag_iserror = 1;
2294 	usb_fifo_wakeup(f);
2295 }
2296 
2297 /*------------------------------------------------------------------------*
2298  *	usb_alloc_symlink
2299  *
2300  * Return values:
2301  * NULL: Failure
2302  * Else: Pointer to symlink entry
2303  *------------------------------------------------------------------------*/
2304 struct usb_symlink *
2305 usb_alloc_symlink(const char *target)
2306 {
2307 	struct usb_symlink *ps;
2308 
2309 	ps = malloc(sizeof(*ps), M_USBDEV, M_WAITOK);
2310 	/* XXX no longer needed */
2311 	strlcpy(ps->src_path, target, sizeof(ps->src_path));
2312 	ps->src_len = strlen(ps->src_path);
2313 	strlcpy(ps->dst_path, target, sizeof(ps->dst_path));
2314 	ps->dst_len = strlen(ps->dst_path);
2315 
2316 	sx_xlock(&usb_sym_lock);
2317 	TAILQ_INSERT_TAIL(&usb_sym_head, ps, sym_entry);
2318 	sx_unlock(&usb_sym_lock);
2319 	return (ps);
2320 }
2321 
2322 /*------------------------------------------------------------------------*
2323  *	usb_free_symlink
2324  *------------------------------------------------------------------------*/
2325 void
2326 usb_free_symlink(struct usb_symlink *ps)
2327 {
2328 	if (ps == NULL) {
2329 		return;
2330 	}
2331 	sx_xlock(&usb_sym_lock);
2332 	TAILQ_REMOVE(&usb_sym_head, ps, sym_entry);
2333 	sx_unlock(&usb_sym_lock);
2334 
2335 	free(ps, M_USBDEV);
2336 }
2337 
2338 /*------------------------------------------------------------------------*
2339  *	usb_read_symlink
2340  *
2341  * Return value:
2342  * 0: Success
2343  * Else: Failure
2344  *------------------------------------------------------------------------*/
2345 int
2346 usb_read_symlink(uint8_t *user_ptr, uint32_t startentry, uint32_t user_len)
2347 {
2348 	struct usb_symlink *ps;
2349 	uint32_t temp;
2350 	uint32_t delta = 0;
2351 	uint8_t len;
2352 	int error = 0;
2353 
2354 	sx_xlock(&usb_sym_lock);
2355 
2356 	TAILQ_FOREACH(ps, &usb_sym_head, sym_entry) {
2357 
2358 		/*
2359 		 * Compute total length of source and destination symlink
2360 		 * strings pluss one length byte and two NUL bytes:
2361 		 */
2362 		temp = ps->src_len + ps->dst_len + 3;
2363 
2364 		if (temp > 255) {
2365 			/*
2366 			 * Skip entry because this length cannot fit
2367 			 * into one byte:
2368 			 */
2369 			continue;
2370 		}
2371 		if (startentry != 0) {
2372 			/* decrement read offset */
2373 			startentry--;
2374 			continue;
2375 		}
2376 		if (temp > user_len) {
2377 			/* out of buffer space */
2378 			break;
2379 		}
2380 		len = temp;
2381 
2382 		/* copy out total length */
2383 
2384 		error = copyout(&len,
2385 		    USB_ADD_BYTES(user_ptr, delta), 1);
2386 		if (error) {
2387 			break;
2388 		}
2389 		delta += 1;
2390 
2391 		/* copy out source string */
2392 
2393 		error = copyout(ps->src_path,
2394 		    USB_ADD_BYTES(user_ptr, delta), ps->src_len);
2395 		if (error) {
2396 			break;
2397 		}
2398 		len = 0;
2399 		delta += ps->src_len;
2400 		error = copyout(&len,
2401 		    USB_ADD_BYTES(user_ptr, delta), 1);
2402 		if (error) {
2403 			break;
2404 		}
2405 		delta += 1;
2406 
2407 		/* copy out destination string */
2408 
2409 		error = copyout(ps->dst_path,
2410 		    USB_ADD_BYTES(user_ptr, delta), ps->dst_len);
2411 		if (error) {
2412 			break;
2413 		}
2414 		len = 0;
2415 		delta += ps->dst_len;
2416 		error = copyout(&len,
2417 		    USB_ADD_BYTES(user_ptr, delta), 1);
2418 		if (error) {
2419 			break;
2420 		}
2421 		delta += 1;
2422 
2423 		user_len -= temp;
2424 	}
2425 
2426 	/* a zero length entry indicates the end */
2427 
2428 	if ((user_len != 0) && (error == 0)) {
2429 
2430 		len = 0;
2431 
2432 		error = copyout(&len,
2433 		    USB_ADD_BYTES(user_ptr, delta), 1);
2434 	}
2435 	sx_unlock(&usb_sym_lock);
2436 	return (error);
2437 }
2438 
2439 void
2440 usb_fifo_set_close_zlp(struct usb_fifo *f, uint8_t onoff)
2441 {
2442 	if (f == NULL)
2443 		return;
2444 
2445 	/* send a Zero Length Packet, ZLP, before close */
2446 	f->flag_short = onoff;
2447 }
2448 
2449 void
2450 usb_fifo_set_write_defrag(struct usb_fifo *f, uint8_t onoff)
2451 {
2452 	if (f == NULL)
2453 		return;
2454 
2455 	/* defrag written data */
2456 	f->flag_write_defrag = onoff;
2457 	/* reset defrag state */
2458 	f->flag_have_fragment = 0;
2459 }
2460 
2461 void *
2462 usb_fifo_softc(struct usb_fifo *f)
2463 {
2464 	return (f->priv_sc0);
2465 }
2466 #endif	/* USB_HAVE_UGEN */
2467