xref: /freebsd/sys/dev/usb/usb_dev.c (revision eaa797943eeac5614edfdc8f6309f332343c3dd2)
1 /* $FreeBSD$ */
2 /*-
3  * Copyright (c) 2006-2008 Hans Petter Selasky. All rights reserved.
4  *
5  * Redistribution and use in source and binary forms, with or without
6  * modification, are permitted provided that the following conditions
7  * are met:
8  * 1. Redistributions of source code must retain the above copyright
9  *    notice, this list of conditions and the following disclaimer.
10  * 2. Redistributions in binary form must reproduce the above copyright
11  *    notice, this list of conditions and the following disclaimer in the
12  *    documentation and/or other materials provided with the distribution.
13  *
14  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24  * SUCH DAMAGE.
25  *
26  *
27  * usb_dev.c - An abstraction layer for creating devices under /dev/...
28  */
29 
30 #ifdef USB_GLOBAL_INCLUDE_FILE
31 #include USB_GLOBAL_INCLUDE_FILE
32 #else
33 #include <sys/stdint.h>
34 #include <sys/stddef.h>
35 #include <sys/param.h>
36 #include <sys/queue.h>
37 #include <sys/types.h>
38 #include <sys/systm.h>
39 #include <sys/kernel.h>
40 #include <sys/bus.h>
41 #include <sys/module.h>
42 #include <sys/lock.h>
43 #include <sys/mutex.h>
44 #include <sys/condvar.h>
45 #include <sys/sysctl.h>
46 #include <sys/sx.h>
47 #include <sys/unistd.h>
48 #include <sys/callout.h>
49 #include <sys/malloc.h>
50 #include <sys/priv.h>
51 #include <sys/vnode.h>
52 #include <sys/conf.h>
53 #include <sys/fcntl.h>
54 
55 #include <dev/usb/usb.h>
56 #include <dev/usb/usb_ioctl.h>
57 #include <dev/usb/usbdi.h>
58 #include <dev/usb/usbdi_util.h>
59 
60 #define	USB_DEBUG_VAR usb_fifo_debug
61 
62 #include <dev/usb/usb_core.h>
63 #include <dev/usb/usb_dev.h>
64 #include <dev/usb/usb_mbuf.h>
65 #include <dev/usb/usb_process.h>
66 #include <dev/usb/usb_device.h>
67 #include <dev/usb/usb_debug.h>
68 #include <dev/usb/usb_busdma.h>
69 #include <dev/usb/usb_generic.h>
70 #include <dev/usb/usb_dynamic.h>
71 #include <dev/usb/usb_util.h>
72 
73 #include <dev/usb/usb_controller.h>
74 #include <dev/usb/usb_bus.h>
75 
76 #include <sys/filio.h>
77 #include <sys/ttycom.h>
78 #include <sys/syscallsubr.h>
79 
80 #include <machine/stdarg.h>
81 #endif			/* USB_GLOBAL_INCLUDE_FILE */
82 
83 #if USB_HAVE_UGEN
84 
85 #ifdef USB_DEBUG
86 static int usb_fifo_debug = 0;
87 
88 static SYSCTL_NODE(_hw_usb, OID_AUTO, dev, CTLFLAG_RW, 0, "USB device");
89 SYSCTL_INT(_hw_usb_dev, OID_AUTO, debug, CTLFLAG_RWTUN,
90     &usb_fifo_debug, 0, "Debug Level");
91 #endif
92 
93 #if ((__FreeBSD_version >= 700001) || (__FreeBSD_version == 0) || \
94      ((__FreeBSD_version >= 600034) && (__FreeBSD_version < 700000)))
95 #define	USB_UCRED struct ucred *ucred,
96 #else
97 #define	USB_UCRED
98 #endif
99 
100 /* prototypes */
101 
102 static int	usb_fifo_open(struct usb_cdev_privdata *,
103 		    struct usb_fifo *, int);
104 static void	usb_fifo_close(struct usb_fifo *, int);
105 static void	usb_dev_init(void *);
106 static void	usb_dev_init_post(void *);
107 static void	usb_dev_uninit(void *);
108 static int	usb_fifo_uiomove(struct usb_fifo *, void *, int,
109 		    struct uio *);
110 static void	usb_fifo_check_methods(struct usb_fifo_methods *);
111 static struct	usb_fifo *usb_fifo_alloc(struct mtx *);
112 static struct	usb_endpoint *usb_dev_get_ep(struct usb_device *, uint8_t,
113 		    uint8_t);
114 static void	usb_loc_fill(struct usb_fs_privdata *,
115 		    struct usb_cdev_privdata *);
116 static void	usb_close(void *);
117 static usb_error_t usb_ref_device(struct usb_cdev_privdata *, struct usb_cdev_refdata *, int);
118 static usb_error_t usb_usb_ref_device(struct usb_cdev_privdata *, struct usb_cdev_refdata *);
119 static void	usb_unref_device(struct usb_cdev_privdata *, struct usb_cdev_refdata *);
120 
121 static d_open_t usb_open;
122 static d_ioctl_t usb_ioctl;
123 static d_read_t usb_read;
124 static d_write_t usb_write;
125 static d_poll_t usb_poll;
126 static d_kqfilter_t usb_kqfilter;
127 
128 static d_ioctl_t usb_static_ioctl;
129 
130 static usb_fifo_open_t usb_fifo_dummy_open;
131 static usb_fifo_close_t usb_fifo_dummy_close;
132 static usb_fifo_ioctl_t usb_fifo_dummy_ioctl;
133 static usb_fifo_cmd_t usb_fifo_dummy_cmd;
134 
135 /* character device structure used for devices (/dev/ugenX.Y and /dev/uXXX) */
136 struct cdevsw usb_devsw = {
137 	.d_version = D_VERSION,
138 	.d_open = usb_open,
139 	.d_ioctl = usb_ioctl,
140 	.d_name = "usbdev",
141 	.d_flags = D_TRACKCLOSE,
142 	.d_read = usb_read,
143 	.d_write = usb_write,
144 	.d_poll = usb_poll,
145 	.d_kqfilter = usb_kqfilter,
146 };
147 
148 static struct cdev* usb_dev = NULL;
149 
150 /* character device structure used for /dev/usb */
151 static struct cdevsw usb_static_devsw = {
152 	.d_version = D_VERSION,
153 	.d_ioctl = usb_static_ioctl,
154 	.d_name = "usb"
155 };
156 
157 static TAILQ_HEAD(, usb_symlink) usb_sym_head;
158 static struct sx usb_sym_lock;
159 
160 struct mtx usb_ref_lock;
161 
162 /*------------------------------------------------------------------------*
163  *	usb_loc_fill
164  *
165  * This is used to fill out a usb_cdev_privdata structure based on the
166  * device's address as contained in usb_fs_privdata.
167  *------------------------------------------------------------------------*/
168 static void
169 usb_loc_fill(struct usb_fs_privdata* pd, struct usb_cdev_privdata *cpd)
170 {
171 	cpd->bus_index = pd->bus_index;
172 	cpd->dev_index = pd->dev_index;
173 	cpd->ep_addr = pd->ep_addr;
174 	cpd->fifo_index = pd->fifo_index;
175 }
176 
177 /*------------------------------------------------------------------------*
178  *	usb_ref_device
179  *
180  * This function is used to atomically refer an USB device by its
181  * device location. If this function returns success the USB device
182  * will not disappear until the USB device is unreferenced.
183  *
184  * Return values:
185  *  0: Success, refcount incremented on the given USB device.
186  *  Else: Failure.
187  *------------------------------------------------------------------------*/
188 static usb_error_t
189 usb_ref_device(struct usb_cdev_privdata *cpd,
190     struct usb_cdev_refdata *crd, int need_uref)
191 {
192 	struct usb_fifo **ppf;
193 	struct usb_fifo *f;
194 
195 	DPRINTFN(2, "cpd=%p need uref=%d\n", cpd, need_uref);
196 
197 	/* clear all refs */
198 	memset(crd, 0, sizeof(*crd));
199 
200 	mtx_lock(&usb_ref_lock);
201 	cpd->bus = devclass_get_softc(usb_devclass_ptr, cpd->bus_index);
202 	if (cpd->bus == NULL) {
203 		DPRINTFN(2, "no bus at %u\n", cpd->bus_index);
204 		goto error;
205 	}
206 	cpd->udev = cpd->bus->devices[cpd->dev_index];
207 	if (cpd->udev == NULL) {
208 		DPRINTFN(2, "no device at %u\n", cpd->dev_index);
209 		goto error;
210 	}
211 	if (cpd->udev->state == USB_STATE_DETACHED &&
212 	    (need_uref != 2)) {
213 		DPRINTFN(2, "device is detached\n");
214 		goto error;
215 	}
216 	if (need_uref) {
217 		DPRINTFN(2, "ref udev - needed\n");
218 
219 		if (cpd->udev->refcount == USB_DEV_REF_MAX) {
220 			DPRINTFN(2, "no dev ref\n");
221 			goto error;
222 		}
223 		cpd->udev->refcount++;
224 
225 		mtx_unlock(&usb_ref_lock);
226 
227 		/*
228 		 * We need to grab the enumeration SX-lock before
229 		 * grabbing the FIFO refs to avoid deadlock at detach!
230 		 */
231 		crd->do_unlock = usbd_enum_lock_sig(cpd->udev);
232 
233 		mtx_lock(&usb_ref_lock);
234 
235 		/*
236 		 * Set "is_uref" after grabbing the default SX lock
237 		 */
238 		crd->is_uref = 1;
239 
240 		/* check for signal */
241 		if (crd->do_unlock > 1) {
242 			crd->do_unlock = 0;
243 			goto error;
244 		}
245 	}
246 
247 	/* check if we are doing an open */
248 	if (cpd->fflags == 0) {
249 		/* use zero defaults */
250 	} else {
251 		/* check for write */
252 		if (cpd->fflags & FWRITE) {
253 			ppf = cpd->udev->fifo;
254 			f = ppf[cpd->fifo_index + USB_FIFO_TX];
255 			crd->txfifo = f;
256 			crd->is_write = 1;	/* ref */
257 			if (f == NULL || f->refcount == USB_FIFO_REF_MAX)
258 				goto error;
259 			if (f->curr_cpd != cpd)
260 				goto error;
261 			/* check if USB-FS is active */
262 			if (f->fs_ep_max != 0) {
263 				crd->is_usbfs = 1;
264 			}
265 		}
266 
267 		/* check for read */
268 		if (cpd->fflags & FREAD) {
269 			ppf = cpd->udev->fifo;
270 			f = ppf[cpd->fifo_index + USB_FIFO_RX];
271 			crd->rxfifo = f;
272 			crd->is_read = 1;	/* ref */
273 			if (f == NULL || f->refcount == USB_FIFO_REF_MAX)
274 				goto error;
275 			if (f->curr_cpd != cpd)
276 				goto error;
277 			/* check if USB-FS is active */
278 			if (f->fs_ep_max != 0) {
279 				crd->is_usbfs = 1;
280 			}
281 		}
282 	}
283 
284 	/* when everything is OK we increment the refcounts */
285 	if (crd->is_write) {
286 		DPRINTFN(2, "ref write\n");
287 		crd->txfifo->refcount++;
288 	}
289 	if (crd->is_read) {
290 		DPRINTFN(2, "ref read\n");
291 		crd->rxfifo->refcount++;
292 	}
293 	mtx_unlock(&usb_ref_lock);
294 
295 	return (0);
296 
297 error:
298 	if (crd->do_unlock)
299 		usbd_enum_unlock(cpd->udev);
300 
301 	if (crd->is_uref) {
302 		if (--(cpd->udev->refcount) == 0)
303 			cv_broadcast(&cpd->udev->ref_cv);
304 	}
305 	mtx_unlock(&usb_ref_lock);
306 	DPRINTFN(2, "fail\n");
307 
308 	/* clear all refs */
309 	memset(crd, 0, sizeof(*crd));
310 
311 	return (USB_ERR_INVAL);
312 }
313 
314 /*------------------------------------------------------------------------*
315  *	usb_usb_ref_device
316  *
317  * This function is used to upgrade an USB reference to include the
318  * USB device reference on a USB location.
319  *
320  * Return values:
321  *  0: Success, refcount incremented on the given USB device.
322  *  Else: Failure.
323  *------------------------------------------------------------------------*/
324 static usb_error_t
325 usb_usb_ref_device(struct usb_cdev_privdata *cpd,
326     struct usb_cdev_refdata *crd)
327 {
328 	/*
329 	 * Check if we already got an USB reference on this location:
330 	 */
331 	if (crd->is_uref)
332 		return (0);		/* success */
333 
334 	/*
335 	 * To avoid deadlock at detach we need to drop the FIFO ref
336 	 * and re-acquire a new ref!
337 	 */
338 	usb_unref_device(cpd, crd);
339 
340 	return (usb_ref_device(cpd, crd, 1 /* need uref */));
341 }
342 
343 /*------------------------------------------------------------------------*
344  *	usb_unref_device
345  *
346  * This function will release the reference count by one unit for the
347  * given USB device.
348  *------------------------------------------------------------------------*/
349 static void
350 usb_unref_device(struct usb_cdev_privdata *cpd,
351     struct usb_cdev_refdata *crd)
352 {
353 
354 	DPRINTFN(2, "cpd=%p is_uref=%d\n", cpd, crd->is_uref);
355 
356 	if (crd->do_unlock)
357 		usbd_enum_unlock(cpd->udev);
358 
359 	mtx_lock(&usb_ref_lock);
360 	if (crd->is_read) {
361 		if (--(crd->rxfifo->refcount) == 0) {
362 			cv_signal(&crd->rxfifo->cv_drain);
363 		}
364 		crd->is_read = 0;
365 	}
366 	if (crd->is_write) {
367 		if (--(crd->txfifo->refcount) == 0) {
368 			cv_signal(&crd->txfifo->cv_drain);
369 		}
370 		crd->is_write = 0;
371 	}
372 	if (crd->is_uref) {
373 		crd->is_uref = 0;
374 		if (--(cpd->udev->refcount) == 0)
375 			cv_broadcast(&cpd->udev->ref_cv);
376 	}
377 	mtx_unlock(&usb_ref_lock);
378 }
379 
380 static struct usb_fifo *
381 usb_fifo_alloc(struct mtx *mtx)
382 {
383 	struct usb_fifo *f;
384 
385 	f = malloc(sizeof(*f), M_USBDEV, M_WAITOK | M_ZERO);
386 	if (f != NULL) {
387 		cv_init(&f->cv_io, "FIFO-IO");
388 		cv_init(&f->cv_drain, "FIFO-DRAIN");
389 		f->priv_mtx = mtx;
390 		f->refcount = 1;
391 		knlist_init_mtx(&f->selinfo.si_note, mtx);
392 	}
393 	return (f);
394 }
395 
396 /*------------------------------------------------------------------------*
397  *	usb_fifo_create
398  *------------------------------------------------------------------------*/
399 static int
400 usb_fifo_create(struct usb_cdev_privdata *cpd,
401     struct usb_cdev_refdata *crd)
402 {
403 	struct usb_device *udev = cpd->udev;
404 	struct usb_fifo *f;
405 	struct usb_endpoint *ep;
406 	uint8_t n;
407 	uint8_t is_tx;
408 	uint8_t is_rx;
409 	uint8_t no_null;
410 	uint8_t is_busy;
411 	int e = cpd->ep_addr;
412 
413 	is_tx = (cpd->fflags & FWRITE) ? 1 : 0;
414 	is_rx = (cpd->fflags & FREAD) ? 1 : 0;
415 	no_null = 1;
416 	is_busy = 0;
417 
418 	/* Preallocated FIFO */
419 	if (e < 0) {
420 		DPRINTFN(5, "Preallocated FIFO\n");
421 		if (is_tx) {
422 			f = udev->fifo[cpd->fifo_index + USB_FIFO_TX];
423 			if (f == NULL)
424 				return (EINVAL);
425 			crd->txfifo = f;
426 		}
427 		if (is_rx) {
428 			f = udev->fifo[cpd->fifo_index + USB_FIFO_RX];
429 			if (f == NULL)
430 				return (EINVAL);
431 			crd->rxfifo = f;
432 		}
433 		return (0);
434 	}
435 
436 	KASSERT(e >= 0 && e <= 15, ("endpoint %d out of range", e));
437 
438 	/* search for a free FIFO slot */
439 	DPRINTFN(5, "Endpoint device, searching for 0x%02x\n", e);
440 	for (n = 0;; n += 2) {
441 
442 		if (n == USB_FIFO_MAX) {
443 			if (no_null) {
444 				no_null = 0;
445 				n = 0;
446 			} else {
447 				/* end of FIFOs reached */
448 				DPRINTFN(5, "out of FIFOs\n");
449 				return (ENOMEM);
450 			}
451 		}
452 		/* Check for TX FIFO */
453 		if (is_tx) {
454 			f = udev->fifo[n + USB_FIFO_TX];
455 			if (f != NULL) {
456 				if (f->dev_ep_index != e) {
457 					/* wrong endpoint index */
458 					continue;
459 				}
460 				if (f->curr_cpd != NULL) {
461 					/* FIFO is opened */
462 					is_busy = 1;
463 					continue;
464 				}
465 			} else if (no_null) {
466 				continue;
467 			}
468 		}
469 		/* Check for RX FIFO */
470 		if (is_rx) {
471 			f = udev->fifo[n + USB_FIFO_RX];
472 			if (f != NULL) {
473 				if (f->dev_ep_index != e) {
474 					/* wrong endpoint index */
475 					continue;
476 				}
477 				if (f->curr_cpd != NULL) {
478 					/* FIFO is opened */
479 					is_busy = 1;
480 					continue;
481 				}
482 			} else if (no_null) {
483 				continue;
484 			}
485 		}
486 		break;
487 	}
488 
489 	if (no_null == 0) {
490 		if (e >= (USB_EP_MAX / 2)) {
491 			/* we don't create any endpoints in this range */
492 			DPRINTFN(5, "ep out of range\n");
493 			return (is_busy ? EBUSY : EINVAL);
494 		}
495 	}
496 
497 	if ((e != 0) && is_busy) {
498 		/*
499 		 * Only the default control endpoint is allowed to be
500 		 * opened multiple times!
501 		 */
502 		DPRINTFN(5, "busy\n");
503 		return (EBUSY);
504 	}
505 
506 	/* Check TX FIFO */
507 	if (is_tx &&
508 	    (udev->fifo[n + USB_FIFO_TX] == NULL)) {
509 		ep = usb_dev_get_ep(udev, e, USB_FIFO_TX);
510 		DPRINTFN(5, "dev_get_endpoint(%d, 0x%x)\n", e, USB_FIFO_TX);
511 		if (ep == NULL) {
512 			DPRINTFN(5, "dev_get_endpoint returned NULL\n");
513 			return (EINVAL);
514 		}
515 		f = usb_fifo_alloc(&udev->device_mtx);
516 		if (f == NULL) {
517 			DPRINTFN(5, "could not alloc tx fifo\n");
518 			return (ENOMEM);
519 		}
520 		/* update some fields */
521 		f->fifo_index = n + USB_FIFO_TX;
522 		f->dev_ep_index = e;
523 		f->priv_sc0 = ep;
524 		f->methods = &usb_ugen_methods;
525 		f->iface_index = ep->iface_index;
526 		f->udev = udev;
527 		mtx_lock(&usb_ref_lock);
528 		udev->fifo[n + USB_FIFO_TX] = f;
529 		mtx_unlock(&usb_ref_lock);
530 	}
531 	/* Check RX FIFO */
532 	if (is_rx &&
533 	    (udev->fifo[n + USB_FIFO_RX] == NULL)) {
534 
535 		ep = usb_dev_get_ep(udev, e, USB_FIFO_RX);
536 		DPRINTFN(5, "dev_get_endpoint(%d, 0x%x)\n", e, USB_FIFO_RX);
537 		if (ep == NULL) {
538 			DPRINTFN(5, "dev_get_endpoint returned NULL\n");
539 			return (EINVAL);
540 		}
541 		f = usb_fifo_alloc(&udev->device_mtx);
542 		if (f == NULL) {
543 			DPRINTFN(5, "could not alloc rx fifo\n");
544 			return (ENOMEM);
545 		}
546 		/* update some fields */
547 		f->fifo_index = n + USB_FIFO_RX;
548 		f->dev_ep_index = e;
549 		f->priv_sc0 = ep;
550 		f->methods = &usb_ugen_methods;
551 		f->iface_index = ep->iface_index;
552 		f->udev = udev;
553 		mtx_lock(&usb_ref_lock);
554 		udev->fifo[n + USB_FIFO_RX] = f;
555 		mtx_unlock(&usb_ref_lock);
556 	}
557 	if (is_tx) {
558 		crd->txfifo = udev->fifo[n + USB_FIFO_TX];
559 	}
560 	if (is_rx) {
561 		crd->rxfifo = udev->fifo[n + USB_FIFO_RX];
562 	}
563 	/* fill out fifo index */
564 	DPRINTFN(5, "fifo index = %d\n", n);
565 	cpd->fifo_index = n;
566 
567 	/* complete */
568 
569 	return (0);
570 }
571 
572 void
573 usb_fifo_free(struct usb_fifo *f)
574 {
575 	uint8_t n;
576 
577 	if (f == NULL) {
578 		/* be NULL safe */
579 		return;
580 	}
581 	/* destroy symlink devices, if any */
582 	for (n = 0; n != 2; n++) {
583 		if (f->symlink[n]) {
584 			usb_free_symlink(f->symlink[n]);
585 			f->symlink[n] = NULL;
586 		}
587 	}
588 	mtx_lock(&usb_ref_lock);
589 
590 	/* delink ourselves to stop calls from userland */
591 	if ((f->fifo_index < USB_FIFO_MAX) &&
592 	    (f->udev != NULL) &&
593 	    (f->udev->fifo[f->fifo_index] == f)) {
594 		f->udev->fifo[f->fifo_index] = NULL;
595 	} else {
596 		DPRINTFN(0, "USB FIFO %p has not been linked\n", f);
597 	}
598 
599 	/* decrease refcount */
600 	f->refcount--;
601 	/* need to wait until all callers have exited */
602 	while (f->refcount != 0) {
603 		mtx_unlock(&usb_ref_lock);	/* avoid LOR */
604 		mtx_lock(f->priv_mtx);
605 		/* prevent write flush, if any */
606 		f->flag_iserror = 1;
607 		/* get I/O thread out of any sleep state */
608 		if (f->flag_sleeping) {
609 			f->flag_sleeping = 0;
610 			cv_broadcast(&f->cv_io);
611 		}
612 		mtx_unlock(f->priv_mtx);
613 		mtx_lock(&usb_ref_lock);
614 
615 		/*
616 		 * Check if the "f->refcount" variable reached zero
617 		 * during the unlocked time before entering wait:
618 		 */
619 		if (f->refcount == 0)
620 			break;
621 
622 		/* wait for sync */
623 		cv_wait(&f->cv_drain, &usb_ref_lock);
624 	}
625 	mtx_unlock(&usb_ref_lock);
626 
627 	/* take care of closing the device here, if any */
628 	usb_fifo_close(f, 0);
629 
630 	cv_destroy(&f->cv_io);
631 	cv_destroy(&f->cv_drain);
632 
633 	knlist_clear(&f->selinfo.si_note, 0);
634 	seldrain(&f->selinfo);
635 	knlist_destroy(&f->selinfo.si_note);
636 
637 	free(f, M_USBDEV);
638 }
639 
640 static struct usb_endpoint *
641 usb_dev_get_ep(struct usb_device *udev, uint8_t ep_index, uint8_t dir)
642 {
643 	struct usb_endpoint *ep;
644 	uint8_t ep_dir;
645 
646 	if (ep_index == 0) {
647 		ep = &udev->ctrl_ep;
648 	} else {
649 		if (dir == USB_FIFO_RX) {
650 			if (udev->flags.usb_mode == USB_MODE_HOST) {
651 				ep_dir = UE_DIR_IN;
652 			} else {
653 				ep_dir = UE_DIR_OUT;
654 			}
655 		} else {
656 			if (udev->flags.usb_mode == USB_MODE_HOST) {
657 				ep_dir = UE_DIR_OUT;
658 			} else {
659 				ep_dir = UE_DIR_IN;
660 			}
661 		}
662 		ep = usbd_get_ep_by_addr(udev, ep_index | ep_dir);
663 	}
664 
665 	if (ep == NULL) {
666 		/* if the endpoint does not exist then return */
667 		return (NULL);
668 	}
669 	if (ep->edesc == NULL) {
670 		/* invalid endpoint */
671 		return (NULL);
672 	}
673 	return (ep);			/* success */
674 }
675 
676 /*------------------------------------------------------------------------*
677  *	usb_fifo_open
678  *
679  * Returns:
680  * 0: Success
681  * Else: Failure
682  *------------------------------------------------------------------------*/
683 static int
684 usb_fifo_open(struct usb_cdev_privdata *cpd,
685     struct usb_fifo *f, int fflags)
686 {
687 	int err;
688 
689 	if (f == NULL) {
690 		/* no FIFO there */
691 		DPRINTFN(2, "no FIFO\n");
692 		return (ENXIO);
693 	}
694 	/* remove FWRITE and FREAD flags */
695 	fflags &= ~(FWRITE | FREAD);
696 
697 	/* set correct file flags */
698 	if ((f->fifo_index & 1) == USB_FIFO_TX) {
699 		fflags |= FWRITE;
700 	} else {
701 		fflags |= FREAD;
702 	}
703 
704 	/* check if we are already opened */
705 	/* we don't need any locks when checking this variable */
706 	if (f->curr_cpd != NULL) {
707 		err = EBUSY;
708 		goto done;
709 	}
710 
711 	/* reset short flag before open */
712 	f->flag_short = 0;
713 
714 	/* call open method */
715 	err = (f->methods->f_open) (f, fflags);
716 	if (err) {
717 		goto done;
718 	}
719 	mtx_lock(f->priv_mtx);
720 
721 	/* reset sleep flag */
722 	f->flag_sleeping = 0;
723 
724 	/* reset error flag */
725 	f->flag_iserror = 0;
726 
727 	/* reset complete flag */
728 	f->flag_iscomplete = 0;
729 
730 	/* reset select flag */
731 	f->flag_isselect = 0;
732 
733 	/* reset flushing flag */
734 	f->flag_flushing = 0;
735 
736 	/* reset ASYNC proc flag */
737 	f->async_p = NULL;
738 
739 	mtx_lock(&usb_ref_lock);
740 	/* flag the fifo as opened to prevent others */
741 	f->curr_cpd = cpd;
742 	mtx_unlock(&usb_ref_lock);
743 
744 	/* reset queue */
745 	usb_fifo_reset(f);
746 
747 	mtx_unlock(f->priv_mtx);
748 done:
749 	return (err);
750 }
751 
752 /*------------------------------------------------------------------------*
753  *	usb_fifo_reset
754  *------------------------------------------------------------------------*/
755 void
756 usb_fifo_reset(struct usb_fifo *f)
757 {
758 	struct usb_mbuf *m;
759 
760 	if (f == NULL) {
761 		return;
762 	}
763 	while (1) {
764 		USB_IF_DEQUEUE(&f->used_q, m);
765 		if (m) {
766 			USB_IF_ENQUEUE(&f->free_q, m);
767 		} else {
768 			break;
769 		}
770 	}
771 	/* reset have fragment flag */
772 	f->flag_have_fragment = 0;
773 }
774 
775 /*------------------------------------------------------------------------*
776  *	usb_fifo_close
777  *------------------------------------------------------------------------*/
778 static void
779 usb_fifo_close(struct usb_fifo *f, int fflags)
780 {
781 	int err;
782 
783 	/* check if we are not opened */
784 	if (f->curr_cpd == NULL) {
785 		/* nothing to do - already closed */
786 		return;
787 	}
788 	mtx_lock(f->priv_mtx);
789 
790 	/* clear current cdev private data pointer */
791 	mtx_lock(&usb_ref_lock);
792 	f->curr_cpd = NULL;
793 	mtx_unlock(&usb_ref_lock);
794 
795 	/* check if we are watched by kevent */
796 	KNOTE_LOCKED(&f->selinfo.si_note, 0);
797 
798 	/* check if we are selected */
799 	if (f->flag_isselect) {
800 		selwakeup(&f->selinfo);
801 		f->flag_isselect = 0;
802 	}
803 	/* check if a thread wants SIGIO */
804 	if (f->async_p != NULL) {
805 		PROC_LOCK(f->async_p);
806 		kern_psignal(f->async_p, SIGIO);
807 		PROC_UNLOCK(f->async_p);
808 		f->async_p = NULL;
809 	}
810 	/* remove FWRITE and FREAD flags */
811 	fflags &= ~(FWRITE | FREAD);
812 
813 	/* flush written data, if any */
814 	if ((f->fifo_index & 1) == USB_FIFO_TX) {
815 
816 		if (!f->flag_iserror) {
817 
818 			/* set flushing flag */
819 			f->flag_flushing = 1;
820 
821 			/* get the last packet in */
822 			if (f->flag_have_fragment) {
823 				struct usb_mbuf *m;
824 				f->flag_have_fragment = 0;
825 				USB_IF_DEQUEUE(&f->free_q, m);
826 				if (m) {
827 					USB_IF_ENQUEUE(&f->used_q, m);
828 				}
829 			}
830 
831 			/* start write transfer, if not already started */
832 			(f->methods->f_start_write) (f);
833 
834 			/* check if flushed already */
835 			while (f->flag_flushing &&
836 			    (!f->flag_iserror)) {
837 				/* wait until all data has been written */
838 				f->flag_sleeping = 1;
839 				err = cv_timedwait_sig(&f->cv_io, f->priv_mtx,
840 				    USB_MS_TO_TICKS(USB_DEFAULT_TIMEOUT));
841 				if (err) {
842 					DPRINTF("signal received\n");
843 					break;
844 				}
845 			}
846 		}
847 		fflags |= FWRITE;
848 
849 		/* stop write transfer, if not already stopped */
850 		(f->methods->f_stop_write) (f);
851 	} else {
852 		fflags |= FREAD;
853 
854 		/* stop write transfer, if not already stopped */
855 		(f->methods->f_stop_read) (f);
856 	}
857 
858 	/* check if we are sleeping */
859 	if (f->flag_sleeping) {
860 		DPRINTFN(2, "Sleeping at close!\n");
861 	}
862 	mtx_unlock(f->priv_mtx);
863 
864 	/* call close method */
865 	(f->methods->f_close) (f, fflags);
866 
867 	DPRINTF("closed\n");
868 }
869 
870 /*------------------------------------------------------------------------*
871  *	usb_open - cdev callback
872  *------------------------------------------------------------------------*/
873 static int
874 usb_open(struct cdev *dev, int fflags, int devtype, struct thread *td)
875 {
876 	struct usb_fs_privdata* pd = (struct usb_fs_privdata*)dev->si_drv1;
877 	struct usb_cdev_refdata refs;
878 	struct usb_cdev_privdata *cpd;
879 	int err, ep;
880 
881 	DPRINTFN(2, "%s fflags=0x%08x\n", devtoname(dev), fflags);
882 
883 	KASSERT(fflags & (FREAD|FWRITE), ("invalid open flags"));
884 	if (((fflags & FREAD) && !(pd->mode & FREAD)) ||
885 	    ((fflags & FWRITE) && !(pd->mode & FWRITE))) {
886 		DPRINTFN(2, "access mode not supported\n");
887 		return (EPERM);
888 	}
889 
890 	cpd = malloc(sizeof(*cpd), M_USBDEV, M_WAITOK | M_ZERO);
891 	ep = cpd->ep_addr = pd->ep_addr;
892 
893 	usb_loc_fill(pd, cpd);
894 	err = usb_ref_device(cpd, &refs, 1);
895 	if (err) {
896 		DPRINTFN(2, "cannot ref device\n");
897 		free(cpd, M_USBDEV);
898 		return (ENXIO);
899 	}
900 	cpd->fflags = fflags;	/* access mode for open lifetime */
901 
902 	/* create FIFOs, if any */
903 	err = usb_fifo_create(cpd, &refs);
904 	/* check for error */
905 	if (err) {
906 		DPRINTFN(2, "cannot create fifo\n");
907 		usb_unref_device(cpd, &refs);
908 		free(cpd, M_USBDEV);
909 		return (err);
910 	}
911 	if (fflags & FREAD) {
912 		err = usb_fifo_open(cpd, refs.rxfifo, fflags);
913 		if (err) {
914 			DPRINTFN(2, "read open failed\n");
915 			usb_unref_device(cpd, &refs);
916 			free(cpd, M_USBDEV);
917 			return (err);
918 		}
919 	}
920 	if (fflags & FWRITE) {
921 		err = usb_fifo_open(cpd, refs.txfifo, fflags);
922 		if (err) {
923 			DPRINTFN(2, "write open failed\n");
924 			if (fflags & FREAD) {
925 				usb_fifo_close(refs.rxfifo, fflags);
926 			}
927 			usb_unref_device(cpd, &refs);
928 			free(cpd, M_USBDEV);
929 			return (err);
930 		}
931 	}
932 	usb_unref_device(cpd, &refs);
933 	devfs_set_cdevpriv(cpd, usb_close);
934 
935 	return (0);
936 }
937 
938 /*------------------------------------------------------------------------*
939  *	usb_close - cdev callback
940  *------------------------------------------------------------------------*/
941 static void
942 usb_close(void *arg)
943 {
944 	struct usb_cdev_refdata refs;
945 	struct usb_cdev_privdata *cpd = arg;
946 	int err;
947 
948 	DPRINTFN(2, "cpd=%p\n", cpd);
949 
950 	err = usb_ref_device(cpd, &refs,
951 	    2 /* uref and allow detached state */);
952 	if (err) {
953 		DPRINTFN(2, "Cannot grab USB reference when "
954 		    "closing USB file handle\n");
955 		goto done;
956 	}
957 	if (cpd->fflags & FREAD) {
958 		usb_fifo_close(refs.rxfifo, cpd->fflags);
959 	}
960 	if (cpd->fflags & FWRITE) {
961 		usb_fifo_close(refs.txfifo, cpd->fflags);
962 	}
963 	usb_unref_device(cpd, &refs);
964 done:
965 	free(cpd, M_USBDEV);
966 }
967 
968 static void
969 usb_dev_init(void *arg)
970 {
971 	mtx_init(&usb_ref_lock, "USB ref mutex", NULL, MTX_DEF);
972 	sx_init(&usb_sym_lock, "USB sym mutex");
973 	TAILQ_INIT(&usb_sym_head);
974 
975 	/* check the UGEN methods */
976 	usb_fifo_check_methods(&usb_ugen_methods);
977 }
978 
979 SYSINIT(usb_dev_init, SI_SUB_KLD, SI_ORDER_FIRST, usb_dev_init, NULL);
980 
981 static void
982 usb_dev_init_post(void *arg)
983 {
984 	/*
985 	 * Create /dev/usb - this is needed for usbconfig(8), which
986 	 * needs a well-known device name to access.
987 	 */
988 	usb_dev = make_dev(&usb_static_devsw, 0, UID_ROOT, GID_OPERATOR,
989 	    0644, USB_DEVICE_NAME);
990 	if (usb_dev == NULL) {
991 		DPRINTFN(0, "Could not create usb bus device\n");
992 	}
993 }
994 
995 SYSINIT(usb_dev_init_post, SI_SUB_KICK_SCHEDULER, SI_ORDER_FIRST, usb_dev_init_post, NULL);
996 
997 static void
998 usb_dev_uninit(void *arg)
999 {
1000 	if (usb_dev != NULL) {
1001 		destroy_dev(usb_dev);
1002 		usb_dev = NULL;
1003 	}
1004 	mtx_destroy(&usb_ref_lock);
1005 	sx_destroy(&usb_sym_lock);
1006 }
1007 
1008 SYSUNINIT(usb_dev_uninit, SI_SUB_KICK_SCHEDULER, SI_ORDER_ANY, usb_dev_uninit, NULL);
1009 
1010 static int
1011 usb_ioctl_f_sub(struct usb_fifo *f, u_long cmd, void *addr,
1012     struct thread *td)
1013 {
1014 	int error = 0;
1015 
1016 	switch (cmd) {
1017 	case FIODTYPE:
1018 		*(int *)addr = 0;	/* character device */
1019 		break;
1020 
1021 	case FIONBIO:
1022 		/* handled by upper FS layer */
1023 		break;
1024 
1025 	case FIOASYNC:
1026 		if (*(int *)addr) {
1027 			if (f->async_p != NULL) {
1028 				error = EBUSY;
1029 				break;
1030 			}
1031 			f->async_p = USB_TD_GET_PROC(td);
1032 		} else {
1033 			f->async_p = NULL;
1034 		}
1035 		break;
1036 
1037 		/* XXX this is not the most general solution */
1038 	case TIOCSPGRP:
1039 		if (f->async_p == NULL) {
1040 			error = EINVAL;
1041 			break;
1042 		}
1043 		if (*(int *)addr != USB_PROC_GET_GID(f->async_p)) {
1044 			error = EPERM;
1045 			break;
1046 		}
1047 		break;
1048 	default:
1049 		return (ENOIOCTL);
1050 	}
1051 	DPRINTFN(3, "cmd 0x%lx = %d\n", cmd, error);
1052 	return (error);
1053 }
1054 
1055 /*------------------------------------------------------------------------*
1056  *	usb_ioctl - cdev callback
1057  *------------------------------------------------------------------------*/
1058 static int
1059 usb_ioctl(struct cdev *dev, u_long cmd, caddr_t addr, int fflag, struct thread* td)
1060 {
1061 	struct usb_cdev_refdata refs;
1062 	struct usb_cdev_privdata* cpd;
1063 	struct usb_fifo *f;
1064 	int fflags;
1065 	int err;
1066 
1067 	DPRINTFN(2, "cmd=0x%lx\n", cmd);
1068 
1069 	err = devfs_get_cdevpriv((void **)&cpd);
1070 	if (err != 0)
1071 		return (err);
1072 
1073 	/*
1074 	 * Performance optimisation: We try to check for IOCTL's that
1075 	 * don't need the USB reference first. Then we grab the USB
1076 	 * reference if we need it!
1077 	 */
1078 	err = usb_ref_device(cpd, &refs, 0 /* no uref */ );
1079 	if (err)
1080 		return (ENXIO);
1081 
1082 	fflags = cpd->fflags;
1083 
1084 	f = NULL;			/* set default value */
1085 	err = ENOIOCTL;			/* set default value */
1086 
1087 	if (fflags & FWRITE) {
1088 		f = refs.txfifo;
1089 		err = usb_ioctl_f_sub(f, cmd, addr, td);
1090 	}
1091 	if (fflags & FREAD) {
1092 		f = refs.rxfifo;
1093 		err = usb_ioctl_f_sub(f, cmd, addr, td);
1094 	}
1095 	KASSERT(f != NULL, ("fifo not found"));
1096 	if (err != ENOIOCTL)
1097 		goto done;
1098 
1099 	err = (f->methods->f_ioctl) (f, cmd, addr, fflags);
1100 
1101 	DPRINTFN(2, "f_ioctl cmd 0x%lx = %d\n", cmd, err);
1102 
1103 	if (err != ENOIOCTL)
1104 		goto done;
1105 
1106 	if (usb_usb_ref_device(cpd, &refs)) {
1107 		/* we lost the reference */
1108 		return (ENXIO);
1109 	}
1110 
1111 	err = (f->methods->f_ioctl_post) (f, cmd, addr, fflags);
1112 
1113 	DPRINTFN(2, "f_ioctl_post cmd 0x%lx = %d\n", cmd, err);
1114 
1115 	if (err == ENOIOCTL)
1116 		err = ENOTTY;
1117 
1118 	if (err)
1119 		goto done;
1120 
1121 	/* Wait for re-enumeration, if any */
1122 
1123 	while (f->udev->re_enumerate_wait != USB_RE_ENUM_DONE) {
1124 
1125 		usb_unref_device(cpd, &refs);
1126 
1127 		usb_pause_mtx(NULL, hz / 128);
1128 
1129 		while (usb_ref_device(cpd, &refs, 1 /* need uref */)) {
1130 			if (usb_ref_device(cpd, &refs, 0)) {
1131 				/* device no longer exists */
1132 				return (ENXIO);
1133 			}
1134 			usb_unref_device(cpd, &refs);
1135 			usb_pause_mtx(NULL, hz / 128);
1136 		}
1137 	}
1138 
1139 done:
1140 	usb_unref_device(cpd, &refs);
1141 	return (err);
1142 }
1143 
1144 static void
1145 usb_filter_detach(struct knote *kn)
1146 {
1147 	struct usb_fifo *f = kn->kn_hook;
1148 	knlist_remove(&f->selinfo.si_note, kn, 0);
1149 }
1150 
1151 static int
1152 usb_filter_write(struct knote *kn, long hint)
1153 {
1154 	struct usb_cdev_privdata* cpd;
1155 	struct usb_fifo *f;
1156 	struct usb_mbuf *m;
1157 
1158 	DPRINTFN(2, "\n");
1159 
1160 	f = kn->kn_hook;
1161 
1162 	USB_MTX_ASSERT(f->priv_mtx, MA_OWNED);
1163 
1164 	cpd = f->curr_cpd;
1165 	if (cpd == NULL) {
1166 		m = (void *)1;
1167 	} else if (f->fs_ep_max == 0) {
1168 		if (f->flag_iserror) {
1169 			/* we got an error */
1170 			m = (void *)1;
1171 		} else {
1172 			if (f->queue_data == NULL) {
1173 				/*
1174 				 * start write transfer, if not
1175 				 * already started
1176 				 */
1177 				(f->methods->f_start_write) (f);
1178 			}
1179 			/* check if any packets are available */
1180 			USB_IF_POLL(&f->free_q, m);
1181 		}
1182 	} else {
1183 		if (f->flag_iscomplete) {
1184 			m = (void *)1;
1185 		} else {
1186 			m = NULL;
1187 		}
1188 	}
1189 	return (m ? 1 : 0);
1190 }
1191 
1192 static int
1193 usb_filter_read(struct knote *kn, long hint)
1194 {
1195 	struct usb_cdev_privdata* cpd;
1196 	struct usb_fifo *f;
1197 	struct usb_mbuf *m;
1198 
1199 	DPRINTFN(2, "\n");
1200 
1201 	f = kn->kn_hook;
1202 
1203 	USB_MTX_ASSERT(f->priv_mtx, MA_OWNED);
1204 
1205 	cpd = f->curr_cpd;
1206 	if (cpd == NULL) {
1207 		m = (void *)1;
1208 	} else if (f->fs_ep_max == 0) {
1209 		if (f->flag_iserror) {
1210 			/* we have an error */
1211 			m = (void *)1;
1212 		} else {
1213 			if (f->queue_data == NULL) {
1214 				/*
1215 				 * start read transfer, if not
1216 				 * already started
1217 				 */
1218 				(f->methods->f_start_read) (f);
1219 			}
1220 			/* check if any packets are available */
1221 			USB_IF_POLL(&f->used_q, m);
1222 
1223 			/* start reading data, if any */
1224 			if (m == NULL)
1225 				(f->methods->f_start_read) (f);
1226 		}
1227 	} else {
1228 		if (f->flag_iscomplete) {
1229 			m = (void *)1;
1230 		} else {
1231 			m = NULL;
1232 		}
1233 	}
1234 	return (m ? 1 : 0);
1235 }
1236 
1237 static struct filterops usb_filtops_write = {
1238 	.f_isfd = 1,
1239 	.f_detach = usb_filter_detach,
1240 	.f_event = usb_filter_write,
1241 };
1242 
1243 static struct filterops usb_filtops_read = {
1244 	.f_isfd = 1,
1245 	.f_detach = usb_filter_detach,
1246 	.f_event = usb_filter_read,
1247 };
1248 
1249 
1250 /* ARGSUSED */
1251 static int
1252 usb_kqfilter(struct cdev* dev, struct knote *kn)
1253 {
1254 	struct usb_cdev_refdata refs;
1255 	struct usb_cdev_privdata* cpd;
1256 	struct usb_fifo *f;
1257 	int fflags;
1258 	int err = EINVAL;
1259 
1260 	DPRINTFN(2, "\n");
1261 
1262 	if (devfs_get_cdevpriv((void **)&cpd) != 0 ||
1263 	    usb_ref_device(cpd, &refs, 0) != 0)
1264 		return (ENXIO);
1265 
1266 	fflags = cpd->fflags;
1267 
1268 	/* Figure out who needs service */
1269 	switch (kn->kn_filter) {
1270 	case EVFILT_WRITE:
1271 		if (fflags & FWRITE) {
1272 			f = refs.txfifo;
1273 			kn->kn_fop = &usb_filtops_write;
1274 			err = 0;
1275 		}
1276 		break;
1277 	case EVFILT_READ:
1278 		if (fflags & FREAD) {
1279 			f = refs.rxfifo;
1280 			kn->kn_fop = &usb_filtops_read;
1281 			err = 0;
1282 		}
1283 		break;
1284 	default:
1285 		err = EOPNOTSUPP;
1286 		break;
1287 	}
1288 
1289 	if (err == 0) {
1290 		kn->kn_hook = f;
1291 		mtx_lock(f->priv_mtx);
1292 		knlist_add(&f->selinfo.si_note, kn, 1);
1293 		mtx_unlock(f->priv_mtx);
1294 	}
1295 
1296 	usb_unref_device(cpd, &refs);
1297 	return (err);
1298 }
1299 
1300 /* ARGSUSED */
1301 static int
1302 usb_poll(struct cdev* dev, int events, struct thread* td)
1303 {
1304 	struct usb_cdev_refdata refs;
1305 	struct usb_cdev_privdata* cpd;
1306 	struct usb_fifo *f;
1307 	struct usb_mbuf *m;
1308 	int fflags, revents;
1309 
1310 	if (devfs_get_cdevpriv((void **)&cpd) != 0 ||
1311 	    usb_ref_device(cpd, &refs, 0) != 0)
1312 		return (events &
1313 		    (POLLHUP|POLLIN|POLLRDNORM|POLLOUT|POLLWRNORM));
1314 
1315 	fflags = cpd->fflags;
1316 
1317 	/* Figure out who needs service */
1318 	revents = 0;
1319 	if ((events & (POLLOUT | POLLWRNORM)) &&
1320 	    (fflags & FWRITE)) {
1321 
1322 		f = refs.txfifo;
1323 
1324 		mtx_lock(f->priv_mtx);
1325 
1326 		if (!refs.is_usbfs) {
1327 			if (f->flag_iserror) {
1328 				/* we got an error */
1329 				m = (void *)1;
1330 			} else {
1331 				if (f->queue_data == NULL) {
1332 					/*
1333 					 * start write transfer, if not
1334 					 * already started
1335 					 */
1336 					(f->methods->f_start_write) (f);
1337 				}
1338 				/* check if any packets are available */
1339 				USB_IF_POLL(&f->free_q, m);
1340 			}
1341 		} else {
1342 			if (f->flag_iscomplete) {
1343 				m = (void *)1;
1344 			} else {
1345 				m = NULL;
1346 			}
1347 		}
1348 
1349 		if (m) {
1350 			revents |= events & (POLLOUT | POLLWRNORM);
1351 		} else {
1352 			f->flag_isselect = 1;
1353 			selrecord(td, &f->selinfo);
1354 		}
1355 
1356 		mtx_unlock(f->priv_mtx);
1357 	}
1358 	if ((events & (POLLIN | POLLRDNORM)) &&
1359 	    (fflags & FREAD)) {
1360 
1361 		f = refs.rxfifo;
1362 
1363 		mtx_lock(f->priv_mtx);
1364 
1365 		if (!refs.is_usbfs) {
1366 			if (f->flag_iserror) {
1367 				/* we have an error */
1368 				m = (void *)1;
1369 			} else {
1370 				if (f->queue_data == NULL) {
1371 					/*
1372 					 * start read transfer, if not
1373 					 * already started
1374 					 */
1375 					(f->methods->f_start_read) (f);
1376 				}
1377 				/* check if any packets are available */
1378 				USB_IF_POLL(&f->used_q, m);
1379 			}
1380 		} else {
1381 			if (f->flag_iscomplete) {
1382 				m = (void *)1;
1383 			} else {
1384 				m = NULL;
1385 			}
1386 		}
1387 
1388 		if (m) {
1389 			revents |= events & (POLLIN | POLLRDNORM);
1390 		} else {
1391 			f->flag_isselect = 1;
1392 			selrecord(td, &f->selinfo);
1393 
1394 			if (!refs.is_usbfs) {
1395 				/* start reading data */
1396 				(f->methods->f_start_read) (f);
1397 			}
1398 		}
1399 
1400 		mtx_unlock(f->priv_mtx);
1401 	}
1402 	usb_unref_device(cpd, &refs);
1403 	return (revents);
1404 }
1405 
1406 static int
1407 usb_read(struct cdev *dev, struct uio *uio, int ioflag)
1408 {
1409 	struct usb_cdev_refdata refs;
1410 	struct usb_cdev_privdata* cpd;
1411 	struct usb_fifo *f;
1412 	struct usb_mbuf *m;
1413 	int fflags;
1414 	int resid;
1415 	int io_len;
1416 	int err;
1417 	uint8_t tr_data = 0;
1418 
1419 	err = devfs_get_cdevpriv((void **)&cpd);
1420 	if (err != 0)
1421 		return (err);
1422 
1423 	err = usb_ref_device(cpd, &refs, 0 /* no uref */ );
1424 	if (err)
1425 		return (ENXIO);
1426 
1427 	fflags = cpd->fflags;
1428 
1429 	f = refs.rxfifo;
1430 	if (f == NULL) {
1431 		/* should not happen */
1432 		usb_unref_device(cpd, &refs);
1433 		return (EPERM);
1434 	}
1435 
1436 	resid = uio->uio_resid;
1437 
1438 	mtx_lock(f->priv_mtx);
1439 
1440 	/* check for permanent read error */
1441 	if (f->flag_iserror) {
1442 		err = EIO;
1443 		goto done;
1444 	}
1445 	/* check if USB-FS interface is active */
1446 	if (refs.is_usbfs) {
1447 		/*
1448 		 * The queue is used for events that should be
1449 		 * retrieved using the "USB_FS_COMPLETE" ioctl.
1450 		 */
1451 		err = EINVAL;
1452 		goto done;
1453 	}
1454 	while (uio->uio_resid > 0) {
1455 
1456 		USB_IF_DEQUEUE(&f->used_q, m);
1457 
1458 		if (m == NULL) {
1459 
1460 			/* start read transfer, if not already started */
1461 
1462 			(f->methods->f_start_read) (f);
1463 
1464 			if (ioflag & IO_NDELAY) {
1465 				if (tr_data) {
1466 					/* return length before error */
1467 					break;
1468 				}
1469 				err = EWOULDBLOCK;
1470 				break;
1471 			}
1472 			DPRINTF("sleeping\n");
1473 
1474 			err = usb_fifo_wait(f);
1475 			if (err) {
1476 				break;
1477 			}
1478 			continue;
1479 		}
1480 		if (f->methods->f_filter_read) {
1481 			/*
1482 			 * Sometimes it is convenient to process data at the
1483 			 * expense of a userland process instead of a kernel
1484 			 * process.
1485 			 */
1486 			(f->methods->f_filter_read) (f, m);
1487 		}
1488 		tr_data = 1;
1489 
1490 		io_len = MIN(m->cur_data_len, uio->uio_resid);
1491 
1492 		DPRINTFN(2, "transfer %d bytes from %p\n",
1493 		    io_len, m->cur_data_ptr);
1494 
1495 		err = usb_fifo_uiomove(f,
1496 		    m->cur_data_ptr, io_len, uio);
1497 
1498 		m->cur_data_len -= io_len;
1499 		m->cur_data_ptr += io_len;
1500 
1501 		if (m->cur_data_len == 0) {
1502 
1503 			uint8_t last_packet;
1504 
1505 			last_packet = m->last_packet;
1506 
1507 			USB_IF_ENQUEUE(&f->free_q, m);
1508 
1509 			if (last_packet) {
1510 				/* keep framing */
1511 				break;
1512 			}
1513 		} else {
1514 			USB_IF_PREPEND(&f->used_q, m);
1515 		}
1516 
1517 		if (err) {
1518 			break;
1519 		}
1520 	}
1521 done:
1522 	mtx_unlock(f->priv_mtx);
1523 
1524 	usb_unref_device(cpd, &refs);
1525 
1526 	return (err);
1527 }
1528 
1529 static int
1530 usb_write(struct cdev *dev, struct uio *uio, int ioflag)
1531 {
1532 	struct usb_cdev_refdata refs;
1533 	struct usb_cdev_privdata* cpd;
1534 	struct usb_fifo *f;
1535 	struct usb_mbuf *m;
1536 	uint8_t *pdata;
1537 	int fflags;
1538 	int resid;
1539 	int io_len;
1540 	int err;
1541 	uint8_t tr_data = 0;
1542 
1543 	DPRINTFN(2, "\n");
1544 
1545 	err = devfs_get_cdevpriv((void **)&cpd);
1546 	if (err != 0)
1547 		return (err);
1548 
1549 	err = usb_ref_device(cpd, &refs, 0 /* no uref */ );
1550 	if (err)
1551 		return (ENXIO);
1552 
1553 	fflags = cpd->fflags;
1554 
1555 	f = refs.txfifo;
1556 	if (f == NULL) {
1557 		/* should not happen */
1558 		usb_unref_device(cpd, &refs);
1559 		return (EPERM);
1560 	}
1561 	resid = uio->uio_resid;
1562 
1563 	mtx_lock(f->priv_mtx);
1564 
1565 	/* check for permanent write error */
1566 	if (f->flag_iserror) {
1567 		err = EIO;
1568 		goto done;
1569 	}
1570 	/* check if USB-FS interface is active */
1571 	if (refs.is_usbfs) {
1572 		/*
1573 		 * The queue is used for events that should be
1574 		 * retrieved using the "USB_FS_COMPLETE" ioctl.
1575 		 */
1576 		err = EINVAL;
1577 		goto done;
1578 	}
1579 	if (f->queue_data == NULL) {
1580 		/* start write transfer, if not already started */
1581 		(f->methods->f_start_write) (f);
1582 	}
1583 	/* we allow writing zero length data */
1584 	do {
1585 		USB_IF_DEQUEUE(&f->free_q, m);
1586 
1587 		if (m == NULL) {
1588 
1589 			if (ioflag & IO_NDELAY) {
1590 				if (tr_data) {
1591 					/* return length before error */
1592 					break;
1593 				}
1594 				err = EWOULDBLOCK;
1595 				break;
1596 			}
1597 			DPRINTF("sleeping\n");
1598 
1599 			err = usb_fifo_wait(f);
1600 			if (err) {
1601 				break;
1602 			}
1603 			continue;
1604 		}
1605 		tr_data = 1;
1606 
1607 		if (f->flag_have_fragment == 0) {
1608 			USB_MBUF_RESET(m);
1609 			io_len = m->cur_data_len;
1610 			pdata = m->cur_data_ptr;
1611 			if (io_len > uio->uio_resid)
1612 				io_len = uio->uio_resid;
1613 			m->cur_data_len = io_len;
1614 		} else {
1615 			io_len = m->max_data_len - m->cur_data_len;
1616 			pdata = m->cur_data_ptr + m->cur_data_len;
1617 			if (io_len > uio->uio_resid)
1618 				io_len = uio->uio_resid;
1619 			m->cur_data_len += io_len;
1620 		}
1621 
1622 		DPRINTFN(2, "transfer %d bytes to %p\n",
1623 		    io_len, pdata);
1624 
1625 		err = usb_fifo_uiomove(f, pdata, io_len, uio);
1626 
1627 		if (err) {
1628 			f->flag_have_fragment = 0;
1629 			USB_IF_ENQUEUE(&f->free_q, m);
1630 			break;
1631 		}
1632 
1633 		/* check if the buffer is ready to be transmitted */
1634 
1635 		if ((f->flag_write_defrag == 0) ||
1636 		    (m->cur_data_len == m->max_data_len)) {
1637 			f->flag_have_fragment = 0;
1638 
1639 			/*
1640 			 * Check for write filter:
1641 			 *
1642 			 * Sometimes it is convenient to process data
1643 			 * at the expense of a userland process
1644 			 * instead of a kernel process.
1645 			 */
1646 			if (f->methods->f_filter_write) {
1647 				(f->methods->f_filter_write) (f, m);
1648 			}
1649 
1650 			/* Put USB mbuf in the used queue */
1651 			USB_IF_ENQUEUE(&f->used_q, m);
1652 
1653 			/* Start writing data, if not already started */
1654 			(f->methods->f_start_write) (f);
1655 		} else {
1656 			/* Wait for more data or close */
1657 			f->flag_have_fragment = 1;
1658 			USB_IF_PREPEND(&f->free_q, m);
1659 		}
1660 
1661 	} while (uio->uio_resid > 0);
1662 done:
1663 	mtx_unlock(f->priv_mtx);
1664 
1665 	usb_unref_device(cpd, &refs);
1666 
1667 	return (err);
1668 }
1669 
1670 int
1671 usb_static_ioctl(struct cdev *dev, u_long cmd, caddr_t data, int fflag,
1672     struct thread *td)
1673 {
1674 	union {
1675 		struct usb_read_dir *urd;
1676 		void* data;
1677 	} u;
1678 	int err;
1679 
1680 	u.data = data;
1681 	switch (cmd) {
1682 		case USB_READ_DIR:
1683 			err = usb_read_symlink(u.urd->urd_data,
1684 			    u.urd->urd_startentry, u.urd->urd_maxlen);
1685 			break;
1686 		case USB_DEV_QUIRK_GET:
1687 		case USB_QUIRK_NAME_GET:
1688 		case USB_DEV_QUIRK_ADD:
1689 		case USB_DEV_QUIRK_REMOVE:
1690 			err = usb_quirk_ioctl_p(cmd, data, fflag, td);
1691 			break;
1692 		case USB_GET_TEMPLATE:
1693 			*(int *)data = usb_template;
1694 			err = 0;
1695 			break;
1696 		case USB_SET_TEMPLATE:
1697 			err = priv_check(curthread, PRIV_DRIVER);
1698 			if (err)
1699 				break;
1700 			usb_template = *(int *)data;
1701 			break;
1702 		default:
1703 			err = ENOTTY;
1704 			break;
1705 	}
1706 	return (err);
1707 }
1708 
1709 static int
1710 usb_fifo_uiomove(struct usb_fifo *f, void *cp,
1711     int n, struct uio *uio)
1712 {
1713 	int error;
1714 
1715 	mtx_unlock(f->priv_mtx);
1716 
1717 	/*
1718 	 * "uiomove()" can sleep so one needs to make a wrapper,
1719 	 * exiting the mutex and checking things:
1720 	 */
1721 	error = uiomove(cp, n, uio);
1722 
1723 	mtx_lock(f->priv_mtx);
1724 
1725 	return (error);
1726 }
1727 
1728 int
1729 usb_fifo_wait(struct usb_fifo *f)
1730 {
1731 	int err;
1732 
1733 	USB_MTX_ASSERT(f->priv_mtx, MA_OWNED);
1734 
1735 	if (f->flag_iserror) {
1736 		/* we are gone */
1737 		return (EIO);
1738 	}
1739 	f->flag_sleeping = 1;
1740 
1741 	err = cv_wait_sig(&f->cv_io, f->priv_mtx);
1742 
1743 	if (f->flag_iserror) {
1744 		/* we are gone */
1745 		err = EIO;
1746 	}
1747 	return (err);
1748 }
1749 
1750 void
1751 usb_fifo_signal(struct usb_fifo *f)
1752 {
1753 	if (f->flag_sleeping) {
1754 		f->flag_sleeping = 0;
1755 		cv_broadcast(&f->cv_io);
1756 	}
1757 }
1758 
1759 void
1760 usb_fifo_wakeup(struct usb_fifo *f)
1761 {
1762 	usb_fifo_signal(f);
1763 
1764 	KNOTE_LOCKED(&f->selinfo.si_note, 0);
1765 
1766 	if (f->flag_isselect) {
1767 		selwakeup(&f->selinfo);
1768 		f->flag_isselect = 0;
1769 	}
1770 	if (f->async_p != NULL) {
1771 		PROC_LOCK(f->async_p);
1772 		kern_psignal(f->async_p, SIGIO);
1773 		PROC_UNLOCK(f->async_p);
1774 	}
1775 }
1776 
1777 static int
1778 usb_fifo_dummy_open(struct usb_fifo *fifo, int fflags)
1779 {
1780 	return (0);
1781 }
1782 
1783 static void
1784 usb_fifo_dummy_close(struct usb_fifo *fifo, int fflags)
1785 {
1786 	return;
1787 }
1788 
1789 static int
1790 usb_fifo_dummy_ioctl(struct usb_fifo *fifo, u_long cmd, void *addr, int fflags)
1791 {
1792 	return (ENOIOCTL);
1793 }
1794 
1795 static void
1796 usb_fifo_dummy_cmd(struct usb_fifo *fifo)
1797 {
1798 	fifo->flag_flushing = 0;	/* not flushing */
1799 }
1800 
1801 static void
1802 usb_fifo_check_methods(struct usb_fifo_methods *pm)
1803 {
1804 	/* check that all callback functions are OK */
1805 
1806 	if (pm->f_open == NULL)
1807 		pm->f_open = &usb_fifo_dummy_open;
1808 
1809 	if (pm->f_close == NULL)
1810 		pm->f_close = &usb_fifo_dummy_close;
1811 
1812 	if (pm->f_ioctl == NULL)
1813 		pm->f_ioctl = &usb_fifo_dummy_ioctl;
1814 
1815 	if (pm->f_ioctl_post == NULL)
1816 		pm->f_ioctl_post = &usb_fifo_dummy_ioctl;
1817 
1818 	if (pm->f_start_read == NULL)
1819 		pm->f_start_read = &usb_fifo_dummy_cmd;
1820 
1821 	if (pm->f_stop_read == NULL)
1822 		pm->f_stop_read = &usb_fifo_dummy_cmd;
1823 
1824 	if (pm->f_start_write == NULL)
1825 		pm->f_start_write = &usb_fifo_dummy_cmd;
1826 
1827 	if (pm->f_stop_write == NULL)
1828 		pm->f_stop_write = &usb_fifo_dummy_cmd;
1829 }
1830 
1831 /*------------------------------------------------------------------------*
1832  *	usb_fifo_attach
1833  *
1834  * The following function will create a duplex FIFO.
1835  *
1836  * Return values:
1837  * 0: Success.
1838  * Else: Failure.
1839  *------------------------------------------------------------------------*/
1840 int
1841 usb_fifo_attach(struct usb_device *udev, void *priv_sc,
1842     struct mtx *priv_mtx, struct usb_fifo_methods *pm,
1843     struct usb_fifo_sc *f_sc, uint16_t unit, int16_t subunit,
1844     uint8_t iface_index, uid_t uid, gid_t gid, int mode)
1845 {
1846 	struct usb_fifo *f_tx;
1847 	struct usb_fifo *f_rx;
1848 	char devname[32];
1849 	uint8_t n;
1850 
1851 	f_sc->fp[USB_FIFO_TX] = NULL;
1852 	f_sc->fp[USB_FIFO_RX] = NULL;
1853 
1854 	if (pm == NULL)
1855 		return (EINVAL);
1856 
1857 	/* check the methods */
1858 	usb_fifo_check_methods(pm);
1859 
1860 	if (priv_mtx == NULL)
1861 		priv_mtx = &Giant;
1862 
1863 	/* search for a free FIFO slot */
1864 	for (n = 0;; n += 2) {
1865 
1866 		if (n == USB_FIFO_MAX) {
1867 			/* end of FIFOs reached */
1868 			return (ENOMEM);
1869 		}
1870 		/* Check for TX FIFO */
1871 		if (udev->fifo[n + USB_FIFO_TX] != NULL) {
1872 			continue;
1873 		}
1874 		/* Check for RX FIFO */
1875 		if (udev->fifo[n + USB_FIFO_RX] != NULL) {
1876 			continue;
1877 		}
1878 		break;
1879 	}
1880 
1881 	f_tx = usb_fifo_alloc(priv_mtx);
1882 	f_rx = usb_fifo_alloc(priv_mtx);
1883 
1884 	if ((f_tx == NULL) || (f_rx == NULL)) {
1885 		usb_fifo_free(f_tx);
1886 		usb_fifo_free(f_rx);
1887 		return (ENOMEM);
1888 	}
1889 	/* initialise FIFO structures */
1890 
1891 	f_tx->fifo_index = n + USB_FIFO_TX;
1892 	f_tx->dev_ep_index = -1;
1893 	f_tx->priv_sc0 = priv_sc;
1894 	f_tx->methods = pm;
1895 	f_tx->iface_index = iface_index;
1896 	f_tx->udev = udev;
1897 
1898 	f_rx->fifo_index = n + USB_FIFO_RX;
1899 	f_rx->dev_ep_index = -1;
1900 	f_rx->priv_sc0 = priv_sc;
1901 	f_rx->methods = pm;
1902 	f_rx->iface_index = iface_index;
1903 	f_rx->udev = udev;
1904 
1905 	f_sc->fp[USB_FIFO_TX] = f_tx;
1906 	f_sc->fp[USB_FIFO_RX] = f_rx;
1907 
1908 	mtx_lock(&usb_ref_lock);
1909 	udev->fifo[f_tx->fifo_index] = f_tx;
1910 	udev->fifo[f_rx->fifo_index] = f_rx;
1911 	mtx_unlock(&usb_ref_lock);
1912 
1913 	for (n = 0; n != 4; n++) {
1914 
1915 		if (pm->basename[n] == NULL) {
1916 			continue;
1917 		}
1918 		if (subunit < 0) {
1919 			if (snprintf(devname, sizeof(devname),
1920 			    "%s%u%s", pm->basename[n],
1921 			    unit, pm->postfix[n] ?
1922 			    pm->postfix[n] : "")) {
1923 				/* ignore */
1924 			}
1925 		} else {
1926 			if (snprintf(devname, sizeof(devname),
1927 			    "%s%u.%d%s", pm->basename[n],
1928 			    unit, subunit, pm->postfix[n] ?
1929 			    pm->postfix[n] : "")) {
1930 				/* ignore */
1931 			}
1932 		}
1933 
1934 		/*
1935 		 * Distribute the symbolic links into two FIFO structures:
1936 		 */
1937 		if (n & 1) {
1938 			f_rx->symlink[n / 2] =
1939 			    usb_alloc_symlink(devname);
1940 		} else {
1941 			f_tx->symlink[n / 2] =
1942 			    usb_alloc_symlink(devname);
1943 		}
1944 
1945 		/* Create the device */
1946 		f_sc->dev = usb_make_dev(udev, devname, -1,
1947 		    f_tx->fifo_index & f_rx->fifo_index,
1948 		    FREAD|FWRITE, uid, gid, mode);
1949 	}
1950 
1951 	DPRINTFN(2, "attached %p/%p\n", f_tx, f_rx);
1952 	return (0);
1953 }
1954 
1955 /*------------------------------------------------------------------------*
1956  *	usb_fifo_alloc_buffer
1957  *
1958  * Return values:
1959  * 0: Success
1960  * Else failure
1961  *------------------------------------------------------------------------*/
1962 int
1963 usb_fifo_alloc_buffer(struct usb_fifo *f, usb_size_t bufsize,
1964     uint16_t nbuf)
1965 {
1966 	usb_fifo_free_buffer(f);
1967 
1968 	/* allocate an endpoint */
1969 	f->free_q.ifq_maxlen = nbuf;
1970 	f->used_q.ifq_maxlen = nbuf;
1971 
1972 	f->queue_data = usb_alloc_mbufs(
1973 	    M_USBDEV, &f->free_q, bufsize, nbuf);
1974 
1975 	if ((f->queue_data == NULL) && bufsize && nbuf) {
1976 		return (ENOMEM);
1977 	}
1978 	return (0);			/* success */
1979 }
1980 
1981 /*------------------------------------------------------------------------*
1982  *	usb_fifo_free_buffer
1983  *
1984  * This function will free the buffers associated with a FIFO. This
1985  * function can be called multiple times in a row.
1986  *------------------------------------------------------------------------*/
1987 void
1988 usb_fifo_free_buffer(struct usb_fifo *f)
1989 {
1990 	if (f->queue_data) {
1991 		/* free old buffer */
1992 		free(f->queue_data, M_USBDEV);
1993 		f->queue_data = NULL;
1994 	}
1995 	/* reset queues */
1996 
1997 	memset(&f->free_q, 0, sizeof(f->free_q));
1998 	memset(&f->used_q, 0, sizeof(f->used_q));
1999 }
2000 
2001 void
2002 usb_fifo_detach(struct usb_fifo_sc *f_sc)
2003 {
2004 	if (f_sc == NULL) {
2005 		return;
2006 	}
2007 	usb_fifo_free(f_sc->fp[USB_FIFO_TX]);
2008 	usb_fifo_free(f_sc->fp[USB_FIFO_RX]);
2009 
2010 	f_sc->fp[USB_FIFO_TX] = NULL;
2011 	f_sc->fp[USB_FIFO_RX] = NULL;
2012 
2013 	usb_destroy_dev(f_sc->dev);
2014 
2015 	f_sc->dev = NULL;
2016 
2017 	DPRINTFN(2, "detached %p\n", f_sc);
2018 }
2019 
2020 usb_size_t
2021 usb_fifo_put_bytes_max(struct usb_fifo *f)
2022 {
2023 	struct usb_mbuf *m;
2024 	usb_size_t len;
2025 
2026 	USB_IF_POLL(&f->free_q, m);
2027 
2028 	if (m) {
2029 		len = m->max_data_len;
2030 	} else {
2031 		len = 0;
2032 	}
2033 	return (len);
2034 }
2035 
2036 /*------------------------------------------------------------------------*
2037  *	usb_fifo_put_data
2038  *
2039  * what:
2040  *  0 - normal operation
2041  *  1 - set last packet flag to enforce framing
2042  *------------------------------------------------------------------------*/
2043 void
2044 usb_fifo_put_data(struct usb_fifo *f, struct usb_page_cache *pc,
2045     usb_frlength_t offset, usb_frlength_t len, uint8_t what)
2046 {
2047 	struct usb_mbuf *m;
2048 	usb_frlength_t io_len;
2049 
2050 	while (len || (what == 1)) {
2051 
2052 		USB_IF_DEQUEUE(&f->free_q, m);
2053 
2054 		if (m) {
2055 			USB_MBUF_RESET(m);
2056 
2057 			io_len = MIN(len, m->cur_data_len);
2058 
2059 			usbd_copy_out(pc, offset, m->cur_data_ptr, io_len);
2060 
2061 			m->cur_data_len = io_len;
2062 			offset += io_len;
2063 			len -= io_len;
2064 
2065 			if ((len == 0) && (what == 1)) {
2066 				m->last_packet = 1;
2067 			}
2068 			USB_IF_ENQUEUE(&f->used_q, m);
2069 
2070 			usb_fifo_wakeup(f);
2071 
2072 			if ((len == 0) || (what == 1)) {
2073 				break;
2074 			}
2075 		} else {
2076 			break;
2077 		}
2078 	}
2079 }
2080 
2081 void
2082 usb_fifo_put_data_linear(struct usb_fifo *f, void *ptr,
2083     usb_size_t len, uint8_t what)
2084 {
2085 	struct usb_mbuf *m;
2086 	usb_size_t io_len;
2087 
2088 	while (len || (what == 1)) {
2089 
2090 		USB_IF_DEQUEUE(&f->free_q, m);
2091 
2092 		if (m) {
2093 			USB_MBUF_RESET(m);
2094 
2095 			io_len = MIN(len, m->cur_data_len);
2096 
2097 			memcpy(m->cur_data_ptr, ptr, io_len);
2098 
2099 			m->cur_data_len = io_len;
2100 			ptr = USB_ADD_BYTES(ptr, io_len);
2101 			len -= io_len;
2102 
2103 			if ((len == 0) && (what == 1)) {
2104 				m->last_packet = 1;
2105 			}
2106 			USB_IF_ENQUEUE(&f->used_q, m);
2107 
2108 			usb_fifo_wakeup(f);
2109 
2110 			if ((len == 0) || (what == 1)) {
2111 				break;
2112 			}
2113 		} else {
2114 			break;
2115 		}
2116 	}
2117 }
2118 
2119 uint8_t
2120 usb_fifo_put_data_buffer(struct usb_fifo *f, void *ptr, usb_size_t len)
2121 {
2122 	struct usb_mbuf *m;
2123 
2124 	USB_IF_DEQUEUE(&f->free_q, m);
2125 
2126 	if (m) {
2127 		m->cur_data_len = len;
2128 		m->cur_data_ptr = ptr;
2129 		USB_IF_ENQUEUE(&f->used_q, m);
2130 		usb_fifo_wakeup(f);
2131 		return (1);
2132 	}
2133 	return (0);
2134 }
2135 
2136 void
2137 usb_fifo_put_data_error(struct usb_fifo *f)
2138 {
2139 	f->flag_iserror = 1;
2140 	usb_fifo_wakeup(f);
2141 }
2142 
2143 /*------------------------------------------------------------------------*
2144  *	usb_fifo_get_data
2145  *
2146  * what:
2147  *  0 - normal operation
2148  *  1 - only get one "usb_mbuf"
2149  *
2150  * returns:
2151  *  0 - no more data
2152  *  1 - data in buffer
2153  *------------------------------------------------------------------------*/
2154 uint8_t
2155 usb_fifo_get_data(struct usb_fifo *f, struct usb_page_cache *pc,
2156     usb_frlength_t offset, usb_frlength_t len, usb_frlength_t *actlen,
2157     uint8_t what)
2158 {
2159 	struct usb_mbuf *m;
2160 	usb_frlength_t io_len;
2161 	uint8_t tr_data = 0;
2162 
2163 	actlen[0] = 0;
2164 
2165 	while (1) {
2166 
2167 		USB_IF_DEQUEUE(&f->used_q, m);
2168 
2169 		if (m) {
2170 
2171 			tr_data = 1;
2172 
2173 			io_len = MIN(len, m->cur_data_len);
2174 
2175 			usbd_copy_in(pc, offset, m->cur_data_ptr, io_len);
2176 
2177 			len -= io_len;
2178 			offset += io_len;
2179 			actlen[0] += io_len;
2180 			m->cur_data_ptr += io_len;
2181 			m->cur_data_len -= io_len;
2182 
2183 			if ((m->cur_data_len == 0) || (what == 1)) {
2184 				USB_IF_ENQUEUE(&f->free_q, m);
2185 
2186 				usb_fifo_wakeup(f);
2187 
2188 				if (what == 1) {
2189 					break;
2190 				}
2191 			} else {
2192 				USB_IF_PREPEND(&f->used_q, m);
2193 			}
2194 		} else {
2195 
2196 			if (tr_data) {
2197 				/* wait for data to be written out */
2198 				break;
2199 			}
2200 			if (f->flag_flushing) {
2201 				/* check if we should send a short packet */
2202 				if (f->flag_short != 0) {
2203 					f->flag_short = 0;
2204 					tr_data = 1;
2205 					break;
2206 				}
2207 				/* flushing complete */
2208 				f->flag_flushing = 0;
2209 				usb_fifo_wakeup(f);
2210 			}
2211 			break;
2212 		}
2213 		if (len == 0) {
2214 			break;
2215 		}
2216 	}
2217 	return (tr_data);
2218 }
2219 
2220 uint8_t
2221 usb_fifo_get_data_linear(struct usb_fifo *f, void *ptr,
2222     usb_size_t len, usb_size_t *actlen, uint8_t what)
2223 {
2224 	struct usb_mbuf *m;
2225 	usb_size_t io_len;
2226 	uint8_t tr_data = 0;
2227 
2228 	actlen[0] = 0;
2229 
2230 	while (1) {
2231 
2232 		USB_IF_DEQUEUE(&f->used_q, m);
2233 
2234 		if (m) {
2235 
2236 			tr_data = 1;
2237 
2238 			io_len = MIN(len, m->cur_data_len);
2239 
2240 			memcpy(ptr, m->cur_data_ptr, io_len);
2241 
2242 			len -= io_len;
2243 			ptr = USB_ADD_BYTES(ptr, io_len);
2244 			actlen[0] += io_len;
2245 			m->cur_data_ptr += io_len;
2246 			m->cur_data_len -= io_len;
2247 
2248 			if ((m->cur_data_len == 0) || (what == 1)) {
2249 				USB_IF_ENQUEUE(&f->free_q, m);
2250 
2251 				usb_fifo_wakeup(f);
2252 
2253 				if (what == 1) {
2254 					break;
2255 				}
2256 			} else {
2257 				USB_IF_PREPEND(&f->used_q, m);
2258 			}
2259 		} else {
2260 
2261 			if (tr_data) {
2262 				/* wait for data to be written out */
2263 				break;
2264 			}
2265 			if (f->flag_flushing) {
2266 				/* check if we should send a short packet */
2267 				if (f->flag_short != 0) {
2268 					f->flag_short = 0;
2269 					tr_data = 1;
2270 					break;
2271 				}
2272 				/* flushing complete */
2273 				f->flag_flushing = 0;
2274 				usb_fifo_wakeup(f);
2275 			}
2276 			break;
2277 		}
2278 		if (len == 0) {
2279 			break;
2280 		}
2281 	}
2282 	return (tr_data);
2283 }
2284 
2285 uint8_t
2286 usb_fifo_get_data_buffer(struct usb_fifo *f, void **pptr, usb_size_t *plen)
2287 {
2288 	struct usb_mbuf *m;
2289 
2290 	USB_IF_POLL(&f->used_q, m);
2291 
2292 	if (m) {
2293 		*plen = m->cur_data_len;
2294 		*pptr = m->cur_data_ptr;
2295 
2296 		return (1);
2297 	}
2298 	return (0);
2299 }
2300 
2301 void
2302 usb_fifo_get_data_error(struct usb_fifo *f)
2303 {
2304 	f->flag_iserror = 1;
2305 	usb_fifo_wakeup(f);
2306 }
2307 
2308 /*------------------------------------------------------------------------*
2309  *	usb_alloc_symlink
2310  *
2311  * Return values:
2312  * NULL: Failure
2313  * Else: Pointer to symlink entry
2314  *------------------------------------------------------------------------*/
2315 struct usb_symlink *
2316 usb_alloc_symlink(const char *target)
2317 {
2318 	struct usb_symlink *ps;
2319 
2320 	ps = malloc(sizeof(*ps), M_USBDEV, M_WAITOK);
2321 	if (ps == NULL) {
2322 		return (ps);
2323 	}
2324 	/* XXX no longer needed */
2325 	strlcpy(ps->src_path, target, sizeof(ps->src_path));
2326 	ps->src_len = strlen(ps->src_path);
2327 	strlcpy(ps->dst_path, target, sizeof(ps->dst_path));
2328 	ps->dst_len = strlen(ps->dst_path);
2329 
2330 	sx_xlock(&usb_sym_lock);
2331 	TAILQ_INSERT_TAIL(&usb_sym_head, ps, sym_entry);
2332 	sx_unlock(&usb_sym_lock);
2333 	return (ps);
2334 }
2335 
2336 /*------------------------------------------------------------------------*
2337  *	usb_free_symlink
2338  *------------------------------------------------------------------------*/
2339 void
2340 usb_free_symlink(struct usb_symlink *ps)
2341 {
2342 	if (ps == NULL) {
2343 		return;
2344 	}
2345 	sx_xlock(&usb_sym_lock);
2346 	TAILQ_REMOVE(&usb_sym_head, ps, sym_entry);
2347 	sx_unlock(&usb_sym_lock);
2348 
2349 	free(ps, M_USBDEV);
2350 }
2351 
2352 /*------------------------------------------------------------------------*
2353  *	usb_read_symlink
2354  *
2355  * Return value:
2356  * 0: Success
2357  * Else: Failure
2358  *------------------------------------------------------------------------*/
2359 int
2360 usb_read_symlink(uint8_t *user_ptr, uint32_t startentry, uint32_t user_len)
2361 {
2362 	struct usb_symlink *ps;
2363 	uint32_t temp;
2364 	uint32_t delta = 0;
2365 	uint8_t len;
2366 	int error = 0;
2367 
2368 	sx_xlock(&usb_sym_lock);
2369 
2370 	TAILQ_FOREACH(ps, &usb_sym_head, sym_entry) {
2371 
2372 		/*
2373 		 * Compute total length of source and destination symlink
2374 		 * strings pluss one length byte and two NUL bytes:
2375 		 */
2376 		temp = ps->src_len + ps->dst_len + 3;
2377 
2378 		if (temp > 255) {
2379 			/*
2380 			 * Skip entry because this length cannot fit
2381 			 * into one byte:
2382 			 */
2383 			continue;
2384 		}
2385 		if (startentry != 0) {
2386 			/* decrement read offset */
2387 			startentry--;
2388 			continue;
2389 		}
2390 		if (temp > user_len) {
2391 			/* out of buffer space */
2392 			break;
2393 		}
2394 		len = temp;
2395 
2396 		/* copy out total length */
2397 
2398 		error = copyout(&len,
2399 		    USB_ADD_BYTES(user_ptr, delta), 1);
2400 		if (error) {
2401 			break;
2402 		}
2403 		delta += 1;
2404 
2405 		/* copy out source string */
2406 
2407 		error = copyout(ps->src_path,
2408 		    USB_ADD_BYTES(user_ptr, delta), ps->src_len);
2409 		if (error) {
2410 			break;
2411 		}
2412 		len = 0;
2413 		delta += ps->src_len;
2414 		error = copyout(&len,
2415 		    USB_ADD_BYTES(user_ptr, delta), 1);
2416 		if (error) {
2417 			break;
2418 		}
2419 		delta += 1;
2420 
2421 		/* copy out destination string */
2422 
2423 		error = copyout(ps->dst_path,
2424 		    USB_ADD_BYTES(user_ptr, delta), ps->dst_len);
2425 		if (error) {
2426 			break;
2427 		}
2428 		len = 0;
2429 		delta += ps->dst_len;
2430 		error = copyout(&len,
2431 		    USB_ADD_BYTES(user_ptr, delta), 1);
2432 		if (error) {
2433 			break;
2434 		}
2435 		delta += 1;
2436 
2437 		user_len -= temp;
2438 	}
2439 
2440 	/* a zero length entry indicates the end */
2441 
2442 	if ((user_len != 0) && (error == 0)) {
2443 
2444 		len = 0;
2445 
2446 		error = copyout(&len,
2447 		    USB_ADD_BYTES(user_ptr, delta), 1);
2448 	}
2449 	sx_unlock(&usb_sym_lock);
2450 	return (error);
2451 }
2452 
2453 void
2454 usb_fifo_set_close_zlp(struct usb_fifo *f, uint8_t onoff)
2455 {
2456 	if (f == NULL)
2457 		return;
2458 
2459 	/* send a Zero Length Packet, ZLP, before close */
2460 	f->flag_short = onoff;
2461 }
2462 
2463 void
2464 usb_fifo_set_write_defrag(struct usb_fifo *f, uint8_t onoff)
2465 {
2466 	if (f == NULL)
2467 		return;
2468 
2469 	/* defrag written data */
2470 	f->flag_write_defrag = onoff;
2471 	/* reset defrag state */
2472 	f->flag_have_fragment = 0;
2473 }
2474 
2475 void *
2476 usb_fifo_softc(struct usb_fifo *f)
2477 {
2478 	return (f->priv_sc0);
2479 }
2480 #endif	/* USB_HAVE_UGEN */
2481