xref: /freebsd/sys/dev/usb/usb_dev.c (revision f5fd950e35c962bad0aa31fdc4b4052e13207893)
1 /* $FreeBSD$ */
2 /*-
3  * Copyright (c) 2006-2008 Hans Petter Selasky. All rights reserved.
4  *
5  * Redistribution and use in source and binary forms, with or without
6  * modification, are permitted provided that the following conditions
7  * are met:
8  * 1. Redistributions of source code must retain the above copyright
9  *    notice, this list of conditions and the following disclaimer.
10  * 2. Redistributions in binary form must reproduce the above copyright
11  *    notice, this list of conditions and the following disclaimer in the
12  *    documentation and/or other materials provided with the distribution.
13  *
14  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24  * SUCH DAMAGE.
25  *
26  *
27  * usb_dev.c - An abstraction layer for creating devices under /dev/...
28  */
29 
30 #ifdef USB_GLOBAL_INCLUDE_FILE
31 #include USB_GLOBAL_INCLUDE_FILE
32 #else
33 #include <sys/stdint.h>
34 #include <sys/stddef.h>
35 #include <sys/param.h>
36 #include <sys/queue.h>
37 #include <sys/types.h>
38 #include <sys/systm.h>
39 #include <sys/kernel.h>
40 #include <sys/bus.h>
41 #include <sys/module.h>
42 #include <sys/lock.h>
43 #include <sys/mutex.h>
44 #include <sys/condvar.h>
45 #include <sys/sysctl.h>
46 #include <sys/sx.h>
47 #include <sys/unistd.h>
48 #include <sys/callout.h>
49 #include <sys/malloc.h>
50 #include <sys/priv.h>
51 #include <sys/vnode.h>
52 #include <sys/conf.h>
53 #include <sys/fcntl.h>
54 
55 #include <dev/usb/usb.h>
56 #include <dev/usb/usb_ioctl.h>
57 #include <dev/usb/usbdi.h>
58 #include <dev/usb/usbdi_util.h>
59 
60 #define	USB_DEBUG_VAR usb_fifo_debug
61 
62 #include <dev/usb/usb_core.h>
63 #include <dev/usb/usb_dev.h>
64 #include <dev/usb/usb_mbuf.h>
65 #include <dev/usb/usb_process.h>
66 #include <dev/usb/usb_device.h>
67 #include <dev/usb/usb_debug.h>
68 #include <dev/usb/usb_busdma.h>
69 #include <dev/usb/usb_generic.h>
70 #include <dev/usb/usb_dynamic.h>
71 #include <dev/usb/usb_util.h>
72 
73 #include <dev/usb/usb_controller.h>
74 #include <dev/usb/usb_bus.h>
75 
76 #include <sys/filio.h>
77 #include <sys/ttycom.h>
78 #include <sys/syscallsubr.h>
79 
80 #include <machine/stdarg.h>
81 #endif			/* USB_GLOBAL_INCLUDE_FILE */
82 
83 #if USB_HAVE_UGEN
84 
85 #ifdef USB_DEBUG
86 static int usb_fifo_debug = 0;
87 
88 static SYSCTL_NODE(_hw_usb, OID_AUTO, dev, CTLFLAG_RW, 0, "USB device");
89 SYSCTL_INT(_hw_usb_dev, OID_AUTO, debug, CTLFLAG_RW | CTLFLAG_TUN,
90     &usb_fifo_debug, 0, "Debug Level");
91 TUNABLE_INT("hw.usb.dev.debug", &usb_fifo_debug);
92 #endif
93 
94 #if ((__FreeBSD_version >= 700001) || (__FreeBSD_version == 0) || \
95      ((__FreeBSD_version >= 600034) && (__FreeBSD_version < 700000)))
96 #define	USB_UCRED struct ucred *ucred,
97 #else
98 #define	USB_UCRED
99 #endif
100 
101 /* prototypes */
102 
103 static int	usb_fifo_open(struct usb_cdev_privdata *,
104 		    struct usb_fifo *, int);
105 static void	usb_fifo_close(struct usb_fifo *, int);
106 static void	usb_dev_init(void *);
107 static void	usb_dev_init_post(void *);
108 static void	usb_dev_uninit(void *);
109 static int	usb_fifo_uiomove(struct usb_fifo *, void *, int,
110 		    struct uio *);
111 static void	usb_fifo_check_methods(struct usb_fifo_methods *);
112 static struct	usb_fifo *usb_fifo_alloc(struct mtx *);
113 static struct	usb_endpoint *usb_dev_get_ep(struct usb_device *, uint8_t,
114 		    uint8_t);
115 static void	usb_loc_fill(struct usb_fs_privdata *,
116 		    struct usb_cdev_privdata *);
117 static void	usb_close(void *);
118 static usb_error_t usb_ref_device(struct usb_cdev_privdata *, struct usb_cdev_refdata *, int);
119 static usb_error_t usb_usb_ref_device(struct usb_cdev_privdata *, struct usb_cdev_refdata *);
120 static void	usb_unref_device(struct usb_cdev_privdata *, struct usb_cdev_refdata *);
121 
122 static d_open_t usb_open;
123 static d_ioctl_t usb_ioctl;
124 static d_read_t usb_read;
125 static d_write_t usb_write;
126 static d_poll_t usb_poll;
127 static d_kqfilter_t usb_kqfilter;
128 
129 static d_ioctl_t usb_static_ioctl;
130 
131 static usb_fifo_open_t usb_fifo_dummy_open;
132 static usb_fifo_close_t usb_fifo_dummy_close;
133 static usb_fifo_ioctl_t usb_fifo_dummy_ioctl;
134 static usb_fifo_cmd_t usb_fifo_dummy_cmd;
135 
136 /* character device structure used for devices (/dev/ugenX.Y and /dev/uXXX) */
137 struct cdevsw usb_devsw = {
138 	.d_version = D_VERSION,
139 	.d_open = usb_open,
140 	.d_ioctl = usb_ioctl,
141 	.d_name = "usbdev",
142 	.d_flags = D_TRACKCLOSE,
143 	.d_read = usb_read,
144 	.d_write = usb_write,
145 	.d_poll = usb_poll,
146 	.d_kqfilter = usb_kqfilter,
147 };
148 
149 static struct cdev* usb_dev = NULL;
150 
151 /* character device structure used for /dev/usb */
152 static struct cdevsw usb_static_devsw = {
153 	.d_version = D_VERSION,
154 	.d_ioctl = usb_static_ioctl,
155 	.d_name = "usb"
156 };
157 
158 static TAILQ_HEAD(, usb_symlink) usb_sym_head;
159 static struct sx usb_sym_lock;
160 
161 struct mtx usb_ref_lock;
162 
163 /*------------------------------------------------------------------------*
164  *	usb_loc_fill
165  *
166  * This is used to fill out a usb_cdev_privdata structure based on the
167  * device's address as contained in usb_fs_privdata.
168  *------------------------------------------------------------------------*/
169 static void
170 usb_loc_fill(struct usb_fs_privdata* pd, struct usb_cdev_privdata *cpd)
171 {
172 	cpd->bus_index = pd->bus_index;
173 	cpd->dev_index = pd->dev_index;
174 	cpd->ep_addr = pd->ep_addr;
175 	cpd->fifo_index = pd->fifo_index;
176 }
177 
178 /*------------------------------------------------------------------------*
179  *	usb_ref_device
180  *
181  * This function is used to atomically refer an USB device by its
182  * device location. If this function returns success the USB device
183  * will not dissappear until the USB device is unreferenced.
184  *
185  * Return values:
186  *  0: Success, refcount incremented on the given USB device.
187  *  Else: Failure.
188  *------------------------------------------------------------------------*/
189 static usb_error_t
190 usb_ref_device(struct usb_cdev_privdata *cpd,
191     struct usb_cdev_refdata *crd, int need_uref)
192 {
193 	struct usb_fifo **ppf;
194 	struct usb_fifo *f;
195 
196 	DPRINTFN(2, "cpd=%p need uref=%d\n", cpd, need_uref);
197 
198 	/* clear all refs */
199 	memset(crd, 0, sizeof(*crd));
200 
201 	mtx_lock(&usb_ref_lock);
202 	cpd->bus = devclass_get_softc(usb_devclass_ptr, cpd->bus_index);
203 	if (cpd->bus == NULL) {
204 		DPRINTFN(2, "no bus at %u\n", cpd->bus_index);
205 		goto error;
206 	}
207 	cpd->udev = cpd->bus->devices[cpd->dev_index];
208 	if (cpd->udev == NULL) {
209 		DPRINTFN(2, "no device at %u\n", cpd->dev_index);
210 		goto error;
211 	}
212 	if (cpd->udev->state == USB_STATE_DETACHED &&
213 	    (need_uref != 2)) {
214 		DPRINTFN(2, "device is detached\n");
215 		goto error;
216 	}
217 	if (cpd->udev->refcount == USB_DEV_REF_MAX) {
218 		DPRINTFN(2, "no dev ref\n");
219 		goto error;
220 	}
221 	if (need_uref) {
222 		DPRINTFN(2, "ref udev - needed\n");
223 		cpd->udev->refcount++;
224 
225 		mtx_unlock(&usb_ref_lock);
226 
227 		/*
228 		 * We need to grab the enumeration SX-lock before
229 		 * grabbing the FIFO refs to avoid deadlock at detach!
230 		 */
231 		crd->do_unlock = usbd_enum_lock(cpd->udev);
232 
233 		mtx_lock(&usb_ref_lock);
234 
235 		/*
236 		 * Set "is_uref" after grabbing the default SX lock
237 		 */
238 		crd->is_uref = 1;
239 	}
240 
241 	/* check if we are doing an open */
242 	if (cpd->fflags == 0) {
243 		/* use zero defaults */
244 	} else {
245 		/* check for write */
246 		if (cpd->fflags & FWRITE) {
247 			ppf = cpd->udev->fifo;
248 			f = ppf[cpd->fifo_index + USB_FIFO_TX];
249 			crd->txfifo = f;
250 			crd->is_write = 1;	/* ref */
251 			if (f == NULL || f->refcount == USB_FIFO_REF_MAX)
252 				goto error;
253 			if (f->curr_cpd != cpd)
254 				goto error;
255 			/* check if USB-FS is active */
256 			if (f->fs_ep_max != 0) {
257 				crd->is_usbfs = 1;
258 			}
259 		}
260 
261 		/* check for read */
262 		if (cpd->fflags & FREAD) {
263 			ppf = cpd->udev->fifo;
264 			f = ppf[cpd->fifo_index + USB_FIFO_RX];
265 			crd->rxfifo = f;
266 			crd->is_read = 1;	/* ref */
267 			if (f == NULL || f->refcount == USB_FIFO_REF_MAX)
268 				goto error;
269 			if (f->curr_cpd != cpd)
270 				goto error;
271 			/* check if USB-FS is active */
272 			if (f->fs_ep_max != 0) {
273 				crd->is_usbfs = 1;
274 			}
275 		}
276 	}
277 
278 	/* when everything is OK we increment the refcounts */
279 	if (crd->is_write) {
280 		DPRINTFN(2, "ref write\n");
281 		crd->txfifo->refcount++;
282 	}
283 	if (crd->is_read) {
284 		DPRINTFN(2, "ref read\n");
285 		crd->rxfifo->refcount++;
286 	}
287 	mtx_unlock(&usb_ref_lock);
288 
289 	return (0);
290 
291 error:
292 	if (crd->do_unlock)
293 		usbd_enum_unlock(cpd->udev);
294 
295 	if (crd->is_uref) {
296 		if (--(cpd->udev->refcount) == 0) {
297 			cv_signal(&cpd->udev->ref_cv);
298 		}
299 	}
300 	mtx_unlock(&usb_ref_lock);
301 	DPRINTFN(2, "fail\n");
302 	return (USB_ERR_INVAL);
303 }
304 
305 /*------------------------------------------------------------------------*
306  *	usb_usb_ref_device
307  *
308  * This function is used to upgrade an USB reference to include the
309  * USB device reference on a USB location.
310  *
311  * Return values:
312  *  0: Success, refcount incremented on the given USB device.
313  *  Else: Failure.
314  *------------------------------------------------------------------------*/
315 static usb_error_t
316 usb_usb_ref_device(struct usb_cdev_privdata *cpd,
317     struct usb_cdev_refdata *crd)
318 {
319 	/*
320 	 * Check if we already got an USB reference on this location:
321 	 */
322 	if (crd->is_uref)
323 		return (0);		/* success */
324 
325 	/*
326 	 * To avoid deadlock at detach we need to drop the FIFO ref
327 	 * and re-acquire a new ref!
328 	 */
329 	usb_unref_device(cpd, crd);
330 
331 	return (usb_ref_device(cpd, crd, 1 /* need uref */));
332 }
333 
334 /*------------------------------------------------------------------------*
335  *	usb_unref_device
336  *
337  * This function will release the reference count by one unit for the
338  * given USB device.
339  *------------------------------------------------------------------------*/
340 static void
341 usb_unref_device(struct usb_cdev_privdata *cpd,
342     struct usb_cdev_refdata *crd)
343 {
344 
345 	DPRINTFN(2, "cpd=%p is_uref=%d\n", cpd, crd->is_uref);
346 
347 	if (crd->do_unlock)
348 		usbd_enum_unlock(cpd->udev);
349 
350 	mtx_lock(&usb_ref_lock);
351 	if (crd->is_read) {
352 		if (--(crd->rxfifo->refcount) == 0) {
353 			cv_signal(&crd->rxfifo->cv_drain);
354 		}
355 		crd->is_read = 0;
356 	}
357 	if (crd->is_write) {
358 		if (--(crd->txfifo->refcount) == 0) {
359 			cv_signal(&crd->txfifo->cv_drain);
360 		}
361 		crd->is_write = 0;
362 	}
363 	if (crd->is_uref) {
364 		if (--(cpd->udev->refcount) == 0) {
365 			cv_signal(&cpd->udev->ref_cv);
366 		}
367 		crd->is_uref = 0;
368 	}
369 	mtx_unlock(&usb_ref_lock);
370 }
371 
372 static struct usb_fifo *
373 usb_fifo_alloc(struct mtx *mtx)
374 {
375 	struct usb_fifo *f;
376 
377 	f = malloc(sizeof(*f), M_USBDEV, M_WAITOK | M_ZERO);
378 	if (f != NULL) {
379 		cv_init(&f->cv_io, "FIFO-IO");
380 		cv_init(&f->cv_drain, "FIFO-DRAIN");
381 		f->priv_mtx = mtx;
382 		f->refcount = 1;
383 		knlist_init_mtx(&f->selinfo.si_note, mtx);
384 	}
385 	return (f);
386 }
387 
388 /*------------------------------------------------------------------------*
389  *	usb_fifo_create
390  *------------------------------------------------------------------------*/
391 static int
392 usb_fifo_create(struct usb_cdev_privdata *cpd,
393     struct usb_cdev_refdata *crd)
394 {
395 	struct usb_device *udev = cpd->udev;
396 	struct usb_fifo *f;
397 	struct usb_endpoint *ep;
398 	uint8_t n;
399 	uint8_t is_tx;
400 	uint8_t is_rx;
401 	uint8_t no_null;
402 	uint8_t is_busy;
403 	int e = cpd->ep_addr;
404 
405 	is_tx = (cpd->fflags & FWRITE) ? 1 : 0;
406 	is_rx = (cpd->fflags & FREAD) ? 1 : 0;
407 	no_null = 1;
408 	is_busy = 0;
409 
410 	/* Preallocated FIFO */
411 	if (e < 0) {
412 		DPRINTFN(5, "Preallocated FIFO\n");
413 		if (is_tx) {
414 			f = udev->fifo[cpd->fifo_index + USB_FIFO_TX];
415 			if (f == NULL)
416 				return (EINVAL);
417 			crd->txfifo = f;
418 		}
419 		if (is_rx) {
420 			f = udev->fifo[cpd->fifo_index + USB_FIFO_RX];
421 			if (f == NULL)
422 				return (EINVAL);
423 			crd->rxfifo = f;
424 		}
425 		return (0);
426 	}
427 
428 	KASSERT(e >= 0 && e <= 15, ("endpoint %d out of range", e));
429 
430 	/* search for a free FIFO slot */
431 	DPRINTFN(5, "Endpoint device, searching for 0x%02x\n", e);
432 	for (n = 0;; n += 2) {
433 
434 		if (n == USB_FIFO_MAX) {
435 			if (no_null) {
436 				no_null = 0;
437 				n = 0;
438 			} else {
439 				/* end of FIFOs reached */
440 				DPRINTFN(5, "out of FIFOs\n");
441 				return (ENOMEM);
442 			}
443 		}
444 		/* Check for TX FIFO */
445 		if (is_tx) {
446 			f = udev->fifo[n + USB_FIFO_TX];
447 			if (f != NULL) {
448 				if (f->dev_ep_index != e) {
449 					/* wrong endpoint index */
450 					continue;
451 				}
452 				if (f->curr_cpd != NULL) {
453 					/* FIFO is opened */
454 					is_busy = 1;
455 					continue;
456 				}
457 			} else if (no_null) {
458 				continue;
459 			}
460 		}
461 		/* Check for RX FIFO */
462 		if (is_rx) {
463 			f = udev->fifo[n + USB_FIFO_RX];
464 			if (f != NULL) {
465 				if (f->dev_ep_index != e) {
466 					/* wrong endpoint index */
467 					continue;
468 				}
469 				if (f->curr_cpd != NULL) {
470 					/* FIFO is opened */
471 					is_busy = 1;
472 					continue;
473 				}
474 			} else if (no_null) {
475 				continue;
476 			}
477 		}
478 		break;
479 	}
480 
481 	if (no_null == 0) {
482 		if (e >= (USB_EP_MAX / 2)) {
483 			/* we don't create any endpoints in this range */
484 			DPRINTFN(5, "ep out of range\n");
485 			return (is_busy ? EBUSY : EINVAL);
486 		}
487 	}
488 
489 	if ((e != 0) && is_busy) {
490 		/*
491 		 * Only the default control endpoint is allowed to be
492 		 * opened multiple times!
493 		 */
494 		DPRINTFN(5, "busy\n");
495 		return (EBUSY);
496 	}
497 
498 	/* Check TX FIFO */
499 	if (is_tx &&
500 	    (udev->fifo[n + USB_FIFO_TX] == NULL)) {
501 		ep = usb_dev_get_ep(udev, e, USB_FIFO_TX);
502 		DPRINTFN(5, "dev_get_endpoint(%d, 0x%x)\n", e, USB_FIFO_TX);
503 		if (ep == NULL) {
504 			DPRINTFN(5, "dev_get_endpoint returned NULL\n");
505 			return (EINVAL);
506 		}
507 		f = usb_fifo_alloc(&udev->device_mtx);
508 		if (f == NULL) {
509 			DPRINTFN(5, "could not alloc tx fifo\n");
510 			return (ENOMEM);
511 		}
512 		/* update some fields */
513 		f->fifo_index = n + USB_FIFO_TX;
514 		f->dev_ep_index = e;
515 		f->priv_sc0 = ep;
516 		f->methods = &usb_ugen_methods;
517 		f->iface_index = ep->iface_index;
518 		f->udev = udev;
519 		mtx_lock(&usb_ref_lock);
520 		udev->fifo[n + USB_FIFO_TX] = f;
521 		mtx_unlock(&usb_ref_lock);
522 	}
523 	/* Check RX FIFO */
524 	if (is_rx &&
525 	    (udev->fifo[n + USB_FIFO_RX] == NULL)) {
526 
527 		ep = usb_dev_get_ep(udev, e, USB_FIFO_RX);
528 		DPRINTFN(5, "dev_get_endpoint(%d, 0x%x)\n", e, USB_FIFO_RX);
529 		if (ep == NULL) {
530 			DPRINTFN(5, "dev_get_endpoint returned NULL\n");
531 			return (EINVAL);
532 		}
533 		f = usb_fifo_alloc(&udev->device_mtx);
534 		if (f == NULL) {
535 			DPRINTFN(5, "could not alloc rx fifo\n");
536 			return (ENOMEM);
537 		}
538 		/* update some fields */
539 		f->fifo_index = n + USB_FIFO_RX;
540 		f->dev_ep_index = e;
541 		f->priv_sc0 = ep;
542 		f->methods = &usb_ugen_methods;
543 		f->iface_index = ep->iface_index;
544 		f->udev = udev;
545 		mtx_lock(&usb_ref_lock);
546 		udev->fifo[n + USB_FIFO_RX] = f;
547 		mtx_unlock(&usb_ref_lock);
548 	}
549 	if (is_tx) {
550 		crd->txfifo = udev->fifo[n + USB_FIFO_TX];
551 	}
552 	if (is_rx) {
553 		crd->rxfifo = udev->fifo[n + USB_FIFO_RX];
554 	}
555 	/* fill out fifo index */
556 	DPRINTFN(5, "fifo index = %d\n", n);
557 	cpd->fifo_index = n;
558 
559 	/* complete */
560 
561 	return (0);
562 }
563 
564 void
565 usb_fifo_free(struct usb_fifo *f)
566 {
567 	uint8_t n;
568 
569 	if (f == NULL) {
570 		/* be NULL safe */
571 		return;
572 	}
573 	/* destroy symlink devices, if any */
574 	for (n = 0; n != 2; n++) {
575 		if (f->symlink[n]) {
576 			usb_free_symlink(f->symlink[n]);
577 			f->symlink[n] = NULL;
578 		}
579 	}
580 	mtx_lock(&usb_ref_lock);
581 
582 	/* delink ourselves to stop calls from userland */
583 	if ((f->fifo_index < USB_FIFO_MAX) &&
584 	    (f->udev != NULL) &&
585 	    (f->udev->fifo[f->fifo_index] == f)) {
586 		f->udev->fifo[f->fifo_index] = NULL;
587 	} else {
588 		DPRINTFN(0, "USB FIFO %p has not been linked\n", f);
589 	}
590 
591 	/* decrease refcount */
592 	f->refcount--;
593 	/* prevent any write flush */
594 	f->flag_iserror = 1;
595 	/* need to wait until all callers have exited */
596 	while (f->refcount != 0) {
597 		mtx_unlock(&usb_ref_lock);	/* avoid LOR */
598 		mtx_lock(f->priv_mtx);
599 		/* get I/O thread out of any sleep state */
600 		if (f->flag_sleeping) {
601 			f->flag_sleeping = 0;
602 			cv_broadcast(&f->cv_io);
603 		}
604 		mtx_unlock(f->priv_mtx);
605 		mtx_lock(&usb_ref_lock);
606 
607 		/*
608 		 * Check if the "f->refcount" variable reached zero
609 		 * during the unlocked time before entering wait:
610 		 */
611 		if (f->refcount == 0)
612 			break;
613 
614 		/* wait for sync */
615 		cv_wait(&f->cv_drain, &usb_ref_lock);
616 	}
617 	mtx_unlock(&usb_ref_lock);
618 
619 	/* take care of closing the device here, if any */
620 	usb_fifo_close(f, 0);
621 
622 	cv_destroy(&f->cv_io);
623 	cv_destroy(&f->cv_drain);
624 
625 	knlist_clear(&f->selinfo.si_note, 0);
626 	seldrain(&f->selinfo);
627 	knlist_destroy(&f->selinfo.si_note);
628 
629 	free(f, M_USBDEV);
630 }
631 
632 static struct usb_endpoint *
633 usb_dev_get_ep(struct usb_device *udev, uint8_t ep_index, uint8_t dir)
634 {
635 	struct usb_endpoint *ep;
636 	uint8_t ep_dir;
637 
638 	if (ep_index == 0) {
639 		ep = &udev->ctrl_ep;
640 	} else {
641 		if (dir == USB_FIFO_RX) {
642 			if (udev->flags.usb_mode == USB_MODE_HOST) {
643 				ep_dir = UE_DIR_IN;
644 			} else {
645 				ep_dir = UE_DIR_OUT;
646 			}
647 		} else {
648 			if (udev->flags.usb_mode == USB_MODE_HOST) {
649 				ep_dir = UE_DIR_OUT;
650 			} else {
651 				ep_dir = UE_DIR_IN;
652 			}
653 		}
654 		ep = usbd_get_ep_by_addr(udev, ep_index | ep_dir);
655 	}
656 
657 	if (ep == NULL) {
658 		/* if the endpoint does not exist then return */
659 		return (NULL);
660 	}
661 	if (ep->edesc == NULL) {
662 		/* invalid endpoint */
663 		return (NULL);
664 	}
665 	return (ep);			/* success */
666 }
667 
668 /*------------------------------------------------------------------------*
669  *	usb_fifo_open
670  *
671  * Returns:
672  * 0: Success
673  * Else: Failure
674  *------------------------------------------------------------------------*/
675 static int
676 usb_fifo_open(struct usb_cdev_privdata *cpd,
677     struct usb_fifo *f, int fflags)
678 {
679 	int err;
680 
681 	if (f == NULL) {
682 		/* no FIFO there */
683 		DPRINTFN(2, "no FIFO\n");
684 		return (ENXIO);
685 	}
686 	/* remove FWRITE and FREAD flags */
687 	fflags &= ~(FWRITE | FREAD);
688 
689 	/* set correct file flags */
690 	if ((f->fifo_index & 1) == USB_FIFO_TX) {
691 		fflags |= FWRITE;
692 	} else {
693 		fflags |= FREAD;
694 	}
695 
696 	/* check if we are already opened */
697 	/* we don't need any locks when checking this variable */
698 	if (f->curr_cpd != NULL) {
699 		err = EBUSY;
700 		goto done;
701 	}
702 
703 	/* reset short flag before open */
704 	f->flag_short = 0;
705 
706 	/* call open method */
707 	err = (f->methods->f_open) (f, fflags);
708 	if (err) {
709 		goto done;
710 	}
711 	mtx_lock(f->priv_mtx);
712 
713 	/* reset sleep flag */
714 	f->flag_sleeping = 0;
715 
716 	/* reset error flag */
717 	f->flag_iserror = 0;
718 
719 	/* reset complete flag */
720 	f->flag_iscomplete = 0;
721 
722 	/* reset select flag */
723 	f->flag_isselect = 0;
724 
725 	/* reset flushing flag */
726 	f->flag_flushing = 0;
727 
728 	/* reset ASYNC proc flag */
729 	f->async_p = NULL;
730 
731 	mtx_lock(&usb_ref_lock);
732 	/* flag the fifo as opened to prevent others */
733 	f->curr_cpd = cpd;
734 	mtx_unlock(&usb_ref_lock);
735 
736 	/* reset queue */
737 	usb_fifo_reset(f);
738 
739 	mtx_unlock(f->priv_mtx);
740 done:
741 	return (err);
742 }
743 
744 /*------------------------------------------------------------------------*
745  *	usb_fifo_reset
746  *------------------------------------------------------------------------*/
747 void
748 usb_fifo_reset(struct usb_fifo *f)
749 {
750 	struct usb_mbuf *m;
751 
752 	if (f == NULL) {
753 		return;
754 	}
755 	while (1) {
756 		USB_IF_DEQUEUE(&f->used_q, m);
757 		if (m) {
758 			USB_IF_ENQUEUE(&f->free_q, m);
759 		} else {
760 			break;
761 		}
762 	}
763 	/* reset have fragment flag */
764 	f->flag_have_fragment = 0;
765 }
766 
767 /*------------------------------------------------------------------------*
768  *	usb_fifo_close
769  *------------------------------------------------------------------------*/
770 static void
771 usb_fifo_close(struct usb_fifo *f, int fflags)
772 {
773 	int err;
774 
775 	/* check if we are not opened */
776 	if (f->curr_cpd == NULL) {
777 		/* nothing to do - already closed */
778 		return;
779 	}
780 	mtx_lock(f->priv_mtx);
781 
782 	/* clear current cdev private data pointer */
783 	mtx_lock(&usb_ref_lock);
784 	f->curr_cpd = NULL;
785 	mtx_unlock(&usb_ref_lock);
786 
787 	/* check if we are watched by kevent */
788 	KNOTE_LOCKED(&f->selinfo.si_note, 0);
789 
790 	/* check if we are selected */
791 	if (f->flag_isselect) {
792 		selwakeup(&f->selinfo);
793 		f->flag_isselect = 0;
794 	}
795 	/* check if a thread wants SIGIO */
796 	if (f->async_p != NULL) {
797 		PROC_LOCK(f->async_p);
798 		kern_psignal(f->async_p, SIGIO);
799 		PROC_UNLOCK(f->async_p);
800 		f->async_p = NULL;
801 	}
802 	/* remove FWRITE and FREAD flags */
803 	fflags &= ~(FWRITE | FREAD);
804 
805 	/* flush written data, if any */
806 	if ((f->fifo_index & 1) == USB_FIFO_TX) {
807 
808 		if (!f->flag_iserror) {
809 
810 			/* set flushing flag */
811 			f->flag_flushing = 1;
812 
813 			/* get the last packet in */
814 			if (f->flag_have_fragment) {
815 				struct usb_mbuf *m;
816 				f->flag_have_fragment = 0;
817 				USB_IF_DEQUEUE(&f->free_q, m);
818 				if (m) {
819 					USB_IF_ENQUEUE(&f->used_q, m);
820 				}
821 			}
822 
823 			/* start write transfer, if not already started */
824 			(f->methods->f_start_write) (f);
825 
826 			/* check if flushed already */
827 			while (f->flag_flushing &&
828 			    (!f->flag_iserror)) {
829 				/* wait until all data has been written */
830 				f->flag_sleeping = 1;
831 				err = cv_wait_sig(&f->cv_io, f->priv_mtx);
832 				if (err) {
833 					DPRINTF("signal received\n");
834 					break;
835 				}
836 			}
837 		}
838 		fflags |= FWRITE;
839 
840 		/* stop write transfer, if not already stopped */
841 		(f->methods->f_stop_write) (f);
842 	} else {
843 		fflags |= FREAD;
844 
845 		/* stop write transfer, if not already stopped */
846 		(f->methods->f_stop_read) (f);
847 	}
848 
849 	/* check if we are sleeping */
850 	if (f->flag_sleeping) {
851 		DPRINTFN(2, "Sleeping at close!\n");
852 	}
853 	mtx_unlock(f->priv_mtx);
854 
855 	/* call close method */
856 	(f->methods->f_close) (f, fflags);
857 
858 	DPRINTF("closed\n");
859 }
860 
861 /*------------------------------------------------------------------------*
862  *	usb_open - cdev callback
863  *------------------------------------------------------------------------*/
864 static int
865 usb_open(struct cdev *dev, int fflags, int devtype, struct thread *td)
866 {
867 	struct usb_fs_privdata* pd = (struct usb_fs_privdata*)dev->si_drv1;
868 	struct usb_cdev_refdata refs;
869 	struct usb_cdev_privdata *cpd;
870 	int err, ep;
871 
872 	DPRINTFN(2, "%s fflags=0x%08x\n", devtoname(dev), fflags);
873 
874 	KASSERT(fflags & (FREAD|FWRITE), ("invalid open flags"));
875 	if (((fflags & FREAD) && !(pd->mode & FREAD)) ||
876 	    ((fflags & FWRITE) && !(pd->mode & FWRITE))) {
877 		DPRINTFN(2, "access mode not supported\n");
878 		return (EPERM);
879 	}
880 
881 	cpd = malloc(sizeof(*cpd), M_USBDEV, M_WAITOK | M_ZERO);
882 	ep = cpd->ep_addr = pd->ep_addr;
883 
884 	usb_loc_fill(pd, cpd);
885 	err = usb_ref_device(cpd, &refs, 1);
886 	if (err) {
887 		DPRINTFN(2, "cannot ref device\n");
888 		free(cpd, M_USBDEV);
889 		return (ENXIO);
890 	}
891 	cpd->fflags = fflags;	/* access mode for open lifetime */
892 
893 	/* create FIFOs, if any */
894 	err = usb_fifo_create(cpd, &refs);
895 	/* check for error */
896 	if (err) {
897 		DPRINTFN(2, "cannot create fifo\n");
898 		usb_unref_device(cpd, &refs);
899 		free(cpd, M_USBDEV);
900 		return (err);
901 	}
902 	if (fflags & FREAD) {
903 		err = usb_fifo_open(cpd, refs.rxfifo, fflags);
904 		if (err) {
905 			DPRINTFN(2, "read open failed\n");
906 			usb_unref_device(cpd, &refs);
907 			free(cpd, M_USBDEV);
908 			return (err);
909 		}
910 	}
911 	if (fflags & FWRITE) {
912 		err = usb_fifo_open(cpd, refs.txfifo, fflags);
913 		if (err) {
914 			DPRINTFN(2, "write open failed\n");
915 			if (fflags & FREAD) {
916 				usb_fifo_close(refs.rxfifo, fflags);
917 			}
918 			usb_unref_device(cpd, &refs);
919 			free(cpd, M_USBDEV);
920 			return (err);
921 		}
922 	}
923 	usb_unref_device(cpd, &refs);
924 	devfs_set_cdevpriv(cpd, usb_close);
925 
926 	return (0);
927 }
928 
929 /*------------------------------------------------------------------------*
930  *	usb_close - cdev callback
931  *------------------------------------------------------------------------*/
932 static void
933 usb_close(void *arg)
934 {
935 	struct usb_cdev_refdata refs;
936 	struct usb_cdev_privdata *cpd = arg;
937 	int err;
938 
939 	DPRINTFN(2, "cpd=%p\n", cpd);
940 
941 	err = usb_ref_device(cpd, &refs,
942 	    2 /* uref and allow detached state */);
943 	if (err) {
944 		DPRINTFN(2, "Cannot grab USB reference when "
945 		    "closing USB file handle\n");
946 		goto done;
947 	}
948 	if (cpd->fflags & FREAD) {
949 		usb_fifo_close(refs.rxfifo, cpd->fflags);
950 	}
951 	if (cpd->fflags & FWRITE) {
952 		usb_fifo_close(refs.txfifo, cpd->fflags);
953 	}
954 	usb_unref_device(cpd, &refs);
955 done:
956 	free(cpd, M_USBDEV);
957 }
958 
959 static void
960 usb_dev_init(void *arg)
961 {
962 	mtx_init(&usb_ref_lock, "USB ref mutex", NULL, MTX_DEF);
963 	sx_init(&usb_sym_lock, "USB sym mutex");
964 	TAILQ_INIT(&usb_sym_head);
965 
966 	/* check the UGEN methods */
967 	usb_fifo_check_methods(&usb_ugen_methods);
968 }
969 
970 SYSINIT(usb_dev_init, SI_SUB_KLD, SI_ORDER_FIRST, usb_dev_init, NULL);
971 
972 static void
973 usb_dev_init_post(void *arg)
974 {
975 	/*
976 	 * Create /dev/usb - this is needed for usbconfig(8), which
977 	 * needs a well-known device name to access.
978 	 */
979 	usb_dev = make_dev(&usb_static_devsw, 0, UID_ROOT, GID_OPERATOR,
980 	    0644, USB_DEVICE_NAME);
981 	if (usb_dev == NULL) {
982 		DPRINTFN(0, "Could not create usb bus device\n");
983 	}
984 }
985 
986 SYSINIT(usb_dev_init_post, SI_SUB_KICK_SCHEDULER, SI_ORDER_FIRST, usb_dev_init_post, NULL);
987 
988 static void
989 usb_dev_uninit(void *arg)
990 {
991 	if (usb_dev != NULL) {
992 		destroy_dev(usb_dev);
993 		usb_dev = NULL;
994 	}
995 	mtx_destroy(&usb_ref_lock);
996 	sx_destroy(&usb_sym_lock);
997 }
998 
999 SYSUNINIT(usb_dev_uninit, SI_SUB_KICK_SCHEDULER, SI_ORDER_ANY, usb_dev_uninit, NULL);
1000 
1001 static int
1002 usb_ioctl_f_sub(struct usb_fifo *f, u_long cmd, void *addr,
1003     struct thread *td)
1004 {
1005 	int error = 0;
1006 
1007 	switch (cmd) {
1008 	case FIODTYPE:
1009 		*(int *)addr = 0;	/* character device */
1010 		break;
1011 
1012 	case FIONBIO:
1013 		/* handled by upper FS layer */
1014 		break;
1015 
1016 	case FIOASYNC:
1017 		if (*(int *)addr) {
1018 			if (f->async_p != NULL) {
1019 				error = EBUSY;
1020 				break;
1021 			}
1022 			f->async_p = USB_TD_GET_PROC(td);
1023 		} else {
1024 			f->async_p = NULL;
1025 		}
1026 		break;
1027 
1028 		/* XXX this is not the most general solution */
1029 	case TIOCSPGRP:
1030 		if (f->async_p == NULL) {
1031 			error = EINVAL;
1032 			break;
1033 		}
1034 		if (*(int *)addr != USB_PROC_GET_GID(f->async_p)) {
1035 			error = EPERM;
1036 			break;
1037 		}
1038 		break;
1039 	default:
1040 		return (ENOIOCTL);
1041 	}
1042 	DPRINTFN(3, "cmd 0x%lx = %d\n", cmd, error);
1043 	return (error);
1044 }
1045 
1046 /*------------------------------------------------------------------------*
1047  *	usb_ioctl - cdev callback
1048  *------------------------------------------------------------------------*/
1049 static int
1050 usb_ioctl(struct cdev *dev, u_long cmd, caddr_t addr, int fflag, struct thread* td)
1051 {
1052 	struct usb_cdev_refdata refs;
1053 	struct usb_cdev_privdata* cpd;
1054 	struct usb_fifo *f;
1055 	int fflags;
1056 	int err;
1057 
1058 	DPRINTFN(2, "cmd=0x%lx\n", cmd);
1059 
1060 	err = devfs_get_cdevpriv((void **)&cpd);
1061 	if (err != 0)
1062 		return (err);
1063 
1064 	/*
1065 	 * Performance optimisation: We try to check for IOCTL's that
1066 	 * don't need the USB reference first. Then we grab the USB
1067 	 * reference if we need it!
1068 	 */
1069 	err = usb_ref_device(cpd, &refs, 0 /* no uref */ );
1070 	if (err)
1071 		return (ENXIO);
1072 
1073 	fflags = cpd->fflags;
1074 
1075 	f = NULL;			/* set default value */
1076 	err = ENOIOCTL;			/* set default value */
1077 
1078 	if (fflags & FWRITE) {
1079 		f = refs.txfifo;
1080 		err = usb_ioctl_f_sub(f, cmd, addr, td);
1081 	}
1082 	if (fflags & FREAD) {
1083 		f = refs.rxfifo;
1084 		err = usb_ioctl_f_sub(f, cmd, addr, td);
1085 	}
1086 	KASSERT(f != NULL, ("fifo not found"));
1087 	if (err != ENOIOCTL)
1088 		goto done;
1089 
1090 	err = (f->methods->f_ioctl) (f, cmd, addr, fflags);
1091 
1092 	DPRINTFN(2, "f_ioctl cmd 0x%lx = %d\n", cmd, err);
1093 
1094 	if (err != ENOIOCTL)
1095 		goto done;
1096 
1097 	if (usb_usb_ref_device(cpd, &refs)) {
1098 		err = ENXIO;
1099 		goto done;
1100 	}
1101 
1102 	err = (f->methods->f_ioctl_post) (f, cmd, addr, fflags);
1103 
1104 	DPRINTFN(2, "f_ioctl_post cmd 0x%lx = %d\n", cmd, err);
1105 
1106 	if (err == ENOIOCTL)
1107 		err = ENOTTY;
1108 
1109 	if (err)
1110 		goto done;
1111 
1112 	/* Wait for re-enumeration, if any */
1113 
1114 	while (f->udev->re_enumerate_wait != USB_RE_ENUM_DONE) {
1115 
1116 		usb_unref_device(cpd, &refs);
1117 
1118 		usb_pause_mtx(NULL, hz / 128);
1119 
1120 		if (usb_ref_device(cpd, &refs, 1 /* need uref */)) {
1121 			err = ENXIO;
1122 			goto done;
1123 		}
1124 	}
1125 
1126 done:
1127 	usb_unref_device(cpd, &refs);
1128 	return (err);
1129 }
1130 
1131 static void
1132 usb_filter_detach(struct knote *kn)
1133 {
1134 	struct usb_fifo *f = kn->kn_hook;
1135 	knlist_remove(&f->selinfo.si_note, kn, 0);
1136 }
1137 
1138 static int
1139 usb_filter_write(struct knote *kn, long hint)
1140 {
1141 	struct usb_cdev_privdata* cpd;
1142 	struct usb_fifo *f;
1143 	struct usb_mbuf *m;
1144 
1145 	DPRINTFN(2, "\n");
1146 
1147 	f = kn->kn_hook;
1148 
1149 	mtx_assert(f->priv_mtx, MA_OWNED);
1150 
1151 	cpd = f->curr_cpd;
1152 	if (cpd == NULL) {
1153 		m = (void *)1;
1154 	} else if (f->fs_ep_max == 0) {
1155 		if (f->flag_iserror) {
1156 			/* we got an error */
1157 			m = (void *)1;
1158 		} else {
1159 			if (f->queue_data == NULL) {
1160 				/*
1161 				 * start write transfer, if not
1162 				 * already started
1163 				 */
1164 				(f->methods->f_start_write) (f);
1165 			}
1166 			/* check if any packets are available */
1167 			USB_IF_POLL(&f->free_q, m);
1168 		}
1169 	} else {
1170 		if (f->flag_iscomplete) {
1171 			m = (void *)1;
1172 		} else {
1173 			m = NULL;
1174 		}
1175 	}
1176 	return (m ? 1 : 0);
1177 }
1178 
1179 static int
1180 usb_filter_read(struct knote *kn, long hint)
1181 {
1182 	struct usb_cdev_privdata* cpd;
1183 	struct usb_fifo *f;
1184 	struct usb_mbuf *m;
1185 
1186 	DPRINTFN(2, "\n");
1187 
1188 	f = kn->kn_hook;
1189 
1190 	mtx_assert(f->priv_mtx, MA_OWNED);
1191 
1192 	cpd = f->curr_cpd;
1193 	if (cpd == NULL) {
1194 		m = (void *)1;
1195 	} else if (f->fs_ep_max == 0) {
1196 		if (f->flag_iserror) {
1197 			/* we have an error */
1198 			m = (void *)1;
1199 		} else {
1200 			if (f->queue_data == NULL) {
1201 				/*
1202 				 * start read transfer, if not
1203 				 * already started
1204 				 */
1205 				(f->methods->f_start_read) (f);
1206 			}
1207 			/* check if any packets are available */
1208 			USB_IF_POLL(&f->used_q, m);
1209 
1210 			/* start reading data, if any */
1211 			if (m == NULL)
1212 				(f->methods->f_start_read) (f);
1213 		}
1214 	} else {
1215 		if (f->flag_iscomplete) {
1216 			m = (void *)1;
1217 		} else {
1218 			m = NULL;
1219 		}
1220 	}
1221 	return (m ? 1 : 0);
1222 }
1223 
1224 static struct filterops usb_filtops_write = {
1225 	.f_isfd = 1,
1226 	.f_detach = usb_filter_detach,
1227 	.f_event = usb_filter_write,
1228 };
1229 
1230 static struct filterops usb_filtops_read = {
1231 	.f_isfd = 1,
1232 	.f_detach = usb_filter_detach,
1233 	.f_event = usb_filter_read,
1234 };
1235 
1236 
1237 /* ARGSUSED */
1238 static int
1239 usb_kqfilter(struct cdev* dev, struct knote *kn)
1240 {
1241 	struct usb_cdev_refdata refs;
1242 	struct usb_cdev_privdata* cpd;
1243 	struct usb_fifo *f;
1244 	int fflags;
1245 	int err = EINVAL;
1246 
1247 	DPRINTFN(2, "\n");
1248 
1249 	if (devfs_get_cdevpriv((void **)&cpd) != 0 ||
1250 	    usb_ref_device(cpd, &refs, 0) != 0)
1251 		return (ENXIO);
1252 
1253 	fflags = cpd->fflags;
1254 
1255 	/* Figure out who needs service */
1256 	switch (kn->kn_filter) {
1257 	case EVFILT_WRITE:
1258 		if (fflags & FWRITE) {
1259 			f = refs.txfifo;
1260 			kn->kn_fop = &usb_filtops_write;
1261 			err = 0;
1262 		}
1263 		break;
1264 	case EVFILT_READ:
1265 		if (fflags & FREAD) {
1266 			f = refs.rxfifo;
1267 			kn->kn_fop = &usb_filtops_read;
1268 			err = 0;
1269 		}
1270 		break;
1271 	default:
1272 		err = EOPNOTSUPP;
1273 		break;
1274 	}
1275 
1276 	if (err == 0) {
1277 		kn->kn_hook = f;
1278 		mtx_lock(f->priv_mtx);
1279 		knlist_add(&f->selinfo.si_note, kn, 1);
1280 		mtx_unlock(f->priv_mtx);
1281 	}
1282 
1283 	usb_unref_device(cpd, &refs);
1284 	return (err);
1285 }
1286 
1287 /* ARGSUSED */
1288 static int
1289 usb_poll(struct cdev* dev, int events, struct thread* td)
1290 {
1291 	struct usb_cdev_refdata refs;
1292 	struct usb_cdev_privdata* cpd;
1293 	struct usb_fifo *f;
1294 	struct usb_mbuf *m;
1295 	int fflags, revents;
1296 
1297 	if (devfs_get_cdevpriv((void **)&cpd) != 0 ||
1298 	    usb_ref_device(cpd, &refs, 0) != 0)
1299 		return (events &
1300 		    (POLLHUP|POLLIN|POLLRDNORM|POLLOUT|POLLWRNORM));
1301 
1302 	fflags = cpd->fflags;
1303 
1304 	/* Figure out who needs service */
1305 	revents = 0;
1306 	if ((events & (POLLOUT | POLLWRNORM)) &&
1307 	    (fflags & FWRITE)) {
1308 
1309 		f = refs.txfifo;
1310 
1311 		mtx_lock(f->priv_mtx);
1312 
1313 		if (!refs.is_usbfs) {
1314 			if (f->flag_iserror) {
1315 				/* we got an error */
1316 				m = (void *)1;
1317 			} else {
1318 				if (f->queue_data == NULL) {
1319 					/*
1320 					 * start write transfer, if not
1321 					 * already started
1322 					 */
1323 					(f->methods->f_start_write) (f);
1324 				}
1325 				/* check if any packets are available */
1326 				USB_IF_POLL(&f->free_q, m);
1327 			}
1328 		} else {
1329 			if (f->flag_iscomplete) {
1330 				m = (void *)1;
1331 			} else {
1332 				m = NULL;
1333 			}
1334 		}
1335 
1336 		if (m) {
1337 			revents |= events & (POLLOUT | POLLWRNORM);
1338 		} else {
1339 			f->flag_isselect = 1;
1340 			selrecord(td, &f->selinfo);
1341 		}
1342 
1343 		mtx_unlock(f->priv_mtx);
1344 	}
1345 	if ((events & (POLLIN | POLLRDNORM)) &&
1346 	    (fflags & FREAD)) {
1347 
1348 		f = refs.rxfifo;
1349 
1350 		mtx_lock(f->priv_mtx);
1351 
1352 		if (!refs.is_usbfs) {
1353 			if (f->flag_iserror) {
1354 				/* we have an error */
1355 				m = (void *)1;
1356 			} else {
1357 				if (f->queue_data == NULL) {
1358 					/*
1359 					 * start read transfer, if not
1360 					 * already started
1361 					 */
1362 					(f->methods->f_start_read) (f);
1363 				}
1364 				/* check if any packets are available */
1365 				USB_IF_POLL(&f->used_q, m);
1366 			}
1367 		} else {
1368 			if (f->flag_iscomplete) {
1369 				m = (void *)1;
1370 			} else {
1371 				m = NULL;
1372 			}
1373 		}
1374 
1375 		if (m) {
1376 			revents |= events & (POLLIN | POLLRDNORM);
1377 		} else {
1378 			f->flag_isselect = 1;
1379 			selrecord(td, &f->selinfo);
1380 
1381 			if (!refs.is_usbfs) {
1382 				/* start reading data */
1383 				(f->methods->f_start_read) (f);
1384 			}
1385 		}
1386 
1387 		mtx_unlock(f->priv_mtx);
1388 	}
1389 	usb_unref_device(cpd, &refs);
1390 	return (revents);
1391 }
1392 
1393 static int
1394 usb_read(struct cdev *dev, struct uio *uio, int ioflag)
1395 {
1396 	struct usb_cdev_refdata refs;
1397 	struct usb_cdev_privdata* cpd;
1398 	struct usb_fifo *f;
1399 	struct usb_mbuf *m;
1400 	int fflags;
1401 	int resid;
1402 	int io_len;
1403 	int err;
1404 	uint8_t tr_data = 0;
1405 
1406 	err = devfs_get_cdevpriv((void **)&cpd);
1407 	if (err != 0)
1408 		return (err);
1409 
1410 	err = usb_ref_device(cpd, &refs, 0 /* no uref */ );
1411 	if (err) {
1412 		return (ENXIO);
1413 	}
1414 	fflags = cpd->fflags;
1415 
1416 	f = refs.rxfifo;
1417 	if (f == NULL) {
1418 		/* should not happen */
1419 		usb_unref_device(cpd, &refs);
1420 		return (EPERM);
1421 	}
1422 
1423 	resid = uio->uio_resid;
1424 
1425 	mtx_lock(f->priv_mtx);
1426 
1427 	/* check for permanent read error */
1428 	if (f->flag_iserror) {
1429 		err = EIO;
1430 		goto done;
1431 	}
1432 	/* check if USB-FS interface is active */
1433 	if (refs.is_usbfs) {
1434 		/*
1435 		 * The queue is used for events that should be
1436 		 * retrieved using the "USB_FS_COMPLETE" ioctl.
1437 		 */
1438 		err = EINVAL;
1439 		goto done;
1440 	}
1441 	while (uio->uio_resid > 0) {
1442 
1443 		USB_IF_DEQUEUE(&f->used_q, m);
1444 
1445 		if (m == NULL) {
1446 
1447 			/* start read transfer, if not already started */
1448 
1449 			(f->methods->f_start_read) (f);
1450 
1451 			if (ioflag & IO_NDELAY) {
1452 				if (tr_data) {
1453 					/* return length before error */
1454 					break;
1455 				}
1456 				err = EWOULDBLOCK;
1457 				break;
1458 			}
1459 			DPRINTF("sleeping\n");
1460 
1461 			err = usb_fifo_wait(f);
1462 			if (err) {
1463 				break;
1464 			}
1465 			continue;
1466 		}
1467 		if (f->methods->f_filter_read) {
1468 			/*
1469 			 * Sometimes it is convenient to process data at the
1470 			 * expense of a userland process instead of a kernel
1471 			 * process.
1472 			 */
1473 			(f->methods->f_filter_read) (f, m);
1474 		}
1475 		tr_data = 1;
1476 
1477 		io_len = MIN(m->cur_data_len, uio->uio_resid);
1478 
1479 		DPRINTFN(2, "transfer %d bytes from %p\n",
1480 		    io_len, m->cur_data_ptr);
1481 
1482 		err = usb_fifo_uiomove(f,
1483 		    m->cur_data_ptr, io_len, uio);
1484 
1485 		m->cur_data_len -= io_len;
1486 		m->cur_data_ptr += io_len;
1487 
1488 		if (m->cur_data_len == 0) {
1489 
1490 			uint8_t last_packet;
1491 
1492 			last_packet = m->last_packet;
1493 
1494 			USB_IF_ENQUEUE(&f->free_q, m);
1495 
1496 			if (last_packet) {
1497 				/* keep framing */
1498 				break;
1499 			}
1500 		} else {
1501 			USB_IF_PREPEND(&f->used_q, m);
1502 		}
1503 
1504 		if (err) {
1505 			break;
1506 		}
1507 	}
1508 done:
1509 	mtx_unlock(f->priv_mtx);
1510 
1511 	usb_unref_device(cpd, &refs);
1512 
1513 	return (err);
1514 }
1515 
1516 static int
1517 usb_write(struct cdev *dev, struct uio *uio, int ioflag)
1518 {
1519 	struct usb_cdev_refdata refs;
1520 	struct usb_cdev_privdata* cpd;
1521 	struct usb_fifo *f;
1522 	struct usb_mbuf *m;
1523 	uint8_t *pdata;
1524 	int fflags;
1525 	int resid;
1526 	int io_len;
1527 	int err;
1528 	uint8_t tr_data = 0;
1529 
1530 	DPRINTFN(2, "\n");
1531 
1532 	err = devfs_get_cdevpriv((void **)&cpd);
1533 	if (err != 0)
1534 		return (err);
1535 
1536 	err = usb_ref_device(cpd, &refs, 0 /* no uref */ );
1537 	if (err) {
1538 		return (ENXIO);
1539 	}
1540 	fflags = cpd->fflags;
1541 
1542 	f = refs.txfifo;
1543 	if (f == NULL) {
1544 		/* should not happen */
1545 		usb_unref_device(cpd, &refs);
1546 		return (EPERM);
1547 	}
1548 	resid = uio->uio_resid;
1549 
1550 	mtx_lock(f->priv_mtx);
1551 
1552 	/* check for permanent write error */
1553 	if (f->flag_iserror) {
1554 		err = EIO;
1555 		goto done;
1556 	}
1557 	/* check if USB-FS interface is active */
1558 	if (refs.is_usbfs) {
1559 		/*
1560 		 * The queue is used for events that should be
1561 		 * retrieved using the "USB_FS_COMPLETE" ioctl.
1562 		 */
1563 		err = EINVAL;
1564 		goto done;
1565 	}
1566 	if (f->queue_data == NULL) {
1567 		/* start write transfer, if not already started */
1568 		(f->methods->f_start_write) (f);
1569 	}
1570 	/* we allow writing zero length data */
1571 	do {
1572 		USB_IF_DEQUEUE(&f->free_q, m);
1573 
1574 		if (m == NULL) {
1575 
1576 			if (ioflag & IO_NDELAY) {
1577 				if (tr_data) {
1578 					/* return length before error */
1579 					break;
1580 				}
1581 				err = EWOULDBLOCK;
1582 				break;
1583 			}
1584 			DPRINTF("sleeping\n");
1585 
1586 			err = usb_fifo_wait(f);
1587 			if (err) {
1588 				break;
1589 			}
1590 			continue;
1591 		}
1592 		tr_data = 1;
1593 
1594 		if (f->flag_have_fragment == 0) {
1595 			USB_MBUF_RESET(m);
1596 			io_len = m->cur_data_len;
1597 			pdata = m->cur_data_ptr;
1598 			if (io_len > uio->uio_resid)
1599 				io_len = uio->uio_resid;
1600 			m->cur_data_len = io_len;
1601 		} else {
1602 			io_len = m->max_data_len - m->cur_data_len;
1603 			pdata = m->cur_data_ptr + m->cur_data_len;
1604 			if (io_len > uio->uio_resid)
1605 				io_len = uio->uio_resid;
1606 			m->cur_data_len += io_len;
1607 		}
1608 
1609 		DPRINTFN(2, "transfer %d bytes to %p\n",
1610 		    io_len, pdata);
1611 
1612 		err = usb_fifo_uiomove(f, pdata, io_len, uio);
1613 
1614 		if (err) {
1615 			f->flag_have_fragment = 0;
1616 			USB_IF_ENQUEUE(&f->free_q, m);
1617 			break;
1618 		}
1619 
1620 		/* check if the buffer is ready to be transmitted */
1621 
1622 		if ((f->flag_write_defrag == 0) ||
1623 		    (m->cur_data_len == m->max_data_len)) {
1624 			f->flag_have_fragment = 0;
1625 
1626 			/*
1627 			 * Check for write filter:
1628 			 *
1629 			 * Sometimes it is convenient to process data
1630 			 * at the expense of a userland process
1631 			 * instead of a kernel process.
1632 			 */
1633 			if (f->methods->f_filter_write) {
1634 				(f->methods->f_filter_write) (f, m);
1635 			}
1636 
1637 			/* Put USB mbuf in the used queue */
1638 			USB_IF_ENQUEUE(&f->used_q, m);
1639 
1640 			/* Start writing data, if not already started */
1641 			(f->methods->f_start_write) (f);
1642 		} else {
1643 			/* Wait for more data or close */
1644 			f->flag_have_fragment = 1;
1645 			USB_IF_PREPEND(&f->free_q, m);
1646 		}
1647 
1648 	} while (uio->uio_resid > 0);
1649 done:
1650 	mtx_unlock(f->priv_mtx);
1651 
1652 	usb_unref_device(cpd, &refs);
1653 
1654 	return (err);
1655 }
1656 
1657 int
1658 usb_static_ioctl(struct cdev *dev, u_long cmd, caddr_t data, int fflag,
1659     struct thread *td)
1660 {
1661 	union {
1662 		struct usb_read_dir *urd;
1663 		void* data;
1664 	} u;
1665 	int err;
1666 
1667 	u.data = data;
1668 	switch (cmd) {
1669 		case USB_READ_DIR:
1670 			err = usb_read_symlink(u.urd->urd_data,
1671 			    u.urd->urd_startentry, u.urd->urd_maxlen);
1672 			break;
1673 		case USB_DEV_QUIRK_GET:
1674 		case USB_QUIRK_NAME_GET:
1675 		case USB_DEV_QUIRK_ADD:
1676 		case USB_DEV_QUIRK_REMOVE:
1677 			err = usb_quirk_ioctl_p(cmd, data, fflag, td);
1678 			break;
1679 		case USB_GET_TEMPLATE:
1680 			*(int *)data = usb_template;
1681 			err = 0;
1682 			break;
1683 		case USB_SET_TEMPLATE:
1684 			err = priv_check(curthread, PRIV_DRIVER);
1685 			if (err)
1686 				break;
1687 			usb_template = *(int *)data;
1688 			break;
1689 		default:
1690 			err = ENOTTY;
1691 			break;
1692 	}
1693 	return (err);
1694 }
1695 
1696 static int
1697 usb_fifo_uiomove(struct usb_fifo *f, void *cp,
1698     int n, struct uio *uio)
1699 {
1700 	int error;
1701 
1702 	mtx_unlock(f->priv_mtx);
1703 
1704 	/*
1705 	 * "uiomove()" can sleep so one needs to make a wrapper,
1706 	 * exiting the mutex and checking things:
1707 	 */
1708 	error = uiomove(cp, n, uio);
1709 
1710 	mtx_lock(f->priv_mtx);
1711 
1712 	return (error);
1713 }
1714 
1715 int
1716 usb_fifo_wait(struct usb_fifo *f)
1717 {
1718 	int err;
1719 
1720 	mtx_assert(f->priv_mtx, MA_OWNED);
1721 
1722 	if (f->flag_iserror) {
1723 		/* we are gone */
1724 		return (EIO);
1725 	}
1726 	f->flag_sleeping = 1;
1727 
1728 	err = cv_wait_sig(&f->cv_io, f->priv_mtx);
1729 
1730 	if (f->flag_iserror) {
1731 		/* we are gone */
1732 		err = EIO;
1733 	}
1734 	return (err);
1735 }
1736 
1737 void
1738 usb_fifo_signal(struct usb_fifo *f)
1739 {
1740 	if (f->flag_sleeping) {
1741 		f->flag_sleeping = 0;
1742 		cv_broadcast(&f->cv_io);
1743 	}
1744 }
1745 
1746 void
1747 usb_fifo_wakeup(struct usb_fifo *f)
1748 {
1749 	usb_fifo_signal(f);
1750 
1751 	KNOTE_LOCKED(&f->selinfo.si_note, 0);
1752 
1753 	if (f->flag_isselect) {
1754 		selwakeup(&f->selinfo);
1755 		f->flag_isselect = 0;
1756 	}
1757 	if (f->async_p != NULL) {
1758 		PROC_LOCK(f->async_p);
1759 		kern_psignal(f->async_p, SIGIO);
1760 		PROC_UNLOCK(f->async_p);
1761 	}
1762 }
1763 
1764 static int
1765 usb_fifo_dummy_open(struct usb_fifo *fifo, int fflags)
1766 {
1767 	return (0);
1768 }
1769 
1770 static void
1771 usb_fifo_dummy_close(struct usb_fifo *fifo, int fflags)
1772 {
1773 	return;
1774 }
1775 
1776 static int
1777 usb_fifo_dummy_ioctl(struct usb_fifo *fifo, u_long cmd, void *addr, int fflags)
1778 {
1779 	return (ENOIOCTL);
1780 }
1781 
1782 static void
1783 usb_fifo_dummy_cmd(struct usb_fifo *fifo)
1784 {
1785 	fifo->flag_flushing = 0;	/* not flushing */
1786 }
1787 
1788 static void
1789 usb_fifo_check_methods(struct usb_fifo_methods *pm)
1790 {
1791 	/* check that all callback functions are OK */
1792 
1793 	if (pm->f_open == NULL)
1794 		pm->f_open = &usb_fifo_dummy_open;
1795 
1796 	if (pm->f_close == NULL)
1797 		pm->f_close = &usb_fifo_dummy_close;
1798 
1799 	if (pm->f_ioctl == NULL)
1800 		pm->f_ioctl = &usb_fifo_dummy_ioctl;
1801 
1802 	if (pm->f_ioctl_post == NULL)
1803 		pm->f_ioctl_post = &usb_fifo_dummy_ioctl;
1804 
1805 	if (pm->f_start_read == NULL)
1806 		pm->f_start_read = &usb_fifo_dummy_cmd;
1807 
1808 	if (pm->f_stop_read == NULL)
1809 		pm->f_stop_read = &usb_fifo_dummy_cmd;
1810 
1811 	if (pm->f_start_write == NULL)
1812 		pm->f_start_write = &usb_fifo_dummy_cmd;
1813 
1814 	if (pm->f_stop_write == NULL)
1815 		pm->f_stop_write = &usb_fifo_dummy_cmd;
1816 }
1817 
1818 /*------------------------------------------------------------------------*
1819  *	usb_fifo_attach
1820  *
1821  * The following function will create a duplex FIFO.
1822  *
1823  * Return values:
1824  * 0: Success.
1825  * Else: Failure.
1826  *------------------------------------------------------------------------*/
1827 int
1828 usb_fifo_attach(struct usb_device *udev, void *priv_sc,
1829     struct mtx *priv_mtx, struct usb_fifo_methods *pm,
1830     struct usb_fifo_sc *f_sc, uint16_t unit, int16_t subunit,
1831     uint8_t iface_index, uid_t uid, gid_t gid, int mode)
1832 {
1833 	struct usb_fifo *f_tx;
1834 	struct usb_fifo *f_rx;
1835 	char devname[32];
1836 	uint8_t n;
1837 
1838 	f_sc->fp[USB_FIFO_TX] = NULL;
1839 	f_sc->fp[USB_FIFO_RX] = NULL;
1840 
1841 	if (pm == NULL)
1842 		return (EINVAL);
1843 
1844 	/* check the methods */
1845 	usb_fifo_check_methods(pm);
1846 
1847 	if (priv_mtx == NULL)
1848 		priv_mtx = &Giant;
1849 
1850 	/* search for a free FIFO slot */
1851 	for (n = 0;; n += 2) {
1852 
1853 		if (n == USB_FIFO_MAX) {
1854 			/* end of FIFOs reached */
1855 			return (ENOMEM);
1856 		}
1857 		/* Check for TX FIFO */
1858 		if (udev->fifo[n + USB_FIFO_TX] != NULL) {
1859 			continue;
1860 		}
1861 		/* Check for RX FIFO */
1862 		if (udev->fifo[n + USB_FIFO_RX] != NULL) {
1863 			continue;
1864 		}
1865 		break;
1866 	}
1867 
1868 	f_tx = usb_fifo_alloc(priv_mtx);
1869 	f_rx = usb_fifo_alloc(priv_mtx);
1870 
1871 	if ((f_tx == NULL) || (f_rx == NULL)) {
1872 		usb_fifo_free(f_tx);
1873 		usb_fifo_free(f_rx);
1874 		return (ENOMEM);
1875 	}
1876 	/* initialise FIFO structures */
1877 
1878 	f_tx->fifo_index = n + USB_FIFO_TX;
1879 	f_tx->dev_ep_index = -1;
1880 	f_tx->priv_sc0 = priv_sc;
1881 	f_tx->methods = pm;
1882 	f_tx->iface_index = iface_index;
1883 	f_tx->udev = udev;
1884 
1885 	f_rx->fifo_index = n + USB_FIFO_RX;
1886 	f_rx->dev_ep_index = -1;
1887 	f_rx->priv_sc0 = priv_sc;
1888 	f_rx->methods = pm;
1889 	f_rx->iface_index = iface_index;
1890 	f_rx->udev = udev;
1891 
1892 	f_sc->fp[USB_FIFO_TX] = f_tx;
1893 	f_sc->fp[USB_FIFO_RX] = f_rx;
1894 
1895 	mtx_lock(&usb_ref_lock);
1896 	udev->fifo[f_tx->fifo_index] = f_tx;
1897 	udev->fifo[f_rx->fifo_index] = f_rx;
1898 	mtx_unlock(&usb_ref_lock);
1899 
1900 	for (n = 0; n != 4; n++) {
1901 
1902 		if (pm->basename[n] == NULL) {
1903 			continue;
1904 		}
1905 		if (subunit < 0) {
1906 			if (snprintf(devname, sizeof(devname),
1907 			    "%s%u%s", pm->basename[n],
1908 			    unit, pm->postfix[n] ?
1909 			    pm->postfix[n] : "")) {
1910 				/* ignore */
1911 			}
1912 		} else {
1913 			if (snprintf(devname, sizeof(devname),
1914 			    "%s%u.%d%s", pm->basename[n],
1915 			    unit, subunit, pm->postfix[n] ?
1916 			    pm->postfix[n] : "")) {
1917 				/* ignore */
1918 			}
1919 		}
1920 
1921 		/*
1922 		 * Distribute the symbolic links into two FIFO structures:
1923 		 */
1924 		if (n & 1) {
1925 			f_rx->symlink[n / 2] =
1926 			    usb_alloc_symlink(devname);
1927 		} else {
1928 			f_tx->symlink[n / 2] =
1929 			    usb_alloc_symlink(devname);
1930 		}
1931 
1932 		/* Create the device */
1933 		f_sc->dev = usb_make_dev(udev, devname, -1,
1934 		    f_tx->fifo_index & f_rx->fifo_index,
1935 		    FREAD|FWRITE, uid, gid, mode);
1936 	}
1937 
1938 	DPRINTFN(2, "attached %p/%p\n", f_tx, f_rx);
1939 	return (0);
1940 }
1941 
1942 /*------------------------------------------------------------------------*
1943  *	usb_fifo_alloc_buffer
1944  *
1945  * Return values:
1946  * 0: Success
1947  * Else failure
1948  *------------------------------------------------------------------------*/
1949 int
1950 usb_fifo_alloc_buffer(struct usb_fifo *f, usb_size_t bufsize,
1951     uint16_t nbuf)
1952 {
1953 	usb_fifo_free_buffer(f);
1954 
1955 	/* allocate an endpoint */
1956 	f->free_q.ifq_maxlen = nbuf;
1957 	f->used_q.ifq_maxlen = nbuf;
1958 
1959 	f->queue_data = usb_alloc_mbufs(
1960 	    M_USBDEV, &f->free_q, bufsize, nbuf);
1961 
1962 	if ((f->queue_data == NULL) && bufsize && nbuf) {
1963 		return (ENOMEM);
1964 	}
1965 	return (0);			/* success */
1966 }
1967 
1968 /*------------------------------------------------------------------------*
1969  *	usb_fifo_free_buffer
1970  *
1971  * This function will free the buffers associated with a FIFO. This
1972  * function can be called multiple times in a row.
1973  *------------------------------------------------------------------------*/
1974 void
1975 usb_fifo_free_buffer(struct usb_fifo *f)
1976 {
1977 	if (f->queue_data) {
1978 		/* free old buffer */
1979 		free(f->queue_data, M_USBDEV);
1980 		f->queue_data = NULL;
1981 	}
1982 	/* reset queues */
1983 
1984 	memset(&f->free_q, 0, sizeof(f->free_q));
1985 	memset(&f->used_q, 0, sizeof(f->used_q));
1986 }
1987 
1988 void
1989 usb_fifo_detach(struct usb_fifo_sc *f_sc)
1990 {
1991 	if (f_sc == NULL) {
1992 		return;
1993 	}
1994 	usb_fifo_free(f_sc->fp[USB_FIFO_TX]);
1995 	usb_fifo_free(f_sc->fp[USB_FIFO_RX]);
1996 
1997 	f_sc->fp[USB_FIFO_TX] = NULL;
1998 	f_sc->fp[USB_FIFO_RX] = NULL;
1999 
2000 	usb_destroy_dev(f_sc->dev);
2001 
2002 	f_sc->dev = NULL;
2003 
2004 	DPRINTFN(2, "detached %p\n", f_sc);
2005 }
2006 
2007 usb_size_t
2008 usb_fifo_put_bytes_max(struct usb_fifo *f)
2009 {
2010 	struct usb_mbuf *m;
2011 	usb_size_t len;
2012 
2013 	USB_IF_POLL(&f->free_q, m);
2014 
2015 	if (m) {
2016 		len = m->max_data_len;
2017 	} else {
2018 		len = 0;
2019 	}
2020 	return (len);
2021 }
2022 
2023 /*------------------------------------------------------------------------*
2024  *	usb_fifo_put_data
2025  *
2026  * what:
2027  *  0 - normal operation
2028  *  1 - set last packet flag to enforce framing
2029  *------------------------------------------------------------------------*/
2030 void
2031 usb_fifo_put_data(struct usb_fifo *f, struct usb_page_cache *pc,
2032     usb_frlength_t offset, usb_frlength_t len, uint8_t what)
2033 {
2034 	struct usb_mbuf *m;
2035 	usb_frlength_t io_len;
2036 
2037 	while (len || (what == 1)) {
2038 
2039 		USB_IF_DEQUEUE(&f->free_q, m);
2040 
2041 		if (m) {
2042 			USB_MBUF_RESET(m);
2043 
2044 			io_len = MIN(len, m->cur_data_len);
2045 
2046 			usbd_copy_out(pc, offset, m->cur_data_ptr, io_len);
2047 
2048 			m->cur_data_len = io_len;
2049 			offset += io_len;
2050 			len -= io_len;
2051 
2052 			if ((len == 0) && (what == 1)) {
2053 				m->last_packet = 1;
2054 			}
2055 			USB_IF_ENQUEUE(&f->used_q, m);
2056 
2057 			usb_fifo_wakeup(f);
2058 
2059 			if ((len == 0) || (what == 1)) {
2060 				break;
2061 			}
2062 		} else {
2063 			break;
2064 		}
2065 	}
2066 }
2067 
2068 void
2069 usb_fifo_put_data_linear(struct usb_fifo *f, void *ptr,
2070     usb_size_t len, uint8_t what)
2071 {
2072 	struct usb_mbuf *m;
2073 	usb_size_t io_len;
2074 
2075 	while (len || (what == 1)) {
2076 
2077 		USB_IF_DEQUEUE(&f->free_q, m);
2078 
2079 		if (m) {
2080 			USB_MBUF_RESET(m);
2081 
2082 			io_len = MIN(len, m->cur_data_len);
2083 
2084 			memcpy(m->cur_data_ptr, ptr, io_len);
2085 
2086 			m->cur_data_len = io_len;
2087 			ptr = USB_ADD_BYTES(ptr, io_len);
2088 			len -= io_len;
2089 
2090 			if ((len == 0) && (what == 1)) {
2091 				m->last_packet = 1;
2092 			}
2093 			USB_IF_ENQUEUE(&f->used_q, m);
2094 
2095 			usb_fifo_wakeup(f);
2096 
2097 			if ((len == 0) || (what == 1)) {
2098 				break;
2099 			}
2100 		} else {
2101 			break;
2102 		}
2103 	}
2104 }
2105 
2106 uint8_t
2107 usb_fifo_put_data_buffer(struct usb_fifo *f, void *ptr, usb_size_t len)
2108 {
2109 	struct usb_mbuf *m;
2110 
2111 	USB_IF_DEQUEUE(&f->free_q, m);
2112 
2113 	if (m) {
2114 		m->cur_data_len = len;
2115 		m->cur_data_ptr = ptr;
2116 		USB_IF_ENQUEUE(&f->used_q, m);
2117 		usb_fifo_wakeup(f);
2118 		return (1);
2119 	}
2120 	return (0);
2121 }
2122 
2123 void
2124 usb_fifo_put_data_error(struct usb_fifo *f)
2125 {
2126 	f->flag_iserror = 1;
2127 	usb_fifo_wakeup(f);
2128 }
2129 
2130 /*------------------------------------------------------------------------*
2131  *	usb_fifo_get_data
2132  *
2133  * what:
2134  *  0 - normal operation
2135  *  1 - only get one "usb_mbuf"
2136  *
2137  * returns:
2138  *  0 - no more data
2139  *  1 - data in buffer
2140  *------------------------------------------------------------------------*/
2141 uint8_t
2142 usb_fifo_get_data(struct usb_fifo *f, struct usb_page_cache *pc,
2143     usb_frlength_t offset, usb_frlength_t len, usb_frlength_t *actlen,
2144     uint8_t what)
2145 {
2146 	struct usb_mbuf *m;
2147 	usb_frlength_t io_len;
2148 	uint8_t tr_data = 0;
2149 
2150 	actlen[0] = 0;
2151 
2152 	while (1) {
2153 
2154 		USB_IF_DEQUEUE(&f->used_q, m);
2155 
2156 		if (m) {
2157 
2158 			tr_data = 1;
2159 
2160 			io_len = MIN(len, m->cur_data_len);
2161 
2162 			usbd_copy_in(pc, offset, m->cur_data_ptr, io_len);
2163 
2164 			len -= io_len;
2165 			offset += io_len;
2166 			actlen[0] += io_len;
2167 			m->cur_data_ptr += io_len;
2168 			m->cur_data_len -= io_len;
2169 
2170 			if ((m->cur_data_len == 0) || (what == 1)) {
2171 				USB_IF_ENQUEUE(&f->free_q, m);
2172 
2173 				usb_fifo_wakeup(f);
2174 
2175 				if (what == 1) {
2176 					break;
2177 				}
2178 			} else {
2179 				USB_IF_PREPEND(&f->used_q, m);
2180 			}
2181 		} else {
2182 
2183 			if (tr_data) {
2184 				/* wait for data to be written out */
2185 				break;
2186 			}
2187 			if (f->flag_flushing) {
2188 				/* check if we should send a short packet */
2189 				if (f->flag_short != 0) {
2190 					f->flag_short = 0;
2191 					tr_data = 1;
2192 					break;
2193 				}
2194 				/* flushing complete */
2195 				f->flag_flushing = 0;
2196 				usb_fifo_wakeup(f);
2197 			}
2198 			break;
2199 		}
2200 		if (len == 0) {
2201 			break;
2202 		}
2203 	}
2204 	return (tr_data);
2205 }
2206 
2207 uint8_t
2208 usb_fifo_get_data_linear(struct usb_fifo *f, void *ptr,
2209     usb_size_t len, usb_size_t *actlen, uint8_t what)
2210 {
2211 	struct usb_mbuf *m;
2212 	usb_size_t io_len;
2213 	uint8_t tr_data = 0;
2214 
2215 	actlen[0] = 0;
2216 
2217 	while (1) {
2218 
2219 		USB_IF_DEQUEUE(&f->used_q, m);
2220 
2221 		if (m) {
2222 
2223 			tr_data = 1;
2224 
2225 			io_len = MIN(len, m->cur_data_len);
2226 
2227 			memcpy(ptr, m->cur_data_ptr, io_len);
2228 
2229 			len -= io_len;
2230 			ptr = USB_ADD_BYTES(ptr, io_len);
2231 			actlen[0] += io_len;
2232 			m->cur_data_ptr += io_len;
2233 			m->cur_data_len -= io_len;
2234 
2235 			if ((m->cur_data_len == 0) || (what == 1)) {
2236 				USB_IF_ENQUEUE(&f->free_q, m);
2237 
2238 				usb_fifo_wakeup(f);
2239 
2240 				if (what == 1) {
2241 					break;
2242 				}
2243 			} else {
2244 				USB_IF_PREPEND(&f->used_q, m);
2245 			}
2246 		} else {
2247 
2248 			if (tr_data) {
2249 				/* wait for data to be written out */
2250 				break;
2251 			}
2252 			if (f->flag_flushing) {
2253 				/* check if we should send a short packet */
2254 				if (f->flag_short != 0) {
2255 					f->flag_short = 0;
2256 					tr_data = 1;
2257 					break;
2258 				}
2259 				/* flushing complete */
2260 				f->flag_flushing = 0;
2261 				usb_fifo_wakeup(f);
2262 			}
2263 			break;
2264 		}
2265 		if (len == 0) {
2266 			break;
2267 		}
2268 	}
2269 	return (tr_data);
2270 }
2271 
2272 uint8_t
2273 usb_fifo_get_data_buffer(struct usb_fifo *f, void **pptr, usb_size_t *plen)
2274 {
2275 	struct usb_mbuf *m;
2276 
2277 	USB_IF_POLL(&f->used_q, m);
2278 
2279 	if (m) {
2280 		*plen = m->cur_data_len;
2281 		*pptr = m->cur_data_ptr;
2282 
2283 		return (1);
2284 	}
2285 	return (0);
2286 }
2287 
2288 void
2289 usb_fifo_get_data_error(struct usb_fifo *f)
2290 {
2291 	f->flag_iserror = 1;
2292 	usb_fifo_wakeup(f);
2293 }
2294 
2295 /*------------------------------------------------------------------------*
2296  *	usb_alloc_symlink
2297  *
2298  * Return values:
2299  * NULL: Failure
2300  * Else: Pointer to symlink entry
2301  *------------------------------------------------------------------------*/
2302 struct usb_symlink *
2303 usb_alloc_symlink(const char *target)
2304 {
2305 	struct usb_symlink *ps;
2306 
2307 	ps = malloc(sizeof(*ps), M_USBDEV, M_WAITOK);
2308 	if (ps == NULL) {
2309 		return (ps);
2310 	}
2311 	/* XXX no longer needed */
2312 	strlcpy(ps->src_path, target, sizeof(ps->src_path));
2313 	ps->src_len = strlen(ps->src_path);
2314 	strlcpy(ps->dst_path, target, sizeof(ps->dst_path));
2315 	ps->dst_len = strlen(ps->dst_path);
2316 
2317 	sx_xlock(&usb_sym_lock);
2318 	TAILQ_INSERT_TAIL(&usb_sym_head, ps, sym_entry);
2319 	sx_unlock(&usb_sym_lock);
2320 	return (ps);
2321 }
2322 
2323 /*------------------------------------------------------------------------*
2324  *	usb_free_symlink
2325  *------------------------------------------------------------------------*/
2326 void
2327 usb_free_symlink(struct usb_symlink *ps)
2328 {
2329 	if (ps == NULL) {
2330 		return;
2331 	}
2332 	sx_xlock(&usb_sym_lock);
2333 	TAILQ_REMOVE(&usb_sym_head, ps, sym_entry);
2334 	sx_unlock(&usb_sym_lock);
2335 
2336 	free(ps, M_USBDEV);
2337 }
2338 
2339 /*------------------------------------------------------------------------*
2340  *	usb_read_symlink
2341  *
2342  * Return value:
2343  * 0: Success
2344  * Else: Failure
2345  *------------------------------------------------------------------------*/
2346 int
2347 usb_read_symlink(uint8_t *user_ptr, uint32_t startentry, uint32_t user_len)
2348 {
2349 	struct usb_symlink *ps;
2350 	uint32_t temp;
2351 	uint32_t delta = 0;
2352 	uint8_t len;
2353 	int error = 0;
2354 
2355 	sx_xlock(&usb_sym_lock);
2356 
2357 	TAILQ_FOREACH(ps, &usb_sym_head, sym_entry) {
2358 
2359 		/*
2360 		 * Compute total length of source and destination symlink
2361 		 * strings pluss one length byte and two NUL bytes:
2362 		 */
2363 		temp = ps->src_len + ps->dst_len + 3;
2364 
2365 		if (temp > 255) {
2366 			/*
2367 			 * Skip entry because this length cannot fit
2368 			 * into one byte:
2369 			 */
2370 			continue;
2371 		}
2372 		if (startentry != 0) {
2373 			/* decrement read offset */
2374 			startentry--;
2375 			continue;
2376 		}
2377 		if (temp > user_len) {
2378 			/* out of buffer space */
2379 			break;
2380 		}
2381 		len = temp;
2382 
2383 		/* copy out total length */
2384 
2385 		error = copyout(&len,
2386 		    USB_ADD_BYTES(user_ptr, delta), 1);
2387 		if (error) {
2388 			break;
2389 		}
2390 		delta += 1;
2391 
2392 		/* copy out source string */
2393 
2394 		error = copyout(ps->src_path,
2395 		    USB_ADD_BYTES(user_ptr, delta), ps->src_len);
2396 		if (error) {
2397 			break;
2398 		}
2399 		len = 0;
2400 		delta += ps->src_len;
2401 		error = copyout(&len,
2402 		    USB_ADD_BYTES(user_ptr, delta), 1);
2403 		if (error) {
2404 			break;
2405 		}
2406 		delta += 1;
2407 
2408 		/* copy out destination string */
2409 
2410 		error = copyout(ps->dst_path,
2411 		    USB_ADD_BYTES(user_ptr, delta), ps->dst_len);
2412 		if (error) {
2413 			break;
2414 		}
2415 		len = 0;
2416 		delta += ps->dst_len;
2417 		error = copyout(&len,
2418 		    USB_ADD_BYTES(user_ptr, delta), 1);
2419 		if (error) {
2420 			break;
2421 		}
2422 		delta += 1;
2423 
2424 		user_len -= temp;
2425 	}
2426 
2427 	/* a zero length entry indicates the end */
2428 
2429 	if ((user_len != 0) && (error == 0)) {
2430 
2431 		len = 0;
2432 
2433 		error = copyout(&len,
2434 		    USB_ADD_BYTES(user_ptr, delta), 1);
2435 	}
2436 	sx_unlock(&usb_sym_lock);
2437 	return (error);
2438 }
2439 
2440 void
2441 usb_fifo_set_close_zlp(struct usb_fifo *f, uint8_t onoff)
2442 {
2443 	if (f == NULL)
2444 		return;
2445 
2446 	/* send a Zero Length Packet, ZLP, before close */
2447 	f->flag_short = onoff;
2448 }
2449 
2450 void
2451 usb_fifo_set_write_defrag(struct usb_fifo *f, uint8_t onoff)
2452 {
2453 	if (f == NULL)
2454 		return;
2455 
2456 	/* defrag written data */
2457 	f->flag_write_defrag = onoff;
2458 	/* reset defrag state */
2459 	f->flag_have_fragment = 0;
2460 }
2461 
2462 void *
2463 usb_fifo_softc(struct usb_fifo *f)
2464 {
2465 	return (f->priv_sc0);
2466 }
2467 #endif	/* USB_HAVE_UGEN */
2468