xref: /freebsd/sys/dev/usb/storage/cfumass.c (revision 5bb3134a8c21cb87b30e135ef168483f0333dabb)
1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3  *
4  * Copyright (c) 2016 The FreeBSD Foundation
5  *
6  * This software was developed by Edward Tomasz Napierala under sponsorship
7  * from the FreeBSD Foundation.
8  *
9  * Redistribution and use in source and binary forms, with or without
10  * modification, are permitted provided that the following conditions
11  * are met:
12  * 1. Redistributions of source code must retain the above copyright
13  *    notice, this list of conditions and the following disclaimer.
14  * 2. Redistributions in binary form must reproduce the above copyright
15  *    notice, this list of conditions and the following disclaimer in the
16  *    documentation and/or other materials provided with the distribution.
17  *
18  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
19  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
22  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28  * SUCH DAMAGE.
29  *
30  */
31 /*
32  * USB Mass Storage Class Bulk-Only (BBB) Transport target.
33  *
34  * http://www.usb.org/developers/docs/devclass_docs/usbmassbulk_10.pdf
35  *
36  * This code implements the USB Mass Storage frontend driver for the CAM
37  * Target Layer (ctl(4)) subsystem.
38  */
39 
40 #include <sys/cdefs.h>
41 __FBSDID("$FreeBSD$");
42 
43 #include <sys/param.h>
44 #include <sys/bus.h>
45 #include <sys/kernel.h>
46 #include <sys/lock.h>
47 #include <sys/module.h>
48 #include <sys/mutex.h>
49 #include <sys/refcount.h>
50 #include <sys/stdint.h>
51 #include <sys/sysctl.h>
52 #include <sys/systm.h>
53 
54 #include <dev/usb/usb.h>
55 #include <dev/usb/usbdi.h>
56 #include "usbdevs.h"
57 #include "usb_if.h"
58 
59 #include <cam/scsi/scsi_all.h>
60 #include <cam/scsi/scsi_da.h>
61 #include <cam/ctl/ctl_io.h>
62 #include <cam/ctl/ctl.h>
63 #include <cam/ctl/ctl_backend.h>
64 #include <cam/ctl/ctl_error.h>
65 #include <cam/ctl/ctl_frontend.h>
66 #include <cam/ctl/ctl_debug.h>
67 #include <cam/ctl/ctl_ha.h>
68 #include <cam/ctl/ctl_ioctl.h>
69 #include <cam/ctl/ctl_private.h>
70 
71 SYSCTL_NODE(_hw_usb, OID_AUTO, cfumass, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
72     "CAM Target Layer USB Mass Storage Frontend");
73 static int debug = 1;
74 SYSCTL_INT(_hw_usb_cfumass, OID_AUTO, debug, CTLFLAG_RWTUN,
75     &debug, 1, "Enable debug messages");
76 static int max_lun = 0;
77 SYSCTL_INT(_hw_usb_cfumass, OID_AUTO, max_lun, CTLFLAG_RWTUN,
78     &max_lun, 1, "Maximum advertised LUN number");
79 static int ignore_stop = 1;
80 SYSCTL_INT(_hw_usb_cfumass, OID_AUTO, ignore_stop, CTLFLAG_RWTUN,
81     &ignore_stop, 1, "Ignore START STOP UNIT with START and LOEJ bits cleared");
82 
83 /*
84  * The driver uses a single, global CTL port.  It could create its ports
85  * in cfumass_attach() instead, but that would make it impossible to specify
86  * "port cfumass0" in ctl.conf(5), as the port generally wouldn't exist
87  * at the time ctld(8) gets run.
88  */
89 struct ctl_port	cfumass_port;
90 bool		cfumass_port_online;
91 volatile u_int	cfumass_refcount;
92 
93 #ifndef CFUMASS_BULK_SIZE
94 #define	CFUMASS_BULK_SIZE	(1U << 17)	/* bytes */
95 #endif
96 
97 /*
98  * USB transfer definitions.
99  */
100 #define	CFUMASS_T_COMMAND	0
101 #define	CFUMASS_T_DATA_OUT	1
102 #define	CFUMASS_T_DATA_IN	2
103 #define	CFUMASS_T_STATUS	3
104 #define	CFUMASS_T_MAX		4
105 
106 /*
107  * USB interface specific control requests.
108  */
109 #define	UR_RESET	0xff	/* Bulk-Only Mass Storage Reset */
110 #define	UR_GET_MAX_LUN	0xfe	/* Get Max LUN */
111 
112 /*
113  * Command Block Wrapper.
114  */
115 struct cfumass_cbw_t {
116 	uDWord	dCBWSignature;
117 #define	CBWSIGNATURE		0x43425355 /* "USBC" */
118 	uDWord	dCBWTag;
119 	uDWord	dCBWDataTransferLength;
120 	uByte	bCBWFlags;
121 #define	CBWFLAGS_OUT		0x00
122 #define	CBWFLAGS_IN		0x80
123 	uByte	bCBWLUN;
124 	uByte	bCDBLength;
125 #define	CBWCBLENGTH		16
126 	uByte	CBWCB[CBWCBLENGTH];
127 } __packed;
128 
129 #define	CFUMASS_CBW_SIZE	31
130 CTASSERT(sizeof(struct cfumass_cbw_t) == CFUMASS_CBW_SIZE);
131 
132 /*
133  * Command Status Wrapper.
134  */
135 struct cfumass_csw_t {
136 	uDWord	dCSWSignature;
137 #define	CSWSIGNATURE		0x53425355 /* "USBS" */
138 	uDWord	dCSWTag;
139 	uDWord	dCSWDataResidue;
140 	uByte	bCSWStatus;
141 #define	CSWSTATUS_GOOD		0x0
142 #define	CSWSTATUS_FAILED	0x1
143 #define	CSWSTATUS_PHASE		0x2
144 } __packed;
145 
146 #define	CFUMASS_CSW_SIZE	13
147 CTASSERT(sizeof(struct cfumass_csw_t) == CFUMASS_CSW_SIZE);
148 
149 struct cfumass_softc {
150 	device_t		sc_dev;
151 	struct usb_device	*sc_udev;
152 	struct usb_xfer		*sc_xfer[CFUMASS_T_MAX];
153 
154 	struct cfumass_cbw_t *sc_cbw;
155 	struct cfumass_csw_t *sc_csw;
156 
157 	struct mtx	sc_mtx;
158 	int		sc_online;
159 	int		sc_ctl_initid;
160 
161 	/*
162 	 * This is used to communicate between CTL callbacks
163 	 * and USB callbacks; basically, it holds the state
164 	 * for the current command ("the" command, since there
165 	 * is no queueing in USB Mass Storage).
166 	 */
167 	bool		sc_current_stalled;
168 
169 	/*
170 	 * The following are set upon receiving a SCSI command.
171 	 */
172 	int		sc_current_tag;
173 	int		sc_current_transfer_length;
174 	int		sc_current_flags;
175 
176 	/*
177 	 * The following are set in ctl_datamove().
178 	 */
179 	int		sc_current_residue;
180 	union ctl_io	*sc_ctl_io;
181 
182 	/*
183 	 * The following is set in cfumass_done().
184 	 */
185 	int		sc_current_status;
186 
187 	/*
188 	 * Number of requests queued to CTL.
189 	 */
190 	volatile u_int	sc_queued;
191 };
192 
193 /*
194  * USB interface.
195  */
196 static device_probe_t		cfumass_probe;
197 static device_attach_t		cfumass_attach;
198 static device_detach_t		cfumass_detach;
199 static device_suspend_t		cfumass_suspend;
200 static device_resume_t		cfumass_resume;
201 static usb_handle_request_t	cfumass_handle_request;
202 
203 static usb_callback_t		cfumass_t_command_callback;
204 static usb_callback_t		cfumass_t_data_callback;
205 static usb_callback_t		cfumass_t_status_callback;
206 
207 static device_method_t cfumass_methods[] = {
208 	/* USB interface. */
209 	DEVMETHOD(usb_handle_request, cfumass_handle_request),
210 
211 	/* Device interface. */
212 	DEVMETHOD(device_probe, cfumass_probe),
213 	DEVMETHOD(device_attach, cfumass_attach),
214 	DEVMETHOD(device_detach, cfumass_detach),
215 	DEVMETHOD(device_suspend, cfumass_suspend),
216 	DEVMETHOD(device_resume, cfumass_resume),
217 
218 	DEVMETHOD_END
219 };
220 
221 static driver_t cfumass_driver = {
222 	.name = "cfumass",
223 	.methods = cfumass_methods,
224 	.size = sizeof(struct cfumass_softc),
225 };
226 
227 static devclass_t cfumass_devclass;
228 
229 DRIVER_MODULE(cfumass, uhub, cfumass_driver, cfumass_devclass, NULL, 0);
230 MODULE_VERSION(cfumass, 0);
231 MODULE_DEPEND(cfumass, usb, 1, 1, 1);
232 MODULE_DEPEND(cfumass, usb_template, 1, 1, 1);
233 
234 static struct usb_config cfumass_config[CFUMASS_T_MAX] = {
235 	[CFUMASS_T_COMMAND] = {
236 		.type = UE_BULK,
237 		.endpoint = UE_ADDR_ANY,
238 		.direction = UE_DIR_OUT,
239 		.bufsize = sizeof(struct cfumass_cbw_t),
240 		.callback = &cfumass_t_command_callback,
241 		.usb_mode = USB_MODE_DEVICE,
242 	},
243 
244 	[CFUMASS_T_DATA_OUT] = {
245 		.type = UE_BULK,
246 		.endpoint = UE_ADDR_ANY,
247 		.direction = UE_DIR_OUT,
248 		.bufsize = CFUMASS_BULK_SIZE,
249 		.flags = {.proxy_buffer = 1, .short_xfer_ok = 1,
250 		    .ext_buffer = 1},
251 		.callback = &cfumass_t_data_callback,
252 		.usb_mode = USB_MODE_DEVICE,
253 	},
254 
255 	[CFUMASS_T_DATA_IN] = {
256 		.type = UE_BULK,
257 		.endpoint = UE_ADDR_ANY,
258 		.direction = UE_DIR_IN,
259 		.bufsize = CFUMASS_BULK_SIZE,
260 		.flags = {.proxy_buffer = 1, .short_xfer_ok = 1,
261 		    .ext_buffer = 1},
262 		.callback = &cfumass_t_data_callback,
263 		.usb_mode = USB_MODE_DEVICE,
264 	},
265 
266 	[CFUMASS_T_STATUS] = {
267 		.type = UE_BULK,
268 		.endpoint = UE_ADDR_ANY,
269 		.direction = UE_DIR_IN,
270 		.bufsize = sizeof(struct cfumass_csw_t),
271 		.flags = {.short_xfer_ok = 1},
272 		.callback = &cfumass_t_status_callback,
273 		.usb_mode = USB_MODE_DEVICE,
274 	},
275 };
276 
277 /*
278  * CTL frontend interface.
279  */
280 static int	cfumass_init(void);
281 static int	cfumass_shutdown(void);
282 static void	cfumass_online(void *arg);
283 static void	cfumass_offline(void *arg);
284 static void	cfumass_datamove(union ctl_io *io);
285 static void	cfumass_done(union ctl_io *io);
286 
287 static struct ctl_frontend cfumass_frontend = {
288 	.name = "umass",
289 	.init = cfumass_init,
290 	.shutdown = cfumass_shutdown,
291 };
292 CTL_FRONTEND_DECLARE(ctlcfumass, cfumass_frontend);
293 
294 #define	CFUMASS_DEBUG(S, X, ...)					\
295 	do {								\
296 		if (debug > 1) {					\
297 			device_printf(S->sc_dev, "%s: " X "\n",		\
298 			    __func__, ## __VA_ARGS__);			\
299 		}							\
300 	} while (0)
301 
302 #define	CFUMASS_WARN(S, X, ...)						\
303 	do {								\
304 		if (debug > 0) {					\
305 			device_printf(S->sc_dev, "WARNING: %s: " X "\n",\
306 			    __func__, ## __VA_ARGS__);			\
307 		}							\
308 	} while (0)
309 
310 #define CFUMASS_LOCK(X)		mtx_lock(&X->sc_mtx)
311 #define CFUMASS_UNLOCK(X)	mtx_unlock(&X->sc_mtx)
312 
313 static void	cfumass_transfer_start(struct cfumass_softc *sc,
314 		    uint8_t xfer_index);
315 static void	cfumass_terminate(struct cfumass_softc *sc);
316 
317 static int
318 cfumass_probe(device_t dev)
319 {
320 	struct usb_attach_arg *uaa;
321 	struct usb_interface_descriptor *id;
322 
323 	uaa = device_get_ivars(dev);
324 
325 	if (uaa->usb_mode != USB_MODE_DEVICE)
326 		return (ENXIO);
327 
328 	/*
329 	 * Check for a compliant device.
330 	 */
331 	id = usbd_get_interface_descriptor(uaa->iface);
332 	if ((id == NULL) ||
333 	    (id->bInterfaceClass != UICLASS_MASS) ||
334 	    (id->bInterfaceSubClass != UISUBCLASS_SCSI) ||
335 	    (id->bInterfaceProtocol != UIPROTO_MASS_BBB)) {
336 		return (ENXIO);
337 	}
338 
339 	return (BUS_PROBE_GENERIC);
340 }
341 
342 static int
343 cfumass_attach(device_t dev)
344 {
345 	struct cfumass_softc *sc;
346 	struct usb_attach_arg *uaa;
347 	int error;
348 
349 	sc = device_get_softc(dev);
350 	uaa = device_get_ivars(dev);
351 
352 	sc->sc_dev = dev;
353 	sc->sc_udev = uaa->device;
354 
355 	CFUMASS_DEBUG(sc, "go");
356 
357 	usbd_set_power_mode(uaa->device, USB_POWER_MODE_SAVE);
358 	device_set_usb_desc(dev);
359 
360 	mtx_init(&sc->sc_mtx, "cfumass", NULL, MTX_DEF);
361 	refcount_acquire(&cfumass_refcount);
362 
363 	error = usbd_transfer_setup(uaa->device,
364 	    &uaa->info.bIfaceIndex, sc->sc_xfer, cfumass_config,
365 	    CFUMASS_T_MAX, sc, &sc->sc_mtx);
366 	if (error != 0) {
367 		CFUMASS_WARN(sc, "usbd_transfer_setup() failed: %s",
368 		    usbd_errstr(error));
369 		refcount_release(&cfumass_refcount);
370 		return (ENXIO);
371 	}
372 
373 	sc->sc_cbw =
374 	    usbd_xfer_get_frame_buffer(sc->sc_xfer[CFUMASS_T_COMMAND], 0);
375 	sc->sc_csw =
376 	    usbd_xfer_get_frame_buffer(sc->sc_xfer[CFUMASS_T_STATUS], 0);
377 
378 	sc->sc_ctl_initid = ctl_add_initiator(&cfumass_port, -1, 0, NULL);
379 	if (sc->sc_ctl_initid < 0) {
380 		CFUMASS_WARN(sc, "ctl_add_initiator() failed with error %d",
381 		    sc->sc_ctl_initid);
382 		usbd_transfer_unsetup(sc->sc_xfer, CFUMASS_T_MAX);
383 		refcount_release(&cfumass_refcount);
384 		return (ENXIO);
385 	}
386 
387 	refcount_init(&sc->sc_queued, 0);
388 
389 	CFUMASS_LOCK(sc);
390 	cfumass_transfer_start(sc, CFUMASS_T_COMMAND);
391 	CFUMASS_UNLOCK(sc);
392 
393 	return (0);
394 }
395 
396 static int
397 cfumass_detach(device_t dev)
398 {
399 	struct cfumass_softc *sc;
400 	int error;
401 
402 	sc = device_get_softc(dev);
403 
404 	CFUMASS_DEBUG(sc, "go");
405 
406 	CFUMASS_LOCK(sc);
407 	cfumass_terminate(sc);
408 	CFUMASS_UNLOCK(sc);
409 	usbd_transfer_unsetup(sc->sc_xfer, CFUMASS_T_MAX);
410 
411 	if (sc->sc_ctl_initid != -1) {
412 		error = ctl_remove_initiator(&cfumass_port, sc->sc_ctl_initid);
413 		if (error != 0) {
414 			CFUMASS_WARN(sc, "ctl_remove_initiator() failed "
415 			    "with error %d", error);
416 		}
417 		sc->sc_ctl_initid = -1;
418 	}
419 
420 	mtx_destroy(&sc->sc_mtx);
421 	refcount_release(&cfumass_refcount);
422 
423 	return (0);
424 }
425 
426 static int
427 cfumass_suspend(device_t dev)
428 {
429 	struct cfumass_softc *sc;
430 
431 	sc = device_get_softc(dev);
432 	CFUMASS_DEBUG(sc, "go");
433 
434 	return (0);
435 }
436 
437 static int
438 cfumass_resume(device_t dev)
439 {
440 	struct cfumass_softc *sc;
441 
442 	sc = device_get_softc(dev);
443 	CFUMASS_DEBUG(sc, "go");
444 
445 	return (0);
446 }
447 
448 static void
449 cfumass_transfer_start(struct cfumass_softc *sc, uint8_t xfer_index)
450 {
451 
452 	usbd_transfer_start(sc->sc_xfer[xfer_index]);
453 }
454 
455 static void
456 cfumass_transfer_stop_and_drain(struct cfumass_softc *sc, uint8_t xfer_index)
457 {
458 
459 	usbd_transfer_stop(sc->sc_xfer[xfer_index]);
460 	CFUMASS_UNLOCK(sc);
461 	usbd_transfer_drain(sc->sc_xfer[xfer_index]);
462 	CFUMASS_LOCK(sc);
463 }
464 
465 static void
466 cfumass_terminate(struct cfumass_softc *sc)
467 {
468 	int last;
469 
470 	for (;;) {
471 		cfumass_transfer_stop_and_drain(sc, CFUMASS_T_COMMAND);
472 		cfumass_transfer_stop_and_drain(sc, CFUMASS_T_DATA_IN);
473 		cfumass_transfer_stop_and_drain(sc, CFUMASS_T_DATA_OUT);
474 
475 		if (sc->sc_ctl_io != NULL) {
476 			CFUMASS_DEBUG(sc, "terminating CTL transfer");
477 			ctl_set_data_phase_error(&sc->sc_ctl_io->scsiio);
478 			ctl_datamove_done(sc->sc_ctl_io, false);
479 			sc->sc_ctl_io = NULL;
480 		}
481 
482 		cfumass_transfer_stop_and_drain(sc, CFUMASS_T_STATUS);
483 
484 		refcount_acquire(&sc->sc_queued);
485 		last = refcount_release(&sc->sc_queued);
486 		if (last != 0)
487 			break;
488 
489 		CFUMASS_DEBUG(sc, "%d CTL tasks pending", sc->sc_queued);
490 		msleep(__DEVOLATILE(void *, &sc->sc_queued), &sc->sc_mtx,
491 		    0, "cfumass_reset", hz / 100);
492 	}
493 }
494 
495 static int
496 cfumass_handle_request(device_t dev,
497     const void *preq, void **pptr, uint16_t *plen,
498     uint16_t offset, uint8_t *pstate)
499 {
500 	static uint8_t max_lun_tmp;
501 	struct cfumass_softc *sc;
502 	const struct usb_device_request *req;
503 	uint8_t is_complete;
504 
505 	sc = device_get_softc(dev);
506 	req = preq;
507 	is_complete = *pstate;
508 
509 	CFUMASS_DEBUG(sc, "go");
510 
511 	if (is_complete)
512 		return (ENXIO);
513 
514 	if ((req->bmRequestType == UT_WRITE_CLASS_INTERFACE) &&
515 	    (req->bRequest == UR_RESET)) {
516 		CFUMASS_WARN(sc, "received Bulk-Only Mass Storage Reset");
517 		*plen = 0;
518 
519 		CFUMASS_LOCK(sc);
520 		cfumass_terminate(sc);
521 		cfumass_transfer_start(sc, CFUMASS_T_COMMAND);
522 		CFUMASS_UNLOCK(sc);
523 
524 		CFUMASS_DEBUG(sc, "Bulk-Only Mass Storage Reset done");
525 		return (0);
526 	}
527 
528 	if ((req->bmRequestType == UT_READ_CLASS_INTERFACE) &&
529 	    (req->bRequest == UR_GET_MAX_LUN)) {
530 		CFUMASS_DEBUG(sc, "received Get Max LUN");
531 		if (offset == 0) {
532 			*plen = 1;
533 			/*
534 			 * The protocol doesn't support LUN numbers higher
535 			 * than 15.  Also, some initiators (namely Windows XP
536 			 * SP3 Version 2002) can't properly query the number
537 			 * of LUNs, resulting in inaccessible "fake" ones - thus
538 			 * the default limit of one LUN.
539 			 */
540 			if (max_lun < 0 || max_lun > 15) {
541 				CFUMASS_WARN(sc,
542 				    "invalid hw.usb.cfumass.max_lun, must be "
543 				    "between 0 and 15; defaulting to 0");
544 				max_lun_tmp = 0;
545 			} else {
546 				max_lun_tmp = max_lun;
547 			}
548 			*pptr = &max_lun_tmp;
549 		} else {
550 			*plen = 0;
551 		}
552 		return (0);
553 	}
554 
555 	return (ENXIO);
556 }
557 
558 static int
559 cfumass_quirk(struct cfumass_softc *sc, unsigned char *cdb, int cdb_len)
560 {
561 	struct scsi_start_stop_unit *sssu;
562 
563 	switch (cdb[0]) {
564 	case START_STOP_UNIT:
565 		/*
566 		 * Some initiators - eg OSX, Darwin Kernel Version 15.6.0,
567 		 * root:xnu-3248.60.11~2/RELEASE_X86_64 - attempt to stop
568 		 * the unit on eject, but fail to start it when it's plugged
569 		 * back.  Just ignore the command.
570 		 */
571 
572 		if (cdb_len < sizeof(*sssu)) {
573 			CFUMASS_DEBUG(sc, "received START STOP UNIT with "
574 			    "bCDBLength %d, should be %zd",
575 			    cdb_len, sizeof(*sssu));
576 			break;
577 		}
578 
579 		sssu = (struct scsi_start_stop_unit *)cdb;
580 		if ((sssu->how & SSS_PC_MASK) != 0)
581 			break;
582 
583 		if ((sssu->how & SSS_START) != 0)
584 			break;
585 
586 		if ((sssu->how & SSS_LOEJ) != 0)
587 			break;
588 
589 		if (ignore_stop == 0) {
590 			break;
591 		} else if (ignore_stop == 1) {
592 			CFUMASS_WARN(sc, "ignoring START STOP UNIT request");
593 		} else {
594 			CFUMASS_DEBUG(sc, "ignoring START STOP UNIT request");
595 		}
596 
597 		sc->sc_current_status = 0;
598 		cfumass_transfer_start(sc, CFUMASS_T_STATUS);
599 
600 		return (1);
601 	default:
602 		break;
603 	}
604 
605 	return (0);
606 }
607 
608 static void
609 cfumass_t_command_callback(struct usb_xfer *xfer, usb_error_t usb_error)
610 {
611 	struct cfumass_softc *sc;
612 	uint32_t signature;
613 	union ctl_io *io;
614 	int error = 0;
615 
616 	sc = usbd_xfer_softc(xfer);
617 
618 	KASSERT(sc->sc_ctl_io == NULL,
619 	    ("sc_ctl_io is %p, should be NULL", sc->sc_ctl_io));
620 
621 	switch (USB_GET_STATE(xfer)) {
622 	case USB_ST_TRANSFERRED:
623 		CFUMASS_DEBUG(sc, "USB_ST_TRANSFERRED");
624 
625 		signature = UGETDW(sc->sc_cbw->dCBWSignature);
626 		if (signature != CBWSIGNATURE) {
627 			CFUMASS_WARN(sc, "wrong dCBWSignature 0x%08x, "
628 			    "should be 0x%08x", signature, CBWSIGNATURE);
629 			break;
630 		}
631 
632 		if (sc->sc_cbw->bCDBLength <= 0 ||
633 		    sc->sc_cbw->bCDBLength > sizeof(sc->sc_cbw->CBWCB)) {
634 			CFUMASS_WARN(sc, "invalid bCDBLength %d, should be <= %zd",
635 			    sc->sc_cbw->bCDBLength, sizeof(sc->sc_cbw->CBWCB));
636 			break;
637 		}
638 
639 		sc->sc_current_stalled = false;
640 		sc->sc_current_status = 0;
641 		sc->sc_current_tag = UGETDW(sc->sc_cbw->dCBWTag);
642 		sc->sc_current_transfer_length =
643 		    UGETDW(sc->sc_cbw->dCBWDataTransferLength);
644 		sc->sc_current_flags = sc->sc_cbw->bCBWFlags;
645 
646 		/*
647 		 * Make sure to report proper residue if the datamove wasn't
648 		 * required, or wasn't called due to SCSI error.
649 		 */
650 		sc->sc_current_residue = sc->sc_current_transfer_length;
651 
652 		if (cfumass_quirk(sc,
653 		    sc->sc_cbw->CBWCB, sc->sc_cbw->bCDBLength) != 0)
654 			break;
655 
656 		if (!cfumass_port_online) {
657 			CFUMASS_DEBUG(sc, "cfumass port is offline; stalling");
658 			usbd_xfer_set_stall(xfer);
659 			break;
660 		}
661 
662 		/*
663 		 * Those CTL functions cannot be called with mutex held.
664 		 */
665 		CFUMASS_UNLOCK(sc);
666 		io = ctl_alloc_io(cfumass_port.ctl_pool_ref);
667 		ctl_zero_io(io);
668 		io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr = sc;
669 		io->io_hdr.io_type = CTL_IO_SCSI;
670 		io->io_hdr.nexus.initid = sc->sc_ctl_initid;
671 		io->io_hdr.nexus.targ_port = cfumass_port.targ_port;
672 		io->io_hdr.nexus.targ_lun = ctl_decode_lun(sc->sc_cbw->bCBWLUN);
673 		io->scsiio.tag_num = UGETDW(sc->sc_cbw->dCBWTag);
674 		io->scsiio.tag_type = CTL_TAG_UNTAGGED;
675 		io->scsiio.cdb_len = sc->sc_cbw->bCDBLength;
676 		memcpy(io->scsiio.cdb, sc->sc_cbw->CBWCB, sc->sc_cbw->bCDBLength);
677 		refcount_acquire(&sc->sc_queued);
678 		error = ctl_queue(io);
679 		if (error != CTL_RETVAL_COMPLETE) {
680 			CFUMASS_WARN(sc,
681 			    "ctl_queue() failed; error %d; stalling", error);
682 			ctl_free_io(io);
683 			refcount_release(&sc->sc_queued);
684 			CFUMASS_LOCK(sc);
685 			usbd_xfer_set_stall(xfer);
686 			break;
687 		}
688 
689 		CFUMASS_LOCK(sc);
690 		break;
691 
692 	case USB_ST_SETUP:
693 tr_setup:
694 		CFUMASS_DEBUG(sc, "USB_ST_SETUP");
695 
696 		usbd_xfer_set_frame_len(xfer, 0, sizeof(*sc->sc_cbw));
697 		usbd_transfer_submit(xfer);
698 		break;
699 
700 	default:
701 		if (usb_error == USB_ERR_CANCELLED) {
702 			CFUMASS_DEBUG(sc, "USB_ERR_CANCELLED");
703 			break;
704 		}
705 
706 		CFUMASS_DEBUG(sc, "USB_ST_ERROR: %s", usbd_errstr(usb_error));
707 
708 		goto tr_setup;
709 	}
710 }
711 
712 static void
713 cfumass_t_data_callback(struct usb_xfer *xfer, usb_error_t usb_error)
714 {
715 	struct cfumass_softc *sc = usbd_xfer_softc(xfer);
716 	union ctl_io *io = sc->sc_ctl_io;
717 	uint32_t max_bulk;
718 	struct ctl_sg_entry sg_entry, *sglist;
719 	int actlen, sumlen, sg_count;
720 
721 	switch (USB_GET_STATE(xfer)) {
722 	case USB_ST_TRANSFERRED:
723 		CFUMASS_DEBUG(sc, "USB_ST_TRANSFERRED");
724 
725 		usbd_xfer_status(xfer, &actlen, &sumlen, NULL, NULL);
726 		sc->sc_current_residue -= actlen;
727 		io->scsiio.ext_data_filled += actlen;
728 		io->scsiio.kern_data_resid -= actlen;
729 		if (actlen < sumlen ||
730 		    sc->sc_current_residue == 0 ||
731 		    io->scsiio.kern_data_resid == 0) {
732 			sc->sc_ctl_io = NULL;
733 			ctl_datamove_done(io, false);
734 			break;
735 		}
736 		/* FALLTHROUGH */
737 
738 	case USB_ST_SETUP:
739 tr_setup:
740 		CFUMASS_DEBUG(sc, "USB_ST_SETUP");
741 
742 		if (io->scsiio.kern_sg_entries > 0) {
743 			sglist = (struct ctl_sg_entry *)io->scsiio.kern_data_ptr;
744 			sg_count = io->scsiio.kern_sg_entries;
745 		} else {
746 			sglist = &sg_entry;
747 			sglist->addr = io->scsiio.kern_data_ptr;
748 			sglist->len = io->scsiio.kern_data_len;
749 			sg_count = 1;
750 		}
751 
752 		sumlen = io->scsiio.ext_data_filled -
753 		    io->scsiio.kern_rel_offset;
754 		while (sumlen >= sglist->len && sg_count > 0) {
755 			sumlen -= sglist->len;
756 			sglist++;
757 			sg_count--;
758 		}
759 		KASSERT(sg_count > 0, ("Run out of S/G list entries"));
760 
761 		max_bulk = usbd_xfer_max_len(xfer);
762 		actlen = min(sglist->len - sumlen, max_bulk);
763 		actlen = min(actlen, sc->sc_current_transfer_length -
764 		    io->scsiio.ext_data_filled);
765 		CFUMASS_DEBUG(sc, "requested %d, done %d, max_bulk %d, "
766 		    "segment %zd => transfer %d",
767 		    sc->sc_current_transfer_length, io->scsiio.ext_data_filled,
768 		    max_bulk, sglist->len - sumlen, actlen);
769 
770 		usbd_xfer_set_frame_data(xfer, 0,
771 		    (uint8_t *)sglist->addr + sumlen, actlen);
772 		usbd_transfer_submit(xfer);
773 		break;
774 
775 	default:
776 		if (usb_error == USB_ERR_CANCELLED) {
777 			CFUMASS_DEBUG(sc, "USB_ERR_CANCELLED");
778 			break;
779 		}
780 		CFUMASS_DEBUG(sc, "USB_ST_ERROR: %s", usbd_errstr(usb_error));
781 		goto tr_setup;
782 	}
783 }
784 
785 static void
786 cfumass_t_status_callback(struct usb_xfer *xfer, usb_error_t usb_error)
787 {
788 	struct cfumass_softc *sc;
789 
790 	sc = usbd_xfer_softc(xfer);
791 
792 	KASSERT(sc->sc_ctl_io == NULL,
793 	    ("sc_ctl_io is %p, should be NULL", sc->sc_ctl_io));
794 
795 	switch (USB_GET_STATE(xfer)) {
796 	case USB_ST_TRANSFERRED:
797 		CFUMASS_DEBUG(sc, "USB_ST_TRANSFERRED");
798 
799 		cfumass_transfer_start(sc, CFUMASS_T_COMMAND);
800 		break;
801 
802 	case USB_ST_SETUP:
803 tr_setup:
804 		CFUMASS_DEBUG(sc, "USB_ST_SETUP");
805 
806 		if (sc->sc_current_residue > 0 && !sc->sc_current_stalled) {
807 			CFUMASS_DEBUG(sc, "non-zero residue, stalling");
808 			usbd_xfer_set_stall(xfer);
809 			sc->sc_current_stalled = true;
810 		}
811 
812 		USETDW(sc->sc_csw->dCSWSignature, CSWSIGNATURE);
813 		USETDW(sc->sc_csw->dCSWTag, sc->sc_current_tag);
814 		USETDW(sc->sc_csw->dCSWDataResidue, sc->sc_current_residue);
815 		sc->sc_csw->bCSWStatus = sc->sc_current_status;
816 
817 		usbd_xfer_set_frame_len(xfer, 0, sizeof(*sc->sc_csw));
818 		usbd_transfer_submit(xfer);
819 		break;
820 
821 	default:
822 		if (usb_error == USB_ERR_CANCELLED) {
823 			CFUMASS_DEBUG(sc, "USB_ERR_CANCELLED");
824 			break;
825 		}
826 
827 		CFUMASS_DEBUG(sc, "USB_ST_ERROR: %s",
828 		    usbd_errstr(usb_error));
829 
830 		goto tr_setup;
831 	}
832 }
833 
834 static void
835 cfumass_online(void *arg __unused)
836 {
837 
838 	cfumass_port_online = true;
839 }
840 
841 static void
842 cfumass_offline(void *arg __unused)
843 {
844 
845 	cfumass_port_online = false;
846 }
847 
848 static void
849 cfumass_datamove(union ctl_io *io)
850 {
851 	struct cfumass_softc *sc;
852 
853 	sc = io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr;
854 
855 	CFUMASS_DEBUG(sc, "go");
856 
857 	CFUMASS_LOCK(sc);
858 
859 	KASSERT(sc->sc_ctl_io == NULL,
860 	    ("sc_ctl_io is %p, should be NULL", sc->sc_ctl_io));
861 	sc->sc_ctl_io = io;
862 
863 	if ((io->io_hdr.flags & CTL_FLAG_DATA_MASK) == CTL_FLAG_DATA_IN) {
864 		/*
865 		 * Verify that CTL wants us to send the data in the direction
866 		 * expected by the initiator.
867 		 */
868 		if (sc->sc_current_flags != CBWFLAGS_IN) {
869 			CFUMASS_WARN(sc, "wrong bCBWFlags 0x%x, should be 0x%x",
870 			    sc->sc_current_flags, CBWFLAGS_IN);
871 			goto fail;
872 		}
873 
874 		cfumass_transfer_start(sc, CFUMASS_T_DATA_IN);
875 	} else {
876 		if (sc->sc_current_flags != CBWFLAGS_OUT) {
877 			CFUMASS_WARN(sc, "wrong bCBWFlags 0x%x, should be 0x%x",
878 			    sc->sc_current_flags, CBWFLAGS_OUT);
879 			goto fail;
880 		}
881 
882 		cfumass_transfer_start(sc, CFUMASS_T_DATA_OUT);
883 	}
884 
885 	CFUMASS_UNLOCK(sc);
886 	return;
887 
888 fail:
889 	ctl_set_data_phase_error(&io->scsiio);
890 	ctl_datamove_done(io, true);
891 	sc->sc_ctl_io = NULL;
892 }
893 
894 static void
895 cfumass_done(union ctl_io *io)
896 {
897 	struct cfumass_softc *sc;
898 
899 	sc = io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr;
900 
901 	CFUMASS_DEBUG(sc, "go");
902 
903 	KASSERT(((io->io_hdr.status & CTL_STATUS_MASK) != CTL_STATUS_NONE),
904 	    ("invalid CTL status %#x", io->io_hdr.status));
905 	KASSERT(sc->sc_ctl_io == NULL,
906 	    ("sc_ctl_io is %p, should be NULL", sc->sc_ctl_io));
907 
908 	if (io->io_hdr.io_type == CTL_IO_TASK &&
909 	    io->taskio.task_action == CTL_TASK_I_T_NEXUS_RESET) {
910 		/*
911 		 * Implicit task termination has just completed; nothing to do.
912 		 */
913 		ctl_free_io(io);
914 		return;
915 	}
916 
917 	/*
918 	 * Do not return status for aborted commands.
919 	 * There are exceptions, but none supported by CTL yet.
920 	 */
921 	if (((io->io_hdr.flags & CTL_FLAG_ABORT) &&
922 	     (io->io_hdr.flags & CTL_FLAG_ABORT_STATUS) == 0) ||
923 	    (io->io_hdr.flags & CTL_FLAG_STATUS_SENT)) {
924 		ctl_free_io(io);
925 		return;
926 	}
927 
928 	if ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_SUCCESS)
929 		sc->sc_current_status = 0;
930 	else
931 		sc->sc_current_status = 1;
932 
933 	/* XXX: How should we report BUSY, RESERVATION CONFLICT, etc? */
934 	if ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_SCSI_ERROR &&
935 	    io->scsiio.scsi_status == SCSI_STATUS_CHECK_COND)
936 		ctl_queue_sense(io);
937 	else
938 		ctl_free_io(io);
939 
940 	CFUMASS_LOCK(sc);
941 	cfumass_transfer_start(sc, CFUMASS_T_STATUS);
942 	CFUMASS_UNLOCK(sc);
943 
944 	refcount_release(&sc->sc_queued);
945 }
946 
947 int
948 cfumass_init(void)
949 {
950 	int error;
951 
952 	cfumass_port.frontend = &cfumass_frontend;
953 	cfumass_port.port_type = CTL_PORT_UMASS;
954 	cfumass_port.num_requested_ctl_io = 1;
955 	cfumass_port.port_name = "cfumass";
956 	cfumass_port.physical_port = 0;
957 	cfumass_port.virtual_port = 0;
958 	cfumass_port.port_online = cfumass_online;
959 	cfumass_port.port_offline = cfumass_offline;
960 	cfumass_port.onoff_arg = NULL;
961 	cfumass_port.fe_datamove = cfumass_datamove;
962 	cfumass_port.fe_done = cfumass_done;
963 	cfumass_port.targ_port = -1;
964 
965 	error = ctl_port_register(&cfumass_port);
966 	if (error != 0) {
967 		printf("%s: ctl_port_register() failed "
968 		    "with error %d", __func__, error);
969 	}
970 
971 	cfumass_port_online = true;
972 	refcount_init(&cfumass_refcount, 0);
973 
974 	return (error);
975 }
976 
977 int
978 cfumass_shutdown(void)
979 {
980 	int error;
981 
982 	if (cfumass_refcount > 0) {
983 		if (debug > 1) {
984 			printf("%s: still have %u attachments; "
985 			    "returning EBUSY\n", __func__, cfumass_refcount);
986 		}
987 		return (EBUSY);
988 	}
989 
990 	error = ctl_port_deregister(&cfumass_port);
991 	if (error != 0) {
992 		printf("%s: ctl_port_deregister() failed "
993 		    "with error %d\n", __func__, error);
994 	}
995 
996 	return (error);
997 }
998