xref: /freebsd/sys/cam/ctl/scsi_ctl.c (revision 31ba4ce8898f9dfa5e7f054fdbc26e50a599a6e3)
1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3  *
4  * Copyright (c) 2008, 2009 Silicon Graphics International Corp.
5  * Copyright (c) 2014-2015 Alexander Motin <mav@FreeBSD.org>
6  * All rights reserved.
7  *
8  * Redistribution and use in source and binary forms, with or without
9  * modification, are permitted provided that the following conditions
10  * are met:
11  * 1. Redistributions of source code must retain the above copyright
12  *    notice, this list of conditions, and the following disclaimer,
13  *    without modification.
14  * 2. Redistributions in binary form must reproduce at minimum a disclaimer
15  *    substantially similar to the "NO WARRANTY" disclaimer below
16  *    ("Disclaimer") and any redistribution must be conditioned upon
17  *    including a substantially similar Disclaimer requirement for further
18  *    binary redistribution.
19  *
20  * NO WARRANTY
21  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTIBILITY AND FITNESS FOR
24  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25  * HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
26  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
27  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
28  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
29  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
30  * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
31  * POSSIBILITY OF SUCH DAMAGES.
32  *
33  * $Id: //depot/users/kenm/FreeBSD-test2/sys/cam/ctl/scsi_ctl.c#4 $
34  */
35 /*
36  * Peripheral driver interface between CAM and CTL (CAM Target Layer).
37  *
38  * Author: Ken Merry <ken@FreeBSD.org>
39  */
40 
41 #include <sys/cdefs.h>
42 __FBSDID("$FreeBSD$");
43 
44 #include <sys/param.h>
45 #include <sys/queue.h>
46 #include <sys/systm.h>
47 #include <sys/kernel.h>
48 #include <sys/lock.h>
49 #include <sys/mutex.h>
50 #include <sys/condvar.h>
51 #include <sys/malloc.h>
52 #include <sys/bus.h>
53 #include <sys/endian.h>
54 #include <sys/sbuf.h>
55 #include <sys/sysctl.h>
56 #include <sys/types.h>
57 #include <sys/systm.h>
58 #include <sys/taskqueue.h>
59 #include <machine/bus.h>
60 
61 #include <cam/cam.h>
62 #include <cam/cam_ccb.h>
63 #include <cam/cam_periph.h>
64 #include <cam/cam_queue.h>
65 #include <cam/cam_xpt_periph.h>
66 #include <cam/cam_debug.h>
67 #include <cam/cam_sim.h>
68 #include <cam/cam_xpt.h>
69 
70 #include <cam/scsi/scsi_all.h>
71 #include <cam/scsi/scsi_message.h>
72 
73 #include <cam/ctl/ctl_io.h>
74 #include <cam/ctl/ctl.h>
75 #include <cam/ctl/ctl_frontend.h>
76 #include <cam/ctl/ctl_util.h>
77 #include <cam/ctl/ctl_error.h>
78 
79 struct ctlfe_softc {
80 	struct ctl_port	port;
81 	path_id_t	path_id;
82 	target_id_t	target_id;
83 	uint32_t	hba_misc;
84 	u_int		maxio;
85 	struct cam_sim *sim;
86 	char		port_name[DEV_IDLEN];
87 	struct mtx	lun_softc_mtx;
88 	STAILQ_HEAD(, ctlfe_lun_softc) lun_softc_list;
89 	STAILQ_ENTRY(ctlfe_softc) links;
90 };
91 
92 STAILQ_HEAD(, ctlfe_softc) ctlfe_softc_list;
93 struct mtx ctlfe_list_mtx;
94 static char ctlfe_mtx_desc[] = "ctlfelist";
95 
96 typedef enum {
97 	CTLFE_LUN_NONE		= 0x00,
98 	CTLFE_LUN_WILDCARD	= 0x01
99 } ctlfe_lun_flags;
100 
101 struct ctlfe_lun_softc {
102 	struct ctlfe_softc *parent_softc;
103 	struct cam_periph *periph;
104 	ctlfe_lun_flags flags;
105 	int	 ctios_sent;		/* Number of active CTIOs */
106 	int	 refcount;		/* Number of active xpt_action() */
107 	int	 atios_alloced;		/* Number of ATIOs not freed */
108 	int	 inots_alloced;		/* Number of INOTs not freed */
109 	struct task	refdrain_task;
110 	STAILQ_HEAD(, ccb_hdr) work_queue;
111 	LIST_HEAD(, ccb_hdr) atio_list;	/* List of ATIOs queued to SIM. */
112 	LIST_HEAD(, ccb_hdr) inot_list;	/* List of INOTs queued to SIM. */
113 	STAILQ_ENTRY(ctlfe_lun_softc) links;
114 };
115 
116 typedef enum {
117 	CTLFE_CMD_NONE		= 0x00,
118 	CTLFE_CMD_PIECEWISE	= 0x01
119 } ctlfe_cmd_flags;
120 
121 struct ctlfe_cmd_info {
122 	int cur_transfer_index;
123 	size_t cur_transfer_off;
124 	ctlfe_cmd_flags flags;
125 	/*
126 	 * XXX KDM struct bus_dma_segment is 8 bytes on i386, and 16
127 	 * bytes on amd64.  So with 32 elements, this is 256 bytes on
128 	 * i386 and 512 bytes on amd64.
129 	 */
130 #define CTLFE_MAX_SEGS	32
131 	bus_dma_segment_t cam_sglist[CTLFE_MAX_SEGS];
132 };
133 
134 /*
135  * When we register the adapter/bus, request that this many ctl_ios be
136  * allocated.  This should be the maximum supported by the adapter, but we
137  * currently don't have a way to get that back from the path inquiry.
138  * XXX KDM add that to the path inquiry.
139  */
140 #define	CTLFE_REQ_CTL_IO	4096
141 /*
142  * Number of Accept Target I/O CCBs to allocate and queue down to the
143  * adapter per LUN.
144  * XXX KDM should this be controlled by CTL?
145  */
146 #define	CTLFE_ATIO_PER_LUN	1024
147 /*
148  * Number of Immediate Notify CCBs (used for aborts, resets, etc.) to
149  * allocate and queue down to the adapter per LUN.
150  * XXX KDM should this be controlled by CTL?
151  */
152 #define	CTLFE_IN_PER_LUN	1024
153 
154 /*
155  * Timeout (in seconds) on CTIO CCB doing DMA or sending status
156  */
157 #define	CTLFE_TIMEOUT	5
158 
159 /*
160  * Turn this on to enable extra debugging prints.
161  */
162 #if 0
163 #define	CTLFE_DEBUG
164 #endif
165 
166 MALLOC_DEFINE(M_CTLFE, "CAM CTL FE", "CAM CTL FE interface");
167 
168 #define	io_ptr		ppriv_ptr0
169 
170 /* This is only used in the CTIO */
171 #define	ccb_atio	ppriv_ptr1
172 
173 #define PRIV_CCB(io)	((io)->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptrs[0])
174 #define PRIV_INFO(io)	((io)->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptrs[1])
175 
176 static int		ctlfeinitialize(void);
177 static int		ctlfeshutdown(void);
178 static periph_init_t	ctlfeperiphinit;
179 static periph_deinit_t	ctlfeperiphdeinit;
180 static void		ctlfeasync(void *callback_arg, uint32_t code,
181 				   struct cam_path *path, void *arg);
182 static periph_ctor_t	ctlferegister;
183 static periph_oninv_t	ctlfeoninvalidate;
184 static periph_dtor_t	ctlfecleanup;
185 static periph_start_t	ctlfestart;
186 static void		ctlfedone(struct cam_periph *periph,
187 				  union ccb *done_ccb);
188 
189 static void 		ctlfe_onoffline(void *arg, int online);
190 static void 		ctlfe_online(void *arg);
191 static void 		ctlfe_offline(void *arg);
192 static int 		ctlfe_lun_enable(void *arg, int lun_id);
193 static int 		ctlfe_lun_disable(void *arg, int lun_id);
194 static void		ctlfe_dump_sim(struct cam_sim *sim);
195 static void		ctlfe_dump_queue(struct ctlfe_lun_softc *softc);
196 static void 		ctlfe_datamove(union ctl_io *io);
197 static void 		ctlfe_done(union ctl_io *io);
198 static void 		ctlfe_dump(void);
199 static void		ctlfe_free_ccb(struct cam_periph *periph,
200 			    union ccb *ccb);
201 static void		ctlfe_requeue_ccb(struct cam_periph *periph,
202 			    union ccb *ccb, int unlock);
203 
204 static struct periph_driver ctlfe_driver =
205 {
206 	ctlfeperiphinit, "ctl",
207 	TAILQ_HEAD_INITIALIZER(ctlfe_driver.units), /*generation*/ 0,
208 	CAM_PERIPH_DRV_EARLY,
209 	ctlfeperiphdeinit
210 };
211 
212 static struct ctl_frontend ctlfe_frontend =
213 {
214 	.name = "camtgt",
215 	.init = ctlfeinitialize,
216 	.fe_dump = ctlfe_dump,
217 	.shutdown = ctlfeshutdown,
218 };
219 CTL_FRONTEND_DECLARE(ctlfe, ctlfe_frontend);
220 
221 static int
222 ctlfeinitialize(void)
223 {
224 
225 	STAILQ_INIT(&ctlfe_softc_list);
226 	mtx_init(&ctlfe_list_mtx, ctlfe_mtx_desc, NULL, MTX_DEF);
227 	periphdriver_register(&ctlfe_driver);
228 	return (0);
229 }
230 
231 static int
232 ctlfeshutdown(void)
233 {
234 	int error;
235 
236 	error = periphdriver_unregister(&ctlfe_driver);
237 	if (error != 0)
238 		return (error);
239 	mtx_destroy(&ctlfe_list_mtx);
240 	return (0);
241 }
242 
243 static void
244 ctlfeperiphinit(void)
245 {
246 	cam_status status;
247 
248 	status = xpt_register_async(AC_PATH_REGISTERED | AC_PATH_DEREGISTERED |
249 				    AC_CONTRACT, ctlfeasync, NULL, NULL);
250 	if (status != CAM_REQ_CMP) {
251 		printf("ctl: Failed to attach async callback due to CAM "
252 		       "status 0x%x!\n", status);
253 	}
254 }
255 
256 static int
257 ctlfeperiphdeinit(void)
258 {
259 
260 	/* XXX: It would be good to tear down active ports here. */
261 	if (!TAILQ_EMPTY(&ctlfe_driver.units))
262 		return (EBUSY);
263 	xpt_register_async(0, ctlfeasync, NULL, NULL);
264 	return (0);
265 }
266 
267 static void
268 ctlfeasync(void *callback_arg, uint32_t code, struct cam_path *path, void *arg)
269 {
270 	struct ctlfe_softc *softc;
271 
272 #ifdef CTLFEDEBUG
273 	printf("%s: entered\n", __func__);
274 #endif
275 
276 	mtx_lock(&ctlfe_list_mtx);
277 	STAILQ_FOREACH(softc, &ctlfe_softc_list, links) {
278 		if (softc->path_id == xpt_path_path_id(path))
279 			break;
280 	}
281 	mtx_unlock(&ctlfe_list_mtx);
282 
283 	/*
284 	 * When a new path gets registered, and it is capable of target
285 	 * mode, go ahead and attach.  Later on, we may need to be more
286 	 * selective, but for now this will be sufficient.
287  	 */
288 	switch (code) {
289 	case AC_PATH_REGISTERED: {
290 		struct ctl_port *port;
291 		struct ccb_pathinq *cpi;
292 		int retval;
293 
294 		cpi = (struct ccb_pathinq *)arg;
295 
296 		/* Don't attach if it doesn't support target mode */
297 		if ((cpi->target_sprt & PIT_PROCESSOR) == 0) {
298 #ifdef CTLFEDEBUG
299 			printf("%s: SIM %s%d doesn't support target mode\n",
300 			       __func__, cpi->dev_name, cpi->unit_number);
301 #endif
302 			break;
303 		}
304 
305 		if (softc != NULL) {
306 #ifdef CTLFEDEBUG
307 			printf("%s: CTL port for CAM path %u already exists\n",
308 			       __func__, xpt_path_path_id(path));
309 #endif
310 			break;
311 		}
312 
313 		/*
314 		 * We're in an interrupt context here, so we have to
315 		 * use M_NOWAIT.  Of course this means trouble if we
316 		 * can't allocate memory.
317 		 */
318 		softc = malloc(sizeof(*softc), M_CTLFE, M_NOWAIT | M_ZERO);
319 		if (softc == NULL) {
320 			printf("%s: unable to malloc %zd bytes for softc\n",
321 			       __func__, sizeof(*softc));
322 			return;
323 		}
324 
325 		softc->path_id = cpi->ccb_h.path_id;
326 		softc->target_id = cpi->initiator_id;
327 		softc->sim = xpt_path_sim(path);
328 		softc->hba_misc = cpi->hba_misc;
329 		if (cpi->maxio != 0)
330 			softc->maxio = cpi->maxio;
331 		else
332 			softc->maxio = DFLTPHYS;
333 		mtx_init(&softc->lun_softc_mtx, "LUN softc mtx", NULL, MTX_DEF);
334 		STAILQ_INIT(&softc->lun_softc_list);
335 
336 		port = &softc->port;
337 		port->frontend = &ctlfe_frontend;
338 
339 		/*
340 		 * XXX KDM should we be more accurate here ?
341 		 */
342 		if (cpi->transport == XPORT_FC)
343 			port->port_type = CTL_PORT_FC;
344 		else if (cpi->transport == XPORT_SAS)
345 			port->port_type = CTL_PORT_SAS;
346 		else
347 			port->port_type = CTL_PORT_SCSI;
348 
349 		/* XXX KDM what should the real number be here? */
350 		port->num_requested_ctl_io = CTLFE_REQ_CTL_IO;
351 		snprintf(softc->port_name, sizeof(softc->port_name),
352 			 "%s%d", cpi->dev_name, cpi->unit_number);
353 		/*
354 		 * XXX KDM it would be nice to allocate storage in the
355 		 * frontend structure itself.
356 	 	 */
357 		port->port_name = softc->port_name;
358 		port->physical_port = cpi->bus_id;
359 		port->virtual_port = 0;
360 		port->port_online = ctlfe_online;
361 		port->port_offline = ctlfe_offline;
362 		port->onoff_arg = softc;
363 		port->lun_enable = ctlfe_lun_enable;
364 		port->lun_disable = ctlfe_lun_disable;
365 		port->targ_lun_arg = softc;
366 		port->fe_datamove = ctlfe_datamove;
367 		port->fe_done = ctlfe_done;
368 		port->targ_port = -1;
369 
370 		retval = ctl_port_register(port);
371 		if (retval != 0) {
372 			printf("%s: ctl_port_register() failed with "
373 			       "error %d!\n", __func__, retval);
374 			mtx_destroy(&softc->lun_softc_mtx);
375 			free(softc, M_CTLFE);
376 			break;
377 		} else {
378 			mtx_lock(&ctlfe_list_mtx);
379 			STAILQ_INSERT_TAIL(&ctlfe_softc_list, softc, links);
380 			mtx_unlock(&ctlfe_list_mtx);
381 		}
382 
383 		break;
384 	}
385 	case AC_PATH_DEREGISTERED: {
386 		if (softc != NULL) {
387 			/*
388 			 * XXX KDM are we certain at this point that there
389 			 * are no outstanding commands for this frontend?
390 			 */
391 			mtx_lock(&ctlfe_list_mtx);
392 			STAILQ_REMOVE(&ctlfe_softc_list, softc, ctlfe_softc,
393 			    links);
394 			mtx_unlock(&ctlfe_list_mtx);
395 			ctl_port_deregister(&softc->port);
396 			mtx_destroy(&softc->lun_softc_mtx);
397 			free(softc, M_CTLFE);
398 		}
399 		break;
400 	}
401 	case AC_CONTRACT: {
402 		struct ac_contract *ac;
403 
404 		ac = (struct ac_contract *)arg;
405 
406 		switch (ac->contract_number) {
407 		case AC_CONTRACT_DEV_CHG: {
408 			struct ac_device_changed *dev_chg;
409 			int retval;
410 
411 			dev_chg = (struct ac_device_changed *)ac->contract_data;
412 
413 			printf("%s: WWPN %#jx port 0x%06x path %u target %u %s\n",
414 			       __func__, dev_chg->wwpn, dev_chg->port,
415 			       xpt_path_path_id(path), dev_chg->target,
416 			       (dev_chg->arrived == 0) ?  "left" : "arrived");
417 
418 			if (softc == NULL) {
419 				printf("%s: CTL port for CAM path %u not "
420 				       "found!\n", __func__,
421 				       xpt_path_path_id(path));
422 				break;
423 			}
424 			if (dev_chg->arrived != 0) {
425 				retval = ctl_add_initiator(&softc->port,
426 				    dev_chg->target, dev_chg->wwpn, NULL);
427 			} else {
428 				retval = ctl_remove_initiator(&softc->port,
429 				    dev_chg->target);
430 			}
431 
432 			if (retval < 0) {
433 				printf("%s: could not %s port %d iid %u "
434 				       "WWPN %#jx!\n", __func__,
435 				       (dev_chg->arrived != 0) ? "add" :
436 				       "remove", softc->port.targ_port,
437 				       dev_chg->target,
438 				       (uintmax_t)dev_chg->wwpn);
439 			}
440 			break;
441 		}
442 		default:
443 			printf("%s: unsupported contract number %ju\n",
444 			       __func__, (uintmax_t)ac->contract_number);
445 			break;
446 		}
447 		break;
448 	}
449 	default:
450 		break;
451 	}
452 }
453 
454 static cam_status
455 ctlferegister(struct cam_periph *periph, void *arg)
456 {
457 	struct ctlfe_softc *bus_softc;
458 	struct ctlfe_lun_softc *softc;
459 	union ccb ccb;
460 	cam_status status;
461 	int i, acstatus;
462 
463 	softc = (struct ctlfe_lun_softc *)arg;
464 	bus_softc = softc->parent_softc;
465 
466 	STAILQ_INIT(&softc->work_queue);
467 	LIST_INIT(&softc->atio_list);
468 	LIST_INIT(&softc->inot_list);
469 	softc->periph = periph;
470 	periph->softc = softc;
471 
472 	/* Increase device openings to maximum for the SIM. */
473 	if (bus_softc->sim->max_tagged_dev_openings >
474 	    bus_softc->sim->max_dev_openings) {
475 		cam_release_devq(periph->path,
476 		    /*relsim_flags*/RELSIM_ADJUST_OPENINGS,
477 		    /*openings*/bus_softc->sim->max_tagged_dev_openings,
478 		    /*timeout*/0,
479 		    /*getcount_only*/1);
480 	}
481 
482 	memset(&ccb, 0, sizeof(ccb));
483 	xpt_setup_ccb(&ccb.ccb_h, periph->path, CAM_PRIORITY_NONE);
484 	ccb.ccb_h.func_code = XPT_EN_LUN;
485 	ccb.cel.grp6_len = 0;
486 	ccb.cel.grp7_len = 0;
487 	ccb.cel.enable = 1;
488 	xpt_action(&ccb);
489 	status = (ccb.ccb_h.status & CAM_STATUS_MASK);
490 	if (status != CAM_REQ_CMP) {
491 		xpt_print(periph->path, "%s: Enable LUN failed, status 0x%x\n",
492 			  __func__, ccb.ccb_h.status);
493 		return (status);
494 	}
495 
496 	status = CAM_REQ_CMP;
497 
498 	for (i = 0; i < CTLFE_ATIO_PER_LUN; i++) {
499 		union ccb *new_ccb;
500 		union ctl_io *new_io;
501 		struct ctlfe_cmd_info *cmd_info;
502 
503 		new_ccb = (union ccb *)malloc(sizeof(*new_ccb), M_CTLFE,
504 					      M_ZERO|M_NOWAIT);
505 		if (new_ccb == NULL) {
506 			status = CAM_RESRC_UNAVAIL;
507 			break;
508 		}
509 		new_io = ctl_alloc_io_nowait(bus_softc->port.ctl_pool_ref);
510 		if (new_io == NULL) {
511 			free(new_ccb, M_CTLFE);
512 			status = CAM_RESRC_UNAVAIL;
513 			break;
514 		}
515 		cmd_info = malloc(sizeof(*cmd_info), M_CTLFE,
516 		    M_ZERO | M_NOWAIT);
517 		if (cmd_info == NULL) {
518 			ctl_free_io(new_io);
519 			free(new_ccb, M_CTLFE);
520 			status = CAM_RESRC_UNAVAIL;
521 			break;
522 		}
523 		PRIV_INFO(new_io) = cmd_info;
524 		softc->atios_alloced++;
525 		new_ccb->ccb_h.io_ptr = new_io;
526 		LIST_INSERT_HEAD(&softc->atio_list, &new_ccb->ccb_h, periph_links.le);
527 
528 		xpt_setup_ccb(&new_ccb->ccb_h, periph->path, CAM_PRIORITY_NONE);
529 		new_ccb->ccb_h.func_code = XPT_ACCEPT_TARGET_IO;
530 		new_ccb->ccb_h.cbfcnp = ctlfedone;
531 		new_ccb->ccb_h.flags |= CAM_UNLOCKED;
532 		xpt_action(new_ccb);
533 		status = new_ccb->ccb_h.status;
534 		if ((status & CAM_STATUS_MASK) != CAM_REQ_INPROG) {
535 			free(cmd_info, M_CTLFE);
536 			ctl_free_io(new_io);
537 			free(new_ccb, M_CTLFE);
538 			break;
539 		}
540 	}
541 
542 	acstatus = cam_periph_acquire(periph);
543 	if (acstatus != 0) {
544 		xpt_print(periph->path, "%s: could not acquire reference "
545 			  "count, status = %#x\n", __func__, acstatus);
546 		return (CAM_REQ_CMP_ERR);
547 	}
548 
549 	if (i == 0) {
550 		xpt_print(periph->path, "%s: could not allocate ATIO CCBs, "
551 			  "status 0x%x\n", __func__, status);
552 		return (CAM_REQ_CMP_ERR);
553 	}
554 
555 	for (i = 0; i < CTLFE_IN_PER_LUN; i++) {
556 		union ccb *new_ccb;
557 		union ctl_io *new_io;
558 
559 		new_ccb = (union ccb *)malloc(sizeof(*new_ccb), M_CTLFE,
560 					      M_ZERO|M_NOWAIT);
561 		if (new_ccb == NULL) {
562 			status = CAM_RESRC_UNAVAIL;
563 			break;
564 		}
565 		new_io = ctl_alloc_io_nowait(bus_softc->port.ctl_pool_ref);
566 		if (new_io == NULL) {
567 			free(new_ccb, M_CTLFE);
568 			status = CAM_RESRC_UNAVAIL;
569 			break;
570 		}
571 		softc->inots_alloced++;
572 		new_ccb->ccb_h.io_ptr = new_io;
573 		LIST_INSERT_HEAD(&softc->inot_list, &new_ccb->ccb_h, periph_links.le);
574 
575 		xpt_setup_ccb(&new_ccb->ccb_h, periph->path, CAM_PRIORITY_NONE);
576 		new_ccb->ccb_h.func_code = XPT_IMMEDIATE_NOTIFY;
577 		new_ccb->ccb_h.cbfcnp = ctlfedone;
578 		new_ccb->ccb_h.flags |= CAM_UNLOCKED;
579 		xpt_action(new_ccb);
580 		status = new_ccb->ccb_h.status;
581 		if ((status & CAM_STATUS_MASK) != CAM_REQ_INPROG) {
582 			/*
583 			 * Note that we don't free the CCB here.  If the
584 			 * status is not CAM_REQ_INPROG, then we're
585 			 * probably talking to a SIM that says it is
586 			 * target-capable but doesn't support the
587 			 * XPT_IMMEDIATE_NOTIFY CCB.  i.e. it supports the
588 			 * older API.  In that case, it'll call xpt_done()
589 			 * on the CCB, and we need to free it in our done
590 			 * routine as a result.
591 			 */
592 			break;
593 		}
594 	}
595 	if ((i == 0)
596 	 || (status != CAM_REQ_INPROG)) {
597 		xpt_print(periph->path, "%s: could not allocate immediate "
598 			  "notify CCBs, status 0x%x\n", __func__, status);
599 		return (CAM_REQ_CMP_ERR);
600 	}
601 	mtx_lock(&bus_softc->lun_softc_mtx);
602 	STAILQ_INSERT_TAIL(&bus_softc->lun_softc_list, softc, links);
603 	mtx_unlock(&bus_softc->lun_softc_mtx);
604 	return (CAM_REQ_CMP);
605 }
606 
607 static void
608 ctlfeoninvalidate(struct cam_periph *periph)
609 {
610 	struct ctlfe_lun_softc *softc = (struct ctlfe_lun_softc *)periph->softc;
611 	struct ctlfe_softc *bus_softc;
612 	union ccb ccb;
613 	struct ccb_hdr *hdr;
614 	cam_status status;
615 
616 	/* Abort all ATIOs and INOTs queued to SIM. */
617 	memset(&ccb, 0, sizeof(ccb));
618 	xpt_setup_ccb(&ccb.ccb_h, periph->path, CAM_PRIORITY_NONE);
619 	ccb.ccb_h.func_code = XPT_ABORT;
620 	LIST_FOREACH(hdr, &softc->atio_list, periph_links.le) {
621 		ccb.cab.abort_ccb = (union ccb *)hdr;
622 		xpt_action(&ccb);
623 	}
624 	LIST_FOREACH(hdr, &softc->inot_list, periph_links.le) {
625 		ccb.cab.abort_ccb = (union ccb *)hdr;
626 		xpt_action(&ccb);
627 	}
628 
629 	/* Disable the LUN in SIM. */
630 	ccb.ccb_h.func_code = XPT_EN_LUN;
631 	ccb.cel.grp6_len = 0;
632 	ccb.cel.grp7_len = 0;
633 	ccb.cel.enable = 0;
634 	xpt_action(&ccb);
635 	status = (ccb.ccb_h.status & CAM_STATUS_MASK);
636 	if (status != CAM_REQ_CMP) {
637 		xpt_print(periph->path, "%s: Disable LUN failed, status 0x%x\n",
638 			  __func__, ccb.ccb_h.status);
639 		/*
640 		 * XXX KDM what do we do now?
641 		 */
642 	}
643 
644 	bus_softc = softc->parent_softc;
645 	mtx_lock(&bus_softc->lun_softc_mtx);
646 	STAILQ_REMOVE(&bus_softc->lun_softc_list, softc, ctlfe_lun_softc, links);
647 	mtx_unlock(&bus_softc->lun_softc_mtx);
648 }
649 
650 static void
651 ctlfecleanup(struct cam_periph *periph)
652 {
653 	struct ctlfe_lun_softc *softc;
654 
655 	softc = (struct ctlfe_lun_softc *)periph->softc;
656 
657 	KASSERT(softc->ctios_sent == 0, ("%s: ctios_sent %d != 0",
658 	    __func__, softc->ctios_sent));
659 	KASSERT(softc->refcount == 0, ("%s: refcount %d != 0",
660 	    __func__, softc->refcount));
661 	KASSERT(softc->atios_alloced == 0, ("%s: atios_alloced %d != 0",
662 	    __func__, softc->atios_alloced));
663 	KASSERT(softc->inots_alloced == 0, ("%s: inots_alloced %d != 0",
664 	    __func__, softc->inots_alloced));
665 
666 	free(softc, M_CTLFE);
667 }
668 
669 static void
670 ctlfedata(struct ctlfe_lun_softc *softc, union ctl_io *io,
671     ccb_flags *flags, uint8_t **data_ptr, uint32_t *dxfer_len,
672     u_int16_t *sglist_cnt)
673 {
674 	struct ctlfe_softc *bus_softc;
675 	struct ctlfe_cmd_info *cmd_info;
676 	struct ctl_sg_entry *ctl_sglist;
677 	bus_dma_segment_t *cam_sglist;
678 	size_t off;
679 	int i, idx;
680 
681 	cmd_info = PRIV_INFO(io);
682 	bus_softc = softc->parent_softc;
683 
684 	/*
685 	 * Set the direction, relative to the initiator.
686 	 */
687 	*flags &= ~CAM_DIR_MASK;
688 	if ((io->io_hdr.flags & CTL_FLAG_DATA_MASK) == CTL_FLAG_DATA_IN)
689 		*flags |= CAM_DIR_IN;
690 	else
691 		*flags |= CAM_DIR_OUT;
692 
693 	*flags &= ~CAM_DATA_MASK;
694 	idx = cmd_info->cur_transfer_index;
695 	off = cmd_info->cur_transfer_off;
696 	cmd_info->flags &= ~CTLFE_CMD_PIECEWISE;
697 	if (io->scsiio.kern_sg_entries == 0) {	/* No S/G list. */
698 
699 		/* One time shift for SRR offset. */
700 		off += io->scsiio.ext_data_filled;
701 		io->scsiio.ext_data_filled = 0;
702 
703 		*data_ptr = io->scsiio.kern_data_ptr + off;
704 		if (io->scsiio.kern_data_len - off <= bus_softc->maxio) {
705 			*dxfer_len = io->scsiio.kern_data_len - off;
706 		} else {
707 			*dxfer_len = bus_softc->maxio;
708 			cmd_info->cur_transfer_off += bus_softc->maxio;
709 			cmd_info->flags |= CTLFE_CMD_PIECEWISE;
710 		}
711 		*sglist_cnt = 0;
712 
713 		if (io->io_hdr.flags & CTL_FLAG_BUS_ADDR)
714 			*flags |= CAM_DATA_PADDR;
715 		else
716 			*flags |= CAM_DATA_VADDR;
717 	} else {	/* S/G list with physical or virtual pointers. */
718 		ctl_sglist = (struct ctl_sg_entry *)io->scsiio.kern_data_ptr;
719 
720 		/* One time shift for SRR offset. */
721 		while (io->scsiio.ext_data_filled >= ctl_sglist[idx].len - off) {
722 			io->scsiio.ext_data_filled -= ctl_sglist[idx].len - off;
723 			idx++;
724 			off = 0;
725 		}
726 		off += io->scsiio.ext_data_filled;
727 		io->scsiio.ext_data_filled = 0;
728 
729 		cam_sglist = cmd_info->cam_sglist;
730 		*dxfer_len = 0;
731 		for (i = 0; i < io->scsiio.kern_sg_entries - idx; i++) {
732 			cam_sglist[i].ds_addr = (bus_addr_t)(uintptr_t)ctl_sglist[i + idx].addr + off;
733 			if (ctl_sglist[i + idx].len - off <= bus_softc->maxio - *dxfer_len) {
734 				cam_sglist[i].ds_len = ctl_sglist[idx + i].len - off;
735 				*dxfer_len += cam_sglist[i].ds_len;
736 			} else {
737 				cam_sglist[i].ds_len = bus_softc->maxio - *dxfer_len;
738 				cmd_info->cur_transfer_index = idx + i;
739 				cmd_info->cur_transfer_off = cam_sglist[i].ds_len + off;
740 				cmd_info->flags |= CTLFE_CMD_PIECEWISE;
741 				*dxfer_len += cam_sglist[i].ds_len;
742 				if (ctl_sglist[i].len != 0)
743 					i++;
744 				break;
745 			}
746 			if (i == (CTLFE_MAX_SEGS - 1) &&
747 			    idx + i < (io->scsiio.kern_sg_entries - 1)) {
748 				cmd_info->cur_transfer_index = idx + i + 1;
749 				cmd_info->cur_transfer_off = 0;
750 				cmd_info->flags |= CTLFE_CMD_PIECEWISE;
751 				i++;
752 				break;
753 			}
754 			off = 0;
755 		}
756 		*sglist_cnt = i;
757 		if (io->io_hdr.flags & CTL_FLAG_BUS_ADDR)
758 			*flags |= CAM_DATA_SG_PADDR;
759 		else
760 			*flags |= CAM_DATA_SG;
761 		*data_ptr = (uint8_t *)cam_sglist;
762 	}
763 }
764 
765 static void
766 ctlfestart(struct cam_periph *periph, union ccb *start_ccb)
767 {
768 	struct ctlfe_lun_softc *softc;
769 	struct ctlfe_cmd_info *cmd_info;
770 	struct ccb_hdr *ccb_h;
771 	struct ccb_accept_tio *atio;
772 	struct ccb_scsiio *csio;
773 	uint8_t *data_ptr;
774 	uint32_t dxfer_len;
775 	ccb_flags flags;
776 	union ctl_io *io;
777 	uint8_t scsi_status;
778 
779 	softc = (struct ctlfe_lun_softc *)periph->softc;
780 
781 next:
782 	/* Take the ATIO off the work queue */
783 	ccb_h = STAILQ_FIRST(&softc->work_queue);
784 	if (ccb_h == NULL) {
785 		xpt_release_ccb(start_ccb);
786 		return;
787 	}
788 	STAILQ_REMOVE_HEAD(&softc->work_queue, periph_links.stqe);
789 	atio = (struct ccb_accept_tio *)ccb_h;
790 	io = (union ctl_io *)ccb_h->io_ptr;
791 	csio = &start_ccb->csio;
792 
793 	flags = atio->ccb_h.flags &
794 		(CAM_DIS_DISCONNECT|CAM_TAG_ACTION_VALID|CAM_DIR_MASK);
795 	cmd_info = PRIV_INFO(io);
796 	cmd_info->cur_transfer_index = 0;
797 	cmd_info->cur_transfer_off = 0;
798 	cmd_info->flags = 0;
799 
800 	if (io->io_hdr.flags & CTL_FLAG_DMA_QUEUED) {
801 		/*
802 		 * Datamove call, we need to setup the S/G list.
803 		 */
804 		ctlfedata(softc, io, &flags, &data_ptr, &dxfer_len,
805 		    &csio->sglist_cnt);
806 	} else {
807 		/*
808 		 * We're done, send status back.
809 		 */
810 		if ((io->io_hdr.flags & CTL_FLAG_ABORT) &&
811 		    (io->io_hdr.flags & CTL_FLAG_ABORT_STATUS) == 0) {
812 			io->io_hdr.flags &= ~CTL_FLAG_STATUS_QUEUED;
813 
814 			/* Tell the SIM that we've aborted this ATIO */
815 #ifdef CTLFEDEBUG
816 			printf("%s: tag %04x abort\n", __func__, atio->tag_id);
817 #endif
818 			KASSERT(atio->ccb_h.func_code == XPT_ACCEPT_TARGET_IO,
819 			    ("func_code %#x is not ATIO", atio->ccb_h.func_code));
820 			start_ccb->ccb_h.func_code = XPT_ABORT;
821 			start_ccb->cab.abort_ccb = (union ccb *)atio;
822 			xpt_action(start_ccb);
823 
824 			ctlfe_requeue_ccb(periph, (union ccb *)atio,
825 			    /* unlock */0);
826 
827 			/* XPT_ABORT is not queued, so we can take next I/O. */
828 			goto next;
829 		}
830 		data_ptr = NULL;
831 		dxfer_len = 0;
832 		csio->sglist_cnt = 0;
833 	}
834 	scsi_status = 0;
835 	if ((io->io_hdr.flags & CTL_FLAG_STATUS_QUEUED) &&
836 	    (cmd_info->flags & CTLFE_CMD_PIECEWISE) == 0 &&
837 	    ((io->io_hdr.flags & CTL_FLAG_DMA_QUEUED) == 0 ||
838 	     io->io_hdr.status == CTL_SUCCESS)) {
839 		flags |= CAM_SEND_STATUS;
840 		scsi_status = io->scsiio.scsi_status;
841 		csio->sense_len = io->scsiio.sense_len;
842 #ifdef CTLFEDEBUG
843 		printf("%s: tag %04x status %x\n", __func__,
844 		       atio->tag_id, io->io_hdr.status);
845 #endif
846 		if (csio->sense_len != 0) {
847 			csio->sense_data = io->scsiio.sense_data;
848 			flags |= CAM_SEND_SENSE;
849 		}
850 	}
851 
852 #ifdef CTLFEDEBUG
853 	printf("%s: %s: tag %04x flags %x ptr %p len %u\n", __func__,
854 	       (flags & CAM_SEND_STATUS) ? "done" : "datamove",
855 	       atio->tag_id, flags, data_ptr, dxfer_len);
856 #endif
857 
858 	/*
859 	 * Valid combinations:
860 	 *  - CAM_SEND_STATUS, CAM_DATA_SG = 0, dxfer_len = 0,
861 	 *    sglist_cnt = 0
862 	 *  - CAM_SEND_STATUS = 0, CAM_DATA_SG = 0, dxfer_len != 0,
863 	 *    sglist_cnt = 0
864 	 *  - CAM_SEND_STATUS = 0, CAM_DATA_SG, dxfer_len != 0,
865 	 *    sglist_cnt != 0
866 	 */
867 #ifdef CTLFEDEBUG
868 	if (((flags & CAM_SEND_STATUS)
869 	  && (((flags & CAM_DATA_SG) != 0)
870 	   || (dxfer_len != 0)
871 	   || (csio->sglist_cnt != 0)))
872 	 || (((flags & CAM_SEND_STATUS) == 0)
873 	  && (dxfer_len == 0))
874 	 || ((flags & CAM_DATA_SG)
875 	  && (csio->sglist_cnt == 0))
876 	 || (((flags & CAM_DATA_SG) == 0)
877 	  && (csio->sglist_cnt != 0))) {
878 		printf("%s: tag %04x cdb %02x flags %#x dxfer_len "
879 		       "%d sg %u\n", __func__, atio->tag_id,
880 		       atio_cdb_ptr(atio)[0], flags, dxfer_len,
881 		       csio->sglist_cnt);
882 		printf("%s: tag %04x io status %#x\n", __func__,
883 		       atio->tag_id, io->io_hdr.status);
884 	}
885 #endif
886 	cam_fill_ctio(csio,
887 		      /*retries*/ 2,
888 		      ctlfedone,
889 		      flags,
890 		      (flags & CAM_TAG_ACTION_VALID) ? MSG_SIMPLE_Q_TAG : 0,
891 		      atio->tag_id,
892 		      atio->init_id,
893 		      scsi_status,
894 		      /*data_ptr*/ data_ptr,
895 		      /*dxfer_len*/ dxfer_len,
896 		      /*timeout*/ CTLFE_TIMEOUT * 1000);
897 	start_ccb->ccb_h.flags |= CAM_UNLOCKED;
898 	start_ccb->ccb_h.ccb_atio = atio;
899 	if (io->io_hdr.flags & CTL_FLAG_DMA_QUEUED)
900 		io->io_hdr.flags |= CTL_FLAG_DMA_INPROG;
901 	io->io_hdr.flags &= ~(CTL_FLAG_DMA_QUEUED | CTL_FLAG_STATUS_QUEUED);
902 
903 	softc->ctios_sent++;
904 	softc->refcount++;
905 	cam_periph_unlock(periph);
906 	xpt_action(start_ccb);
907 	cam_periph_lock(periph);
908 	softc->refcount--;
909 
910 	/*
911 	 * If we still have work to do, ask for another CCB.
912 	 */
913 	if (!STAILQ_EMPTY(&softc->work_queue))
914 		xpt_schedule(periph, CAM_PRIORITY_NORMAL);
915 }
916 
917 static void
918 ctlfe_drain(void *context, int pending)
919 {
920 	struct cam_periph *periph = context;
921 	struct ctlfe_lun_softc *softc = periph->softc;
922 
923 	cam_periph_lock(periph);
924 	while (softc->refcount != 0) {
925 		cam_periph_sleep(periph, &softc->refcount, PRIBIO,
926 		    "ctlfe_drain", 1);
927 	}
928 	cam_periph_unlock(periph);
929 	cam_periph_release(periph);
930 }
931 
932 static void
933 ctlfe_free_ccb(struct cam_periph *periph, union ccb *ccb)
934 {
935 	struct ctlfe_lun_softc *softc;
936 	union ctl_io *io;
937 	struct ctlfe_cmd_info *cmd_info;
938 
939 	softc = (struct ctlfe_lun_softc *)periph->softc;
940 	io = ccb->ccb_h.io_ptr;
941 
942 	switch (ccb->ccb_h.func_code) {
943 	case XPT_ACCEPT_TARGET_IO:
944 		softc->atios_alloced--;
945 		cmd_info = PRIV_INFO(io);
946 		free(cmd_info, M_CTLFE);
947 		break;
948 	case XPT_IMMEDIATE_NOTIFY:
949 	case XPT_NOTIFY_ACKNOWLEDGE:
950 		softc->inots_alloced--;
951 		break;
952 	default:
953 		break;
954 	}
955 
956 	ctl_free_io(io);
957 	free(ccb, M_CTLFE);
958 
959 	KASSERT(softc->atios_alloced >= 0, ("%s: atios_alloced %d < 0",
960 	    __func__, softc->atios_alloced));
961 	KASSERT(softc->inots_alloced >= 0, ("%s: inots_alloced %d < 0",
962 	    __func__, softc->inots_alloced));
963 
964 	/*
965 	 * If we have received all of our CCBs, we can release our
966 	 * reference on the peripheral driver.  It will probably go away
967 	 * now.
968 	 */
969 	if (softc->atios_alloced == 0 && softc->inots_alloced == 0) {
970 		if (softc->refcount == 0) {
971 			cam_periph_release_locked(periph);
972 		} else {
973 			TASK_INIT(&softc->refdrain_task, 0, ctlfe_drain, periph);
974 			taskqueue_enqueue(taskqueue_thread,
975 			    &softc->refdrain_task);
976 		}
977 	}
978 }
979 
980 /*
981  * Send the ATIO/INOT back to the SIM, or free it if periph was invalidated.
982  */
983 static void
984 ctlfe_requeue_ccb(struct cam_periph *periph, union ccb *ccb, int unlock)
985 {
986 	struct ctlfe_lun_softc *softc;
987 	struct mtx *mtx;
988 
989 	if (periph->flags & CAM_PERIPH_INVALID) {
990 		mtx = cam_periph_mtx(periph);
991 		ctlfe_free_ccb(periph, ccb);
992 		if (unlock)
993 			mtx_unlock(mtx);
994 		return;
995 	}
996 	softc = (struct ctlfe_lun_softc *)periph->softc;
997 	if (ccb->ccb_h.func_code == XPT_ACCEPT_TARGET_IO)
998 		LIST_INSERT_HEAD(&softc->atio_list, &ccb->ccb_h, periph_links.le);
999 	else
1000 		LIST_INSERT_HEAD(&softc->inot_list, &ccb->ccb_h, periph_links.le);
1001 	if (unlock)
1002 		cam_periph_unlock(periph);
1003 
1004 	/*
1005 	 * For a wildcard attachment, commands can come in with a specific
1006 	 * target/lun.  Reset the target and LUN fields back to the wildcard
1007 	 * values before we send them back down to the SIM.
1008 	 */
1009 	xpt_setup_ccb_flags(&ccb->ccb_h, periph->path, CAM_PRIORITY_NONE,
1010 	    ccb->ccb_h.flags);
1011 
1012 	xpt_action(ccb);
1013 }
1014 
1015 static int
1016 ctlfe_adjust_cdb(struct ccb_accept_tio *atio, uint32_t offset)
1017 {
1018 	uint64_t lba;
1019 	uint32_t num_blocks, nbc;
1020 	uint8_t *cmdbyt = atio_cdb_ptr(atio);
1021 
1022 	nbc = offset >> 9;	/* ASSUMING 512 BYTE BLOCKS */
1023 
1024 	switch (cmdbyt[0]) {
1025 	case READ_6:
1026 	case WRITE_6:
1027 	{
1028 		struct scsi_rw_6 *cdb = (struct scsi_rw_6 *)cmdbyt;
1029 		lba = scsi_3btoul(cdb->addr);
1030 		lba &= 0x1fffff;
1031 		num_blocks = cdb->length;
1032 		if (num_blocks == 0)
1033 			num_blocks = 256;
1034 		lba += nbc;
1035 		num_blocks -= nbc;
1036 		scsi_ulto3b(lba, cdb->addr);
1037 		cdb->length = num_blocks;
1038 		break;
1039 	}
1040 	case READ_10:
1041 	case WRITE_10:
1042 	{
1043 		struct scsi_rw_10 *cdb = (struct scsi_rw_10 *)cmdbyt;
1044 		lba = scsi_4btoul(cdb->addr);
1045 		num_blocks = scsi_2btoul(cdb->length);
1046 		lba += nbc;
1047 		num_blocks -= nbc;
1048 		scsi_ulto4b(lba, cdb->addr);
1049 		scsi_ulto2b(num_blocks, cdb->length);
1050 		break;
1051 	}
1052 	case READ_12:
1053 	case WRITE_12:
1054 	{
1055 		struct scsi_rw_12 *cdb = (struct scsi_rw_12 *)cmdbyt;
1056 		lba = scsi_4btoul(cdb->addr);
1057 		num_blocks = scsi_4btoul(cdb->length);
1058 		lba += nbc;
1059 		num_blocks -= nbc;
1060 		scsi_ulto4b(lba, cdb->addr);
1061 		scsi_ulto4b(num_blocks, cdb->length);
1062 		break;
1063 	}
1064 	case READ_16:
1065 	case WRITE_16:
1066 	{
1067 		struct scsi_rw_16 *cdb = (struct scsi_rw_16 *)cmdbyt;
1068 		lba = scsi_8btou64(cdb->addr);
1069 		num_blocks = scsi_4btoul(cdb->length);
1070 		lba += nbc;
1071 		num_blocks -= nbc;
1072 		scsi_u64to8b(lba, cdb->addr);
1073 		scsi_ulto4b(num_blocks, cdb->length);
1074 		break;
1075 	}
1076 	default:
1077 		return -1;
1078 	}
1079 	return (0);
1080 }
1081 
1082 static void
1083 ctlfedone(struct cam_periph *periph, union ccb *done_ccb)
1084 {
1085 	struct ctlfe_lun_softc *softc;
1086 	struct ctlfe_softc *bus_softc;
1087 	struct ctlfe_cmd_info *cmd_info;
1088 	struct ccb_accept_tio *atio = NULL;
1089 	union ctl_io *io = NULL;
1090 	struct mtx *mtx;
1091 	cam_status status;
1092 
1093 	KASSERT((done_ccb->ccb_h.flags & CAM_UNLOCKED) != 0,
1094 	    ("CCB in ctlfedone() without CAM_UNLOCKED flag"));
1095 #ifdef CTLFE_DEBUG
1096 	printf("%s: entered, func_code = %#x\n", __func__,
1097 	       done_ccb->ccb_h.func_code);
1098 #endif
1099 
1100 	/*
1101 	 * At this point CTL has no known use case for device queue freezes.
1102 	 * In case some SIM think different -- drop its freeze right here.
1103 	 */
1104 	if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0) {
1105 		cam_release_devq(periph->path,
1106 				 /*relsim_flags*/0,
1107 				 /*reduction*/0,
1108 				 /*timeout*/0,
1109 				 /*getcount_only*/0);
1110 		done_ccb->ccb_h.status &= ~CAM_DEV_QFRZN;
1111 	}
1112 
1113 	softc = (struct ctlfe_lun_softc *)periph->softc;
1114 	bus_softc = softc->parent_softc;
1115 	mtx = cam_periph_mtx(periph);
1116 	mtx_lock(mtx);
1117 
1118 	switch (done_ccb->ccb_h.func_code) {
1119 	case XPT_ACCEPT_TARGET_IO: {
1120 		LIST_REMOVE(&done_ccb->ccb_h, periph_links.le);
1121 		atio = &done_ccb->atio;
1122 		status = atio->ccb_h.status & CAM_STATUS_MASK;
1123 		if (status != CAM_CDB_RECVD) {
1124 			ctlfe_free_ccb(periph, done_ccb);
1125 			goto out;
1126 		}
1127 
1128  resubmit:
1129 		/*
1130 		 * Allocate a ctl_io, pass it to CTL, and wait for the
1131 		 * datamove or done.
1132 		 */
1133 		mtx_unlock(mtx);
1134 		io = done_ccb->ccb_h.io_ptr;
1135 		cmd_info = PRIV_INFO(io);
1136 		ctl_zero_io(io);
1137 
1138 		/* Save pointers on both sides */
1139 		PRIV_CCB(io) = done_ccb;
1140 		PRIV_INFO(io) = cmd_info;
1141 		done_ccb->ccb_h.io_ptr = io;
1142 
1143 		/*
1144 		 * Only SCSI I/O comes down this path, resets, etc. come
1145 		 * down the immediate notify path below.
1146 		 */
1147 		io->io_hdr.io_type = CTL_IO_SCSI;
1148 		io->io_hdr.nexus.initid = atio->init_id;
1149 		io->io_hdr.nexus.targ_port = bus_softc->port.targ_port;
1150 		if (bus_softc->hba_misc & PIM_EXTLUNS) {
1151 			io->io_hdr.nexus.targ_lun = ctl_decode_lun(
1152 			    CAM_EXTLUN_BYTE_SWIZZLE(atio->ccb_h.target_lun));
1153 		} else {
1154 			io->io_hdr.nexus.targ_lun = atio->ccb_h.target_lun;
1155 		}
1156 		io->scsiio.priority = atio->priority;
1157 		io->scsiio.tag_num = atio->tag_id;
1158 		switch (atio->tag_action) {
1159 		case CAM_TAG_ACTION_NONE:
1160 			io->scsiio.tag_type = CTL_TAG_UNTAGGED;
1161 			break;
1162 		case MSG_SIMPLE_TASK:
1163 			io->scsiio.tag_type = CTL_TAG_SIMPLE;
1164 			break;
1165 		case MSG_HEAD_OF_QUEUE_TASK:
1166         		io->scsiio.tag_type = CTL_TAG_HEAD_OF_QUEUE;
1167 			break;
1168 		case MSG_ORDERED_TASK:
1169         		io->scsiio.tag_type = CTL_TAG_ORDERED;
1170 			break;
1171 		case MSG_ACA_TASK:
1172 			io->scsiio.tag_type = CTL_TAG_ACA;
1173 			break;
1174 		default:
1175 			io->scsiio.tag_type = CTL_TAG_UNTAGGED;
1176 			printf("%s: unhandled tag type %#x!!\n", __func__,
1177 			       atio->tag_action);
1178 			break;
1179 		}
1180 		if (atio->cdb_len > sizeof(io->scsiio.cdb)) {
1181 			printf("%s: WARNING: CDB len %d > ctl_io space %zd\n",
1182 			       __func__, atio->cdb_len, sizeof(io->scsiio.cdb));
1183 		}
1184 		io->scsiio.cdb_len = min(atio->cdb_len, sizeof(io->scsiio.cdb));
1185 		bcopy(atio_cdb_ptr(atio), io->scsiio.cdb, io->scsiio.cdb_len);
1186 
1187 #ifdef CTLFEDEBUG
1188 		printf("%s: %u:%u:%u: tag %04x CDB %02x\n", __func__,
1189 		        io->io_hdr.nexus.initid,
1190 		        io->io_hdr.nexus.targ_port,
1191 		        io->io_hdr.nexus.targ_lun,
1192 			io->scsiio.tag_num, io->scsiio.cdb[0]);
1193 #endif
1194 
1195 		ctl_queue(io);
1196 		return;
1197 	}
1198 	case XPT_CONT_TARGET_IO: {
1199 		int srr = 0;
1200 		uint32_t srr_off = 0;
1201 
1202 		atio = (struct ccb_accept_tio *)done_ccb->ccb_h.ccb_atio;
1203 		io = (union ctl_io *)atio->ccb_h.io_ptr;
1204 
1205 		softc->ctios_sent--;
1206 #ifdef CTLFEDEBUG
1207 		printf("%s: got XPT_CONT_TARGET_IO tag %#x flags %#x\n",
1208 		       __func__, atio->tag_id, done_ccb->ccb_h.flags);
1209 #endif
1210 		/*
1211 		 * Handle SRR case were the data pointer is pushed back hack
1212 		 */
1213 		if ((done_ccb->ccb_h.status & CAM_STATUS_MASK) == CAM_MESSAGE_RECV
1214 		    && done_ccb->csio.msg_ptr != NULL
1215 		    && done_ccb->csio.msg_ptr[0] == MSG_EXTENDED
1216 		    && done_ccb->csio.msg_ptr[1] == 5
1217        		    && done_ccb->csio.msg_ptr[2] == 0) {
1218 			srr = 1;
1219 			srr_off =
1220 			    (done_ccb->csio.msg_ptr[3] << 24)
1221 			    | (done_ccb->csio.msg_ptr[4] << 16)
1222 			    | (done_ccb->csio.msg_ptr[5] << 8)
1223 			    | (done_ccb->csio.msg_ptr[6]);
1224 		}
1225 
1226 		/*
1227 		 * If we have an SRR and we're still sending data, we
1228 		 * should be able to adjust offsets and cycle again.
1229 		 * It is possible only if offset is from this datamove.
1230 		 */
1231 		if (srr && (io->io_hdr.flags & CTL_FLAG_DMA_INPROG) &&
1232 		    srr_off >= io->scsiio.kern_rel_offset &&
1233 		    srr_off < io->scsiio.kern_rel_offset +
1234 		     io->scsiio.kern_data_len) {
1235 			io->scsiio.kern_data_resid =
1236 			    io->scsiio.kern_rel_offset +
1237 			    io->scsiio.kern_data_len - srr_off;
1238 			io->scsiio.ext_data_filled = srr_off;
1239 			io->scsiio.io_hdr.status = CTL_STATUS_NONE;
1240 			io->io_hdr.flags |= CTL_FLAG_DMA_QUEUED;
1241 			xpt_release_ccb(done_ccb);
1242 			STAILQ_INSERT_HEAD(&softc->work_queue, &atio->ccb_h,
1243 					  periph_links.stqe);
1244 			xpt_schedule(periph, CAM_PRIORITY_NORMAL);
1245 			break;
1246 		}
1247 
1248 		/*
1249 		 * If status was being sent, the back end data is now history.
1250 		 * Hack it up and resubmit a new command with the CDB adjusted.
1251 		 * If the SIM does the right thing, all of the resid math
1252 		 * should work.
1253 		 */
1254 		if (srr && (io->io_hdr.flags & CTL_FLAG_DMA_INPROG) == 0) {
1255 			xpt_release_ccb(done_ccb);
1256 			if (ctlfe_adjust_cdb(atio, srr_off) == 0) {
1257 				done_ccb = (union ccb *)atio;
1258 				goto resubmit;
1259 			}
1260 			/*
1261 			 * Fall through to doom....
1262 			 */
1263 		}
1264 
1265 		if ((done_ccb->ccb_h.flags & CAM_SEND_STATUS) &&
1266 		    (done_ccb->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP)
1267 			io->io_hdr.flags |= CTL_FLAG_STATUS_SENT;
1268 
1269 		/*
1270 		 * If we were sending status back to the initiator, free up
1271 		 * resources.  If we were doing a datamove, call the
1272 		 * datamove done routine.
1273 		 */
1274 		if ((io->io_hdr.flags & CTL_FLAG_DMA_INPROG) == 0) {
1275 			/*
1276 			 * If we asked to send sense data but it wasn't sent,
1277 			 * queue the I/O back to CTL for later REQUEST SENSE.
1278 			 */
1279 			if ((done_ccb->ccb_h.flags & CAM_SEND_SENSE) != 0 &&
1280 			    (done_ccb->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP &&
1281 			    (done_ccb->ccb_h.status & CAM_SENT_SENSE) == 0 &&
1282 			    (io = ctl_alloc_io_nowait(bus_softc->port.ctl_pool_ref)) != NULL) {
1283 				PRIV_INFO(io) = PRIV_INFO(
1284 				    (union ctl_io *)atio->ccb_h.io_ptr);
1285 				ctl_queue_sense(atio->ccb_h.io_ptr);
1286 				atio->ccb_h.io_ptr = io;
1287 			}
1288 
1289 			/* Abort ATIO if CTIO sending status has failed. */
1290 			if ((done_ccb->ccb_h.status & CAM_STATUS_MASK) !=
1291 			    CAM_REQ_CMP) {
1292 				done_ccb->ccb_h.func_code = XPT_ABORT;
1293 				done_ccb->cab.abort_ccb = (union ccb *)atio;
1294 				xpt_action(done_ccb);
1295 			}
1296 
1297 			xpt_release_ccb(done_ccb);
1298 			ctlfe_requeue_ccb(periph, (union ccb *)atio,
1299 			    /* unlock */1);
1300 			return;
1301 		} else {
1302 			struct ctlfe_cmd_info *cmd_info;
1303 			struct ccb_scsiio *csio;
1304 
1305 			csio = &done_ccb->csio;
1306 			cmd_info = PRIV_INFO(io);
1307 
1308 			io->io_hdr.flags &= ~CTL_FLAG_DMA_INPROG;
1309 
1310 			/*
1311 			 * Translate CAM status to CTL status.  Success
1312 			 * does not change the overall, ctl_io status.  In
1313 			 * that case we just set port_status to 0.  If we
1314 			 * have a failure, though, set a data phase error
1315 			 * for the overall ctl_io.
1316 			 */
1317 			switch (done_ccb->ccb_h.status & CAM_STATUS_MASK) {
1318 			case CAM_REQ_CMP:
1319 				io->scsiio.kern_data_resid -=
1320 				    csio->dxfer_len - csio->resid;
1321 				io->io_hdr.port_status = 0;
1322 				break;
1323 			default:
1324 				/*
1325 				 * XXX KDM we probably need to figure out a
1326 				 * standard set of errors that the SIM
1327 				 * drivers should return in the event of a
1328 				 * data transfer failure.  A data phase
1329 				 * error will at least point the user to a
1330 				 * data transfer error of some sort.
1331 				 * Hopefully the SIM printed out some
1332 				 * additional information to give the user
1333 				 * a clue what happened.
1334 				 */
1335 				io->io_hdr.port_status = 0xbad1;
1336 				ctl_set_data_phase_error(&io->scsiio);
1337 				/*
1338 				 * XXX KDM figure out residual.
1339 				 */
1340 				break;
1341 			}
1342 			/*
1343 			 * If we had to break this S/G list into multiple
1344 			 * pieces, figure out where we are in the list, and
1345 			 * continue sending pieces if necessary.
1346 			 */
1347 			if ((cmd_info->flags & CTLFE_CMD_PIECEWISE) &&
1348 			    io->io_hdr.port_status == 0 && csio->resid == 0) {
1349 				ccb_flags flags;
1350 				uint8_t *data_ptr;
1351 				uint32_t dxfer_len;
1352 
1353 				flags = atio->ccb_h.flags &
1354 					(CAM_DIS_DISCONNECT|
1355 					 CAM_TAG_ACTION_VALID);
1356 
1357 				ctlfedata(softc, io, &flags, &data_ptr,
1358 				    &dxfer_len, &csio->sglist_cnt);
1359 
1360 				if (((flags & CAM_SEND_STATUS) == 0)
1361 				 && (dxfer_len == 0)) {
1362 					printf("%s: tag %04x no status or "
1363 					       "len cdb = %02x\n", __func__,
1364 					       atio->tag_id,
1365 					       atio_cdb_ptr(atio)[0]);
1366 					printf("%s: tag %04x io status %#x\n",
1367 					       __func__, atio->tag_id,
1368 					       io->io_hdr.status);
1369 				}
1370 
1371 				cam_fill_ctio(csio,
1372 					      /*retries*/ 2,
1373 					      ctlfedone,
1374 					      flags,
1375 					      (flags & CAM_TAG_ACTION_VALID) ?
1376 					       MSG_SIMPLE_Q_TAG : 0,
1377 					      atio->tag_id,
1378 					      atio->init_id,
1379 					      0,
1380 					      /*data_ptr*/ data_ptr,
1381 					      /*dxfer_len*/ dxfer_len,
1382 					      CTLFE_TIMEOUT * 1000);
1383 
1384 				csio->ccb_h.flags |= CAM_UNLOCKED;
1385 				csio->resid = 0;
1386 				csio->ccb_h.ccb_atio = atio;
1387 				io->io_hdr.flags |= CTL_FLAG_DMA_INPROG;
1388 				softc->ctios_sent++;
1389 				mtx_unlock(mtx);
1390 				xpt_action((union ccb *)csio);
1391 			} else {
1392 				/*
1393 				 * Release the CTIO.  The ATIO will be sent back
1394 				 * down to the SIM once we send status.
1395 				 */
1396 				xpt_release_ccb(done_ccb);
1397 				mtx_unlock(mtx);
1398 
1399 				ctl_datamove_done(io, false);
1400 			}
1401 			return;
1402 		}
1403 		break;
1404 	}
1405 	case XPT_IMMEDIATE_NOTIFY: {
1406 		union ctl_io *io;
1407 		struct ccb_immediate_notify *inot;
1408 		int send_ctl_io;
1409 
1410 		LIST_REMOVE(&done_ccb->ccb_h, periph_links.le);
1411 		inot = &done_ccb->cin1;
1412 		io = done_ccb->ccb_h.io_ptr;
1413 		ctl_zero_io(io);
1414 
1415 		send_ctl_io = 1;
1416 
1417 		io->io_hdr.io_type = CTL_IO_TASK;
1418 		PRIV_CCB(io) = done_ccb;
1419 		inot->ccb_h.io_ptr = io;
1420 		io->io_hdr.nexus.initid = inot->initiator_id;
1421 		io->io_hdr.nexus.targ_port = bus_softc->port.targ_port;
1422 		if (bus_softc->hba_misc & PIM_EXTLUNS) {
1423 			io->io_hdr.nexus.targ_lun = ctl_decode_lun(
1424 			    CAM_EXTLUN_BYTE_SWIZZLE(inot->ccb_h.target_lun));
1425 		} else {
1426 			io->io_hdr.nexus.targ_lun = inot->ccb_h.target_lun;
1427 		}
1428 		/* XXX KDM should this be the tag_id? */
1429 		io->taskio.tag_num = inot->seq_id;
1430 
1431 		status = inot->ccb_h.status & CAM_STATUS_MASK;
1432 		switch (status) {
1433 		case CAM_SCSI_BUS_RESET:
1434 			io->taskio.task_action = CTL_TASK_BUS_RESET;
1435 			break;
1436 		case CAM_BDR_SENT:
1437 			io->taskio.task_action = CTL_TASK_TARGET_RESET;
1438 			break;
1439 		case CAM_MESSAGE_RECV:
1440 			switch (inot->arg) {
1441 			case MSG_ABORT_TASK_SET:
1442 				io->taskio.task_action =
1443 				    CTL_TASK_ABORT_TASK_SET;
1444 				break;
1445 			case MSG_TARGET_RESET:
1446 				io->taskio.task_action = CTL_TASK_TARGET_RESET;
1447 				break;
1448 			case MSG_ABORT_TASK:
1449 				io->taskio.task_action = CTL_TASK_ABORT_TASK;
1450 				break;
1451 			case MSG_LOGICAL_UNIT_RESET:
1452 				io->taskio.task_action = CTL_TASK_LUN_RESET;
1453 				break;
1454 			case MSG_CLEAR_TASK_SET:
1455 				io->taskio.task_action =
1456 				    CTL_TASK_CLEAR_TASK_SET;
1457 				break;
1458 			case MSG_CLEAR_ACA:
1459 				io->taskio.task_action = CTL_TASK_CLEAR_ACA;
1460 				break;
1461 			case MSG_QUERY_TASK:
1462 				io->taskio.task_action = CTL_TASK_QUERY_TASK;
1463 				break;
1464 			case MSG_QUERY_TASK_SET:
1465 				io->taskio.task_action =
1466 				    CTL_TASK_QUERY_TASK_SET;
1467 				break;
1468 			case MSG_QUERY_ASYNC_EVENT:
1469 				io->taskio.task_action =
1470 				    CTL_TASK_QUERY_ASYNC_EVENT;
1471 				break;
1472 			case MSG_NOOP:
1473 				send_ctl_io = 0;
1474 				break;
1475 			default:
1476 				xpt_print(periph->path,
1477 				    "%s: unsupported INOT message 0x%x\n",
1478 				    __func__, inot->arg);
1479 				send_ctl_io = 0;
1480 				break;
1481 			}
1482 			break;
1483 		default:
1484 			xpt_print(periph->path,
1485 			    "%s: unsupported INOT status 0x%x\n",
1486 			    __func__, status);
1487 			/* FALLTHROUGH */
1488 		case CAM_REQ_ABORTED:
1489 		case CAM_REQ_INVALID:
1490 		case CAM_DEV_NOT_THERE:
1491 		case CAM_PROVIDE_FAIL:
1492 			ctlfe_free_ccb(periph, done_ccb);
1493 			goto out;
1494 		}
1495 		mtx_unlock(mtx);
1496 		if (send_ctl_io != 0) {
1497 			ctl_queue(io);
1498 		} else {
1499 			done_ccb->ccb_h.status = CAM_REQ_INPROG;
1500 			done_ccb->ccb_h.func_code = XPT_NOTIFY_ACKNOWLEDGE;
1501 			xpt_action(done_ccb);
1502 		}
1503 		return;
1504 	}
1505 	case XPT_NOTIFY_ACKNOWLEDGE:
1506 		/* Queue this back down to the SIM as an immediate notify. */
1507 		done_ccb->ccb_h.status = CAM_REQ_INPROG;
1508 		done_ccb->ccb_h.func_code = XPT_IMMEDIATE_NOTIFY;
1509 		ctlfe_requeue_ccb(periph, done_ccb, /* unlock */1);
1510 		return;
1511 	case XPT_SET_SIM_KNOB:
1512 	case XPT_GET_SIM_KNOB:
1513 	case XPT_GET_SIM_KNOB_OLD:
1514 		break;
1515 	default:
1516 		panic("%s: unexpected CCB type %#x", __func__,
1517 		      done_ccb->ccb_h.func_code);
1518 		break;
1519 	}
1520 
1521 out:
1522 	mtx_unlock(mtx);
1523 }
1524 
1525 static void
1526 ctlfe_onoffline(void *arg, int online)
1527 {
1528 	struct ctlfe_softc *bus_softc = arg;
1529 	union ccb *ccb;
1530 	cam_status status;
1531 	struct cam_path *path;
1532 	int set_wwnn = 0;
1533 
1534 	status = xpt_create_path(&path, /*periph*/ NULL, bus_softc->path_id,
1535 	    CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD);
1536 	if (status != CAM_REQ_CMP) {
1537 		printf("%s: unable to create path!\n", __func__);
1538 		return;
1539 	}
1540 	ccb = xpt_alloc_ccb();
1541 	xpt_setup_ccb(&ccb->ccb_h, path, CAM_PRIORITY_NONE);
1542 	ccb->ccb_h.func_code = XPT_GET_SIM_KNOB;
1543 	xpt_action(ccb);
1544 
1545 	/* Check whether we should change WWNs. */
1546 	if (online != 0) {
1547 		if ((ccb->knob.xport_specific.valid & KNOB_VALID_ADDRESS) != 0){
1548 			printf("%s: %s current WWNN %#jx\n", __func__,
1549 			       bus_softc->port_name,
1550 			       ccb->knob.xport_specific.fc.wwnn);
1551 			printf("%s: %s current WWPN %#jx\n", __func__,
1552 			       bus_softc->port_name,
1553 			       ccb->knob.xport_specific.fc.wwpn);
1554 
1555 			/*
1556 			 * If the user has specified a WWNN/WWPN, send them
1557 			 * down to the SIM.  Otherwise, record what the SIM
1558 			 * has reported.
1559 			 */
1560 			if (bus_softc->port.wwnn != 0 && bus_softc->port.wwnn
1561 			    != ccb->knob.xport_specific.fc.wwnn) {
1562 				ccb->knob.xport_specific.fc.wwnn =
1563 				    bus_softc->port.wwnn;
1564 				set_wwnn = 1;
1565 			} else {
1566 				ctl_port_set_wwns(&bus_softc->port,
1567 				    true, ccb->knob.xport_specific.fc.wwnn,
1568 				    false, 0);
1569 			}
1570 			if (bus_softc->port.wwpn != 0 && bus_softc->port.wwpn
1571 			     != ccb->knob.xport_specific.fc.wwpn) {
1572 				ccb->knob.xport_specific.fc.wwpn =
1573 				    bus_softc->port.wwpn;
1574 				set_wwnn = 1;
1575 			} else {
1576 				ctl_port_set_wwns(&bus_softc->port,
1577 				    false, 0,
1578 				    true, ccb->knob.xport_specific.fc.wwpn);
1579 			}
1580 		} else {
1581 			printf("%s: %s has no valid WWNN/WWPN\n", __func__,
1582 			       bus_softc->port_name);
1583 			if (bus_softc->port.wwnn != 0) {
1584 				ccb->knob.xport_specific.fc.wwnn =
1585 				    bus_softc->port.wwnn;
1586 				set_wwnn = 1;
1587 			}
1588 			if (bus_softc->port.wwpn != 0) {
1589 				ccb->knob.xport_specific.fc.wwpn =
1590 				    bus_softc->port.wwpn;
1591 				set_wwnn = 1;
1592 			}
1593 		}
1594 	}
1595 	if (set_wwnn) {
1596 		ccb->ccb_h.func_code = XPT_SET_SIM_KNOB;
1597 		ccb->knob.xport_specific.valid = KNOB_VALID_ADDRESS;
1598 		xpt_action(ccb);
1599 		if ((ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP) {
1600 			printf("%s: %s (path id %d) failed set WWNs: %#x\n",
1601 			    __func__, bus_softc->port_name, bus_softc->path_id,
1602 			    ccb->ccb_h.status);
1603 		} else {
1604 			printf("%s: %s new WWNN %#jx\n", __func__,
1605 			       bus_softc->port_name,
1606 			       ccb->knob.xport_specific.fc.wwnn);
1607 			printf("%s: %s new WWPN %#jx\n", __func__,
1608 			       bus_softc->port_name,
1609 			       ccb->knob.xport_specific.fc.wwpn);
1610 		}
1611 	}
1612 
1613 	/* Check whether we should change role. */
1614 	if ((ccb->knob.xport_specific.valid & KNOB_VALID_ROLE) == 0 ||
1615 	    ((online != 0) ^
1616 	    ((ccb->knob.xport_specific.fc.role & KNOB_ROLE_TARGET) != 0)) != 0) {
1617 		ccb->ccb_h.func_code = XPT_SET_SIM_KNOB;
1618 		ccb->knob.xport_specific.valid = KNOB_VALID_ROLE;
1619 		if (online)
1620 			ccb->knob.xport_specific.fc.role |= KNOB_ROLE_TARGET;
1621 		else
1622 			ccb->knob.xport_specific.fc.role &= ~KNOB_ROLE_TARGET;
1623 		xpt_action(ccb);
1624 		if ((ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP) {
1625 			printf("%s: %s (path id %d) failed %s target role: %#x\n",
1626 			    __func__, bus_softc->port_name, bus_softc->path_id,
1627 			    online ? "enable" : "disable", ccb->ccb_h.status);
1628 		} else {
1629 			printf("%s: %s (path id %d) target role %s succeeded\n",
1630 			    __func__, bus_softc->port_name, bus_softc->path_id,
1631 			    online ? "enable" : "disable");
1632 		}
1633 	}
1634 
1635 	xpt_free_path(path);
1636 	xpt_free_ccb(ccb);
1637 }
1638 
1639 static void
1640 ctlfe_online(void *arg)
1641 {
1642 	struct ctlfe_softc *bus_softc;
1643 	struct cam_path *path;
1644 	cam_status status;
1645 	struct ctlfe_lun_softc *lun_softc;
1646 	struct cam_periph *periph;
1647 
1648 	bus_softc = (struct ctlfe_softc *)arg;
1649 
1650 	/*
1651 	 * Create the wildcard LUN before bringing the port online.
1652 	 */
1653 	status = xpt_create_path(&path, /*periph*/ NULL,
1654 				 bus_softc->path_id, CAM_TARGET_WILDCARD,
1655 				 CAM_LUN_WILDCARD);
1656 	if (status != CAM_REQ_CMP) {
1657 		printf("%s: unable to create path for wildcard periph\n",
1658 				__func__);
1659 		return;
1660 	}
1661 
1662 	lun_softc = malloc(sizeof(*lun_softc), M_CTLFE, M_WAITOK | M_ZERO);
1663 
1664 	xpt_path_lock(path);
1665 	periph = cam_periph_find(path, "ctl");
1666 	if (periph != NULL) {
1667 		/* We've already got a periph, no need to alloc a new one. */
1668 		xpt_path_unlock(path);
1669 		xpt_free_path(path);
1670 		free(lun_softc, M_CTLFE);
1671 		return;
1672 	}
1673 	lun_softc->parent_softc = bus_softc;
1674 	lun_softc->flags |= CTLFE_LUN_WILDCARD;
1675 
1676 	status = cam_periph_alloc(ctlferegister,
1677 				  ctlfeoninvalidate,
1678 				  ctlfecleanup,
1679 				  ctlfestart,
1680 				  "ctl",
1681 				  CAM_PERIPH_BIO,
1682 				  path,
1683 				  ctlfeasync,
1684 				  0,
1685 				  lun_softc);
1686 
1687 	if ((status & CAM_STATUS_MASK) != CAM_REQ_CMP) {
1688 		const struct cam_status_entry *entry;
1689 
1690 		entry = cam_fetch_status_entry(status);
1691 		printf("%s: CAM error %s (%#x) returned from "
1692 		       "cam_periph_alloc()\n", __func__, (entry != NULL) ?
1693 		       entry->status_text : "Unknown", status);
1694 		free(lun_softc, M_CTLFE);
1695 	}
1696 
1697 	xpt_path_unlock(path);
1698 	ctlfe_onoffline(arg, /*online*/ 1);
1699 	xpt_free_path(path);
1700 }
1701 
1702 static void
1703 ctlfe_offline(void *arg)
1704 {
1705 	struct ctlfe_softc *bus_softc;
1706 	struct cam_path *path;
1707 	cam_status status;
1708 	struct cam_periph *periph;
1709 
1710 	bus_softc = (struct ctlfe_softc *)arg;
1711 
1712 	ctlfe_onoffline(arg, /*online*/ 0);
1713 
1714 	/*
1715 	 * Disable the wildcard LUN for this port now that we have taken
1716 	 * the port offline.
1717 	 */
1718 	status = xpt_create_path(&path, /*periph*/ NULL,
1719 				 bus_softc->path_id, CAM_TARGET_WILDCARD,
1720 				 CAM_LUN_WILDCARD);
1721 	if (status != CAM_REQ_CMP) {
1722 		printf("%s: unable to create path for wildcard periph\n",
1723 		       __func__);
1724 		return;
1725 	}
1726 	xpt_path_lock(path);
1727 	if ((periph = cam_periph_find(path, "ctl")) != NULL)
1728 		cam_periph_invalidate(periph);
1729 	xpt_path_unlock(path);
1730 	xpt_free_path(path);
1731 }
1732 
1733 /*
1734  * This will get called to enable a LUN on every bus that is attached to
1735  * CTL.  So we only need to create a path/periph for this particular bus.
1736  */
1737 static int
1738 ctlfe_lun_enable(void *arg, int lun_id)
1739 {
1740 	struct ctlfe_softc *bus_softc;
1741 	struct ctlfe_lun_softc *softc;
1742 	struct cam_path *path;
1743 	struct cam_periph *periph;
1744 	cam_status status;
1745 
1746 	bus_softc = (struct ctlfe_softc *)arg;
1747 	if (bus_softc->hba_misc & PIM_EXTLUNS)
1748 		lun_id = CAM_EXTLUN_BYTE_SWIZZLE(ctl_encode_lun(lun_id));
1749 
1750 	status = xpt_create_path(&path, /*periph*/ NULL,
1751 	    bus_softc->path_id, bus_softc->target_id, lun_id);
1752 	/* XXX KDM need some way to return status to CTL here? */
1753 	if (status != CAM_REQ_CMP) {
1754 		printf("%s: could not create path, status %#x\n", __func__,
1755 		       status);
1756 		return (1);
1757 	}
1758 
1759 	softc = malloc(sizeof(*softc), M_CTLFE, M_WAITOK | M_ZERO);
1760 	xpt_path_lock(path);
1761 	periph = cam_periph_find(path, "ctl");
1762 	if (periph != NULL) {
1763 		/* We've already got a periph, no need to alloc a new one. */
1764 		xpt_path_unlock(path);
1765 		xpt_free_path(path);
1766 		free(softc, M_CTLFE);
1767 		return (0);
1768 	}
1769 	softc->parent_softc = bus_softc;
1770 
1771 	status = cam_periph_alloc(ctlferegister,
1772 				  ctlfeoninvalidate,
1773 				  ctlfecleanup,
1774 				  ctlfestart,
1775 				  "ctl",
1776 				  CAM_PERIPH_BIO,
1777 				  path,
1778 				  ctlfeasync,
1779 				  0,
1780 				  softc);
1781 
1782 	if ((status & CAM_STATUS_MASK) != CAM_REQ_CMP) {
1783 		const struct cam_status_entry *entry;
1784 
1785 		entry = cam_fetch_status_entry(status);
1786 		printf("%s: CAM error %s (%#x) returned from "
1787 		       "cam_periph_alloc()\n", __func__, (entry != NULL) ?
1788 		       entry->status_text : "Unknown", status);
1789 		free(softc, M_CTLFE);
1790 	}
1791 
1792 	xpt_path_unlock(path);
1793 	xpt_free_path(path);
1794 	return (0);
1795 }
1796 
1797 /*
1798  * This will get called when the user removes a LUN to disable that LUN
1799  * on every bus that is attached to CTL.
1800  */
1801 static int
1802 ctlfe_lun_disable(void *arg, int lun_id)
1803 {
1804 	struct ctlfe_softc *softc;
1805 	struct ctlfe_lun_softc *lun_softc;
1806 
1807 	softc = (struct ctlfe_softc *)arg;
1808 	if (softc->hba_misc & PIM_EXTLUNS)
1809 		lun_id = CAM_EXTLUN_BYTE_SWIZZLE(ctl_encode_lun(lun_id));
1810 
1811 	mtx_lock(&softc->lun_softc_mtx);
1812 	STAILQ_FOREACH(lun_softc, &softc->lun_softc_list, links) {
1813 		struct cam_path *path;
1814 
1815 		path = lun_softc->periph->path;
1816 
1817 		if ((xpt_path_target_id(path) == softc->target_id)
1818 		 && (xpt_path_lun_id(path) == lun_id)) {
1819 			break;
1820 		}
1821 	}
1822 	if (lun_softc == NULL) {
1823 		mtx_unlock(&softc->lun_softc_mtx);
1824 		printf("%s: can't find lun %d\n", __func__, lun_id);
1825 		return (1);
1826 	}
1827 	cam_periph_acquire(lun_softc->periph);
1828 	mtx_unlock(&softc->lun_softc_mtx);
1829 
1830 	cam_periph_lock(lun_softc->periph);
1831 	cam_periph_invalidate(lun_softc->periph);
1832 	cam_periph_unlock(lun_softc->periph);
1833 	cam_periph_release(lun_softc->periph);
1834 	return (0);
1835 }
1836 
1837 static void
1838 ctlfe_dump_sim(struct cam_sim *sim)
1839 {
1840 
1841 	printf("%s%d: max dev openings: %d, max tagged dev openings: %d\n",
1842 	    sim->sim_name, sim->unit_number, sim->max_dev_openings,
1843 	    sim->max_tagged_dev_openings);
1844 }
1845 
1846 /*
1847  * Assumes that the SIM lock is held.
1848  */
1849 static void
1850 ctlfe_dump_queue(struct ctlfe_lun_softc *softc)
1851 {
1852 	struct cam_periph *periph = softc->periph;
1853 	struct ccb_hdr *hdr;
1854 	struct ccb_getdevstats cgds;
1855 	int num_items;
1856 
1857 	memset(&cgds, 0, sizeof(cgds));
1858 	xpt_setup_ccb(&cgds.ccb_h, periph->path, CAM_PRIORITY_NORMAL);
1859 	cgds.ccb_h.func_code = XPT_GDEV_STATS;
1860 	xpt_action((union ccb *)&cgds);
1861 	if ((cgds.ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) {
1862 		xpt_print(periph->path, "devq: openings %d, active %d, "
1863 		    "allocated %d, queued %d, held %d\n",
1864 		    cgds.dev_openings, cgds.dev_active, cgds.allocated,
1865 		    cgds.queued, cgds.held);
1866 	}
1867 
1868 	num_items = 0;
1869 
1870 	STAILQ_FOREACH(hdr, &softc->work_queue, periph_links.stqe) {
1871 		union ctl_io *io = hdr->io_ptr;
1872 
1873 		num_items++;
1874 
1875 		/*
1876 		 * Only regular SCSI I/O is put on the work
1877 		 * queue, so we can print sense here.  There may be no
1878 		 * sense if it's no the queue for a DMA, but this serves to
1879 		 * print out the CCB as well.
1880 		 *
1881 		 * XXX KDM switch this over to scsi_sense_print() when
1882 		 * CTL is merged in with CAM.
1883 		 */
1884 		ctl_io_error_print(io, NULL);
1885 
1886 		/*
1887 		 * Print DMA status if we are DMA_QUEUED.
1888 		 */
1889 		if (io->io_hdr.flags & CTL_FLAG_DMA_QUEUED) {
1890 			xpt_print(periph->path,
1891 			    "Total %u, Current %u, Resid %u\n",
1892 			    io->scsiio.kern_total_len,
1893 			    io->scsiio.kern_data_len,
1894 			    io->scsiio.kern_data_resid);
1895 		}
1896 	}
1897 
1898 	xpt_print(periph->path, "%d requests waiting for CCBs\n", num_items);
1899 	xpt_print(periph->path, "%d CTIOs outstanding\n", softc->ctios_sent);
1900 }
1901 
1902 /*
1903  * Datamove/done routine called by CTL.  Put ourselves on the queue to
1904  * receive a CCB from CAM so we can queue the continue I/O request down
1905  * to the adapter.
1906  */
1907 static void
1908 ctlfe_datamove(union ctl_io *io)
1909 {
1910 	union ccb *ccb;
1911 	struct cam_periph *periph;
1912 	struct ctlfe_lun_softc *softc;
1913 
1914 	KASSERT(io->io_hdr.io_type == CTL_IO_SCSI,
1915 	    ("%s: unexpected I/O type %x", __func__, io->io_hdr.io_type));
1916 
1917 	io->scsiio.ext_data_filled = 0;
1918 	ccb = PRIV_CCB(io);
1919 	periph = xpt_path_periph(ccb->ccb_h.path);
1920 	cam_periph_lock(periph);
1921 	softc = (struct ctlfe_lun_softc *)periph->softc;
1922 	io->io_hdr.flags |= CTL_FLAG_DMA_QUEUED;
1923 	if ((io->io_hdr.status & CTL_STATUS_MASK) != CTL_STATUS_NONE)
1924 		io->io_hdr.flags |= CTL_FLAG_STATUS_QUEUED;
1925 	STAILQ_INSERT_TAIL(&softc->work_queue, &ccb->ccb_h,
1926 			  periph_links.stqe);
1927 	xpt_schedule(periph, CAM_PRIORITY_NORMAL);
1928 	cam_periph_unlock(periph);
1929 }
1930 
1931 static void
1932 ctlfe_done(union ctl_io *io)
1933 {
1934 	union ccb *ccb;
1935 	struct cam_periph *periph;
1936 	struct ctlfe_lun_softc *softc;
1937 
1938 	ccb = PRIV_CCB(io);
1939 	periph = xpt_path_periph(ccb->ccb_h.path);
1940 	cam_periph_lock(periph);
1941 	softc = (struct ctlfe_lun_softc *)periph->softc;
1942 
1943 	if (io->io_hdr.io_type == CTL_IO_TASK) {
1944 		/*
1945 		 * Send the notify acknowledge down to the SIM, to let it
1946 		 * know we processed the task management command.
1947 		 */
1948 		ccb->ccb_h.status = CAM_REQ_INPROG;
1949 		ccb->ccb_h.func_code = XPT_NOTIFY_ACKNOWLEDGE;
1950 		switch (io->taskio.task_status) {
1951 		case CTL_TASK_FUNCTION_COMPLETE:
1952 			ccb->cna2.arg = CAM_RSP_TMF_COMPLETE;
1953 			break;
1954 		case CTL_TASK_FUNCTION_SUCCEEDED:
1955 			ccb->cna2.arg = CAM_RSP_TMF_SUCCEEDED;
1956 			ccb->ccb_h.flags |= CAM_SEND_STATUS;
1957 			break;
1958 		case CTL_TASK_FUNCTION_REJECTED:
1959 			ccb->cna2.arg = CAM_RSP_TMF_REJECTED;
1960 			ccb->ccb_h.flags |= CAM_SEND_STATUS;
1961 			break;
1962 		case CTL_TASK_LUN_DOES_NOT_EXIST:
1963 			ccb->cna2.arg = CAM_RSP_TMF_INCORRECT_LUN;
1964 			ccb->ccb_h.flags |= CAM_SEND_STATUS;
1965 			break;
1966 		case CTL_TASK_FUNCTION_NOT_SUPPORTED:
1967 			ccb->cna2.arg = CAM_RSP_TMF_FAILED;
1968 			ccb->ccb_h.flags |= CAM_SEND_STATUS;
1969 			break;
1970 		}
1971 		ccb->cna2.arg |= scsi_3btoul(io->taskio.task_resp) << 8;
1972 		xpt_action(ccb);
1973 	} else if (io->io_hdr.flags & CTL_FLAG_STATUS_SENT) {
1974 		ctlfe_requeue_ccb(periph, ccb, /* unlock */1);
1975 		return;
1976 	} else {
1977 		io->io_hdr.flags |= CTL_FLAG_STATUS_QUEUED;
1978 		STAILQ_INSERT_TAIL(&softc->work_queue, &ccb->ccb_h,
1979 				  periph_links.stqe);
1980 		xpt_schedule(periph, CAM_PRIORITY_NORMAL);
1981 	}
1982 
1983 	cam_periph_unlock(periph);
1984 }
1985 
1986 static void
1987 ctlfe_dump(void)
1988 {
1989 	struct ctlfe_softc *bus_softc;
1990 	struct ctlfe_lun_softc *lun_softc;
1991 
1992 	STAILQ_FOREACH(bus_softc, &ctlfe_softc_list, links) {
1993 		ctlfe_dump_sim(bus_softc->sim);
1994 		STAILQ_FOREACH(lun_softc, &bus_softc->lun_softc_list, links)
1995 			ctlfe_dump_queue(lun_softc);
1996 	}
1997 }
1998