xref: /freebsd/sys/cam/ctl/ctl.c (revision a223d3ed90bfe313ce5987d468a25a915d7d1254)
1 /*-
2  * Copyright (c) 2003-2009 Silicon Graphics International Corp.
3  * Copyright (c) 2012 The FreeBSD Foundation
4  * All rights reserved.
5  *
6  * Portions of this software were developed by Edward Tomasz Napierala
7  * under sponsorship from the FreeBSD Foundation.
8  *
9  * Redistribution and use in source and binary forms, with or without
10  * modification, are permitted provided that the following conditions
11  * are met:
12  * 1. Redistributions of source code must retain the above copyright
13  *    notice, this list of conditions, and the following disclaimer,
14  *    without modification.
15  * 2. Redistributions in binary form must reproduce at minimum a disclaimer
16  *    substantially similar to the "NO WARRANTY" disclaimer below
17  *    ("Disclaimer") and any redistribution must be conditioned upon
18  *    including a substantially similar Disclaimer requirement for further
19  *    binary redistribution.
20  *
21  * NO WARRANTY
22  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
23  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
24  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTIBILITY AND FITNESS FOR
25  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
26  * HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
27  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
28  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
29  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
30  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
31  * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
32  * POSSIBILITY OF SUCH DAMAGES.
33  *
34  * $Id: //depot/users/kenm/FreeBSD-test2/sys/cam/ctl/ctl.c#8 $
35  */
36 /*
37  * CAM Target Layer, a SCSI device emulation subsystem.
38  *
39  * Author: Ken Merry <ken@FreeBSD.org>
40  */
41 
42 #define _CTL_C
43 
44 #include <sys/cdefs.h>
45 __FBSDID("$FreeBSD$");
46 
47 #include <sys/param.h>
48 #include <sys/systm.h>
49 #include <sys/kernel.h>
50 #include <sys/types.h>
51 #include <sys/kthread.h>
52 #include <sys/bio.h>
53 #include <sys/fcntl.h>
54 #include <sys/lock.h>
55 #include <sys/module.h>
56 #include <sys/mutex.h>
57 #include <sys/condvar.h>
58 #include <sys/malloc.h>
59 #include <sys/conf.h>
60 #include <sys/ioccom.h>
61 #include <sys/queue.h>
62 #include <sys/sbuf.h>
63 #include <sys/smp.h>
64 #include <sys/endian.h>
65 #include <sys/sysctl.h>
66 
67 #include <cam/cam.h>
68 #include <cam/scsi/scsi_all.h>
69 #include <cam/scsi/scsi_da.h>
70 #include <cam/ctl/ctl_io.h>
71 #include <cam/ctl/ctl.h>
72 #include <cam/ctl/ctl_frontend.h>
73 #include <cam/ctl/ctl_frontend_internal.h>
74 #include <cam/ctl/ctl_util.h>
75 #include <cam/ctl/ctl_backend.h>
76 #include <cam/ctl/ctl_ioctl.h>
77 #include <cam/ctl/ctl_ha.h>
78 #include <cam/ctl/ctl_private.h>
79 #include <cam/ctl/ctl_debug.h>
80 #include <cam/ctl/ctl_scsi_all.h>
81 #include <cam/ctl/ctl_error.h>
82 
83 struct ctl_softc *control_softc = NULL;
84 
85 /*
86  * Size and alignment macros needed for Copan-specific HA hardware.  These
87  * can go away when the HA code is re-written, and uses busdma for any
88  * hardware.
89  */
90 #define	CTL_ALIGN_8B(target, source, type)				\
91 	if (((uint32_t)source & 0x7) != 0)				\
92 		target = (type)(source + (0x8 - ((uint32_t)source & 0x7)));\
93 	else								\
94 		target = (type)source;
95 
96 #define	CTL_SIZE_8B(target, size)					\
97 	if ((size & 0x7) != 0)						\
98 		target = size + (0x8 - (size & 0x7));			\
99 	else								\
100 		target = size;
101 
102 #define CTL_ALIGN_8B_MARGIN	16
103 
104 /*
105  * Template mode pages.
106  */
107 
108 /*
109  * Note that these are default values only.  The actual values will be
110  * filled in when the user does a mode sense.
111  */
112 static struct copan_power_subpage power_page_default = {
113 	/*page_code*/ PWR_PAGE_CODE | SMPH_SPF,
114 	/*subpage*/ PWR_SUBPAGE_CODE,
115 	/*page_length*/ {(sizeof(struct copan_power_subpage) - 4) & 0xff00,
116 			 (sizeof(struct copan_power_subpage) - 4) & 0x00ff},
117 	/*page_version*/ PWR_VERSION,
118 	/* total_luns */ 26,
119 	/* max_active_luns*/ PWR_DFLT_MAX_LUNS,
120 	/*reserved*/ {0, 0, 0, 0, 0, 0, 0, 0, 0,
121 		      0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
122 		      0, 0, 0, 0, 0, 0}
123 };
124 
125 static struct copan_power_subpage power_page_changeable = {
126 	/*page_code*/ PWR_PAGE_CODE | SMPH_SPF,
127 	/*subpage*/ PWR_SUBPAGE_CODE,
128 	/*page_length*/ {(sizeof(struct copan_power_subpage) - 4) & 0xff00,
129 			 (sizeof(struct copan_power_subpage) - 4) & 0x00ff},
130 	/*page_version*/ 0,
131 	/* total_luns */ 0,
132 	/* max_active_luns*/ 0,
133 	/*reserved*/ {0, 0, 0, 0, 0, 0, 0, 0, 0,
134 		      0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
135 		      0, 0, 0, 0, 0, 0}
136 };
137 
138 static struct copan_aps_subpage aps_page_default = {
139 	APS_PAGE_CODE | SMPH_SPF, //page_code
140 	APS_SUBPAGE_CODE, //subpage
141 	{(sizeof(struct copan_aps_subpage) - 4) & 0xff00,
142 	 (sizeof(struct copan_aps_subpage) - 4) & 0x00ff}, //page_length
143 	APS_VERSION, //page_version
144 	0, //lock_active
145 	{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
146 	0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
147 	0, 0, 0, 0, 0} //reserved
148 };
149 
150 static struct copan_aps_subpage aps_page_changeable = {
151 	APS_PAGE_CODE | SMPH_SPF, //page_code
152 	APS_SUBPAGE_CODE, //subpage
153 	{(sizeof(struct copan_aps_subpage) - 4) & 0xff00,
154 	 (sizeof(struct copan_aps_subpage) - 4) & 0x00ff}, //page_length
155 	0, //page_version
156 	0, //lock_active
157 	{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
158 	0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
159 	0, 0, 0, 0, 0} //reserved
160 };
161 
162 static struct copan_debugconf_subpage debugconf_page_default = {
163 	DBGCNF_PAGE_CODE | SMPH_SPF,	/* page_code */
164 	DBGCNF_SUBPAGE_CODE,		/* subpage */
165 	{(sizeof(struct copan_debugconf_subpage) - 4) >> 8,
166 	 (sizeof(struct copan_debugconf_subpage) - 4) >> 0}, /* page_length */
167 	DBGCNF_VERSION,			/* page_version */
168 	{CTL_TIME_IO_DEFAULT_SECS>>8,
169 	 CTL_TIME_IO_DEFAULT_SECS>>0},	/* ctl_time_io_secs */
170 };
171 
172 static struct copan_debugconf_subpage debugconf_page_changeable = {
173 	DBGCNF_PAGE_CODE | SMPH_SPF,	/* page_code */
174 	DBGCNF_SUBPAGE_CODE,		/* subpage */
175 	{(sizeof(struct copan_debugconf_subpage) - 4) >> 8,
176 	 (sizeof(struct copan_debugconf_subpage) - 4) >> 0}, /* page_length */
177 	0,				/* page_version */
178 	{0xff,0xff},			/* ctl_time_io_secs */
179 };
180 
181 static struct scsi_format_page format_page_default = {
182 	/*page_code*/SMS_FORMAT_DEVICE_PAGE,
183 	/*page_length*/sizeof(struct scsi_format_page) - 2,
184 	/*tracks_per_zone*/ {0, 0},
185 	/*alt_sectors_per_zone*/ {0, 0},
186 	/*alt_tracks_per_zone*/ {0, 0},
187 	/*alt_tracks_per_lun*/ {0, 0},
188 	/*sectors_per_track*/ {(CTL_DEFAULT_SECTORS_PER_TRACK >> 8) & 0xff,
189 			        CTL_DEFAULT_SECTORS_PER_TRACK & 0xff},
190 	/*bytes_per_sector*/ {0, 0},
191 	/*interleave*/ {0, 0},
192 	/*track_skew*/ {0, 0},
193 	/*cylinder_skew*/ {0, 0},
194 	/*flags*/ SFP_HSEC,
195 	/*reserved*/ {0, 0, 0}
196 };
197 
198 static struct scsi_format_page format_page_changeable = {
199 	/*page_code*/SMS_FORMAT_DEVICE_PAGE,
200 	/*page_length*/sizeof(struct scsi_format_page) - 2,
201 	/*tracks_per_zone*/ {0, 0},
202 	/*alt_sectors_per_zone*/ {0, 0},
203 	/*alt_tracks_per_zone*/ {0, 0},
204 	/*alt_tracks_per_lun*/ {0, 0},
205 	/*sectors_per_track*/ {0, 0},
206 	/*bytes_per_sector*/ {0, 0},
207 	/*interleave*/ {0, 0},
208 	/*track_skew*/ {0, 0},
209 	/*cylinder_skew*/ {0, 0},
210 	/*flags*/ 0,
211 	/*reserved*/ {0, 0, 0}
212 };
213 
214 static struct scsi_rigid_disk_page rigid_disk_page_default = {
215 	/*page_code*/SMS_RIGID_DISK_PAGE,
216 	/*page_length*/sizeof(struct scsi_rigid_disk_page) - 2,
217 	/*cylinders*/ {0, 0, 0},
218 	/*heads*/ CTL_DEFAULT_HEADS,
219 	/*start_write_precomp*/ {0, 0, 0},
220 	/*start_reduced_current*/ {0, 0, 0},
221 	/*step_rate*/ {0, 0},
222 	/*landing_zone_cylinder*/ {0, 0, 0},
223 	/*rpl*/ SRDP_RPL_DISABLED,
224 	/*rotational_offset*/ 0,
225 	/*reserved1*/ 0,
226 	/*rotation_rate*/ {(CTL_DEFAULT_ROTATION_RATE >> 8) & 0xff,
227 			   CTL_DEFAULT_ROTATION_RATE & 0xff},
228 	/*reserved2*/ {0, 0}
229 };
230 
231 static struct scsi_rigid_disk_page rigid_disk_page_changeable = {
232 	/*page_code*/SMS_RIGID_DISK_PAGE,
233 	/*page_length*/sizeof(struct scsi_rigid_disk_page) - 2,
234 	/*cylinders*/ {0, 0, 0},
235 	/*heads*/ 0,
236 	/*start_write_precomp*/ {0, 0, 0},
237 	/*start_reduced_current*/ {0, 0, 0},
238 	/*step_rate*/ {0, 0},
239 	/*landing_zone_cylinder*/ {0, 0, 0},
240 	/*rpl*/ 0,
241 	/*rotational_offset*/ 0,
242 	/*reserved1*/ 0,
243 	/*rotation_rate*/ {0, 0},
244 	/*reserved2*/ {0, 0}
245 };
246 
247 static struct scsi_caching_page caching_page_default = {
248 	/*page_code*/SMS_CACHING_PAGE,
249 	/*page_length*/sizeof(struct scsi_caching_page) - 2,
250 	/*flags1*/ SCP_DISC | SCP_WCE,
251 	/*ret_priority*/ 0,
252 	/*disable_pf_transfer_len*/ {0xff, 0xff},
253 	/*min_prefetch*/ {0, 0},
254 	/*max_prefetch*/ {0xff, 0xff},
255 	/*max_pf_ceiling*/ {0xff, 0xff},
256 	/*flags2*/ 0,
257 	/*cache_segments*/ 0,
258 	/*cache_seg_size*/ {0, 0},
259 	/*reserved*/ 0,
260 	/*non_cache_seg_size*/ {0, 0, 0}
261 };
262 
263 static struct scsi_caching_page caching_page_changeable = {
264 	/*page_code*/SMS_CACHING_PAGE,
265 	/*page_length*/sizeof(struct scsi_caching_page) - 2,
266 	/*flags1*/ 0,
267 	/*ret_priority*/ 0,
268 	/*disable_pf_transfer_len*/ {0, 0},
269 	/*min_prefetch*/ {0, 0},
270 	/*max_prefetch*/ {0, 0},
271 	/*max_pf_ceiling*/ {0, 0},
272 	/*flags2*/ 0,
273 	/*cache_segments*/ 0,
274 	/*cache_seg_size*/ {0, 0},
275 	/*reserved*/ 0,
276 	/*non_cache_seg_size*/ {0, 0, 0}
277 };
278 
279 static struct scsi_control_page control_page_default = {
280 	/*page_code*/SMS_CONTROL_MODE_PAGE,
281 	/*page_length*/sizeof(struct scsi_control_page) - 2,
282 	/*rlec*/0,
283 	/*queue_flags*/0,
284 	/*eca_and_aen*/0,
285 	/*reserved*/0,
286 	/*aen_holdoff_period*/{0, 0}
287 };
288 
289 static struct scsi_control_page control_page_changeable = {
290 	/*page_code*/SMS_CONTROL_MODE_PAGE,
291 	/*page_length*/sizeof(struct scsi_control_page) - 2,
292 	/*rlec*/SCP_DSENSE,
293 	/*queue_flags*/0,
294 	/*eca_and_aen*/0,
295 	/*reserved*/0,
296 	/*aen_holdoff_period*/{0, 0}
297 };
298 
299 
300 /*
301  * XXX KDM move these into the softc.
302  */
303 static int rcv_sync_msg;
304 static int persis_offset;
305 static uint8_t ctl_pause_rtr;
306 static int     ctl_is_single = 1;
307 static int     index_to_aps_page;
308 
309 SYSCTL_NODE(_kern_cam, OID_AUTO, ctl, CTLFLAG_RD, 0, "CAM Target Layer");
310 static int worker_threads = -1;
311 TUNABLE_INT("kern.cam.ctl.worker_threads", &worker_threads);
312 SYSCTL_INT(_kern_cam_ctl, OID_AUTO, worker_threads, CTLFLAG_RDTUN,
313     &worker_threads, 1, "Number of worker threads");
314 static int verbose = 0;
315 TUNABLE_INT("kern.cam.ctl.verbose", &verbose);
316 SYSCTL_INT(_kern_cam_ctl, OID_AUTO, verbose, CTLFLAG_RWTUN,
317     &verbose, 0, "Show SCSI errors returned to initiator");
318 
319 /*
320  * Serial number (0x80), device id (0x83), supported pages (0x00),
321  * Block limits (0xB0) and Logical Block Provisioning (0xB2)
322  */
323 #define SCSI_EVPD_NUM_SUPPORTED_PAGES	5
324 
325 static void ctl_isc_event_handler(ctl_ha_channel chanel, ctl_ha_event event,
326 				  int param);
327 static void ctl_copy_sense_data(union ctl_ha_msg *src, union ctl_io *dest);
328 static int ctl_init(void);
329 void ctl_shutdown(void);
330 static int ctl_open(struct cdev *dev, int flags, int fmt, struct thread *td);
331 static int ctl_close(struct cdev *dev, int flags, int fmt, struct thread *td);
332 static void ctl_ioctl_online(void *arg);
333 static void ctl_ioctl_offline(void *arg);
334 static int ctl_ioctl_targ_enable(void *arg, struct ctl_id targ_id);
335 static int ctl_ioctl_targ_disable(void *arg, struct ctl_id targ_id);
336 static int ctl_ioctl_lun_enable(void *arg, struct ctl_id targ_id, int lun_id);
337 static int ctl_ioctl_lun_disable(void *arg, struct ctl_id targ_id, int lun_id);
338 static int ctl_ioctl_do_datamove(struct ctl_scsiio *ctsio);
339 static int ctl_serialize_other_sc_cmd(struct ctl_scsiio *ctsio);
340 static int ctl_ioctl_submit_wait(union ctl_io *io);
341 static void ctl_ioctl_datamove(union ctl_io *io);
342 static void ctl_ioctl_done(union ctl_io *io);
343 static void ctl_ioctl_hard_startstop_callback(void *arg,
344 					      struct cfi_metatask *metatask);
345 static void ctl_ioctl_bbrread_callback(void *arg,struct cfi_metatask *metatask);
346 static int ctl_ioctl_fill_ooa(struct ctl_lun *lun, uint32_t *cur_fill_num,
347 			      struct ctl_ooa *ooa_hdr,
348 			      struct ctl_ooa_entry *kern_entries);
349 static int ctl_ioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flag,
350 		     struct thread *td);
351 uint32_t ctl_get_resindex(struct ctl_nexus *nexus);
352 uint32_t ctl_port_idx(int port_num);
353 #ifdef unused
354 static union ctl_io *ctl_malloc_io(ctl_io_type io_type, uint32_t targ_port,
355 				   uint32_t targ_target, uint32_t targ_lun,
356 				   int can_wait);
357 static void ctl_kfree_io(union ctl_io *io);
358 #endif /* unused */
359 static int ctl_alloc_lun(struct ctl_softc *ctl_softc, struct ctl_lun *lun,
360 			 struct ctl_be_lun *be_lun, struct ctl_id target_id);
361 static int ctl_free_lun(struct ctl_lun *lun);
362 static void ctl_create_lun(struct ctl_be_lun *be_lun);
363 /**
364 static void ctl_failover_change_pages(struct ctl_softc *softc,
365 				      struct ctl_scsiio *ctsio, int master);
366 **/
367 
368 static int ctl_do_mode_select(union ctl_io *io);
369 static int ctl_pro_preempt(struct ctl_softc *softc, struct ctl_lun *lun,
370 			   uint64_t res_key, uint64_t sa_res_key,
371 			   uint8_t type, uint32_t residx,
372 			   struct ctl_scsiio *ctsio,
373 			   struct scsi_per_res_out *cdb,
374 			   struct scsi_per_res_out_parms* param);
375 static void ctl_pro_preempt_other(struct ctl_lun *lun,
376 				  union ctl_ha_msg *msg);
377 static void ctl_hndl_per_res_out_on_other_sc(union ctl_ha_msg *msg);
378 static int ctl_inquiry_evpd_supported(struct ctl_scsiio *ctsio, int alloc_len);
379 static int ctl_inquiry_evpd_serial(struct ctl_scsiio *ctsio, int alloc_len);
380 static int ctl_inquiry_evpd_devid(struct ctl_scsiio *ctsio, int alloc_len);
381 static int ctl_inquiry_evpd_block_limits(struct ctl_scsiio *ctsio,
382 					 int alloc_len);
383 static int ctl_inquiry_evpd_lbp(struct ctl_scsiio *ctsio, int alloc_len);
384 static int ctl_inquiry_evpd(struct ctl_scsiio *ctsio);
385 static int ctl_inquiry_std(struct ctl_scsiio *ctsio);
386 static int ctl_get_lba_len(union ctl_io *io, uint64_t *lba, uint32_t *len);
387 static ctl_action ctl_extent_check(union ctl_io *io1, union ctl_io *io2);
388 static ctl_action ctl_check_for_blockage(union ctl_io *pending_io,
389 					 union ctl_io *ooa_io);
390 static ctl_action ctl_check_ooa(struct ctl_lun *lun, union ctl_io *pending_io,
391 				union ctl_io *starting_io);
392 static int ctl_check_blocked(struct ctl_lun *lun);
393 static int ctl_scsiio_lun_check(struct ctl_softc *ctl_softc,
394 				struct ctl_lun *lun,
395 				struct ctl_cmd_entry *entry,
396 				struct ctl_scsiio *ctsio);
397 //static int ctl_check_rtr(union ctl_io *pending_io, struct ctl_softc *softc);
398 static void ctl_failover(void);
399 static int ctl_scsiio_precheck(struct ctl_softc *ctl_softc,
400 			       struct ctl_scsiio *ctsio);
401 static int ctl_scsiio(struct ctl_scsiio *ctsio);
402 
403 static int ctl_bus_reset(struct ctl_softc *ctl_softc, union ctl_io *io);
404 static int ctl_target_reset(struct ctl_softc *ctl_softc, union ctl_io *io,
405 			    ctl_ua_type ua_type);
406 static int ctl_lun_reset(struct ctl_lun *lun, union ctl_io *io,
407 			 ctl_ua_type ua_type);
408 static int ctl_abort_task(union ctl_io *io);
409 static void ctl_run_task(union ctl_io *io);
410 #ifdef CTL_IO_DELAY
411 static void ctl_datamove_timer_wakeup(void *arg);
412 static void ctl_done_timer_wakeup(void *arg);
413 #endif /* CTL_IO_DELAY */
414 
415 static void ctl_send_datamove_done(union ctl_io *io, int have_lock);
416 static void ctl_datamove_remote_write_cb(struct ctl_ha_dt_req *rq);
417 static int ctl_datamove_remote_dm_write_cb(union ctl_io *io);
418 static void ctl_datamove_remote_write(union ctl_io *io);
419 static int ctl_datamove_remote_dm_read_cb(union ctl_io *io);
420 static void ctl_datamove_remote_read_cb(struct ctl_ha_dt_req *rq);
421 static int ctl_datamove_remote_sgl_setup(union ctl_io *io);
422 static int ctl_datamove_remote_xfer(union ctl_io *io, unsigned command,
423 				    ctl_ha_dt_cb callback);
424 static void ctl_datamove_remote_read(union ctl_io *io);
425 static void ctl_datamove_remote(union ctl_io *io);
426 static int ctl_process_done(union ctl_io *io);
427 static void ctl_lun_thread(void *arg);
428 static void ctl_work_thread(void *arg);
429 static void ctl_enqueue_incoming(union ctl_io *io);
430 static void ctl_enqueue_rtr(union ctl_io *io);
431 static void ctl_enqueue_done(union ctl_io *io);
432 static void ctl_enqueue_isc(union ctl_io *io);
433 
434 /*
435  * Load the serialization table.  This isn't very pretty, but is probably
436  * the easiest way to do it.
437  */
438 #include "ctl_ser_table.c"
439 
440 /*
441  * We only need to define open, close and ioctl routines for this driver.
442  */
443 static struct cdevsw ctl_cdevsw = {
444 	.d_version =	D_VERSION,
445 	.d_flags =	0,
446 	.d_open =	ctl_open,
447 	.d_close =	ctl_close,
448 	.d_ioctl =	ctl_ioctl,
449 	.d_name =	"ctl",
450 };
451 
452 
453 MALLOC_DEFINE(M_CTL, "ctlmem", "Memory used for CTL");
454 
455 static int ctl_module_event_handler(module_t, int /*modeventtype_t*/, void *);
456 
457 static moduledata_t ctl_moduledata = {
458 	"ctl",
459 	ctl_module_event_handler,
460 	NULL
461 };
462 
463 DECLARE_MODULE(ctl, ctl_moduledata, SI_SUB_CONFIGURE, SI_ORDER_THIRD);
464 MODULE_VERSION(ctl, 1);
465 
466 static void
467 ctl_isc_handler_finish_xfer(struct ctl_softc *ctl_softc,
468 			    union ctl_ha_msg *msg_info)
469 {
470 	struct ctl_scsiio *ctsio;
471 
472 	if (msg_info->hdr.original_sc == NULL) {
473 		printf("%s: original_sc == NULL!\n", __func__);
474 		/* XXX KDM now what? */
475 		return;
476 	}
477 
478 	ctsio = &msg_info->hdr.original_sc->scsiio;
479 	ctsio->io_hdr.flags |= CTL_FLAG_IO_ACTIVE;
480 	ctsio->io_hdr.msg_type = CTL_MSG_FINISH_IO;
481 	ctsio->io_hdr.status = msg_info->hdr.status;
482 	ctsio->scsi_status = msg_info->scsi.scsi_status;
483 	ctsio->sense_len = msg_info->scsi.sense_len;
484 	ctsio->sense_residual = msg_info->scsi.sense_residual;
485 	ctsio->residual = msg_info->scsi.residual;
486 	memcpy(&ctsio->sense_data, &msg_info->scsi.sense_data,
487 	       sizeof(ctsio->sense_data));
488 	memcpy(&ctsio->io_hdr.ctl_private[CTL_PRIV_LBA_LEN].bytes,
489 	       &msg_info->scsi.lbalen, sizeof(msg_info->scsi.lbalen));
490 	ctl_enqueue_isc((union ctl_io *)ctsio);
491 }
492 
493 static void
494 ctl_isc_handler_finish_ser_only(struct ctl_softc *ctl_softc,
495 				union ctl_ha_msg *msg_info)
496 {
497 	struct ctl_scsiio *ctsio;
498 
499 	if (msg_info->hdr.serializing_sc == NULL) {
500 		printf("%s: serializing_sc == NULL!\n", __func__);
501 		/* XXX KDM now what? */
502 		return;
503 	}
504 
505 	ctsio = &msg_info->hdr.serializing_sc->scsiio;
506 #if 0
507 	/*
508 	 * Attempt to catch the situation where an I/O has
509 	 * been freed, and we're using it again.
510 	 */
511 	if (ctsio->io_hdr.io_type == 0xff) {
512 		union ctl_io *tmp_io;
513 		tmp_io = (union ctl_io *)ctsio;
514 		printf("%s: %p use after free!\n", __func__,
515 		       ctsio);
516 		printf("%s: type %d msg %d cdb %x iptl: "
517 		       "%d:%d:%d:%d tag 0x%04x "
518 		       "flag %#x status %x\n",
519 			__func__,
520 			tmp_io->io_hdr.io_type,
521 			tmp_io->io_hdr.msg_type,
522 			tmp_io->scsiio.cdb[0],
523 			tmp_io->io_hdr.nexus.initid.id,
524 			tmp_io->io_hdr.nexus.targ_port,
525 			tmp_io->io_hdr.nexus.targ_target.id,
526 			tmp_io->io_hdr.nexus.targ_lun,
527 			(tmp_io->io_hdr.io_type ==
528 			CTL_IO_TASK) ?
529 			tmp_io->taskio.tag_num :
530 			tmp_io->scsiio.tag_num,
531 		        tmp_io->io_hdr.flags,
532 			tmp_io->io_hdr.status);
533 	}
534 #endif
535 	ctsio->io_hdr.msg_type = CTL_MSG_FINISH_IO;
536 	ctl_enqueue_isc((union ctl_io *)ctsio);
537 }
538 
539 /*
540  * ISC (Inter Shelf Communication) event handler.  Events from the HA
541  * subsystem come in here.
542  */
543 static void
544 ctl_isc_event_handler(ctl_ha_channel channel, ctl_ha_event event, int param)
545 {
546 	struct ctl_softc *ctl_softc;
547 	union ctl_io *io;
548 	struct ctl_prio *presio;
549 	ctl_ha_status isc_status;
550 
551 	ctl_softc = control_softc;
552 	io = NULL;
553 
554 
555 #if 0
556 	printf("CTL: Isc Msg event %d\n", event);
557 #endif
558 	if (event == CTL_HA_EVT_MSG_RECV) {
559 		union ctl_ha_msg msg_info;
560 
561 		isc_status = ctl_ha_msg_recv(CTL_HA_CHAN_CTL, &msg_info,
562 					     sizeof(msg_info), /*wait*/ 0);
563 #if 0
564 		printf("CTL: msg_type %d\n", msg_info.msg_type);
565 #endif
566 		if (isc_status != 0) {
567 			printf("Error receiving message, status = %d\n",
568 			       isc_status);
569 			return;
570 		}
571 
572 		switch (msg_info.hdr.msg_type) {
573 		case CTL_MSG_SERIALIZE:
574 #if 0
575 			printf("Serialize\n");
576 #endif
577 			io = ctl_alloc_io((void *)ctl_softc->othersc_pool);
578 			if (io == NULL) {
579 				printf("ctl_isc_event_handler: can't allocate "
580 				       "ctl_io!\n");
581 				/* Bad Juju */
582 				/* Need to set busy and send msg back */
583 				msg_info.hdr.msg_type = CTL_MSG_BAD_JUJU;
584 				msg_info.hdr.status = CTL_SCSI_ERROR;
585 				msg_info.scsi.scsi_status = SCSI_STATUS_BUSY;
586 				msg_info.scsi.sense_len = 0;
587 			        if (ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg_info,
588 				    sizeof(msg_info), 0) > CTL_HA_STATUS_SUCCESS){
589 				}
590 				goto bailout;
591 			}
592 			ctl_zero_io(io);
593 			// populate ctsio from msg_info
594 			io->io_hdr.io_type = CTL_IO_SCSI;
595 			io->io_hdr.msg_type = CTL_MSG_SERIALIZE;
596 			io->io_hdr.original_sc = msg_info.hdr.original_sc;
597 #if 0
598 			printf("pOrig %x\n", (int)msg_info.original_sc);
599 #endif
600 			io->io_hdr.flags |= CTL_FLAG_FROM_OTHER_SC |
601 					    CTL_FLAG_IO_ACTIVE;
602 			/*
603 			 * If we're in serialization-only mode, we don't
604 			 * want to go through full done processing.  Thus
605 			 * the COPY flag.
606 			 *
607 			 * XXX KDM add another flag that is more specific.
608 			 */
609 			if (ctl_softc->ha_mode == CTL_HA_MODE_SER_ONLY)
610 				io->io_hdr.flags |= CTL_FLAG_INT_COPY;
611 			io->io_hdr.nexus = msg_info.hdr.nexus;
612 #if 0
613 			printf("targ %d, port %d, iid %d, lun %d\n",
614 			       io->io_hdr.nexus.targ_target.id,
615 			       io->io_hdr.nexus.targ_port,
616 			       io->io_hdr.nexus.initid.id,
617 			       io->io_hdr.nexus.targ_lun);
618 #endif
619 			io->scsiio.tag_num = msg_info.scsi.tag_num;
620 			io->scsiio.tag_type = msg_info.scsi.tag_type;
621 			memcpy(io->scsiio.cdb, msg_info.scsi.cdb,
622 			       CTL_MAX_CDBLEN);
623 			if (ctl_softc->ha_mode == CTL_HA_MODE_XFER) {
624 				struct ctl_cmd_entry *entry;
625 				uint8_t opcode;
626 
627 				opcode = io->scsiio.cdb[0];
628 				entry = &ctl_cmd_table[opcode];
629 				io->io_hdr.flags &= ~CTL_FLAG_DATA_MASK;
630 				io->io_hdr.flags |=
631 					entry->flags & CTL_FLAG_DATA_MASK;
632 			}
633 			ctl_enqueue_isc(io);
634 			break;
635 
636 		/* Performed on the Originating SC, XFER mode only */
637 		case CTL_MSG_DATAMOVE: {
638 			struct ctl_sg_entry *sgl;
639 			int i, j;
640 
641 			io = msg_info.hdr.original_sc;
642 			if (io == NULL) {
643 				printf("%s: original_sc == NULL!\n", __func__);
644 				/* XXX KDM do something here */
645 				break;
646 			}
647 			io->io_hdr.msg_type = CTL_MSG_DATAMOVE;
648 			io->io_hdr.flags |= CTL_FLAG_IO_ACTIVE;
649 			/*
650 			 * Keep track of this, we need to send it back over
651 			 * when the datamove is complete.
652 			 */
653 			io->io_hdr.serializing_sc = msg_info.hdr.serializing_sc;
654 
655 			if (msg_info.dt.sg_sequence == 0) {
656 				/*
657 				 * XXX KDM we use the preallocated S/G list
658 				 * here, but we'll need to change this to
659 				 * dynamic allocation if we need larger S/G
660 				 * lists.
661 				 */
662 				if (msg_info.dt.kern_sg_entries >
663 				    sizeof(io->io_hdr.remote_sglist) /
664 				    sizeof(io->io_hdr.remote_sglist[0])) {
665 					printf("%s: number of S/G entries "
666 					    "needed %u > allocated num %zd\n",
667 					    __func__,
668 					    msg_info.dt.kern_sg_entries,
669 					    sizeof(io->io_hdr.remote_sglist)/
670 					    sizeof(io->io_hdr.remote_sglist[0]));
671 
672 					/*
673 					 * XXX KDM send a message back to
674 					 * the other side to shut down the
675 					 * DMA.  The error will come back
676 					 * through via the normal channel.
677 					 */
678 					break;
679 				}
680 				sgl = io->io_hdr.remote_sglist;
681 				memset(sgl, 0,
682 				       sizeof(io->io_hdr.remote_sglist));
683 
684 				io->scsiio.kern_data_ptr = (uint8_t *)sgl;
685 
686 				io->scsiio.kern_sg_entries =
687 					msg_info.dt.kern_sg_entries;
688 				io->scsiio.rem_sg_entries =
689 					msg_info.dt.kern_sg_entries;
690 				io->scsiio.kern_data_len =
691 					msg_info.dt.kern_data_len;
692 				io->scsiio.kern_total_len =
693 					msg_info.dt.kern_total_len;
694 				io->scsiio.kern_data_resid =
695 					msg_info.dt.kern_data_resid;
696 				io->scsiio.kern_rel_offset =
697 					msg_info.dt.kern_rel_offset;
698 				/*
699 				 * Clear out per-DMA flags.
700 				 */
701 				io->io_hdr.flags &= ~CTL_FLAG_RDMA_MASK;
702 				/*
703 				 * Add per-DMA flags that are set for this
704 				 * particular DMA request.
705 				 */
706 				io->io_hdr.flags |= msg_info.dt.flags &
707 						    CTL_FLAG_RDMA_MASK;
708 			} else
709 				sgl = (struct ctl_sg_entry *)
710 					io->scsiio.kern_data_ptr;
711 
712 			for (i = msg_info.dt.sent_sg_entries, j = 0;
713 			     i < (msg_info.dt.sent_sg_entries +
714 			     msg_info.dt.cur_sg_entries); i++, j++) {
715 				sgl[i].addr = msg_info.dt.sg_list[j].addr;
716 				sgl[i].len = msg_info.dt.sg_list[j].len;
717 
718 #if 0
719 				printf("%s: L: %p,%d -> %p,%d j=%d, i=%d\n",
720 				       __func__,
721 				       msg_info.dt.sg_list[j].addr,
722 				       msg_info.dt.sg_list[j].len,
723 				       sgl[i].addr, sgl[i].len, j, i);
724 #endif
725 			}
726 #if 0
727 			memcpy(&sgl[msg_info.dt.sent_sg_entries],
728 			       msg_info.dt.sg_list,
729 			       sizeof(*sgl) * msg_info.dt.cur_sg_entries);
730 #endif
731 
732 			/*
733 			 * If this is the last piece of the I/O, we've got
734 			 * the full S/G list.  Queue processing in the thread.
735 			 * Otherwise wait for the next piece.
736 			 */
737 			if (msg_info.dt.sg_last != 0)
738 				ctl_enqueue_isc(io);
739 			break;
740 		}
741 		/* Performed on the Serializing (primary) SC, XFER mode only */
742 		case CTL_MSG_DATAMOVE_DONE: {
743 			if (msg_info.hdr.serializing_sc == NULL) {
744 				printf("%s: serializing_sc == NULL!\n",
745 				       __func__);
746 				/* XXX KDM now what? */
747 				break;
748 			}
749 			/*
750 			 * We grab the sense information here in case
751 			 * there was a failure, so we can return status
752 			 * back to the initiator.
753 			 */
754 			io = msg_info.hdr.serializing_sc;
755 			io->io_hdr.msg_type = CTL_MSG_DATAMOVE_DONE;
756 			io->io_hdr.status = msg_info.hdr.status;
757 			io->scsiio.scsi_status = msg_info.scsi.scsi_status;
758 			io->scsiio.sense_len = msg_info.scsi.sense_len;
759 			io->scsiio.sense_residual =msg_info.scsi.sense_residual;
760 			io->io_hdr.port_status = msg_info.scsi.fetd_status;
761 			io->scsiio.residual = msg_info.scsi.residual;
762 			memcpy(&io->scsiio.sense_data,&msg_info.scsi.sense_data,
763 			       sizeof(io->scsiio.sense_data));
764 			ctl_enqueue_isc(io);
765 			break;
766 		}
767 
768 		/* Preformed on Originating SC, SER_ONLY mode */
769 		case CTL_MSG_R2R:
770 			io = msg_info.hdr.original_sc;
771 			if (io == NULL) {
772 				printf("%s: Major Bummer\n", __func__);
773 				return;
774 			} else {
775 #if 0
776 				printf("pOrig %x\n",(int) ctsio);
777 #endif
778 			}
779 			io->io_hdr.msg_type = CTL_MSG_R2R;
780 			io->io_hdr.serializing_sc = msg_info.hdr.serializing_sc;
781 			ctl_enqueue_isc(io);
782 			break;
783 
784 		/*
785 		 * Performed on Serializing(i.e. primary SC) SC in SER_ONLY
786 		 * mode.
787 		 * Performed on the Originating (i.e. secondary) SC in XFER
788 		 * mode
789 		 */
790 		case CTL_MSG_FINISH_IO:
791 			if (ctl_softc->ha_mode == CTL_HA_MODE_XFER)
792 				ctl_isc_handler_finish_xfer(ctl_softc,
793 							    &msg_info);
794 			else
795 				ctl_isc_handler_finish_ser_only(ctl_softc,
796 								&msg_info);
797 			break;
798 
799 		/* Preformed on Originating SC */
800 		case CTL_MSG_BAD_JUJU:
801 			io = msg_info.hdr.original_sc;
802 			if (io == NULL) {
803 				printf("%s: Bad JUJU!, original_sc is NULL!\n",
804 				       __func__);
805 				break;
806 			}
807 			ctl_copy_sense_data(&msg_info, io);
808 			/*
809 			 * IO should have already been cleaned up on other
810 			 * SC so clear this flag so we won't send a message
811 			 * back to finish the IO there.
812 			 */
813 			io->io_hdr.flags &= ~CTL_FLAG_SENT_2OTHER_SC;
814 			io->io_hdr.flags |= CTL_FLAG_IO_ACTIVE;
815 
816 			/* io = msg_info.hdr.serializing_sc; */
817 			io->io_hdr.msg_type = CTL_MSG_BAD_JUJU;
818 			ctl_enqueue_isc(io);
819 			break;
820 
821 		/* Handle resets sent from the other side */
822 		case CTL_MSG_MANAGE_TASKS: {
823 			struct ctl_taskio *taskio;
824 			taskio = (struct ctl_taskio *)ctl_alloc_io(
825 				(void *)ctl_softc->othersc_pool);
826 			if (taskio == NULL) {
827 				printf("ctl_isc_event_handler: can't allocate "
828 				       "ctl_io!\n");
829 				/* Bad Juju */
830 				/* should I just call the proper reset func
831 				   here??? */
832 				goto bailout;
833 			}
834 			ctl_zero_io((union ctl_io *)taskio);
835 			taskio->io_hdr.io_type = CTL_IO_TASK;
836 			taskio->io_hdr.flags |= CTL_FLAG_FROM_OTHER_SC;
837 			taskio->io_hdr.nexus = msg_info.hdr.nexus;
838 			taskio->task_action = msg_info.task.task_action;
839 			taskio->tag_num = msg_info.task.tag_num;
840 			taskio->tag_type = msg_info.task.tag_type;
841 #ifdef CTL_TIME_IO
842 			taskio->io_hdr.start_time = time_uptime;
843 			getbintime(&taskio->io_hdr.start_bt);
844 #if 0
845 			cs_prof_gettime(&taskio->io_hdr.start_ticks);
846 #endif
847 #endif /* CTL_TIME_IO */
848 			ctl_run_task((union ctl_io *)taskio);
849 			break;
850 		}
851 		/* Persistent Reserve action which needs attention */
852 		case CTL_MSG_PERS_ACTION:
853 			presio = (struct ctl_prio *)ctl_alloc_io(
854 				(void *)ctl_softc->othersc_pool);
855 			if (presio == NULL) {
856 				printf("ctl_isc_event_handler: can't allocate "
857 				       "ctl_io!\n");
858 				/* Bad Juju */
859 				/* Need to set busy and send msg back */
860 				goto bailout;
861 			}
862 			ctl_zero_io((union ctl_io *)presio);
863 			presio->io_hdr.msg_type = CTL_MSG_PERS_ACTION;
864 			presio->pr_msg = msg_info.pr;
865 			ctl_enqueue_isc((union ctl_io *)presio);
866 			break;
867 		case CTL_MSG_SYNC_FE:
868 			rcv_sync_msg = 1;
869 			break;
870 		case CTL_MSG_APS_LOCK: {
871 			// It's quicker to execute this then to
872 			// queue it.
873 			struct ctl_lun *lun;
874 			struct ctl_page_index *page_index;
875 			struct copan_aps_subpage *current_sp;
876 			uint32_t targ_lun;
877 
878 			targ_lun = msg_info.hdr.nexus.targ_mapped_lun;
879 			lun = ctl_softc->ctl_luns[targ_lun];
880 			mtx_lock(&lun->lun_lock);
881 			page_index = &lun->mode_pages.index[index_to_aps_page];
882 			current_sp = (struct copan_aps_subpage *)
883 				     (page_index->page_data +
884 				     (page_index->page_len * CTL_PAGE_CURRENT));
885 
886 			current_sp->lock_active = msg_info.aps.lock_flag;
887 			mtx_unlock(&lun->lun_lock);
888 		        break;
889 		}
890 		default:
891 		        printf("How did I get here?\n");
892 		}
893 	} else if (event == CTL_HA_EVT_MSG_SENT) {
894 		if (param != CTL_HA_STATUS_SUCCESS) {
895 			printf("Bad status from ctl_ha_msg_send status %d\n",
896 			       param);
897 		}
898 		return;
899 	} else if (event == CTL_HA_EVT_DISCONNECT) {
900 		printf("CTL: Got a disconnect from Isc\n");
901 		return;
902 	} else {
903 		printf("ctl_isc_event_handler: Unknown event %d\n", event);
904 		return;
905 	}
906 
907 bailout:
908 	return;
909 }
910 
911 static void
912 ctl_copy_sense_data(union ctl_ha_msg *src, union ctl_io *dest)
913 {
914 	struct scsi_sense_data *sense;
915 
916 	sense = &dest->scsiio.sense_data;
917 	bcopy(&src->scsi.sense_data, sense, sizeof(*sense));
918 	dest->scsiio.scsi_status = src->scsi.scsi_status;
919 	dest->scsiio.sense_len = src->scsi.sense_len;
920 	dest->io_hdr.status = src->hdr.status;
921 }
922 
923 static int
924 ctl_init(void)
925 {
926 	struct ctl_softc *softc;
927 	struct ctl_io_pool *internal_pool, *emergency_pool, *other_pool;
928 	struct ctl_frontend *fe;
929         uint8_t sc_id =0;
930 	int i, error, retval;
931 	//int isc_retval;
932 
933 	retval = 0;
934 	ctl_pause_rtr = 0;
935         rcv_sync_msg = 0;
936 
937 	control_softc = malloc(sizeof(*control_softc), M_DEVBUF,
938 			       M_WAITOK | M_ZERO);
939 	softc = control_softc;
940 
941 	softc->dev = make_dev(&ctl_cdevsw, 0, UID_ROOT, GID_OPERATOR, 0600,
942 			      "cam/ctl");
943 
944 	softc->dev->si_drv1 = softc;
945 
946 	/*
947 	 * By default, return a "bad LUN" peripheral qualifier for unknown
948 	 * LUNs.  The user can override this default using the tunable or
949 	 * sysctl.  See the comment in ctl_inquiry_std() for more details.
950 	 */
951 	softc->inquiry_pq_no_lun = 1;
952 	TUNABLE_INT_FETCH("kern.cam.ctl.inquiry_pq_no_lun",
953 			  &softc->inquiry_pq_no_lun);
954 	sysctl_ctx_init(&softc->sysctl_ctx);
955 	softc->sysctl_tree = SYSCTL_ADD_NODE(&softc->sysctl_ctx,
956 		SYSCTL_STATIC_CHILDREN(_kern_cam), OID_AUTO, "ctl",
957 		CTLFLAG_RD, 0, "CAM Target Layer");
958 
959 	if (softc->sysctl_tree == NULL) {
960 		printf("%s: unable to allocate sysctl tree\n", __func__);
961 		destroy_dev(softc->dev);
962 		free(control_softc, M_DEVBUF);
963 		control_softc = NULL;
964 		return (ENOMEM);
965 	}
966 
967 	SYSCTL_ADD_INT(&softc->sysctl_ctx,
968 		       SYSCTL_CHILDREN(softc->sysctl_tree), OID_AUTO,
969 		       "inquiry_pq_no_lun", CTLFLAG_RW,
970 		       &softc->inquiry_pq_no_lun, 0,
971 		       "Report no lun possible for invalid LUNs");
972 
973 	mtx_init(&softc->ctl_lock, "CTL mutex", NULL, MTX_DEF);
974 	mtx_init(&softc->pool_lock, "CTL pool mutex", NULL, MTX_DEF);
975 	softc->open_count = 0;
976 
977 	/*
978 	 * Default to actually sending a SYNCHRONIZE CACHE command down to
979 	 * the drive.
980 	 */
981 	softc->flags = CTL_FLAG_REAL_SYNC;
982 
983 	/*
984 	 * In Copan's HA scheme, the "master" and "slave" roles are
985 	 * figured out through the slot the controller is in.  Although it
986 	 * is an active/active system, someone has to be in charge.
987  	 */
988 #ifdef NEEDTOPORT
989         scmicro_rw(SCMICRO_GET_SHELF_ID, &sc_id);
990 #endif
991 
992         if (sc_id == 0) {
993 		softc->flags |= CTL_FLAG_MASTER_SHELF;
994 		persis_offset = 0;
995 	} else
996 		persis_offset = CTL_MAX_INITIATORS;
997 
998 	/*
999 	 * XXX KDM need to figure out where we want to get our target ID
1000 	 * and WWID.  Is it different on each port?
1001 	 */
1002 	softc->target.id = 0;
1003 	softc->target.wwid[0] = 0x12345678;
1004 	softc->target.wwid[1] = 0x87654321;
1005 	STAILQ_INIT(&softc->lun_list);
1006 	STAILQ_INIT(&softc->pending_lun_queue);
1007 	STAILQ_INIT(&softc->fe_list);
1008 	STAILQ_INIT(&softc->be_list);
1009 	STAILQ_INIT(&softc->io_pools);
1010 
1011 	if (ctl_pool_create(softc, CTL_POOL_INTERNAL, CTL_POOL_ENTRIES_INTERNAL,
1012 			    &internal_pool)!= 0){
1013 		printf("ctl: can't allocate %d entry internal pool, "
1014 		       "exiting\n", CTL_POOL_ENTRIES_INTERNAL);
1015 		return (ENOMEM);
1016 	}
1017 
1018 	if (ctl_pool_create(softc, CTL_POOL_EMERGENCY,
1019 			    CTL_POOL_ENTRIES_EMERGENCY, &emergency_pool) != 0) {
1020 		printf("ctl: can't allocate %d entry emergency pool, "
1021 		       "exiting\n", CTL_POOL_ENTRIES_EMERGENCY);
1022 		ctl_pool_free(internal_pool);
1023 		return (ENOMEM);
1024 	}
1025 
1026 	if (ctl_pool_create(softc, CTL_POOL_4OTHERSC, CTL_POOL_ENTRIES_OTHER_SC,
1027 	                    &other_pool) != 0)
1028 	{
1029 		printf("ctl: can't allocate %d entry other SC pool, "
1030 		       "exiting\n", CTL_POOL_ENTRIES_OTHER_SC);
1031 		ctl_pool_free(internal_pool);
1032 		ctl_pool_free(emergency_pool);
1033 		return (ENOMEM);
1034 	}
1035 
1036 	softc->internal_pool = internal_pool;
1037 	softc->emergency_pool = emergency_pool;
1038 	softc->othersc_pool = other_pool;
1039 
1040 	if (worker_threads <= 0)
1041 		worker_threads = max(1, mp_ncpus / 4);
1042 	if (worker_threads > CTL_MAX_THREADS)
1043 		worker_threads = CTL_MAX_THREADS;
1044 
1045 	for (i = 0; i < worker_threads; i++) {
1046 		struct ctl_thread *thr = &softc->threads[i];
1047 
1048 		mtx_init(&thr->queue_lock, "CTL queue mutex", NULL, MTX_DEF);
1049 		thr->ctl_softc = softc;
1050 		STAILQ_INIT(&thr->incoming_queue);
1051 		STAILQ_INIT(&thr->rtr_queue);
1052 		STAILQ_INIT(&thr->done_queue);
1053 		STAILQ_INIT(&thr->isc_queue);
1054 
1055 		error = kproc_kthread_add(ctl_work_thread, thr,
1056 		    &softc->ctl_proc, &thr->thread, 0, 0, "ctl", "work%d", i);
1057 		if (error != 0) {
1058 			printf("error creating CTL work thread!\n");
1059 			ctl_pool_free(internal_pool);
1060 			ctl_pool_free(emergency_pool);
1061 			ctl_pool_free(other_pool);
1062 			return (error);
1063 		}
1064 	}
1065 	error = kproc_kthread_add(ctl_lun_thread, softc,
1066 	    &softc->ctl_proc, NULL, 0, 0, "ctl", "lun");
1067 	if (error != 0) {
1068 		printf("error creating CTL lun thread!\n");
1069 		ctl_pool_free(internal_pool);
1070 		ctl_pool_free(emergency_pool);
1071 		ctl_pool_free(other_pool);
1072 		return (error);
1073 	}
1074 	if (bootverbose)
1075 		printf("ctl: CAM Target Layer loaded\n");
1076 
1077 	/*
1078 	 * Initialize the initiator and portname mappings
1079 	 */
1080 	memset(softc->wwpn_iid, 0, sizeof(softc->wwpn_iid));
1081 
1082 	/*
1083 	 * Initialize the ioctl front end.
1084 	 */
1085 	fe = &softc->ioctl_info.fe;
1086 	sprintf(softc->ioctl_info.port_name, "CTL ioctl");
1087 	fe->port_type = CTL_PORT_IOCTL;
1088 	fe->num_requested_ctl_io = 100;
1089 	fe->port_name = softc->ioctl_info.port_name;
1090 	fe->port_online = ctl_ioctl_online;
1091 	fe->port_offline = ctl_ioctl_offline;
1092 	fe->onoff_arg = &softc->ioctl_info;
1093 	fe->targ_enable = ctl_ioctl_targ_enable;
1094 	fe->targ_disable = ctl_ioctl_targ_disable;
1095 	fe->lun_enable = ctl_ioctl_lun_enable;
1096 	fe->lun_disable = ctl_ioctl_lun_disable;
1097 	fe->targ_lun_arg = &softc->ioctl_info;
1098 	fe->fe_datamove = ctl_ioctl_datamove;
1099 	fe->fe_done = ctl_ioctl_done;
1100 	fe->max_targets = 15;
1101 	fe->max_target_id = 15;
1102 
1103 	if (ctl_frontend_register(&softc->ioctl_info.fe,
1104 	                  (softc->flags & CTL_FLAG_MASTER_SHELF)) != 0) {
1105 		printf("ctl: ioctl front end registration failed, will "
1106 		       "continue anyway\n");
1107 	}
1108 
1109 #ifdef CTL_IO_DELAY
1110 	if (sizeof(struct callout) > CTL_TIMER_BYTES) {
1111 		printf("sizeof(struct callout) %zd > CTL_TIMER_BYTES %zd\n",
1112 		       sizeof(struct callout), CTL_TIMER_BYTES);
1113 		return (EINVAL);
1114 	}
1115 #endif /* CTL_IO_DELAY */
1116 
1117 	return (0);
1118 }
1119 
1120 void
1121 ctl_shutdown(void)
1122 {
1123 	struct ctl_softc *softc;
1124 	struct ctl_lun *lun, *next_lun;
1125 	struct ctl_io_pool *pool;
1126 
1127 	softc = (struct ctl_softc *)control_softc;
1128 
1129 	if (ctl_frontend_deregister(&softc->ioctl_info.fe) != 0)
1130 		printf("ctl: ioctl front end deregistration failed\n");
1131 
1132 	mtx_lock(&softc->ctl_lock);
1133 
1134 	/*
1135 	 * Free up each LUN.
1136 	 */
1137 	for (lun = STAILQ_FIRST(&softc->lun_list); lun != NULL; lun = next_lun){
1138 		next_lun = STAILQ_NEXT(lun, links);
1139 		ctl_free_lun(lun);
1140 	}
1141 
1142 	mtx_unlock(&softc->ctl_lock);
1143 
1144 	/*
1145 	 * This will rip the rug out from under any FETDs or anyone else
1146 	 * that has a pool allocated.  Since we increment our module
1147 	 * refcount any time someone outside the main CTL module allocates
1148 	 * a pool, we shouldn't have any problems here.  The user won't be
1149 	 * able to unload the CTL module until client modules have
1150 	 * successfully unloaded.
1151 	 */
1152 	while ((pool = STAILQ_FIRST(&softc->io_pools)) != NULL)
1153 		ctl_pool_free(pool);
1154 
1155 #if 0
1156 	ctl_shutdown_thread(softc->work_thread);
1157 	mtx_destroy(&softc->queue_lock);
1158 #endif
1159 
1160 	mtx_destroy(&softc->pool_lock);
1161 	mtx_destroy(&softc->ctl_lock);
1162 
1163 	destroy_dev(softc->dev);
1164 
1165 	sysctl_ctx_free(&softc->sysctl_ctx);
1166 
1167 	free(control_softc, M_DEVBUF);
1168 	control_softc = NULL;
1169 
1170 	if (bootverbose)
1171 		printf("ctl: CAM Target Layer unloaded\n");
1172 }
1173 
1174 static int
1175 ctl_module_event_handler(module_t mod, int what, void *arg)
1176 {
1177 
1178 	switch (what) {
1179 	case MOD_LOAD:
1180 		return (ctl_init());
1181 	case MOD_UNLOAD:
1182 		return (EBUSY);
1183 	default:
1184 		return (EOPNOTSUPP);
1185 	}
1186 }
1187 
1188 /*
1189  * XXX KDM should we do some access checks here?  Bump a reference count to
1190  * prevent a CTL module from being unloaded while someone has it open?
1191  */
1192 static int
1193 ctl_open(struct cdev *dev, int flags, int fmt, struct thread *td)
1194 {
1195 	return (0);
1196 }
1197 
1198 static int
1199 ctl_close(struct cdev *dev, int flags, int fmt, struct thread *td)
1200 {
1201 	return (0);
1202 }
1203 
1204 int
1205 ctl_port_enable(ctl_port_type port_type)
1206 {
1207 	struct ctl_softc *softc;
1208 	struct ctl_frontend *fe;
1209 
1210 	if (ctl_is_single == 0) {
1211 		union ctl_ha_msg msg_info;
1212 		int isc_retval;
1213 
1214 #if 0
1215 		printf("%s: HA mode, synchronizing frontend enable\n",
1216 		        __func__);
1217 #endif
1218 		msg_info.hdr.msg_type = CTL_MSG_SYNC_FE;
1219 	        if ((isc_retval=ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg_info,
1220 		        sizeof(msg_info), 1 )) > CTL_HA_STATUS_SUCCESS) {
1221 			printf("Sync msg send error retval %d\n", isc_retval);
1222 		}
1223 		if (!rcv_sync_msg) {
1224 			isc_retval=ctl_ha_msg_recv(CTL_HA_CHAN_CTL, &msg_info,
1225 			        sizeof(msg_info), 1);
1226 		}
1227 #if 0
1228         	printf("CTL:Frontend Enable\n");
1229 	} else {
1230 		printf("%s: single mode, skipping frontend synchronization\n",
1231 		        __func__);
1232 #endif
1233 	}
1234 
1235 	softc = control_softc;
1236 
1237 	STAILQ_FOREACH(fe, &softc->fe_list, links) {
1238 		if (port_type & fe->port_type)
1239 		{
1240 #if 0
1241 			printf("port %d\n", fe->targ_port);
1242 #endif
1243 			ctl_frontend_online(fe);
1244 		}
1245 	}
1246 
1247 	return (0);
1248 }
1249 
1250 int
1251 ctl_port_disable(ctl_port_type port_type)
1252 {
1253 	struct ctl_softc *softc;
1254 	struct ctl_frontend *fe;
1255 
1256 	softc = control_softc;
1257 
1258 	STAILQ_FOREACH(fe, &softc->fe_list, links) {
1259 		if (port_type & fe->port_type)
1260 			ctl_frontend_offline(fe);
1261 	}
1262 
1263 	return (0);
1264 }
1265 
1266 /*
1267  * Returns 0 for success, 1 for failure.
1268  * Currently the only failure mode is if there aren't enough entries
1269  * allocated.  So, in case of a failure, look at num_entries_dropped,
1270  * reallocate and try again.
1271  */
1272 int
1273 ctl_port_list(struct ctl_port_entry *entries, int num_entries_alloced,
1274 	      int *num_entries_filled, int *num_entries_dropped,
1275 	      ctl_port_type port_type, int no_virtual)
1276 {
1277 	struct ctl_softc *softc;
1278 	struct ctl_frontend *fe;
1279 	int entries_dropped, entries_filled;
1280 	int retval;
1281 	int i;
1282 
1283 	softc = control_softc;
1284 
1285 	retval = 0;
1286 	entries_filled = 0;
1287 	entries_dropped = 0;
1288 
1289 	i = 0;
1290 	mtx_lock(&softc->ctl_lock);
1291 	STAILQ_FOREACH(fe, &softc->fe_list, links) {
1292 		struct ctl_port_entry *entry;
1293 
1294 		if ((fe->port_type & port_type) == 0)
1295 			continue;
1296 
1297 		if ((no_virtual != 0)
1298 		 && (fe->virtual_port != 0))
1299 			continue;
1300 
1301 		if (entries_filled >= num_entries_alloced) {
1302 			entries_dropped++;
1303 			continue;
1304 		}
1305 		entry = &entries[i];
1306 
1307 		entry->port_type = fe->port_type;
1308 		strlcpy(entry->port_name, fe->port_name,
1309 			sizeof(entry->port_name));
1310 		entry->physical_port = fe->physical_port;
1311 		entry->virtual_port = fe->virtual_port;
1312 		entry->wwnn = fe->wwnn;
1313 		entry->wwpn = fe->wwpn;
1314 
1315 		i++;
1316 		entries_filled++;
1317 	}
1318 
1319 	mtx_unlock(&softc->ctl_lock);
1320 
1321 	if (entries_dropped > 0)
1322 		retval = 1;
1323 
1324 	*num_entries_dropped = entries_dropped;
1325 	*num_entries_filled = entries_filled;
1326 
1327 	return (retval);
1328 }
1329 
1330 static void
1331 ctl_ioctl_online(void *arg)
1332 {
1333 	struct ctl_ioctl_info *ioctl_info;
1334 
1335 	ioctl_info = (struct ctl_ioctl_info *)arg;
1336 
1337 	ioctl_info->flags |= CTL_IOCTL_FLAG_ENABLED;
1338 }
1339 
1340 static void
1341 ctl_ioctl_offline(void *arg)
1342 {
1343 	struct ctl_ioctl_info *ioctl_info;
1344 
1345 	ioctl_info = (struct ctl_ioctl_info *)arg;
1346 
1347 	ioctl_info->flags &= ~CTL_IOCTL_FLAG_ENABLED;
1348 }
1349 
1350 /*
1351  * Remove an initiator by port number and initiator ID.
1352  * Returns 0 for success, 1 for failure.
1353  */
1354 int
1355 ctl_remove_initiator(int32_t targ_port, uint32_t iid)
1356 {
1357 	struct ctl_softc *softc;
1358 
1359 	softc = control_softc;
1360 
1361 	mtx_assert(&softc->ctl_lock, MA_NOTOWNED);
1362 
1363 	if ((targ_port < 0)
1364 	 || (targ_port > CTL_MAX_PORTS)) {
1365 		printf("%s: invalid port number %d\n", __func__, targ_port);
1366 		return (1);
1367 	}
1368 	if (iid > CTL_MAX_INIT_PER_PORT) {
1369 		printf("%s: initiator ID %u > maximun %u!\n",
1370 		       __func__, iid, CTL_MAX_INIT_PER_PORT);
1371 		return (1);
1372 	}
1373 
1374 	mtx_lock(&softc->ctl_lock);
1375 
1376 	softc->wwpn_iid[targ_port][iid].in_use = 0;
1377 
1378 	mtx_unlock(&softc->ctl_lock);
1379 
1380 	return (0);
1381 }
1382 
1383 /*
1384  * Add an initiator to the initiator map.
1385  * Returns 0 for success, 1 for failure.
1386  */
1387 int
1388 ctl_add_initiator(uint64_t wwpn, int32_t targ_port, uint32_t iid)
1389 {
1390 	struct ctl_softc *softc;
1391 	int retval;
1392 
1393 	softc = control_softc;
1394 
1395 	mtx_assert(&softc->ctl_lock, MA_NOTOWNED);
1396 
1397 	retval = 0;
1398 
1399 	if ((targ_port < 0)
1400 	 || (targ_port > CTL_MAX_PORTS)) {
1401 		printf("%s: invalid port number %d\n", __func__, targ_port);
1402 		return (1);
1403 	}
1404 	if (iid > CTL_MAX_INIT_PER_PORT) {
1405 		printf("%s: WWPN %#jx initiator ID %u > maximun %u!\n",
1406 		       __func__, wwpn, iid, CTL_MAX_INIT_PER_PORT);
1407 		return (1);
1408 	}
1409 
1410 	mtx_lock(&softc->ctl_lock);
1411 
1412 	if (softc->wwpn_iid[targ_port][iid].in_use != 0) {
1413 		/*
1414 		 * We don't treat this as an error.
1415 		 */
1416 		if (softc->wwpn_iid[targ_port][iid].wwpn == wwpn) {
1417 			printf("%s: port %d iid %u WWPN %#jx arrived again?\n",
1418 			       __func__, targ_port, iid, (uintmax_t)wwpn);
1419 			goto bailout;
1420 		}
1421 
1422 		/*
1423 		 * This is an error, but what do we do about it?  The
1424 		 * driver is telling us we have a new WWPN for this
1425 		 * initiator ID, so we pretty much need to use it.
1426 		 */
1427 		printf("%s: port %d iid %u WWPN %#jx arrived, WWPN %#jx is "
1428 		       "still at that address\n", __func__, targ_port, iid,
1429 		       (uintmax_t)wwpn,
1430 		       (uintmax_t)softc->wwpn_iid[targ_port][iid].wwpn);
1431 
1432 		/*
1433 		 * XXX KDM clear have_ca and ua_pending on each LUN for
1434 		 * this initiator.
1435 		 */
1436 	}
1437 	softc->wwpn_iid[targ_port][iid].in_use = 1;
1438 	softc->wwpn_iid[targ_port][iid].iid = iid;
1439 	softc->wwpn_iid[targ_port][iid].wwpn = wwpn;
1440 	softc->wwpn_iid[targ_port][iid].port = targ_port;
1441 
1442 bailout:
1443 
1444 	mtx_unlock(&softc->ctl_lock);
1445 
1446 	return (retval);
1447 }
1448 
1449 /*
1450  * XXX KDM should we pretend to do something in the target/lun
1451  * enable/disable functions?
1452  */
1453 static int
1454 ctl_ioctl_targ_enable(void *arg, struct ctl_id targ_id)
1455 {
1456 	return (0);
1457 }
1458 
1459 static int
1460 ctl_ioctl_targ_disable(void *arg, struct ctl_id targ_id)
1461 {
1462 	return (0);
1463 }
1464 
1465 static int
1466 ctl_ioctl_lun_enable(void *arg, struct ctl_id targ_id, int lun_id)
1467 {
1468 	return (0);
1469 }
1470 
1471 static int
1472 ctl_ioctl_lun_disable(void *arg, struct ctl_id targ_id, int lun_id)
1473 {
1474 	return (0);
1475 }
1476 
1477 /*
1478  * Data movement routine for the CTL ioctl frontend port.
1479  */
1480 static int
1481 ctl_ioctl_do_datamove(struct ctl_scsiio *ctsio)
1482 {
1483 	struct ctl_sg_entry *ext_sglist, *kern_sglist;
1484 	struct ctl_sg_entry ext_entry, kern_entry;
1485 	int ext_sglen, ext_sg_entries, kern_sg_entries;
1486 	int ext_sg_start, ext_offset;
1487 	int len_to_copy, len_copied;
1488 	int kern_watermark, ext_watermark;
1489 	int ext_sglist_malloced;
1490 	int i, j;
1491 
1492 	ext_sglist_malloced = 0;
1493 	ext_sg_start = 0;
1494 	ext_offset = 0;
1495 
1496 	CTL_DEBUG_PRINT(("ctl_ioctl_do_datamove\n"));
1497 
1498 	/*
1499 	 * If this flag is set, fake the data transfer.
1500 	 */
1501 	if (ctsio->io_hdr.flags & CTL_FLAG_NO_DATAMOVE) {
1502 		ctsio->ext_data_filled = ctsio->ext_data_len;
1503 		goto bailout;
1504 	}
1505 
1506 	/*
1507 	 * To simplify things here, if we have a single buffer, stick it in
1508 	 * a S/G entry and just make it a single entry S/G list.
1509 	 */
1510 	if (ctsio->io_hdr.flags & CTL_FLAG_EDPTR_SGLIST) {
1511 		int len_seen;
1512 
1513 		ext_sglen = ctsio->ext_sg_entries * sizeof(*ext_sglist);
1514 
1515 		ext_sglist = (struct ctl_sg_entry *)malloc(ext_sglen, M_CTL,
1516 							   M_WAITOK);
1517 		ext_sglist_malloced = 1;
1518 		if (copyin(ctsio->ext_data_ptr, ext_sglist,
1519 				   ext_sglen) != 0) {
1520 			ctl_set_internal_failure(ctsio,
1521 						 /*sks_valid*/ 0,
1522 						 /*retry_count*/ 0);
1523 			goto bailout;
1524 		}
1525 		ext_sg_entries = ctsio->ext_sg_entries;
1526 		len_seen = 0;
1527 		for (i = 0; i < ext_sg_entries; i++) {
1528 			if ((len_seen + ext_sglist[i].len) >=
1529 			     ctsio->ext_data_filled) {
1530 				ext_sg_start = i;
1531 				ext_offset = ctsio->ext_data_filled - len_seen;
1532 				break;
1533 			}
1534 			len_seen += ext_sglist[i].len;
1535 		}
1536 	} else {
1537 		ext_sglist = &ext_entry;
1538 		ext_sglist->addr = ctsio->ext_data_ptr;
1539 		ext_sglist->len = ctsio->ext_data_len;
1540 		ext_sg_entries = 1;
1541 		ext_sg_start = 0;
1542 		ext_offset = ctsio->ext_data_filled;
1543 	}
1544 
1545 	if (ctsio->kern_sg_entries > 0) {
1546 		kern_sglist = (struct ctl_sg_entry *)ctsio->kern_data_ptr;
1547 		kern_sg_entries = ctsio->kern_sg_entries;
1548 	} else {
1549 		kern_sglist = &kern_entry;
1550 		kern_sglist->addr = ctsio->kern_data_ptr;
1551 		kern_sglist->len = ctsio->kern_data_len;
1552 		kern_sg_entries = 1;
1553 	}
1554 
1555 
1556 	kern_watermark = 0;
1557 	ext_watermark = ext_offset;
1558 	len_copied = 0;
1559 	for (i = ext_sg_start, j = 0;
1560 	     i < ext_sg_entries && j < kern_sg_entries;) {
1561 		uint8_t *ext_ptr, *kern_ptr;
1562 
1563 		len_to_copy = ctl_min(ext_sglist[i].len - ext_watermark,
1564 				      kern_sglist[j].len - kern_watermark);
1565 
1566 		ext_ptr = (uint8_t *)ext_sglist[i].addr;
1567 		ext_ptr = ext_ptr + ext_watermark;
1568 		if (ctsio->io_hdr.flags & CTL_FLAG_BUS_ADDR) {
1569 			/*
1570 			 * XXX KDM fix this!
1571 			 */
1572 			panic("need to implement bus address support");
1573 #if 0
1574 			kern_ptr = bus_to_virt(kern_sglist[j].addr);
1575 #endif
1576 		} else
1577 			kern_ptr = (uint8_t *)kern_sglist[j].addr;
1578 		kern_ptr = kern_ptr + kern_watermark;
1579 
1580 		kern_watermark += len_to_copy;
1581 		ext_watermark += len_to_copy;
1582 
1583 		if ((ctsio->io_hdr.flags & CTL_FLAG_DATA_MASK) ==
1584 		     CTL_FLAG_DATA_IN) {
1585 			CTL_DEBUG_PRINT(("ctl_ioctl_do_datamove: copying %d "
1586 					 "bytes to user\n", len_to_copy));
1587 			CTL_DEBUG_PRINT(("ctl_ioctl_do_datamove: from %p "
1588 					 "to %p\n", kern_ptr, ext_ptr));
1589 			if (copyout(kern_ptr, ext_ptr, len_to_copy) != 0) {
1590 				ctl_set_internal_failure(ctsio,
1591 							 /*sks_valid*/ 0,
1592 							 /*retry_count*/ 0);
1593 				goto bailout;
1594 			}
1595 		} else {
1596 			CTL_DEBUG_PRINT(("ctl_ioctl_do_datamove: copying %d "
1597 					 "bytes from user\n", len_to_copy));
1598 			CTL_DEBUG_PRINT(("ctl_ioctl_do_datamove: from %p "
1599 					 "to %p\n", ext_ptr, kern_ptr));
1600 			if (copyin(ext_ptr, kern_ptr, len_to_copy)!= 0){
1601 				ctl_set_internal_failure(ctsio,
1602 							 /*sks_valid*/ 0,
1603 							 /*retry_count*/0);
1604 				goto bailout;
1605 			}
1606 		}
1607 
1608 		len_copied += len_to_copy;
1609 
1610 		if (ext_sglist[i].len == ext_watermark) {
1611 			i++;
1612 			ext_watermark = 0;
1613 		}
1614 
1615 		if (kern_sglist[j].len == kern_watermark) {
1616 			j++;
1617 			kern_watermark = 0;
1618 		}
1619 	}
1620 
1621 	ctsio->ext_data_filled += len_copied;
1622 
1623 	CTL_DEBUG_PRINT(("ctl_ioctl_do_datamove: ext_sg_entries: %d, "
1624 			 "kern_sg_entries: %d\n", ext_sg_entries,
1625 			 kern_sg_entries));
1626 	CTL_DEBUG_PRINT(("ctl_ioctl_do_datamove: ext_data_len = %d, "
1627 			 "kern_data_len = %d\n", ctsio->ext_data_len,
1628 			 ctsio->kern_data_len));
1629 
1630 
1631 	/* XXX KDM set residual?? */
1632 bailout:
1633 
1634 	if (ext_sglist_malloced != 0)
1635 		free(ext_sglist, M_CTL);
1636 
1637 	return (CTL_RETVAL_COMPLETE);
1638 }
1639 
1640 /*
1641  * Serialize a command that went down the "wrong" side, and so was sent to
1642  * this controller for execution.  The logic is a little different than the
1643  * standard case in ctl_scsiio_precheck().  Errors in this case need to get
1644  * sent back to the other side, but in the success case, we execute the
1645  * command on this side (XFER mode) or tell the other side to execute it
1646  * (SER_ONLY mode).
1647  */
1648 static int
1649 ctl_serialize_other_sc_cmd(struct ctl_scsiio *ctsio)
1650 {
1651 	struct ctl_softc *ctl_softc;
1652 	union ctl_ha_msg msg_info;
1653 	struct ctl_lun *lun;
1654 	int retval = 0;
1655 	uint32_t targ_lun;
1656 
1657 	ctl_softc = control_softc;
1658 
1659 	targ_lun = ctsio->io_hdr.nexus.targ_mapped_lun;
1660 	lun = ctl_softc->ctl_luns[targ_lun];
1661 	if (lun==NULL)
1662 	{
1663 		/*
1664 		 * Why isn't LUN defined? The other side wouldn't
1665 		 * send a cmd if the LUN is undefined.
1666 		 */
1667 		printf("%s: Bad JUJU!, LUN is NULL!\n", __func__);
1668 
1669 		/* "Logical unit not supported" */
1670 		ctl_set_sense_data(&msg_info.scsi.sense_data,
1671 				   lun,
1672 				   /*sense_format*/SSD_TYPE_NONE,
1673 				   /*current_error*/ 1,
1674 				   /*sense_key*/ SSD_KEY_ILLEGAL_REQUEST,
1675 				   /*asc*/ 0x25,
1676 				   /*ascq*/ 0x00,
1677 				   SSD_ELEM_NONE);
1678 
1679 		msg_info.scsi.sense_len = SSD_FULL_SIZE;
1680 		msg_info.scsi.scsi_status = SCSI_STATUS_CHECK_COND;
1681 		msg_info.hdr.status = CTL_SCSI_ERROR | CTL_AUTOSENSE;
1682 		msg_info.hdr.original_sc = ctsio->io_hdr.original_sc;
1683 		msg_info.hdr.serializing_sc = NULL;
1684 		msg_info.hdr.msg_type = CTL_MSG_BAD_JUJU;
1685 	        if (ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg_info,
1686 				sizeof(msg_info), 0 ) > CTL_HA_STATUS_SUCCESS) {
1687 		}
1688 		return(1);
1689 
1690 	}
1691 
1692 	mtx_lock(&lun->lun_lock);
1693     	TAILQ_INSERT_TAIL(&lun->ooa_queue, &ctsio->io_hdr, ooa_links);
1694 
1695 	switch (ctl_check_ooa(lun, (union ctl_io *)ctsio,
1696 		(union ctl_io *)TAILQ_PREV(&ctsio->io_hdr, ctl_ooaq,
1697 		 ooa_links))) {
1698 	case CTL_ACTION_BLOCK:
1699 		ctsio->io_hdr.flags |= CTL_FLAG_BLOCKED;
1700 		TAILQ_INSERT_TAIL(&lun->blocked_queue, &ctsio->io_hdr,
1701 				  blocked_links);
1702 		break;
1703 	case CTL_ACTION_PASS:
1704 	case CTL_ACTION_SKIP:
1705 		if (ctl_softc->ha_mode == CTL_HA_MODE_XFER) {
1706 			ctsio->io_hdr.flags |= CTL_FLAG_IS_WAS_ON_RTR;
1707 			ctl_enqueue_rtr((union ctl_io *)ctsio);
1708 		} else {
1709 
1710 			/* send msg back to other side */
1711 			msg_info.hdr.original_sc = ctsio->io_hdr.original_sc;
1712 			msg_info.hdr.serializing_sc = (union ctl_io *)ctsio;
1713 			msg_info.hdr.msg_type = CTL_MSG_R2R;
1714 #if 0
1715 			printf("2. pOrig %x\n", (int)msg_info.hdr.original_sc);
1716 #endif
1717 		        if (ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg_info,
1718 			    sizeof(msg_info), 0 ) > CTL_HA_STATUS_SUCCESS) {
1719 			}
1720 		}
1721 		break;
1722 	case CTL_ACTION_OVERLAP:
1723 		/* OVERLAPPED COMMANDS ATTEMPTED */
1724 		ctl_set_sense_data(&msg_info.scsi.sense_data,
1725 				   lun,
1726 				   /*sense_format*/SSD_TYPE_NONE,
1727 				   /*current_error*/ 1,
1728 				   /*sense_key*/ SSD_KEY_ILLEGAL_REQUEST,
1729 				   /*asc*/ 0x4E,
1730 				   /*ascq*/ 0x00,
1731 				   SSD_ELEM_NONE);
1732 
1733 		msg_info.scsi.sense_len = SSD_FULL_SIZE;
1734 		msg_info.scsi.scsi_status = SCSI_STATUS_CHECK_COND;
1735 		msg_info.hdr.status = CTL_SCSI_ERROR | CTL_AUTOSENSE;
1736 		msg_info.hdr.original_sc = ctsio->io_hdr.original_sc;
1737 		msg_info.hdr.serializing_sc = NULL;
1738 		msg_info.hdr.msg_type = CTL_MSG_BAD_JUJU;
1739 #if 0
1740 		printf("BAD JUJU:Major Bummer Overlap\n");
1741 #endif
1742 		TAILQ_REMOVE(&lun->ooa_queue, &ctsio->io_hdr, ooa_links);
1743 		retval = 1;
1744 		if (ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg_info,
1745 		    sizeof(msg_info), 0 ) > CTL_HA_STATUS_SUCCESS) {
1746 		}
1747 		break;
1748 	case CTL_ACTION_OVERLAP_TAG:
1749 		/* TAGGED OVERLAPPED COMMANDS (NN = QUEUE TAG) */
1750 		ctl_set_sense_data(&msg_info.scsi.sense_data,
1751 				   lun,
1752 				   /*sense_format*/SSD_TYPE_NONE,
1753 				   /*current_error*/ 1,
1754 				   /*sense_key*/ SSD_KEY_ILLEGAL_REQUEST,
1755 				   /*asc*/ 0x4D,
1756 				   /*ascq*/ ctsio->tag_num & 0xff,
1757 				   SSD_ELEM_NONE);
1758 
1759 		msg_info.scsi.sense_len = SSD_FULL_SIZE;
1760 		msg_info.scsi.scsi_status = SCSI_STATUS_CHECK_COND;
1761 		msg_info.hdr.status = CTL_SCSI_ERROR | CTL_AUTOSENSE;
1762 		msg_info.hdr.original_sc = ctsio->io_hdr.original_sc;
1763 		msg_info.hdr.serializing_sc = NULL;
1764 		msg_info.hdr.msg_type = CTL_MSG_BAD_JUJU;
1765 #if 0
1766 		printf("BAD JUJU:Major Bummer Overlap Tag\n");
1767 #endif
1768 		TAILQ_REMOVE(&lun->ooa_queue, &ctsio->io_hdr, ooa_links);
1769 		retval = 1;
1770 		if (ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg_info,
1771 		    sizeof(msg_info), 0 ) > CTL_HA_STATUS_SUCCESS) {
1772 		}
1773 		break;
1774 	case CTL_ACTION_ERROR:
1775 	default:
1776 		/* "Internal target failure" */
1777 		ctl_set_sense_data(&msg_info.scsi.sense_data,
1778 				   lun,
1779 				   /*sense_format*/SSD_TYPE_NONE,
1780 				   /*current_error*/ 1,
1781 				   /*sense_key*/ SSD_KEY_HARDWARE_ERROR,
1782 				   /*asc*/ 0x44,
1783 				   /*ascq*/ 0x00,
1784 				   SSD_ELEM_NONE);
1785 
1786 		msg_info.scsi.sense_len = SSD_FULL_SIZE;
1787 		msg_info.scsi.scsi_status = SCSI_STATUS_CHECK_COND;
1788 		msg_info.hdr.status = CTL_SCSI_ERROR | CTL_AUTOSENSE;
1789 		msg_info.hdr.original_sc = ctsio->io_hdr.original_sc;
1790 		msg_info.hdr.serializing_sc = NULL;
1791 		msg_info.hdr.msg_type = CTL_MSG_BAD_JUJU;
1792 #if 0
1793 		printf("BAD JUJU:Major Bummer HW Error\n");
1794 #endif
1795 		TAILQ_REMOVE(&lun->ooa_queue, &ctsio->io_hdr, ooa_links);
1796 		retval = 1;
1797 		if (ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg_info,
1798 		    sizeof(msg_info), 0 ) > CTL_HA_STATUS_SUCCESS) {
1799 		}
1800 		break;
1801 	}
1802 	mtx_unlock(&lun->lun_lock);
1803 	return (retval);
1804 }
1805 
1806 static int
1807 ctl_ioctl_submit_wait(union ctl_io *io)
1808 {
1809 	struct ctl_fe_ioctl_params params;
1810 	ctl_fe_ioctl_state last_state;
1811 	int done, retval;
1812 
1813 	retval = 0;
1814 
1815 	bzero(&params, sizeof(params));
1816 
1817 	mtx_init(&params.ioctl_mtx, "ctliocmtx", NULL, MTX_DEF);
1818 	cv_init(&params.sem, "ctlioccv");
1819 	params.state = CTL_IOCTL_INPROG;
1820 	last_state = params.state;
1821 
1822 	io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr = &params;
1823 
1824 	CTL_DEBUG_PRINT(("ctl_ioctl_submit_wait\n"));
1825 
1826 	/* This shouldn't happen */
1827 	if ((retval = ctl_queue(io)) != CTL_RETVAL_COMPLETE)
1828 		return (retval);
1829 
1830 	done = 0;
1831 
1832 	do {
1833 		mtx_lock(&params.ioctl_mtx);
1834 		/*
1835 		 * Check the state here, and don't sleep if the state has
1836 		 * already changed (i.e. wakeup has already occured, but we
1837 		 * weren't waiting yet).
1838 		 */
1839 		if (params.state == last_state) {
1840 			/* XXX KDM cv_wait_sig instead? */
1841 			cv_wait(&params.sem, &params.ioctl_mtx);
1842 		}
1843 		last_state = params.state;
1844 
1845 		switch (params.state) {
1846 		case CTL_IOCTL_INPROG:
1847 			/* Why did we wake up? */
1848 			/* XXX KDM error here? */
1849 			mtx_unlock(&params.ioctl_mtx);
1850 			break;
1851 		case CTL_IOCTL_DATAMOVE:
1852 			CTL_DEBUG_PRINT(("got CTL_IOCTL_DATAMOVE\n"));
1853 
1854 			/*
1855 			 * change last_state back to INPROG to avoid
1856 			 * deadlock on subsequent data moves.
1857 			 */
1858 			params.state = last_state = CTL_IOCTL_INPROG;
1859 
1860 			mtx_unlock(&params.ioctl_mtx);
1861 			ctl_ioctl_do_datamove(&io->scsiio);
1862 			/*
1863 			 * Note that in some cases, most notably writes,
1864 			 * this will queue the I/O and call us back later.
1865 			 * In other cases, generally reads, this routine
1866 			 * will immediately call back and wake us up,
1867 			 * probably using our own context.
1868 			 */
1869 			io->scsiio.be_move_done(io);
1870 			break;
1871 		case CTL_IOCTL_DONE:
1872 			mtx_unlock(&params.ioctl_mtx);
1873 			CTL_DEBUG_PRINT(("got CTL_IOCTL_DONE\n"));
1874 			done = 1;
1875 			break;
1876 		default:
1877 			mtx_unlock(&params.ioctl_mtx);
1878 			/* XXX KDM error here? */
1879 			break;
1880 		}
1881 	} while (done == 0);
1882 
1883 	mtx_destroy(&params.ioctl_mtx);
1884 	cv_destroy(&params.sem);
1885 
1886 	return (CTL_RETVAL_COMPLETE);
1887 }
1888 
1889 static void
1890 ctl_ioctl_datamove(union ctl_io *io)
1891 {
1892 	struct ctl_fe_ioctl_params *params;
1893 
1894 	params = (struct ctl_fe_ioctl_params *)
1895 		io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr;
1896 
1897 	mtx_lock(&params->ioctl_mtx);
1898 	params->state = CTL_IOCTL_DATAMOVE;
1899 	cv_broadcast(&params->sem);
1900 	mtx_unlock(&params->ioctl_mtx);
1901 }
1902 
1903 static void
1904 ctl_ioctl_done(union ctl_io *io)
1905 {
1906 	struct ctl_fe_ioctl_params *params;
1907 
1908 	params = (struct ctl_fe_ioctl_params *)
1909 		io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr;
1910 
1911 	mtx_lock(&params->ioctl_mtx);
1912 	params->state = CTL_IOCTL_DONE;
1913 	cv_broadcast(&params->sem);
1914 	mtx_unlock(&params->ioctl_mtx);
1915 }
1916 
1917 static void
1918 ctl_ioctl_hard_startstop_callback(void *arg, struct cfi_metatask *metatask)
1919 {
1920 	struct ctl_fe_ioctl_startstop_info *sd_info;
1921 
1922 	sd_info = (struct ctl_fe_ioctl_startstop_info *)arg;
1923 
1924 	sd_info->hs_info.status = metatask->status;
1925 	sd_info->hs_info.total_luns = metatask->taskinfo.startstop.total_luns;
1926 	sd_info->hs_info.luns_complete =
1927 		metatask->taskinfo.startstop.luns_complete;
1928 	sd_info->hs_info.luns_failed = metatask->taskinfo.startstop.luns_failed;
1929 
1930 	cv_broadcast(&sd_info->sem);
1931 }
1932 
1933 static void
1934 ctl_ioctl_bbrread_callback(void *arg, struct cfi_metatask *metatask)
1935 {
1936 	struct ctl_fe_ioctl_bbrread_info *fe_bbr_info;
1937 
1938 	fe_bbr_info = (struct ctl_fe_ioctl_bbrread_info *)arg;
1939 
1940 	mtx_lock(fe_bbr_info->lock);
1941 	fe_bbr_info->bbr_info->status = metatask->status;
1942 	fe_bbr_info->bbr_info->bbr_status = metatask->taskinfo.bbrread.status;
1943 	fe_bbr_info->wakeup_done = 1;
1944 	mtx_unlock(fe_bbr_info->lock);
1945 
1946 	cv_broadcast(&fe_bbr_info->sem);
1947 }
1948 
1949 /*
1950  * Returns 0 for success, errno for failure.
1951  */
1952 static int
1953 ctl_ioctl_fill_ooa(struct ctl_lun *lun, uint32_t *cur_fill_num,
1954 		   struct ctl_ooa *ooa_hdr, struct ctl_ooa_entry *kern_entries)
1955 {
1956 	union ctl_io *io;
1957 	int retval;
1958 
1959 	retval = 0;
1960 
1961 	mtx_lock(&lun->lun_lock);
1962 	for (io = (union ctl_io *)TAILQ_FIRST(&lun->ooa_queue); (io != NULL);
1963 	     (*cur_fill_num)++, io = (union ctl_io *)TAILQ_NEXT(&io->io_hdr,
1964 	     ooa_links)) {
1965 		struct ctl_ooa_entry *entry;
1966 
1967 		/*
1968 		 * If we've got more than we can fit, just count the
1969 		 * remaining entries.
1970 		 */
1971 		if (*cur_fill_num >= ooa_hdr->alloc_num)
1972 			continue;
1973 
1974 		entry = &kern_entries[*cur_fill_num];
1975 
1976 		entry->tag_num = io->scsiio.tag_num;
1977 		entry->lun_num = lun->lun;
1978 #ifdef CTL_TIME_IO
1979 		entry->start_bt = io->io_hdr.start_bt;
1980 #endif
1981 		bcopy(io->scsiio.cdb, entry->cdb, io->scsiio.cdb_len);
1982 		entry->cdb_len = io->scsiio.cdb_len;
1983 		if (io->io_hdr.flags & CTL_FLAG_BLOCKED)
1984 			entry->cmd_flags |= CTL_OOACMD_FLAG_BLOCKED;
1985 
1986 		if (io->io_hdr.flags & CTL_FLAG_DMA_INPROG)
1987 			entry->cmd_flags |= CTL_OOACMD_FLAG_DMA;
1988 
1989 		if (io->io_hdr.flags & CTL_FLAG_ABORT)
1990 			entry->cmd_flags |= CTL_OOACMD_FLAG_ABORT;
1991 
1992 		if (io->io_hdr.flags & CTL_FLAG_IS_WAS_ON_RTR)
1993 			entry->cmd_flags |= CTL_OOACMD_FLAG_RTR;
1994 
1995 		if (io->io_hdr.flags & CTL_FLAG_DMA_QUEUED)
1996 			entry->cmd_flags |= CTL_OOACMD_FLAG_DMA_QUEUED;
1997 	}
1998 	mtx_unlock(&lun->lun_lock);
1999 
2000 	return (retval);
2001 }
2002 
2003 static void *
2004 ctl_copyin_alloc(void *user_addr, int len, char *error_str,
2005 		 size_t error_str_len)
2006 {
2007 	void *kptr;
2008 
2009 	kptr = malloc(len, M_CTL, M_WAITOK | M_ZERO);
2010 
2011 	if (copyin(user_addr, kptr, len) != 0) {
2012 		snprintf(error_str, error_str_len, "Error copying %d bytes "
2013 			 "from user address %p to kernel address %p", len,
2014 			 user_addr, kptr);
2015 		free(kptr, M_CTL);
2016 		return (NULL);
2017 	}
2018 
2019 	return (kptr);
2020 }
2021 
2022 static void
2023 ctl_free_args(int num_be_args, struct ctl_be_arg *be_args)
2024 {
2025 	int i;
2026 
2027 	if (be_args == NULL)
2028 		return;
2029 
2030 	for (i = 0; i < num_be_args; i++) {
2031 		free(be_args[i].kname, M_CTL);
2032 		free(be_args[i].kvalue, M_CTL);
2033 	}
2034 
2035 	free(be_args, M_CTL);
2036 }
2037 
2038 static struct ctl_be_arg *
2039 ctl_copyin_args(int num_be_args, struct ctl_be_arg *be_args,
2040 		char *error_str, size_t error_str_len)
2041 {
2042 	struct ctl_be_arg *args;
2043 	int i;
2044 
2045 	args = ctl_copyin_alloc(be_args, num_be_args * sizeof(*be_args),
2046 				error_str, error_str_len);
2047 
2048 	if (args == NULL)
2049 		goto bailout;
2050 
2051 	for (i = 0; i < num_be_args; i++) {
2052 		args[i].kname = NULL;
2053 		args[i].kvalue = NULL;
2054 	}
2055 
2056 	for (i = 0; i < num_be_args; i++) {
2057 		uint8_t *tmpptr;
2058 
2059 		args[i].kname = ctl_copyin_alloc(args[i].name,
2060 			args[i].namelen, error_str, error_str_len);
2061 		if (args[i].kname == NULL)
2062 			goto bailout;
2063 
2064 		if (args[i].kname[args[i].namelen - 1] != '\0') {
2065 			snprintf(error_str, error_str_len, "Argument %d "
2066 				 "name is not NUL-terminated", i);
2067 			goto bailout;
2068 		}
2069 
2070 		args[i].kvalue = NULL;
2071 
2072 		tmpptr = ctl_copyin_alloc(args[i].value,
2073 			args[i].vallen, error_str, error_str_len);
2074 		if (tmpptr == NULL)
2075 			goto bailout;
2076 
2077 		args[i].kvalue = tmpptr;
2078 
2079 		if ((args[i].flags & CTL_BEARG_ASCII)
2080 		 && (tmpptr[args[i].vallen - 1] != '\0')) {
2081 			snprintf(error_str, error_str_len, "Argument %d "
2082 				 "value is not NUL-terminated", i);
2083 			goto bailout;
2084 		}
2085 	}
2086 
2087 	return (args);
2088 bailout:
2089 
2090 	ctl_free_args(num_be_args, args);
2091 
2092 	return (NULL);
2093 }
2094 
2095 /*
2096  * Escape characters that are illegal or not recommended in XML.
2097  */
2098 int
2099 ctl_sbuf_printf_esc(struct sbuf *sb, char *str)
2100 {
2101 	int retval;
2102 
2103 	retval = 0;
2104 
2105 	for (; *str; str++) {
2106 		switch (*str) {
2107 		case '&':
2108 			retval = sbuf_printf(sb, "&amp;");
2109 			break;
2110 		case '>':
2111 			retval = sbuf_printf(sb, "&gt;");
2112 			break;
2113 		case '<':
2114 			retval = sbuf_printf(sb, "&lt;");
2115 			break;
2116 		default:
2117 			retval = sbuf_putc(sb, *str);
2118 			break;
2119 		}
2120 
2121 		if (retval != 0)
2122 			break;
2123 
2124 	}
2125 
2126 	return (retval);
2127 }
2128 
2129 static int
2130 ctl_ioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flag,
2131 	  struct thread *td)
2132 {
2133 	struct ctl_softc *softc;
2134 	int retval;
2135 
2136 	softc = control_softc;
2137 
2138 	retval = 0;
2139 
2140 	switch (cmd) {
2141 	case CTL_IO: {
2142 		union ctl_io *io;
2143 		void *pool_tmp;
2144 
2145 		/*
2146 		 * If we haven't been "enabled", don't allow any SCSI I/O
2147 		 * to this FETD.
2148 		 */
2149 		if ((softc->ioctl_info.flags & CTL_IOCTL_FLAG_ENABLED) == 0) {
2150 			retval = -EPERM;
2151 			break;
2152 		}
2153 
2154 		io = ctl_alloc_io(softc->ioctl_info.fe.ctl_pool_ref);
2155 		if (io == NULL) {
2156 			printf("ctl_ioctl: can't allocate ctl_io!\n");
2157 			retval = -ENOSPC;
2158 			break;
2159 		}
2160 
2161 		/*
2162 		 * Need to save the pool reference so it doesn't get
2163 		 * spammed by the user's ctl_io.
2164 		 */
2165 		pool_tmp = io->io_hdr.pool;
2166 
2167 		memcpy(io, (void *)addr, sizeof(*io));
2168 
2169 		io->io_hdr.pool = pool_tmp;
2170 		/*
2171 		 * No status yet, so make sure the status is set properly.
2172 		 */
2173 		io->io_hdr.status = CTL_STATUS_NONE;
2174 
2175 		/*
2176 		 * The user sets the initiator ID, target and LUN IDs.
2177 		 */
2178 		io->io_hdr.nexus.targ_port = softc->ioctl_info.fe.targ_port;
2179 		io->io_hdr.flags |= CTL_FLAG_USER_REQ;
2180 		if ((io->io_hdr.io_type == CTL_IO_SCSI)
2181 		 && (io->scsiio.tag_type != CTL_TAG_UNTAGGED))
2182 			io->scsiio.tag_num = softc->ioctl_info.cur_tag_num++;
2183 
2184 		retval = ctl_ioctl_submit_wait(io);
2185 
2186 		if (retval != 0) {
2187 			ctl_free_io(io);
2188 			break;
2189 		}
2190 
2191 		memcpy((void *)addr, io, sizeof(*io));
2192 
2193 		/* return this to our pool */
2194 		ctl_free_io(io);
2195 
2196 		break;
2197 	}
2198 	case CTL_ENABLE_PORT:
2199 	case CTL_DISABLE_PORT:
2200 	case CTL_SET_PORT_WWNS: {
2201 		struct ctl_frontend *fe;
2202 		struct ctl_port_entry *entry;
2203 
2204 		entry = (struct ctl_port_entry *)addr;
2205 
2206 		mtx_lock(&softc->ctl_lock);
2207 		STAILQ_FOREACH(fe, &softc->fe_list, links) {
2208 			int action, done;
2209 
2210 			action = 0;
2211 			done = 0;
2212 
2213 			if ((entry->port_type == CTL_PORT_NONE)
2214 			 && (entry->targ_port == fe->targ_port)) {
2215 				/*
2216 				 * If the user only wants to enable or
2217 				 * disable or set WWNs on a specific port,
2218 				 * do the operation and we're done.
2219 				 */
2220 				action = 1;
2221 				done = 1;
2222 			} else if (entry->port_type & fe->port_type) {
2223 				/*
2224 				 * Compare the user's type mask with the
2225 				 * particular frontend type to see if we
2226 				 * have a match.
2227 				 */
2228 				action = 1;
2229 				done = 0;
2230 
2231 				/*
2232 				 * Make sure the user isn't trying to set
2233 				 * WWNs on multiple ports at the same time.
2234 				 */
2235 				if (cmd == CTL_SET_PORT_WWNS) {
2236 					printf("%s: Can't set WWNs on "
2237 					       "multiple ports\n", __func__);
2238 					retval = EINVAL;
2239 					break;
2240 				}
2241 			}
2242 			if (action != 0) {
2243 				/*
2244 				 * XXX KDM we have to drop the lock here,
2245 				 * because the online/offline operations
2246 				 * can potentially block.  We need to
2247 				 * reference count the frontends so they
2248 				 * can't go away,
2249 				 */
2250 				mtx_unlock(&softc->ctl_lock);
2251 
2252 				if (cmd == CTL_ENABLE_PORT) {
2253 					struct ctl_lun *lun;
2254 
2255 					STAILQ_FOREACH(lun, &softc->lun_list,
2256 						       links) {
2257 						fe->lun_enable(fe->targ_lun_arg,
2258 						    lun->target,
2259 						    lun->lun);
2260 					}
2261 
2262 					ctl_frontend_online(fe);
2263 				} else if (cmd == CTL_DISABLE_PORT) {
2264 					struct ctl_lun *lun;
2265 
2266 					ctl_frontend_offline(fe);
2267 
2268 					STAILQ_FOREACH(lun, &softc->lun_list,
2269 						       links) {
2270 						fe->lun_disable(
2271 						    fe->targ_lun_arg,
2272 						    lun->target,
2273 						    lun->lun);
2274 					}
2275 				}
2276 
2277 				mtx_lock(&softc->ctl_lock);
2278 
2279 				if (cmd == CTL_SET_PORT_WWNS)
2280 					ctl_frontend_set_wwns(fe,
2281 					    (entry->flags & CTL_PORT_WWNN_VALID) ?
2282 					    1 : 0, entry->wwnn,
2283 					    (entry->flags & CTL_PORT_WWPN_VALID) ?
2284 					    1 : 0, entry->wwpn);
2285 			}
2286 			if (done != 0)
2287 				break;
2288 		}
2289 		mtx_unlock(&softc->ctl_lock);
2290 		break;
2291 	}
2292 	case CTL_GET_PORT_LIST: {
2293 		struct ctl_frontend *fe;
2294 		struct ctl_port_list *list;
2295 		int i;
2296 
2297 		list = (struct ctl_port_list *)addr;
2298 
2299 		if (list->alloc_len != (list->alloc_num *
2300 		    sizeof(struct ctl_port_entry))) {
2301 			printf("%s: CTL_GET_PORT_LIST: alloc_len %u != "
2302 			       "alloc_num %u * sizeof(struct ctl_port_entry) "
2303 			       "%zu\n", __func__, list->alloc_len,
2304 			       list->alloc_num, sizeof(struct ctl_port_entry));
2305 			retval = EINVAL;
2306 			break;
2307 		}
2308 		list->fill_len = 0;
2309 		list->fill_num = 0;
2310 		list->dropped_num = 0;
2311 		i = 0;
2312 		mtx_lock(&softc->ctl_lock);
2313 		STAILQ_FOREACH(fe, &softc->fe_list, links) {
2314 			struct ctl_port_entry entry, *list_entry;
2315 
2316 			if (list->fill_num >= list->alloc_num) {
2317 				list->dropped_num++;
2318 				continue;
2319 			}
2320 
2321 			entry.port_type = fe->port_type;
2322 			strlcpy(entry.port_name, fe->port_name,
2323 				sizeof(entry.port_name));
2324 			entry.targ_port = fe->targ_port;
2325 			entry.physical_port = fe->physical_port;
2326 			entry.virtual_port = fe->virtual_port;
2327 			entry.wwnn = fe->wwnn;
2328 			entry.wwpn = fe->wwpn;
2329 			if (fe->status & CTL_PORT_STATUS_ONLINE)
2330 				entry.online = 1;
2331 			else
2332 				entry.online = 0;
2333 
2334 			list_entry = &list->entries[i];
2335 
2336 			retval = copyout(&entry, list_entry, sizeof(entry));
2337 			if (retval != 0) {
2338 				printf("%s: CTL_GET_PORT_LIST: copyout "
2339 				       "returned %d\n", __func__, retval);
2340 				break;
2341 			}
2342 			i++;
2343 			list->fill_num++;
2344 			list->fill_len += sizeof(entry);
2345 		}
2346 		mtx_unlock(&softc->ctl_lock);
2347 
2348 		/*
2349 		 * If this is non-zero, we had a copyout fault, so there's
2350 		 * probably no point in attempting to set the status inside
2351 		 * the structure.
2352 		 */
2353 		if (retval != 0)
2354 			break;
2355 
2356 		if (list->dropped_num > 0)
2357 			list->status = CTL_PORT_LIST_NEED_MORE_SPACE;
2358 		else
2359 			list->status = CTL_PORT_LIST_OK;
2360 		break;
2361 	}
2362 	case CTL_DUMP_OOA: {
2363 		struct ctl_lun *lun;
2364 		union ctl_io *io;
2365 		char printbuf[128];
2366 		struct sbuf sb;
2367 
2368 		mtx_lock(&softc->ctl_lock);
2369 		printf("Dumping OOA queues:\n");
2370 		STAILQ_FOREACH(lun, &softc->lun_list, links) {
2371 			mtx_lock(&lun->lun_lock);
2372 			for (io = (union ctl_io *)TAILQ_FIRST(
2373 			     &lun->ooa_queue); io != NULL;
2374 			     io = (union ctl_io *)TAILQ_NEXT(&io->io_hdr,
2375 			     ooa_links)) {
2376 				sbuf_new(&sb, printbuf, sizeof(printbuf),
2377 					 SBUF_FIXEDLEN);
2378 				sbuf_printf(&sb, "LUN %jd tag 0x%04x%s%s%s%s: ",
2379 					    (intmax_t)lun->lun,
2380 					    io->scsiio.tag_num,
2381 					    (io->io_hdr.flags &
2382 					    CTL_FLAG_BLOCKED) ? "" : " BLOCKED",
2383 					    (io->io_hdr.flags &
2384 					    CTL_FLAG_DMA_INPROG) ? " DMA" : "",
2385 					    (io->io_hdr.flags &
2386 					    CTL_FLAG_ABORT) ? " ABORT" : "",
2387 			                    (io->io_hdr.flags &
2388 		                        CTL_FLAG_IS_WAS_ON_RTR) ? " RTR" : "");
2389 				ctl_scsi_command_string(&io->scsiio, NULL, &sb);
2390 				sbuf_finish(&sb);
2391 				printf("%s\n", sbuf_data(&sb));
2392 			}
2393 			mtx_unlock(&lun->lun_lock);
2394 		}
2395 		printf("OOA queues dump done\n");
2396 		mtx_unlock(&softc->ctl_lock);
2397 		break;
2398 	}
2399 	case CTL_GET_OOA: {
2400 		struct ctl_lun *lun;
2401 		struct ctl_ooa *ooa_hdr;
2402 		struct ctl_ooa_entry *entries;
2403 		uint32_t cur_fill_num;
2404 
2405 		ooa_hdr = (struct ctl_ooa *)addr;
2406 
2407 		if ((ooa_hdr->alloc_len == 0)
2408 		 || (ooa_hdr->alloc_num == 0)) {
2409 			printf("%s: CTL_GET_OOA: alloc len %u and alloc num %u "
2410 			       "must be non-zero\n", __func__,
2411 			       ooa_hdr->alloc_len, ooa_hdr->alloc_num);
2412 			retval = EINVAL;
2413 			break;
2414 		}
2415 
2416 		if (ooa_hdr->alloc_len != (ooa_hdr->alloc_num *
2417 		    sizeof(struct ctl_ooa_entry))) {
2418 			printf("%s: CTL_GET_OOA: alloc len %u must be alloc "
2419 			       "num %d * sizeof(struct ctl_ooa_entry) %zd\n",
2420 			       __func__, ooa_hdr->alloc_len,
2421 			       ooa_hdr->alloc_num,sizeof(struct ctl_ooa_entry));
2422 			retval = EINVAL;
2423 			break;
2424 		}
2425 
2426 		entries = malloc(ooa_hdr->alloc_len, M_CTL, M_WAITOK | M_ZERO);
2427 		if (entries == NULL) {
2428 			printf("%s: could not allocate %d bytes for OOA "
2429 			       "dump\n", __func__, ooa_hdr->alloc_len);
2430 			retval = ENOMEM;
2431 			break;
2432 		}
2433 
2434 		mtx_lock(&softc->ctl_lock);
2435 		if (((ooa_hdr->flags & CTL_OOA_FLAG_ALL_LUNS) == 0)
2436 		 && ((ooa_hdr->lun_num > CTL_MAX_LUNS)
2437 		  || (softc->ctl_luns[ooa_hdr->lun_num] == NULL))) {
2438 			mtx_unlock(&softc->ctl_lock);
2439 			free(entries, M_CTL);
2440 			printf("%s: CTL_GET_OOA: invalid LUN %ju\n",
2441 			       __func__, (uintmax_t)ooa_hdr->lun_num);
2442 			retval = EINVAL;
2443 			break;
2444 		}
2445 
2446 		cur_fill_num = 0;
2447 
2448 		if (ooa_hdr->flags & CTL_OOA_FLAG_ALL_LUNS) {
2449 			STAILQ_FOREACH(lun, &softc->lun_list, links) {
2450 				retval = ctl_ioctl_fill_ooa(lun, &cur_fill_num,
2451 					ooa_hdr, entries);
2452 				if (retval != 0)
2453 					break;
2454 			}
2455 			if (retval != 0) {
2456 				mtx_unlock(&softc->ctl_lock);
2457 				free(entries, M_CTL);
2458 				break;
2459 			}
2460 		} else {
2461 			lun = softc->ctl_luns[ooa_hdr->lun_num];
2462 
2463 			retval = ctl_ioctl_fill_ooa(lun, &cur_fill_num,ooa_hdr,
2464 						    entries);
2465 		}
2466 		mtx_unlock(&softc->ctl_lock);
2467 
2468 		ooa_hdr->fill_num = min(cur_fill_num, ooa_hdr->alloc_num);
2469 		ooa_hdr->fill_len = ooa_hdr->fill_num *
2470 			sizeof(struct ctl_ooa_entry);
2471 		retval = copyout(entries, ooa_hdr->entries, ooa_hdr->fill_len);
2472 		if (retval != 0) {
2473 			printf("%s: error copying out %d bytes for OOA dump\n",
2474 			       __func__, ooa_hdr->fill_len);
2475 		}
2476 
2477 		getbintime(&ooa_hdr->cur_bt);
2478 
2479 		if (cur_fill_num > ooa_hdr->alloc_num) {
2480 			ooa_hdr->dropped_num = cur_fill_num -ooa_hdr->alloc_num;
2481 			ooa_hdr->status = CTL_OOA_NEED_MORE_SPACE;
2482 		} else {
2483 			ooa_hdr->dropped_num = 0;
2484 			ooa_hdr->status = CTL_OOA_OK;
2485 		}
2486 
2487 		free(entries, M_CTL);
2488 		break;
2489 	}
2490 	case CTL_CHECK_OOA: {
2491 		union ctl_io *io;
2492 		struct ctl_lun *lun;
2493 		struct ctl_ooa_info *ooa_info;
2494 
2495 
2496 		ooa_info = (struct ctl_ooa_info *)addr;
2497 
2498 		if (ooa_info->lun_id >= CTL_MAX_LUNS) {
2499 			ooa_info->status = CTL_OOA_INVALID_LUN;
2500 			break;
2501 		}
2502 		mtx_lock(&softc->ctl_lock);
2503 		lun = softc->ctl_luns[ooa_info->lun_id];
2504 		if (lun == NULL) {
2505 			mtx_unlock(&softc->ctl_lock);
2506 			ooa_info->status = CTL_OOA_INVALID_LUN;
2507 			break;
2508 		}
2509 		mtx_lock(&lun->lun_lock);
2510 		mtx_unlock(&softc->ctl_lock);
2511 		ooa_info->num_entries = 0;
2512 		for (io = (union ctl_io *)TAILQ_FIRST(&lun->ooa_queue);
2513 		     io != NULL; io = (union ctl_io *)TAILQ_NEXT(
2514 		     &io->io_hdr, ooa_links)) {
2515 			ooa_info->num_entries++;
2516 		}
2517 		mtx_unlock(&lun->lun_lock);
2518 
2519 		ooa_info->status = CTL_OOA_SUCCESS;
2520 
2521 		break;
2522 	}
2523 	case CTL_HARD_START:
2524 	case CTL_HARD_STOP: {
2525 		struct ctl_fe_ioctl_startstop_info ss_info;
2526 		struct cfi_metatask *metatask;
2527 		struct mtx hs_mtx;
2528 
2529 		mtx_init(&hs_mtx, "HS Mutex", NULL, MTX_DEF);
2530 
2531 		cv_init(&ss_info.sem, "hard start/stop cv" );
2532 
2533 		metatask = cfi_alloc_metatask(/*can_wait*/ 1);
2534 		if (metatask == NULL) {
2535 			retval = ENOMEM;
2536 			mtx_destroy(&hs_mtx);
2537 			break;
2538 		}
2539 
2540 		if (cmd == CTL_HARD_START)
2541 			metatask->tasktype = CFI_TASK_STARTUP;
2542 		else
2543 			metatask->tasktype = CFI_TASK_SHUTDOWN;
2544 
2545 		metatask->callback = ctl_ioctl_hard_startstop_callback;
2546 		metatask->callback_arg = &ss_info;
2547 
2548 		cfi_action(metatask);
2549 
2550 		/* Wait for the callback */
2551 		mtx_lock(&hs_mtx);
2552 		cv_wait_sig(&ss_info.sem, &hs_mtx);
2553 		mtx_unlock(&hs_mtx);
2554 
2555 		/*
2556 		 * All information has been copied from the metatask by the
2557 		 * time cv_broadcast() is called, so we free the metatask here.
2558 		 */
2559 		cfi_free_metatask(metatask);
2560 
2561 		memcpy((void *)addr, &ss_info.hs_info, sizeof(ss_info.hs_info));
2562 
2563 		mtx_destroy(&hs_mtx);
2564 		break;
2565 	}
2566 	case CTL_BBRREAD: {
2567 		struct ctl_bbrread_info *bbr_info;
2568 		struct ctl_fe_ioctl_bbrread_info fe_bbr_info;
2569 		struct mtx bbr_mtx;
2570 		struct cfi_metatask *metatask;
2571 
2572 		bbr_info = (struct ctl_bbrread_info *)addr;
2573 
2574 		bzero(&fe_bbr_info, sizeof(fe_bbr_info));
2575 
2576 		bzero(&bbr_mtx, sizeof(bbr_mtx));
2577 		mtx_init(&bbr_mtx, "BBR Mutex", NULL, MTX_DEF);
2578 
2579 		fe_bbr_info.bbr_info = bbr_info;
2580 		fe_bbr_info.lock = &bbr_mtx;
2581 
2582 		cv_init(&fe_bbr_info.sem, "BBR read cv");
2583 		metatask = cfi_alloc_metatask(/*can_wait*/ 1);
2584 
2585 		if (metatask == NULL) {
2586 			mtx_destroy(&bbr_mtx);
2587 			cv_destroy(&fe_bbr_info.sem);
2588 			retval = ENOMEM;
2589 			break;
2590 		}
2591 		metatask->tasktype = CFI_TASK_BBRREAD;
2592 		metatask->callback = ctl_ioctl_bbrread_callback;
2593 		metatask->callback_arg = &fe_bbr_info;
2594 		metatask->taskinfo.bbrread.lun_num = bbr_info->lun_num;
2595 		metatask->taskinfo.bbrread.lba = bbr_info->lba;
2596 		metatask->taskinfo.bbrread.len = bbr_info->len;
2597 
2598 		cfi_action(metatask);
2599 
2600 		mtx_lock(&bbr_mtx);
2601 		while (fe_bbr_info.wakeup_done == 0)
2602 			cv_wait_sig(&fe_bbr_info.sem, &bbr_mtx);
2603 		mtx_unlock(&bbr_mtx);
2604 
2605 		bbr_info->status = metatask->status;
2606 		bbr_info->bbr_status = metatask->taskinfo.bbrread.status;
2607 		bbr_info->scsi_status = metatask->taskinfo.bbrread.scsi_status;
2608 		memcpy(&bbr_info->sense_data,
2609 		       &metatask->taskinfo.bbrread.sense_data,
2610 		       ctl_min(sizeof(bbr_info->sense_data),
2611 			       sizeof(metatask->taskinfo.bbrread.sense_data)));
2612 
2613 		cfi_free_metatask(metatask);
2614 
2615 		mtx_destroy(&bbr_mtx);
2616 		cv_destroy(&fe_bbr_info.sem);
2617 
2618 		break;
2619 	}
2620 	case CTL_DELAY_IO: {
2621 		struct ctl_io_delay_info *delay_info;
2622 #ifdef CTL_IO_DELAY
2623 		struct ctl_lun *lun;
2624 #endif /* CTL_IO_DELAY */
2625 
2626 		delay_info = (struct ctl_io_delay_info *)addr;
2627 
2628 #ifdef CTL_IO_DELAY
2629 		mtx_lock(&softc->ctl_lock);
2630 
2631 		if ((delay_info->lun_id > CTL_MAX_LUNS)
2632 		 || (softc->ctl_luns[delay_info->lun_id] == NULL)) {
2633 			delay_info->status = CTL_DELAY_STATUS_INVALID_LUN;
2634 		} else {
2635 			lun = softc->ctl_luns[delay_info->lun_id];
2636 			mtx_lock(&lun->lun_lock);
2637 
2638 			delay_info->status = CTL_DELAY_STATUS_OK;
2639 
2640 			switch (delay_info->delay_type) {
2641 			case CTL_DELAY_TYPE_CONT:
2642 				break;
2643 			case CTL_DELAY_TYPE_ONESHOT:
2644 				break;
2645 			default:
2646 				delay_info->status =
2647 					CTL_DELAY_STATUS_INVALID_TYPE;
2648 				break;
2649 			}
2650 
2651 			switch (delay_info->delay_loc) {
2652 			case CTL_DELAY_LOC_DATAMOVE:
2653 				lun->delay_info.datamove_type =
2654 					delay_info->delay_type;
2655 				lun->delay_info.datamove_delay =
2656 					delay_info->delay_secs;
2657 				break;
2658 			case CTL_DELAY_LOC_DONE:
2659 				lun->delay_info.done_type =
2660 					delay_info->delay_type;
2661 				lun->delay_info.done_delay =
2662 					delay_info->delay_secs;
2663 				break;
2664 			default:
2665 				delay_info->status =
2666 					CTL_DELAY_STATUS_INVALID_LOC;
2667 				break;
2668 			}
2669 			mtx_unlock(&lun->lun_lock);
2670 		}
2671 
2672 		mtx_unlock(&softc->ctl_lock);
2673 #else
2674 		delay_info->status = CTL_DELAY_STATUS_NOT_IMPLEMENTED;
2675 #endif /* CTL_IO_DELAY */
2676 		break;
2677 	}
2678 	case CTL_REALSYNC_SET: {
2679 		int *syncstate;
2680 
2681 		syncstate = (int *)addr;
2682 
2683 		mtx_lock(&softc->ctl_lock);
2684 		switch (*syncstate) {
2685 		case 0:
2686 			softc->flags &= ~CTL_FLAG_REAL_SYNC;
2687 			break;
2688 		case 1:
2689 			softc->flags |= CTL_FLAG_REAL_SYNC;
2690 			break;
2691 		default:
2692 			retval = -EINVAL;
2693 			break;
2694 		}
2695 		mtx_unlock(&softc->ctl_lock);
2696 		break;
2697 	}
2698 	case CTL_REALSYNC_GET: {
2699 		int *syncstate;
2700 
2701 		syncstate = (int*)addr;
2702 
2703 		mtx_lock(&softc->ctl_lock);
2704 		if (softc->flags & CTL_FLAG_REAL_SYNC)
2705 			*syncstate = 1;
2706 		else
2707 			*syncstate = 0;
2708 		mtx_unlock(&softc->ctl_lock);
2709 
2710 		break;
2711 	}
2712 	case CTL_SETSYNC:
2713 	case CTL_GETSYNC: {
2714 		struct ctl_sync_info *sync_info;
2715 		struct ctl_lun *lun;
2716 
2717 		sync_info = (struct ctl_sync_info *)addr;
2718 
2719 		mtx_lock(&softc->ctl_lock);
2720 		lun = softc->ctl_luns[sync_info->lun_id];
2721 		if (lun == NULL) {
2722 			mtx_unlock(&softc->ctl_lock);
2723 			sync_info->status = CTL_GS_SYNC_NO_LUN;
2724 		}
2725 		/*
2726 		 * Get or set the sync interval.  We're not bounds checking
2727 		 * in the set case, hopefully the user won't do something
2728 		 * silly.
2729 		 */
2730 		mtx_lock(&lun->lun_lock);
2731 		mtx_unlock(&softc->ctl_lock);
2732 		if (cmd == CTL_GETSYNC)
2733 			sync_info->sync_interval = lun->sync_interval;
2734 		else
2735 			lun->sync_interval = sync_info->sync_interval;
2736 		mtx_unlock(&lun->lun_lock);
2737 
2738 		sync_info->status = CTL_GS_SYNC_OK;
2739 
2740 		break;
2741 	}
2742 	case CTL_GETSTATS: {
2743 		struct ctl_stats *stats;
2744 		struct ctl_lun *lun;
2745 		int i;
2746 
2747 		stats = (struct ctl_stats *)addr;
2748 
2749 		if ((sizeof(struct ctl_lun_io_stats) * softc->num_luns) >
2750 		     stats->alloc_len) {
2751 			stats->status = CTL_SS_NEED_MORE_SPACE;
2752 			stats->num_luns = softc->num_luns;
2753 			break;
2754 		}
2755 		/*
2756 		 * XXX KDM no locking here.  If the LUN list changes,
2757 		 * things can blow up.
2758 		 */
2759 		for (i = 0, lun = STAILQ_FIRST(&softc->lun_list); lun != NULL;
2760 		     i++, lun = STAILQ_NEXT(lun, links)) {
2761 			retval = copyout(&lun->stats, &stats->lun_stats[i],
2762 					 sizeof(lun->stats));
2763 			if (retval != 0)
2764 				break;
2765 		}
2766 		stats->num_luns = softc->num_luns;
2767 		stats->fill_len = sizeof(struct ctl_lun_io_stats) *
2768 				 softc->num_luns;
2769 		stats->status = CTL_SS_OK;
2770 #ifdef CTL_TIME_IO
2771 		stats->flags = CTL_STATS_FLAG_TIME_VALID;
2772 #else
2773 		stats->flags = CTL_STATS_FLAG_NONE;
2774 #endif
2775 		getnanouptime(&stats->timestamp);
2776 		break;
2777 	}
2778 	case CTL_ERROR_INJECT: {
2779 		struct ctl_error_desc *err_desc, *new_err_desc;
2780 		struct ctl_lun *lun;
2781 
2782 		err_desc = (struct ctl_error_desc *)addr;
2783 
2784 		new_err_desc = malloc(sizeof(*new_err_desc), M_CTL,
2785 				      M_WAITOK | M_ZERO);
2786 		bcopy(err_desc, new_err_desc, sizeof(*new_err_desc));
2787 
2788 		mtx_lock(&softc->ctl_lock);
2789 		lun = softc->ctl_luns[err_desc->lun_id];
2790 		if (lun == NULL) {
2791 			mtx_unlock(&softc->ctl_lock);
2792 			printf("%s: CTL_ERROR_INJECT: invalid LUN %ju\n",
2793 			       __func__, (uintmax_t)err_desc->lun_id);
2794 			retval = EINVAL;
2795 			break;
2796 		}
2797 		mtx_lock(&lun->lun_lock);
2798 		mtx_unlock(&softc->ctl_lock);
2799 
2800 		/*
2801 		 * We could do some checking here to verify the validity
2802 		 * of the request, but given the complexity of error
2803 		 * injection requests, the checking logic would be fairly
2804 		 * complex.
2805 		 *
2806 		 * For now, if the request is invalid, it just won't get
2807 		 * executed and might get deleted.
2808 		 */
2809 		STAILQ_INSERT_TAIL(&lun->error_list, new_err_desc, links);
2810 
2811 		/*
2812 		 * XXX KDM check to make sure the serial number is unique,
2813 		 * in case we somehow manage to wrap.  That shouldn't
2814 		 * happen for a very long time, but it's the right thing to
2815 		 * do.
2816 		 */
2817 		new_err_desc->serial = lun->error_serial;
2818 		err_desc->serial = lun->error_serial;
2819 		lun->error_serial++;
2820 
2821 		mtx_unlock(&lun->lun_lock);
2822 		break;
2823 	}
2824 	case CTL_ERROR_INJECT_DELETE: {
2825 		struct ctl_error_desc *delete_desc, *desc, *desc2;
2826 		struct ctl_lun *lun;
2827 		int delete_done;
2828 
2829 		delete_desc = (struct ctl_error_desc *)addr;
2830 		delete_done = 0;
2831 
2832 		mtx_lock(&softc->ctl_lock);
2833 		lun = softc->ctl_luns[delete_desc->lun_id];
2834 		if (lun == NULL) {
2835 			mtx_unlock(&softc->ctl_lock);
2836 			printf("%s: CTL_ERROR_INJECT_DELETE: invalid LUN %ju\n",
2837 			       __func__, (uintmax_t)delete_desc->lun_id);
2838 			retval = EINVAL;
2839 			break;
2840 		}
2841 		mtx_lock(&lun->lun_lock);
2842 		mtx_unlock(&softc->ctl_lock);
2843 		STAILQ_FOREACH_SAFE(desc, &lun->error_list, links, desc2) {
2844 			if (desc->serial != delete_desc->serial)
2845 				continue;
2846 
2847 			STAILQ_REMOVE(&lun->error_list, desc, ctl_error_desc,
2848 				      links);
2849 			free(desc, M_CTL);
2850 			delete_done = 1;
2851 		}
2852 		mtx_unlock(&lun->lun_lock);
2853 		if (delete_done == 0) {
2854 			printf("%s: CTL_ERROR_INJECT_DELETE: can't find "
2855 			       "error serial %ju on LUN %u\n", __func__,
2856 			       delete_desc->serial, delete_desc->lun_id);
2857 			retval = EINVAL;
2858 			break;
2859 		}
2860 		break;
2861 	}
2862 	case CTL_DUMP_STRUCTS: {
2863 		int i, j, k;
2864 		struct ctl_frontend *fe;
2865 
2866 		printf("CTL IID to WWPN map start:\n");
2867 		for (i = 0; i < CTL_MAX_PORTS; i++) {
2868 			for (j = 0; j < CTL_MAX_INIT_PER_PORT; j++) {
2869 				if (softc->wwpn_iid[i][j].in_use == 0)
2870 					continue;
2871 
2872 				printf("port %d iid %u WWPN %#jx\n",
2873 				       softc->wwpn_iid[i][j].port,
2874 				       softc->wwpn_iid[i][j].iid,
2875 				       (uintmax_t)softc->wwpn_iid[i][j].wwpn);
2876 			}
2877 		}
2878 		printf("CTL IID to WWPN map end\n");
2879 		printf("CTL Persistent Reservation information start:\n");
2880 		for (i = 0; i < CTL_MAX_LUNS; i++) {
2881 			struct ctl_lun *lun;
2882 
2883 			lun = softc->ctl_luns[i];
2884 
2885 			if ((lun == NULL)
2886 			 || ((lun->flags & CTL_LUN_DISABLED) != 0))
2887 				continue;
2888 
2889 			for (j = 0; j < (CTL_MAX_PORTS * 2); j++) {
2890 				for (k = 0; k < CTL_MAX_INIT_PER_PORT; k++){
2891 					if (lun->per_res[j+k].registered == 0)
2892 						continue;
2893 					printf("LUN %d port %d iid %d key "
2894 					       "%#jx\n", i, j, k,
2895 					       (uintmax_t)scsi_8btou64(
2896 					       lun->per_res[j+k].res_key.key));
2897 				}
2898 			}
2899 		}
2900 		printf("CTL Persistent Reservation information end\n");
2901 		printf("CTL Frontends:\n");
2902 		/*
2903 		 * XXX KDM calling this without a lock.  We'd likely want
2904 		 * to drop the lock before calling the frontend's dump
2905 		 * routine anyway.
2906 		 */
2907 		STAILQ_FOREACH(fe, &softc->fe_list, links) {
2908 			printf("Frontend %s Type %u pport %d vport %d WWNN "
2909 			       "%#jx WWPN %#jx\n", fe->port_name, fe->port_type,
2910 			       fe->physical_port, fe->virtual_port,
2911 			       (uintmax_t)fe->wwnn, (uintmax_t)fe->wwpn);
2912 
2913 			/*
2914 			 * Frontends are not required to support the dump
2915 			 * routine.
2916 			 */
2917 			if (fe->fe_dump == NULL)
2918 				continue;
2919 
2920 			fe->fe_dump();
2921 		}
2922 		printf("CTL Frontend information end\n");
2923 		break;
2924 	}
2925 	case CTL_LUN_REQ: {
2926 		struct ctl_lun_req *lun_req;
2927 		struct ctl_backend_driver *backend;
2928 
2929 		lun_req = (struct ctl_lun_req *)addr;
2930 
2931 		backend = ctl_backend_find(lun_req->backend);
2932 		if (backend == NULL) {
2933 			lun_req->status = CTL_LUN_ERROR;
2934 			snprintf(lun_req->error_str,
2935 				 sizeof(lun_req->error_str),
2936 				 "Backend \"%s\" not found.",
2937 				 lun_req->backend);
2938 			break;
2939 		}
2940 		if (lun_req->num_be_args > 0) {
2941 			lun_req->kern_be_args = ctl_copyin_args(
2942 				lun_req->num_be_args,
2943 				lun_req->be_args,
2944 				lun_req->error_str,
2945 				sizeof(lun_req->error_str));
2946 			if (lun_req->kern_be_args == NULL) {
2947 				lun_req->status = CTL_LUN_ERROR;
2948 				break;
2949 			}
2950 		}
2951 
2952 		retval = backend->ioctl(dev, cmd, addr, flag, td);
2953 
2954 		if (lun_req->num_be_args > 0) {
2955 			ctl_free_args(lun_req->num_be_args,
2956 				      lun_req->kern_be_args);
2957 		}
2958 		break;
2959 	}
2960 	case CTL_LUN_LIST: {
2961 		struct sbuf *sb;
2962 		struct ctl_lun *lun;
2963 		struct ctl_lun_list *list;
2964 		struct ctl_be_lun_option *opt;
2965 
2966 		list = (struct ctl_lun_list *)addr;
2967 
2968 		/*
2969 		 * Allocate a fixed length sbuf here, based on the length
2970 		 * of the user's buffer.  We could allocate an auto-extending
2971 		 * buffer, and then tell the user how much larger our
2972 		 * amount of data is than his buffer, but that presents
2973 		 * some problems:
2974 		 *
2975 		 * 1.  The sbuf(9) routines use a blocking malloc, and so
2976 		 *     we can't hold a lock while calling them with an
2977 		 *     auto-extending buffer.
2978  		 *
2979 		 * 2.  There is not currently a LUN reference counting
2980 		 *     mechanism, outside of outstanding transactions on
2981 		 *     the LUN's OOA queue.  So a LUN could go away on us
2982 		 *     while we're getting the LUN number, backend-specific
2983 		 *     information, etc.  Thus, given the way things
2984 		 *     currently work, we need to hold the CTL lock while
2985 		 *     grabbing LUN information.
2986 		 *
2987 		 * So, from the user's standpoint, the best thing to do is
2988 		 * allocate what he thinks is a reasonable buffer length,
2989 		 * and then if he gets a CTL_LUN_LIST_NEED_MORE_SPACE error,
2990 		 * double the buffer length and try again.  (And repeat
2991 		 * that until he succeeds.)
2992 		 */
2993 		sb = sbuf_new(NULL, NULL, list->alloc_len, SBUF_FIXEDLEN);
2994 		if (sb == NULL) {
2995 			list->status = CTL_LUN_LIST_ERROR;
2996 			snprintf(list->error_str, sizeof(list->error_str),
2997 				 "Unable to allocate %d bytes for LUN list",
2998 				 list->alloc_len);
2999 			break;
3000 		}
3001 
3002 		sbuf_printf(sb, "<ctllunlist>\n");
3003 
3004 		mtx_lock(&softc->ctl_lock);
3005 		STAILQ_FOREACH(lun, &softc->lun_list, links) {
3006 			mtx_lock(&lun->lun_lock);
3007 			retval = sbuf_printf(sb, "<lun id=\"%ju\">\n",
3008 					     (uintmax_t)lun->lun);
3009 
3010 			/*
3011 			 * Bail out as soon as we see that we've overfilled
3012 			 * the buffer.
3013 			 */
3014 			if (retval != 0)
3015 				break;
3016 
3017 			retval = sbuf_printf(sb, "<backend_type>%s"
3018 					     "</backend_type>\n",
3019 					     (lun->backend == NULL) ?  "none" :
3020 					     lun->backend->name);
3021 
3022 			if (retval != 0)
3023 				break;
3024 
3025 			retval = sbuf_printf(sb, "<lun_type>%d</lun_type>\n",
3026 					     lun->be_lun->lun_type);
3027 
3028 			if (retval != 0)
3029 				break;
3030 
3031 			if (lun->backend == NULL) {
3032 				retval = sbuf_printf(sb, "</lun>\n");
3033 				if (retval != 0)
3034 					break;
3035 				continue;
3036 			}
3037 
3038 			retval = sbuf_printf(sb, "<size>%ju</size>\n",
3039 					     (lun->be_lun->maxlba > 0) ?
3040 					     lun->be_lun->maxlba + 1 : 0);
3041 
3042 			if (retval != 0)
3043 				break;
3044 
3045 			retval = sbuf_printf(sb, "<blocksize>%u</blocksize>\n",
3046 					     lun->be_lun->blocksize);
3047 
3048 			if (retval != 0)
3049 				break;
3050 
3051 			retval = sbuf_printf(sb, "<serial_number>");
3052 
3053 			if (retval != 0)
3054 				break;
3055 
3056 			retval = ctl_sbuf_printf_esc(sb,
3057 						     lun->be_lun->serial_num);
3058 
3059 			if (retval != 0)
3060 				break;
3061 
3062 			retval = sbuf_printf(sb, "</serial_number>\n");
3063 
3064 			if (retval != 0)
3065 				break;
3066 
3067 			retval = sbuf_printf(sb, "<device_id>");
3068 
3069 			if (retval != 0)
3070 				break;
3071 
3072 			retval = ctl_sbuf_printf_esc(sb,lun->be_lun->device_id);
3073 
3074 			if (retval != 0)
3075 				break;
3076 
3077 			retval = sbuf_printf(sb, "</device_id>\n");
3078 
3079 			if (retval != 0)
3080 				break;
3081 
3082 			if (lun->backend->lun_info != NULL) {
3083 				retval = lun->backend->lun_info(lun->be_lun->be_lun, sb);
3084 				if (retval != 0)
3085 					break;
3086 			}
3087 			STAILQ_FOREACH(opt, &lun->be_lun->options, links) {
3088 				retval = sbuf_printf(sb, "<%s>%s</%s>", opt->name, opt->value, opt->name);
3089 				if (retval != 0)
3090 					break;
3091 			}
3092 
3093 			retval = sbuf_printf(sb, "</lun>\n");
3094 
3095 			if (retval != 0)
3096 				break;
3097 			mtx_unlock(&lun->lun_lock);
3098 		}
3099 		if (lun != NULL)
3100 			mtx_unlock(&lun->lun_lock);
3101 		mtx_unlock(&softc->ctl_lock);
3102 
3103 		if ((retval != 0)
3104 		 || ((retval = sbuf_printf(sb, "</ctllunlist>\n")) != 0)) {
3105 			retval = 0;
3106 			sbuf_delete(sb);
3107 			list->status = CTL_LUN_LIST_NEED_MORE_SPACE;
3108 			snprintf(list->error_str, sizeof(list->error_str),
3109 				 "Out of space, %d bytes is too small",
3110 				 list->alloc_len);
3111 			break;
3112 		}
3113 
3114 		sbuf_finish(sb);
3115 
3116 		retval = copyout(sbuf_data(sb), list->lun_xml,
3117 				 sbuf_len(sb) + 1);
3118 
3119 		list->fill_len = sbuf_len(sb) + 1;
3120 		list->status = CTL_LUN_LIST_OK;
3121 		sbuf_delete(sb);
3122 		break;
3123 	}
3124 	case CTL_ISCSI: {
3125 		struct ctl_iscsi *ci;
3126 		struct ctl_frontend *fe;
3127 
3128 		ci = (struct ctl_iscsi *)addr;
3129 
3130 		mtx_lock(&softc->ctl_lock);
3131 		STAILQ_FOREACH(fe, &softc->fe_list, links) {
3132 			if (strcmp(fe->port_name, "iscsi") == 0)
3133 				break;
3134 		}
3135 		mtx_unlock(&softc->ctl_lock);
3136 
3137 		if (fe == NULL) {
3138 			ci->status = CTL_ISCSI_ERROR;
3139 			snprintf(ci->error_str, sizeof(ci->error_str), "Backend \"iscsi\" not found.");
3140 			break;
3141 		}
3142 
3143 		retval = fe->ioctl(dev, cmd, addr, flag, td);
3144 		break;
3145 	}
3146 	default: {
3147 		/* XXX KDM should we fix this? */
3148 #if 0
3149 		struct ctl_backend_driver *backend;
3150 		unsigned int type;
3151 		int found;
3152 
3153 		found = 0;
3154 
3155 		/*
3156 		 * We encode the backend type as the ioctl type for backend
3157 		 * ioctls.  So parse it out here, and then search for a
3158 		 * backend of this type.
3159 		 */
3160 		type = _IOC_TYPE(cmd);
3161 
3162 		STAILQ_FOREACH(backend, &softc->be_list, links) {
3163 			if (backend->type == type) {
3164 				found = 1;
3165 				break;
3166 			}
3167 		}
3168 		if (found == 0) {
3169 			printf("ctl: unknown ioctl command %#lx or backend "
3170 			       "%d\n", cmd, type);
3171 			retval = -EINVAL;
3172 			break;
3173 		}
3174 		retval = backend->ioctl(dev, cmd, addr, flag, td);
3175 #endif
3176 		retval = ENOTTY;
3177 		break;
3178 	}
3179 	}
3180 	return (retval);
3181 }
3182 
3183 uint32_t
3184 ctl_get_initindex(struct ctl_nexus *nexus)
3185 {
3186 	if (nexus->targ_port < CTL_MAX_PORTS)
3187 		return (nexus->initid.id +
3188 			(nexus->targ_port * CTL_MAX_INIT_PER_PORT));
3189 	else
3190 		return (nexus->initid.id +
3191 		       ((nexus->targ_port - CTL_MAX_PORTS) *
3192 			CTL_MAX_INIT_PER_PORT));
3193 }
3194 
3195 uint32_t
3196 ctl_get_resindex(struct ctl_nexus *nexus)
3197 {
3198 	return (nexus->initid.id + (nexus->targ_port * CTL_MAX_INIT_PER_PORT));
3199 }
3200 
3201 uint32_t
3202 ctl_port_idx(int port_num)
3203 {
3204 	if (port_num < CTL_MAX_PORTS)
3205 		return(port_num);
3206 	else
3207 		return(port_num - CTL_MAX_PORTS);
3208 }
3209 
3210 /*
3211  * Note:  This only works for bitmask sizes that are at least 32 bits, and
3212  * that are a power of 2.
3213  */
3214 int
3215 ctl_ffz(uint32_t *mask, uint32_t size)
3216 {
3217 	uint32_t num_chunks, num_pieces;
3218 	int i, j;
3219 
3220 	num_chunks = (size >> 5);
3221 	if (num_chunks == 0)
3222 		num_chunks++;
3223 	num_pieces = ctl_min((sizeof(uint32_t) * 8), size);
3224 
3225 	for (i = 0; i < num_chunks; i++) {
3226 		for (j = 0; j < num_pieces; j++) {
3227 			if ((mask[i] & (1 << j)) == 0)
3228 				return ((i << 5) + j);
3229 		}
3230 	}
3231 
3232 	return (-1);
3233 }
3234 
3235 int
3236 ctl_set_mask(uint32_t *mask, uint32_t bit)
3237 {
3238 	uint32_t chunk, piece;
3239 
3240 	chunk = bit >> 5;
3241 	piece = bit % (sizeof(uint32_t) * 8);
3242 
3243 	if ((mask[chunk] & (1 << piece)) != 0)
3244 		return (-1);
3245 	else
3246 		mask[chunk] |= (1 << piece);
3247 
3248 	return (0);
3249 }
3250 
3251 int
3252 ctl_clear_mask(uint32_t *mask, uint32_t bit)
3253 {
3254 	uint32_t chunk, piece;
3255 
3256 	chunk = bit >> 5;
3257 	piece = bit % (sizeof(uint32_t) * 8);
3258 
3259 	if ((mask[chunk] & (1 << piece)) == 0)
3260 		return (-1);
3261 	else
3262 		mask[chunk] &= ~(1 << piece);
3263 
3264 	return (0);
3265 }
3266 
3267 int
3268 ctl_is_set(uint32_t *mask, uint32_t bit)
3269 {
3270 	uint32_t chunk, piece;
3271 
3272 	chunk = bit >> 5;
3273 	piece = bit % (sizeof(uint32_t) * 8);
3274 
3275 	if ((mask[chunk] & (1 << piece)) == 0)
3276 		return (0);
3277 	else
3278 		return (1);
3279 }
3280 
3281 #ifdef unused
3282 /*
3283  * The bus, target and lun are optional, they can be filled in later.
3284  * can_wait is used to determine whether we can wait on the malloc or not.
3285  */
3286 union ctl_io*
3287 ctl_malloc_io(ctl_io_type io_type, uint32_t targ_port, uint32_t targ_target,
3288 	      uint32_t targ_lun, int can_wait)
3289 {
3290 	union ctl_io *io;
3291 
3292 	if (can_wait)
3293 		io = (union ctl_io *)malloc(sizeof(*io), M_CTL, M_WAITOK);
3294 	else
3295 		io = (union ctl_io *)malloc(sizeof(*io), M_CTL, M_NOWAIT);
3296 
3297 	if (io != NULL) {
3298 		io->io_hdr.io_type = io_type;
3299 		io->io_hdr.targ_port = targ_port;
3300 		/*
3301 		 * XXX KDM this needs to change/go away.  We need to move
3302 		 * to a preallocated pool of ctl_scsiio structures.
3303 		 */
3304 		io->io_hdr.nexus.targ_target.id = targ_target;
3305 		io->io_hdr.nexus.targ_lun = targ_lun;
3306 	}
3307 
3308 	return (io);
3309 }
3310 
3311 void
3312 ctl_kfree_io(union ctl_io *io)
3313 {
3314 	free(io, M_CTL);
3315 }
3316 #endif /* unused */
3317 
3318 /*
3319  * ctl_softc, pool_type, total_ctl_io are passed in.
3320  * npool is passed out.
3321  */
3322 int
3323 ctl_pool_create(struct ctl_softc *ctl_softc, ctl_pool_type pool_type,
3324 		uint32_t total_ctl_io, struct ctl_io_pool **npool)
3325 {
3326 	uint32_t i;
3327 	union ctl_io *cur_io, *next_io;
3328 	struct ctl_io_pool *pool;
3329 	int retval;
3330 
3331 	retval = 0;
3332 
3333 	pool = (struct ctl_io_pool *)malloc(sizeof(*pool), M_CTL,
3334 					    M_NOWAIT | M_ZERO);
3335 	if (pool == NULL) {
3336 		retval = -ENOMEM;
3337 		goto bailout;
3338 	}
3339 
3340 	pool->type = pool_type;
3341 	pool->ctl_softc = ctl_softc;
3342 
3343 	mtx_lock(&ctl_softc->pool_lock);
3344 	pool->id = ctl_softc->cur_pool_id++;
3345 	mtx_unlock(&ctl_softc->pool_lock);
3346 
3347 	pool->flags = CTL_POOL_FLAG_NONE;
3348 	pool->refcount = 1;		/* Reference for validity. */
3349 	STAILQ_INIT(&pool->free_queue);
3350 
3351 	/*
3352 	 * XXX KDM other options here:
3353 	 * - allocate a page at a time
3354 	 * - allocate one big chunk of memory.
3355 	 * Page allocation might work well, but would take a little more
3356 	 * tracking.
3357 	 */
3358 	for (i = 0; i < total_ctl_io; i++) {
3359 		cur_io = (union ctl_io *)malloc(sizeof(*cur_io), M_CTL,
3360 						M_NOWAIT);
3361 		if (cur_io == NULL) {
3362 			retval = ENOMEM;
3363 			break;
3364 		}
3365 		cur_io->io_hdr.pool = pool;
3366 		STAILQ_INSERT_TAIL(&pool->free_queue, &cur_io->io_hdr, links);
3367 		pool->total_ctl_io++;
3368 		pool->free_ctl_io++;
3369 	}
3370 
3371 	if (retval != 0) {
3372 		for (cur_io = (union ctl_io *)STAILQ_FIRST(&pool->free_queue);
3373 		     cur_io != NULL; cur_io = next_io) {
3374 			next_io = (union ctl_io *)STAILQ_NEXT(&cur_io->io_hdr,
3375 							      links);
3376 			STAILQ_REMOVE(&pool->free_queue, &cur_io->io_hdr,
3377 				      ctl_io_hdr, links);
3378 			free(cur_io, M_CTL);
3379 		}
3380 
3381 		free(pool, M_CTL);
3382 		goto bailout;
3383 	}
3384 	mtx_lock(&ctl_softc->pool_lock);
3385 	ctl_softc->num_pools++;
3386 	STAILQ_INSERT_TAIL(&ctl_softc->io_pools, pool, links);
3387 	/*
3388 	 * Increment our usage count if this is an external consumer, so we
3389 	 * can't get unloaded until the external consumer (most likely a
3390 	 * FETD) unloads and frees his pool.
3391 	 *
3392 	 * XXX KDM will this increment the caller's module use count, or
3393 	 * mine?
3394 	 */
3395 #if 0
3396 	if ((pool_type != CTL_POOL_EMERGENCY)
3397 	 && (pool_type != CTL_POOL_INTERNAL)
3398 	 && (pool_type != CTL_POOL_IOCTL)
3399 	 && (pool_type != CTL_POOL_4OTHERSC))
3400 		MOD_INC_USE_COUNT;
3401 #endif
3402 
3403 	mtx_unlock(&ctl_softc->pool_lock);
3404 
3405 	*npool = pool;
3406 
3407 bailout:
3408 
3409 	return (retval);
3410 }
3411 
3412 static int
3413 ctl_pool_acquire(struct ctl_io_pool *pool)
3414 {
3415 
3416 	mtx_assert(&pool->ctl_softc->pool_lock, MA_OWNED);
3417 
3418 	if (pool->flags & CTL_POOL_FLAG_INVALID)
3419 		return (-EINVAL);
3420 
3421 	pool->refcount++;
3422 
3423 	return (0);
3424 }
3425 
3426 static void
3427 ctl_pool_release(struct ctl_io_pool *pool)
3428 {
3429 	struct ctl_softc *ctl_softc = pool->ctl_softc;
3430 	union ctl_io *io;
3431 
3432 	mtx_assert(&ctl_softc->pool_lock, MA_OWNED);
3433 
3434 	if (--pool->refcount != 0)
3435 		return;
3436 
3437 	while ((io = (union ctl_io *)STAILQ_FIRST(&pool->free_queue)) != NULL) {
3438 		STAILQ_REMOVE(&pool->free_queue, &io->io_hdr, ctl_io_hdr,
3439 			      links);
3440 		free(io, M_CTL);
3441 	}
3442 
3443 	STAILQ_REMOVE(&ctl_softc->io_pools, pool, ctl_io_pool, links);
3444 	ctl_softc->num_pools--;
3445 
3446 	/*
3447 	 * XXX KDM will this decrement the caller's usage count or mine?
3448 	 */
3449 #if 0
3450 	if ((pool->type != CTL_POOL_EMERGENCY)
3451 	 && (pool->type != CTL_POOL_INTERNAL)
3452 	 && (pool->type != CTL_POOL_IOCTL))
3453 		MOD_DEC_USE_COUNT;
3454 #endif
3455 
3456 	free(pool, M_CTL);
3457 }
3458 
3459 void
3460 ctl_pool_free(struct ctl_io_pool *pool)
3461 {
3462 	struct ctl_softc *ctl_softc;
3463 
3464 	if (pool == NULL)
3465 		return;
3466 
3467 	ctl_softc = pool->ctl_softc;
3468 	mtx_lock(&ctl_softc->pool_lock);
3469 	pool->flags |= CTL_POOL_FLAG_INVALID;
3470 	ctl_pool_release(pool);
3471 	mtx_unlock(&ctl_softc->pool_lock);
3472 }
3473 
3474 /*
3475  * This routine does not block (except for spinlocks of course).
3476  * It tries to allocate a ctl_io union from the caller's pool as quickly as
3477  * possible.
3478  */
3479 union ctl_io *
3480 ctl_alloc_io(void *pool_ref)
3481 {
3482 	union ctl_io *io;
3483 	struct ctl_softc *ctl_softc;
3484 	struct ctl_io_pool *pool, *npool;
3485 	struct ctl_io_pool *emergency_pool;
3486 
3487 	pool = (struct ctl_io_pool *)pool_ref;
3488 
3489 	if (pool == NULL) {
3490 		printf("%s: pool is NULL\n", __func__);
3491 		return (NULL);
3492 	}
3493 
3494 	emergency_pool = NULL;
3495 
3496 	ctl_softc = pool->ctl_softc;
3497 
3498 	mtx_lock(&ctl_softc->pool_lock);
3499 	/*
3500 	 * First, try to get the io structure from the user's pool.
3501 	 */
3502 	if (ctl_pool_acquire(pool) == 0) {
3503 		io = (union ctl_io *)STAILQ_FIRST(&pool->free_queue);
3504 		if (io != NULL) {
3505 			STAILQ_REMOVE_HEAD(&pool->free_queue, links);
3506 			pool->total_allocated++;
3507 			pool->free_ctl_io--;
3508 			mtx_unlock(&ctl_softc->pool_lock);
3509 			return (io);
3510 		} else
3511 			ctl_pool_release(pool);
3512 	}
3513 	/*
3514 	 * If he doesn't have any io structures left, search for an
3515 	 * emergency pool and grab one from there.
3516 	 */
3517 	STAILQ_FOREACH(npool, &ctl_softc->io_pools, links) {
3518 		if (npool->type != CTL_POOL_EMERGENCY)
3519 			continue;
3520 
3521 		if (ctl_pool_acquire(npool) != 0)
3522 			continue;
3523 
3524 		emergency_pool = npool;
3525 
3526 		io = (union ctl_io *)STAILQ_FIRST(&npool->free_queue);
3527 		if (io != NULL) {
3528 			STAILQ_REMOVE_HEAD(&npool->free_queue, links);
3529 			npool->total_allocated++;
3530 			npool->free_ctl_io--;
3531 			mtx_unlock(&ctl_softc->pool_lock);
3532 			return (io);
3533 		} else
3534 			ctl_pool_release(npool);
3535 	}
3536 
3537 	/* Drop the spinlock before we malloc */
3538 	mtx_unlock(&ctl_softc->pool_lock);
3539 
3540 	/*
3541 	 * The emergency pool (if it exists) didn't have one, so try an
3542 	 * atomic (i.e. nonblocking) malloc and see if we get lucky.
3543 	 */
3544 	io = (union ctl_io *)malloc(sizeof(*io), M_CTL, M_NOWAIT);
3545 	if (io != NULL) {
3546 		/*
3547 		 * If the emergency pool exists but is empty, add this
3548 		 * ctl_io to its list when it gets freed.
3549 		 */
3550 		if (emergency_pool != NULL) {
3551 			mtx_lock(&ctl_softc->pool_lock);
3552 			if (ctl_pool_acquire(emergency_pool) == 0) {
3553 				io->io_hdr.pool = emergency_pool;
3554 				emergency_pool->total_ctl_io++;
3555 				/*
3556 				 * Need to bump this, otherwise
3557 				 * total_allocated and total_freed won't
3558 				 * match when we no longer have anything
3559 				 * outstanding.
3560 				 */
3561 				emergency_pool->total_allocated++;
3562 			}
3563 			mtx_unlock(&ctl_softc->pool_lock);
3564 		} else
3565 			io->io_hdr.pool = NULL;
3566 	}
3567 
3568 	return (io);
3569 }
3570 
3571 void
3572 ctl_free_io(union ctl_io *io)
3573 {
3574 	if (io == NULL)
3575 		return;
3576 
3577 	/*
3578 	 * If this ctl_io has a pool, return it to that pool.
3579 	 */
3580 	if (io->io_hdr.pool != NULL) {
3581 		struct ctl_io_pool *pool;
3582 
3583 		pool = (struct ctl_io_pool *)io->io_hdr.pool;
3584 		mtx_lock(&pool->ctl_softc->pool_lock);
3585 		io->io_hdr.io_type = 0xff;
3586 		STAILQ_INSERT_TAIL(&pool->free_queue, &io->io_hdr, links);
3587 		pool->total_freed++;
3588 		pool->free_ctl_io++;
3589 		ctl_pool_release(pool);
3590 		mtx_unlock(&pool->ctl_softc->pool_lock);
3591 	} else {
3592 		/*
3593 		 * Otherwise, just free it.  We probably malloced it and
3594 		 * the emergency pool wasn't available.
3595 		 */
3596 		free(io, M_CTL);
3597 	}
3598 
3599 }
3600 
3601 void
3602 ctl_zero_io(union ctl_io *io)
3603 {
3604 	void *pool_ref;
3605 
3606 	if (io == NULL)
3607 		return;
3608 
3609 	/*
3610 	 * May need to preserve linked list pointers at some point too.
3611 	 */
3612 	pool_ref = io->io_hdr.pool;
3613 
3614 	memset(io, 0, sizeof(*io));
3615 
3616 	io->io_hdr.pool = pool_ref;
3617 }
3618 
3619 /*
3620  * This routine is currently used for internal copies of ctl_ios that need
3621  * to persist for some reason after we've already returned status to the
3622  * FETD.  (Thus the flag set.)
3623  *
3624  * XXX XXX
3625  * Note that this makes a blind copy of all fields in the ctl_io, except
3626  * for the pool reference.  This includes any memory that has been
3627  * allocated!  That memory will no longer be valid after done has been
3628  * called, so this would be VERY DANGEROUS for command that actually does
3629  * any reads or writes.  Right now (11/7/2005), this is only used for immediate
3630  * start and stop commands, which don't transfer any data, so this is not a
3631  * problem.  If it is used for anything else, the caller would also need to
3632  * allocate data buffer space and this routine would need to be modified to
3633  * copy the data buffer(s) as well.
3634  */
3635 void
3636 ctl_copy_io(union ctl_io *src, union ctl_io *dest)
3637 {
3638 	void *pool_ref;
3639 
3640 	if ((src == NULL)
3641 	 || (dest == NULL))
3642 		return;
3643 
3644 	/*
3645 	 * May need to preserve linked list pointers at some point too.
3646 	 */
3647 	pool_ref = dest->io_hdr.pool;
3648 
3649 	memcpy(dest, src, ctl_min(sizeof(*src), sizeof(*dest)));
3650 
3651 	dest->io_hdr.pool = pool_ref;
3652 	/*
3653 	 * We need to know that this is an internal copy, and doesn't need
3654 	 * to get passed back to the FETD that allocated it.
3655 	 */
3656 	dest->io_hdr.flags |= CTL_FLAG_INT_COPY;
3657 }
3658 
3659 #ifdef NEEDTOPORT
3660 static void
3661 ctl_update_power_subpage(struct copan_power_subpage *page)
3662 {
3663 	int num_luns, num_partitions, config_type;
3664 	struct ctl_softc *softc;
3665 	cs_BOOL_t aor_present, shelf_50pct_power;
3666 	cs_raidset_personality_t rs_type;
3667 	int max_active_luns;
3668 
3669 	softc = control_softc;
3670 
3671 	/* subtract out the processor LUN */
3672 	num_luns = softc->num_luns - 1;
3673 	/*
3674 	 * Default to 7 LUNs active, which was the only number we allowed
3675 	 * in the past.
3676 	 */
3677 	max_active_luns = 7;
3678 
3679 	num_partitions = config_GetRsPartitionInfo();
3680 	config_type = config_GetConfigType();
3681 	shelf_50pct_power = config_GetShelfPowerMode();
3682 	aor_present = config_IsAorRsPresent();
3683 
3684 	rs_type = ddb_GetRsRaidType(1);
3685 	if ((rs_type != CS_RAIDSET_PERSONALITY_RAID5)
3686 	 && (rs_type != CS_RAIDSET_PERSONALITY_RAID1)) {
3687 		EPRINT(0, "Unsupported RS type %d!", rs_type);
3688 	}
3689 
3690 
3691 	page->total_luns = num_luns;
3692 
3693 	switch (config_type) {
3694 	case 40:
3695 		/*
3696 		 * In a 40 drive configuration, it doesn't matter what DC
3697 		 * cards we have, whether we have AOR enabled or not,
3698 		 * partitioning or not, or what type of RAIDset we have.
3699 		 * In that scenario, we can power up every LUN we present
3700 		 * to the user.
3701 		 */
3702 		max_active_luns = num_luns;
3703 
3704 		break;
3705 	case 64:
3706 		if (shelf_50pct_power == CS_FALSE) {
3707 			/* 25% power */
3708 			if (aor_present == CS_TRUE) {
3709 				if (rs_type ==
3710 				     CS_RAIDSET_PERSONALITY_RAID5) {
3711 					max_active_luns = 7;
3712 				} else if (rs_type ==
3713 					 CS_RAIDSET_PERSONALITY_RAID1){
3714 					max_active_luns = 14;
3715 				} else {
3716 					/* XXX KDM now what?? */
3717 				}
3718 			} else {
3719 				if (rs_type ==
3720 				     CS_RAIDSET_PERSONALITY_RAID5) {
3721 					max_active_luns = 8;
3722 				} else if (rs_type ==
3723 					 CS_RAIDSET_PERSONALITY_RAID1){
3724 					max_active_luns = 16;
3725 				} else {
3726 					/* XXX KDM now what?? */
3727 				}
3728 			}
3729 		} else {
3730 			/* 50% power */
3731 			/*
3732 			 * With 50% power in a 64 drive configuration, we
3733 			 * can power all LUNs we present.
3734 			 */
3735 			max_active_luns = num_luns;
3736 		}
3737 		break;
3738 	case 112:
3739 		if (shelf_50pct_power == CS_FALSE) {
3740 			/* 25% power */
3741 			if (aor_present == CS_TRUE) {
3742 				if (rs_type ==
3743 				     CS_RAIDSET_PERSONALITY_RAID5) {
3744 					max_active_luns = 7;
3745 				} else if (rs_type ==
3746 					 CS_RAIDSET_PERSONALITY_RAID1){
3747 					max_active_luns = 14;
3748 				} else {
3749 					/* XXX KDM now what?? */
3750 				}
3751 			} else {
3752 				if (rs_type ==
3753 				     CS_RAIDSET_PERSONALITY_RAID5) {
3754 					max_active_luns = 8;
3755 				} else if (rs_type ==
3756 					 CS_RAIDSET_PERSONALITY_RAID1){
3757 					max_active_luns = 16;
3758 				} else {
3759 					/* XXX KDM now what?? */
3760 				}
3761 			}
3762 		} else {
3763 			/* 50% power */
3764 			if (aor_present == CS_TRUE) {
3765 				if (rs_type ==
3766 				     CS_RAIDSET_PERSONALITY_RAID5) {
3767 					max_active_luns = 14;
3768 				} else if (rs_type ==
3769 					 CS_RAIDSET_PERSONALITY_RAID1){
3770 					/*
3771 					 * We're assuming here that disk
3772 					 * caching is enabled, and so we're
3773 					 * able to power up half of each
3774 					 * LUN, and cache all writes.
3775 					 */
3776 					max_active_luns = num_luns;
3777 				} else {
3778 					/* XXX KDM now what?? */
3779 				}
3780 			} else {
3781 				if (rs_type ==
3782 				     CS_RAIDSET_PERSONALITY_RAID5) {
3783 					max_active_luns = 15;
3784 				} else if (rs_type ==
3785 					 CS_RAIDSET_PERSONALITY_RAID1){
3786 					max_active_luns = 30;
3787 				} else {
3788 					/* XXX KDM now what?? */
3789 				}
3790 			}
3791 		}
3792 		break;
3793 	default:
3794 		/*
3795 		 * In this case, we have an unknown configuration, so we
3796 		 * just use the default from above.
3797 		 */
3798 		break;
3799 	}
3800 
3801 	page->max_active_luns = max_active_luns;
3802 #if 0
3803 	printk("%s: total_luns = %d, max_active_luns = %d\n", __func__,
3804 	       page->total_luns, page->max_active_luns);
3805 #endif
3806 }
3807 #endif /* NEEDTOPORT */
3808 
3809 /*
3810  * This routine could be used in the future to load default and/or saved
3811  * mode page parameters for a particuar lun.
3812  */
3813 static int
3814 ctl_init_page_index(struct ctl_lun *lun)
3815 {
3816 	int i;
3817 	struct ctl_page_index *page_index;
3818 	struct ctl_softc *softc;
3819 
3820 	memcpy(&lun->mode_pages.index, page_index_template,
3821 	       sizeof(page_index_template));
3822 
3823 	softc = lun->ctl_softc;
3824 
3825 	for (i = 0; i < CTL_NUM_MODE_PAGES; i++) {
3826 
3827 		page_index = &lun->mode_pages.index[i];
3828 		/*
3829 		 * If this is a disk-only mode page, there's no point in
3830 		 * setting it up.  For some pages, we have to have some
3831 		 * basic information about the disk in order to calculate the
3832 		 * mode page data.
3833 		 */
3834 		if ((lun->be_lun->lun_type != T_DIRECT)
3835 		 && (page_index->page_flags & CTL_PAGE_FLAG_DISK_ONLY))
3836 			continue;
3837 
3838 		switch (page_index->page_code & SMPH_PC_MASK) {
3839 		case SMS_FORMAT_DEVICE_PAGE: {
3840 			struct scsi_format_page *format_page;
3841 
3842 			if (page_index->subpage != SMS_SUBPAGE_PAGE_0)
3843 				panic("subpage is incorrect!");
3844 
3845 			/*
3846 			 * Sectors per track are set above.  Bytes per
3847 			 * sector need to be set here on a per-LUN basis.
3848 			 */
3849 			memcpy(&lun->mode_pages.format_page[CTL_PAGE_CURRENT],
3850 			       &format_page_default,
3851 			       sizeof(format_page_default));
3852 			memcpy(&lun->mode_pages.format_page[
3853 			       CTL_PAGE_CHANGEABLE], &format_page_changeable,
3854 			       sizeof(format_page_changeable));
3855 			memcpy(&lun->mode_pages.format_page[CTL_PAGE_DEFAULT],
3856 			       &format_page_default,
3857 			       sizeof(format_page_default));
3858 			memcpy(&lun->mode_pages.format_page[CTL_PAGE_SAVED],
3859 			       &format_page_default,
3860 			       sizeof(format_page_default));
3861 
3862 			format_page = &lun->mode_pages.format_page[
3863 				CTL_PAGE_CURRENT];
3864 			scsi_ulto2b(lun->be_lun->blocksize,
3865 				    format_page->bytes_per_sector);
3866 
3867 			format_page = &lun->mode_pages.format_page[
3868 				CTL_PAGE_DEFAULT];
3869 			scsi_ulto2b(lun->be_lun->blocksize,
3870 				    format_page->bytes_per_sector);
3871 
3872 			format_page = &lun->mode_pages.format_page[
3873 				CTL_PAGE_SAVED];
3874 			scsi_ulto2b(lun->be_lun->blocksize,
3875 				    format_page->bytes_per_sector);
3876 
3877 			page_index->page_data =
3878 				(uint8_t *)lun->mode_pages.format_page;
3879 			break;
3880 		}
3881 		case SMS_RIGID_DISK_PAGE: {
3882 			struct scsi_rigid_disk_page *rigid_disk_page;
3883 			uint32_t sectors_per_cylinder;
3884 			uint64_t cylinders;
3885 #ifndef	__XSCALE__
3886 			int shift;
3887 #endif /* !__XSCALE__ */
3888 
3889 			if (page_index->subpage != SMS_SUBPAGE_PAGE_0)
3890 				panic("invalid subpage value %d",
3891 				      page_index->subpage);
3892 
3893 			/*
3894 			 * Rotation rate and sectors per track are set
3895 			 * above.  We calculate the cylinders here based on
3896 			 * capacity.  Due to the number of heads and
3897 			 * sectors per track we're using, smaller arrays
3898 			 * may turn out to have 0 cylinders.  Linux and
3899 			 * FreeBSD don't pay attention to these mode pages
3900 			 * to figure out capacity, but Solaris does.  It
3901 			 * seems to deal with 0 cylinders just fine, and
3902 			 * works out a fake geometry based on the capacity.
3903 			 */
3904 			memcpy(&lun->mode_pages.rigid_disk_page[
3905 			       CTL_PAGE_CURRENT], &rigid_disk_page_default,
3906 			       sizeof(rigid_disk_page_default));
3907 			memcpy(&lun->mode_pages.rigid_disk_page[
3908 			       CTL_PAGE_CHANGEABLE],&rigid_disk_page_changeable,
3909 			       sizeof(rigid_disk_page_changeable));
3910 			memcpy(&lun->mode_pages.rigid_disk_page[
3911 			       CTL_PAGE_DEFAULT], &rigid_disk_page_default,
3912 			       sizeof(rigid_disk_page_default));
3913 			memcpy(&lun->mode_pages.rigid_disk_page[
3914 			       CTL_PAGE_SAVED], &rigid_disk_page_default,
3915 			       sizeof(rigid_disk_page_default));
3916 
3917 			sectors_per_cylinder = CTL_DEFAULT_SECTORS_PER_TRACK *
3918 				CTL_DEFAULT_HEADS;
3919 
3920 			/*
3921 			 * The divide method here will be more accurate,
3922 			 * probably, but results in floating point being
3923 			 * used in the kernel on i386 (__udivdi3()).  On the
3924 			 * XScale, though, __udivdi3() is implemented in
3925 			 * software.
3926 			 *
3927 			 * The shift method for cylinder calculation is
3928 			 * accurate if sectors_per_cylinder is a power of
3929 			 * 2.  Otherwise it might be slightly off -- you
3930 			 * might have a bit of a truncation problem.
3931 			 */
3932 #ifdef	__XSCALE__
3933 			cylinders = (lun->be_lun->maxlba + 1) /
3934 				sectors_per_cylinder;
3935 #else
3936 			for (shift = 31; shift > 0; shift--) {
3937 				if (sectors_per_cylinder & (1 << shift))
3938 					break;
3939 			}
3940 			cylinders = (lun->be_lun->maxlba + 1) >> shift;
3941 #endif
3942 
3943 			/*
3944 			 * We've basically got 3 bytes, or 24 bits for the
3945 			 * cylinder size in the mode page.  If we're over,
3946 			 * just round down to 2^24.
3947 			 */
3948 			if (cylinders > 0xffffff)
3949 				cylinders = 0xffffff;
3950 
3951 			rigid_disk_page = &lun->mode_pages.rigid_disk_page[
3952 				CTL_PAGE_CURRENT];
3953 			scsi_ulto3b(cylinders, rigid_disk_page->cylinders);
3954 
3955 			rigid_disk_page = &lun->mode_pages.rigid_disk_page[
3956 				CTL_PAGE_DEFAULT];
3957 			scsi_ulto3b(cylinders, rigid_disk_page->cylinders);
3958 
3959 			rigid_disk_page = &lun->mode_pages.rigid_disk_page[
3960 				CTL_PAGE_SAVED];
3961 			scsi_ulto3b(cylinders, rigid_disk_page->cylinders);
3962 
3963 			page_index->page_data =
3964 				(uint8_t *)lun->mode_pages.rigid_disk_page;
3965 			break;
3966 		}
3967 		case SMS_CACHING_PAGE: {
3968 
3969 			if (page_index->subpage != SMS_SUBPAGE_PAGE_0)
3970 				panic("invalid subpage value %d",
3971 				      page_index->subpage);
3972 			/*
3973 			 * Defaults should be okay here, no calculations
3974 			 * needed.
3975 			 */
3976 			memcpy(&lun->mode_pages.caching_page[CTL_PAGE_CURRENT],
3977 			       &caching_page_default,
3978 			       sizeof(caching_page_default));
3979 			memcpy(&lun->mode_pages.caching_page[
3980 			       CTL_PAGE_CHANGEABLE], &caching_page_changeable,
3981 			       sizeof(caching_page_changeable));
3982 			memcpy(&lun->mode_pages.caching_page[CTL_PAGE_DEFAULT],
3983 			       &caching_page_default,
3984 			       sizeof(caching_page_default));
3985 			memcpy(&lun->mode_pages.caching_page[CTL_PAGE_SAVED],
3986 			       &caching_page_default,
3987 			       sizeof(caching_page_default));
3988 			page_index->page_data =
3989 				(uint8_t *)lun->mode_pages.caching_page;
3990 			break;
3991 		}
3992 		case SMS_CONTROL_MODE_PAGE: {
3993 
3994 			if (page_index->subpage != SMS_SUBPAGE_PAGE_0)
3995 				panic("invalid subpage value %d",
3996 				      page_index->subpage);
3997 
3998 			/*
3999 			 * Defaults should be okay here, no calculations
4000 			 * needed.
4001 			 */
4002 			memcpy(&lun->mode_pages.control_page[CTL_PAGE_CURRENT],
4003 			       &control_page_default,
4004 			       sizeof(control_page_default));
4005 			memcpy(&lun->mode_pages.control_page[
4006 			       CTL_PAGE_CHANGEABLE], &control_page_changeable,
4007 			       sizeof(control_page_changeable));
4008 			memcpy(&lun->mode_pages.control_page[CTL_PAGE_DEFAULT],
4009 			       &control_page_default,
4010 			       sizeof(control_page_default));
4011 			memcpy(&lun->mode_pages.control_page[CTL_PAGE_SAVED],
4012 			       &control_page_default,
4013 			       sizeof(control_page_default));
4014 			page_index->page_data =
4015 				(uint8_t *)lun->mode_pages.control_page;
4016 			break;
4017 
4018 		}
4019 		case SMS_VENDOR_SPECIFIC_PAGE:{
4020 			switch (page_index->subpage) {
4021 			case PWR_SUBPAGE_CODE: {
4022 				struct copan_power_subpage *current_page,
4023 							   *saved_page;
4024 
4025 				memcpy(&lun->mode_pages.power_subpage[
4026 				       CTL_PAGE_CURRENT],
4027 				       &power_page_default,
4028 				       sizeof(power_page_default));
4029 				memcpy(&lun->mode_pages.power_subpage[
4030 				       CTL_PAGE_CHANGEABLE],
4031 				       &power_page_changeable,
4032 				       sizeof(power_page_changeable));
4033 				memcpy(&lun->mode_pages.power_subpage[
4034 				       CTL_PAGE_DEFAULT],
4035 				       &power_page_default,
4036 				       sizeof(power_page_default));
4037 				memcpy(&lun->mode_pages.power_subpage[
4038 				       CTL_PAGE_SAVED],
4039 				       &power_page_default,
4040 				       sizeof(power_page_default));
4041 				page_index->page_data =
4042 				    (uint8_t *)lun->mode_pages.power_subpage;
4043 
4044 				current_page = (struct copan_power_subpage *)
4045 					(page_index->page_data +
4046 					 (page_index->page_len *
4047 					  CTL_PAGE_CURRENT));
4048 			        saved_page = (struct copan_power_subpage *)
4049 				        (page_index->page_data +
4050 					 (page_index->page_len *
4051 					  CTL_PAGE_SAVED));
4052 				break;
4053 			}
4054 			case APS_SUBPAGE_CODE: {
4055 				struct copan_aps_subpage *current_page,
4056 							 *saved_page;
4057 
4058 				// This gets set multiple times but
4059 				// it should always be the same. It's
4060 				// only done during init so who cares.
4061 				index_to_aps_page = i;
4062 
4063 				memcpy(&lun->mode_pages.aps_subpage[
4064 				       CTL_PAGE_CURRENT],
4065 				       &aps_page_default,
4066 				       sizeof(aps_page_default));
4067 				memcpy(&lun->mode_pages.aps_subpage[
4068 				       CTL_PAGE_CHANGEABLE],
4069 				       &aps_page_changeable,
4070 				       sizeof(aps_page_changeable));
4071 				memcpy(&lun->mode_pages.aps_subpage[
4072 				       CTL_PAGE_DEFAULT],
4073 				       &aps_page_default,
4074 				       sizeof(aps_page_default));
4075 				memcpy(&lun->mode_pages.aps_subpage[
4076 				       CTL_PAGE_SAVED],
4077 				       &aps_page_default,
4078 				       sizeof(aps_page_default));
4079 				page_index->page_data =
4080 					(uint8_t *)lun->mode_pages.aps_subpage;
4081 
4082 				current_page = (struct copan_aps_subpage *)
4083 					(page_index->page_data +
4084 					 (page_index->page_len *
4085 					  CTL_PAGE_CURRENT));
4086 				saved_page = (struct copan_aps_subpage *)
4087 					(page_index->page_data +
4088 					 (page_index->page_len *
4089 					  CTL_PAGE_SAVED));
4090 				break;
4091 			}
4092 			case DBGCNF_SUBPAGE_CODE: {
4093 				struct copan_debugconf_subpage *current_page,
4094 							       *saved_page;
4095 
4096 				memcpy(&lun->mode_pages.debugconf_subpage[
4097 				       CTL_PAGE_CURRENT],
4098 				       &debugconf_page_default,
4099 				       sizeof(debugconf_page_default));
4100 				memcpy(&lun->mode_pages.debugconf_subpage[
4101 				       CTL_PAGE_CHANGEABLE],
4102 				       &debugconf_page_changeable,
4103 				       sizeof(debugconf_page_changeable));
4104 				memcpy(&lun->mode_pages.debugconf_subpage[
4105 				       CTL_PAGE_DEFAULT],
4106 				       &debugconf_page_default,
4107 				       sizeof(debugconf_page_default));
4108 				memcpy(&lun->mode_pages.debugconf_subpage[
4109 				       CTL_PAGE_SAVED],
4110 				       &debugconf_page_default,
4111 				       sizeof(debugconf_page_default));
4112 				page_index->page_data =
4113 					(uint8_t *)lun->mode_pages.debugconf_subpage;
4114 
4115 				current_page = (struct copan_debugconf_subpage *)
4116 					(page_index->page_data +
4117 					 (page_index->page_len *
4118 					  CTL_PAGE_CURRENT));
4119 				saved_page = (struct copan_debugconf_subpage *)
4120 					(page_index->page_data +
4121 					 (page_index->page_len *
4122 					  CTL_PAGE_SAVED));
4123 				break;
4124 			}
4125 			default:
4126 				panic("invalid subpage value %d",
4127 				      page_index->subpage);
4128 				break;
4129 			}
4130    			break;
4131 		}
4132 		default:
4133 			panic("invalid page value %d",
4134 			      page_index->page_code & SMPH_PC_MASK);
4135 			break;
4136     	}
4137 	}
4138 
4139 	return (CTL_RETVAL_COMPLETE);
4140 }
4141 
4142 /*
4143  * LUN allocation.
4144  *
4145  * Requirements:
4146  * - caller allocates and zeros LUN storage, or passes in a NULL LUN if he
4147  *   wants us to allocate the LUN and he can block.
4148  * - ctl_softc is always set
4149  * - be_lun is set if the LUN has a backend (needed for disk LUNs)
4150  *
4151  * Returns 0 for success, non-zero (errno) for failure.
4152  */
4153 static int
4154 ctl_alloc_lun(struct ctl_softc *ctl_softc, struct ctl_lun *ctl_lun,
4155 	      struct ctl_be_lun *const be_lun, struct ctl_id target_id)
4156 {
4157 	struct ctl_lun *nlun, *lun;
4158 	struct ctl_frontend *fe;
4159 	int lun_number, i, lun_malloced;
4160 
4161 	if (be_lun == NULL)
4162 		return (EINVAL);
4163 
4164 	/*
4165 	 * We currently only support Direct Access or Processor LUN types.
4166 	 */
4167 	switch (be_lun->lun_type) {
4168 	case T_DIRECT:
4169 		break;
4170 	case T_PROCESSOR:
4171 		break;
4172 	case T_SEQUENTIAL:
4173 	case T_CHANGER:
4174 	default:
4175 		be_lun->lun_config_status(be_lun->be_lun,
4176 					  CTL_LUN_CONFIG_FAILURE);
4177 		break;
4178 	}
4179 	if (ctl_lun == NULL) {
4180 		lun = malloc(sizeof(*lun), M_CTL, M_WAITOK);
4181 		lun_malloced = 1;
4182 	} else {
4183 		lun_malloced = 0;
4184 		lun = ctl_lun;
4185 	}
4186 
4187 	memset(lun, 0, sizeof(*lun));
4188 	if (lun_malloced)
4189 		lun->flags = CTL_LUN_MALLOCED;
4190 
4191 	mtx_lock(&ctl_softc->ctl_lock);
4192 	/*
4193 	 * See if the caller requested a particular LUN number.  If so, see
4194 	 * if it is available.  Otherwise, allocate the first available LUN.
4195 	 */
4196 	if (be_lun->flags & CTL_LUN_FLAG_ID_REQ) {
4197 		if ((be_lun->req_lun_id > (CTL_MAX_LUNS - 1))
4198 		 || (ctl_is_set(ctl_softc->ctl_lun_mask, be_lun->req_lun_id))) {
4199 			mtx_unlock(&ctl_softc->ctl_lock);
4200 			if (be_lun->req_lun_id > (CTL_MAX_LUNS - 1)) {
4201 				printf("ctl: requested LUN ID %d is higher "
4202 				       "than CTL_MAX_LUNS - 1 (%d)\n",
4203 				       be_lun->req_lun_id, CTL_MAX_LUNS - 1);
4204 			} else {
4205 				/*
4206 				 * XXX KDM return an error, or just assign
4207 				 * another LUN ID in this case??
4208 				 */
4209 				printf("ctl: requested LUN ID %d is already "
4210 				       "in use\n", be_lun->req_lun_id);
4211 			}
4212 			if (lun->flags & CTL_LUN_MALLOCED)
4213 				free(lun, M_CTL);
4214 			be_lun->lun_config_status(be_lun->be_lun,
4215 						  CTL_LUN_CONFIG_FAILURE);
4216 			return (ENOSPC);
4217 		}
4218 		lun_number = be_lun->req_lun_id;
4219 	} else {
4220 		lun_number = ctl_ffz(ctl_softc->ctl_lun_mask, CTL_MAX_LUNS);
4221 		if (lun_number == -1) {
4222 			mtx_unlock(&ctl_softc->ctl_lock);
4223 			printf("ctl: can't allocate LUN on target %ju, out of "
4224 			       "LUNs\n", (uintmax_t)target_id.id);
4225 			if (lun->flags & CTL_LUN_MALLOCED)
4226 				free(lun, M_CTL);
4227 			be_lun->lun_config_status(be_lun->be_lun,
4228 						  CTL_LUN_CONFIG_FAILURE);
4229 			return (ENOSPC);
4230 		}
4231 	}
4232 	ctl_set_mask(ctl_softc->ctl_lun_mask, lun_number);
4233 
4234 	mtx_init(&lun->lun_lock, "CTL LUN", NULL, MTX_DEF);
4235 	lun->target = target_id;
4236 	lun->lun = lun_number;
4237 	lun->be_lun = be_lun;
4238 	/*
4239 	 * The processor LUN is always enabled.  Disk LUNs come on line
4240 	 * disabled, and must be enabled by the backend.
4241 	 */
4242 	lun->flags |= CTL_LUN_DISABLED;
4243 	lun->backend = be_lun->be;
4244 	be_lun->ctl_lun = lun;
4245 	be_lun->lun_id = lun_number;
4246 	atomic_add_int(&be_lun->be->num_luns, 1);
4247 	if (be_lun->flags & CTL_LUN_FLAG_POWERED_OFF)
4248 		lun->flags |= CTL_LUN_STOPPED;
4249 
4250 	if (be_lun->flags & CTL_LUN_FLAG_INOPERABLE)
4251 		lun->flags |= CTL_LUN_INOPERABLE;
4252 
4253 	if (be_lun->flags & CTL_LUN_FLAG_PRIMARY)
4254 		lun->flags |= CTL_LUN_PRIMARY_SC;
4255 
4256 	lun->ctl_softc = ctl_softc;
4257 	TAILQ_INIT(&lun->ooa_queue);
4258 	TAILQ_INIT(&lun->blocked_queue);
4259 	STAILQ_INIT(&lun->error_list);
4260 
4261 	/*
4262 	 * Initialize the mode page index.
4263 	 */
4264 	ctl_init_page_index(lun);
4265 
4266 	/*
4267 	 * Set the poweron UA for all initiators on this LUN only.
4268 	 */
4269 	for (i = 0; i < CTL_MAX_INITIATORS; i++)
4270 		lun->pending_sense[i].ua_pending = CTL_UA_POWERON;
4271 
4272 	/*
4273 	 * Now, before we insert this lun on the lun list, set the lun
4274 	 * inventory changed UA for all other luns.
4275 	 */
4276 	STAILQ_FOREACH(nlun, &ctl_softc->lun_list, links) {
4277 		for (i = 0; i < CTL_MAX_INITIATORS; i++) {
4278 			nlun->pending_sense[i].ua_pending |= CTL_UA_LUN_CHANGE;
4279 		}
4280 	}
4281 
4282 	STAILQ_INSERT_TAIL(&ctl_softc->lun_list, lun, links);
4283 
4284 	ctl_softc->ctl_luns[lun_number] = lun;
4285 
4286 	ctl_softc->num_luns++;
4287 
4288 	/* Setup statistics gathering */
4289 	lun->stats.device_type = be_lun->lun_type;
4290 	lun->stats.lun_number = lun_number;
4291 	if (lun->stats.device_type == T_DIRECT)
4292 		lun->stats.blocksize = be_lun->blocksize;
4293 	else
4294 		lun->stats.flags = CTL_LUN_STATS_NO_BLOCKSIZE;
4295 	for (i = 0;i < CTL_MAX_PORTS;i++)
4296 		lun->stats.ports[i].targ_port = i;
4297 
4298 	mtx_unlock(&ctl_softc->ctl_lock);
4299 
4300 	lun->be_lun->lun_config_status(lun->be_lun->be_lun, CTL_LUN_CONFIG_OK);
4301 
4302 	/*
4303 	 * Run through each registered FETD and bring it online if it isn't
4304 	 * already.  Enable the target ID if it hasn't been enabled, and
4305 	 * enable this particular LUN.
4306 	 */
4307 	STAILQ_FOREACH(fe, &ctl_softc->fe_list, links) {
4308 		int retval;
4309 
4310 		/*
4311 		 * XXX KDM this only works for ONE TARGET ID.  We'll need
4312 		 * to do things differently if we go to a multiple target
4313 		 * ID scheme.
4314 		 */
4315 		if ((fe->status & CTL_PORT_STATUS_TARG_ONLINE) == 0) {
4316 
4317 			retval = fe->targ_enable(fe->targ_lun_arg, target_id);
4318 			if (retval != 0) {
4319 				printf("ctl_alloc_lun: FETD %s port %d "
4320 				       "returned error %d for targ_enable on "
4321 				       "target %ju\n", fe->port_name,
4322 				       fe->targ_port, retval,
4323 				       (uintmax_t)target_id.id);
4324 			} else
4325 				fe->status |= CTL_PORT_STATUS_TARG_ONLINE;
4326 		}
4327 
4328 		retval = fe->lun_enable(fe->targ_lun_arg, target_id,lun_number);
4329 		if (retval != 0) {
4330 			printf("ctl_alloc_lun: FETD %s port %d returned error "
4331 			       "%d for lun_enable on target %ju lun %d\n",
4332 			       fe->port_name, fe->targ_port, retval,
4333 			       (uintmax_t)target_id.id, lun_number);
4334 		} else
4335 			fe->status |= CTL_PORT_STATUS_LUN_ONLINE;
4336 	}
4337 	return (0);
4338 }
4339 
4340 /*
4341  * Delete a LUN.
4342  * Assumptions:
4343  * - LUN has already been marked invalid and any pending I/O has been taken
4344  *   care of.
4345  */
4346 static int
4347 ctl_free_lun(struct ctl_lun *lun)
4348 {
4349 	struct ctl_softc *softc;
4350 #if 0
4351 	struct ctl_frontend *fe;
4352 #endif
4353 	struct ctl_lun *nlun;
4354 	int i;
4355 
4356 	softc = lun->ctl_softc;
4357 
4358 	mtx_assert(&softc->ctl_lock, MA_OWNED);
4359 
4360 	STAILQ_REMOVE(&softc->lun_list, lun, ctl_lun, links);
4361 
4362 	ctl_clear_mask(softc->ctl_lun_mask, lun->lun);
4363 
4364 	softc->ctl_luns[lun->lun] = NULL;
4365 
4366 	if (!TAILQ_EMPTY(&lun->ooa_queue))
4367 		panic("Freeing a LUN %p with outstanding I/O!!\n", lun);
4368 
4369 	softc->num_luns--;
4370 
4371 	/*
4372 	 * XXX KDM this scheme only works for a single target/multiple LUN
4373 	 * setup.  It needs to be revamped for a multiple target scheme.
4374 	 *
4375 	 * XXX KDM this results in fe->lun_disable() getting called twice,
4376 	 * once when ctl_disable_lun() is called, and a second time here.
4377 	 * We really need to re-think the LUN disable semantics.  There
4378 	 * should probably be several steps/levels to LUN removal:
4379 	 *  - disable
4380 	 *  - invalidate
4381 	 *  - free
4382  	 *
4383 	 * Right now we only have a disable method when communicating to
4384 	 * the front end ports, at least for individual LUNs.
4385 	 */
4386 #if 0
4387 	STAILQ_FOREACH(fe, &softc->fe_list, links) {
4388 		int retval;
4389 
4390 		retval = fe->lun_disable(fe->targ_lun_arg, lun->target,
4391 					 lun->lun);
4392 		if (retval != 0) {
4393 			printf("ctl_free_lun: FETD %s port %d returned error "
4394 			       "%d for lun_disable on target %ju lun %jd\n",
4395 			       fe->port_name, fe->targ_port, retval,
4396 			       (uintmax_t)lun->target.id, (intmax_t)lun->lun);
4397 		}
4398 
4399 		if (STAILQ_FIRST(&softc->lun_list) == NULL) {
4400 			fe->status &= ~CTL_PORT_STATUS_LUN_ONLINE;
4401 
4402 			retval = fe->targ_disable(fe->targ_lun_arg,lun->target);
4403 			if (retval != 0) {
4404 				printf("ctl_free_lun: FETD %s port %d "
4405 				       "returned error %d for targ_disable on "
4406 				       "target %ju\n", fe->port_name,
4407 				       fe->targ_port, retval,
4408 				       (uintmax_t)lun->target.id);
4409 			} else
4410 				fe->status &= ~CTL_PORT_STATUS_TARG_ONLINE;
4411 
4412 			if ((fe->status & CTL_PORT_STATUS_TARG_ONLINE) != 0)
4413 				continue;
4414 
4415 #if 0
4416 			fe->port_offline(fe->onoff_arg);
4417 			fe->status &= ~CTL_PORT_STATUS_ONLINE;
4418 #endif
4419 		}
4420 	}
4421 #endif
4422 
4423 	/*
4424 	 * Tell the backend to free resources, if this LUN has a backend.
4425 	 */
4426 	atomic_subtract_int(&lun->be_lun->be->num_luns, 1);
4427 	lun->be_lun->lun_shutdown(lun->be_lun->be_lun);
4428 
4429 	mtx_destroy(&lun->lun_lock);
4430 	if (lun->flags & CTL_LUN_MALLOCED)
4431 		free(lun, M_CTL);
4432 
4433 	STAILQ_FOREACH(nlun, &softc->lun_list, links) {
4434 		for (i = 0; i < CTL_MAX_INITIATORS; i++) {
4435 			nlun->pending_sense[i].ua_pending |= CTL_UA_LUN_CHANGE;
4436 		}
4437 	}
4438 
4439 	return (0);
4440 }
4441 
4442 static void
4443 ctl_create_lun(struct ctl_be_lun *be_lun)
4444 {
4445 	struct ctl_softc *ctl_softc;
4446 
4447 	ctl_softc = control_softc;
4448 
4449 	/*
4450 	 * ctl_alloc_lun() should handle all potential failure cases.
4451 	 */
4452 	ctl_alloc_lun(ctl_softc, NULL, be_lun, ctl_softc->target);
4453 }
4454 
4455 int
4456 ctl_add_lun(struct ctl_be_lun *be_lun)
4457 {
4458 	struct ctl_softc *ctl_softc = control_softc;
4459 
4460 	mtx_lock(&ctl_softc->ctl_lock);
4461 	STAILQ_INSERT_TAIL(&ctl_softc->pending_lun_queue, be_lun, links);
4462 	mtx_unlock(&ctl_softc->ctl_lock);
4463 	wakeup(&ctl_softc->pending_lun_queue);
4464 
4465 	return (0);
4466 }
4467 
4468 int
4469 ctl_enable_lun(struct ctl_be_lun *be_lun)
4470 {
4471 	struct ctl_softc *ctl_softc;
4472 	struct ctl_frontend *fe, *nfe;
4473 	struct ctl_lun *lun;
4474 	int retval;
4475 
4476 	ctl_softc = control_softc;
4477 
4478 	lun = (struct ctl_lun *)be_lun->ctl_lun;
4479 
4480 	mtx_lock(&ctl_softc->ctl_lock);
4481 	mtx_lock(&lun->lun_lock);
4482 	if ((lun->flags & CTL_LUN_DISABLED) == 0) {
4483 		/*
4484 		 * eh?  Why did we get called if the LUN is already
4485 		 * enabled?
4486 		 */
4487 		mtx_unlock(&lun->lun_lock);
4488 		mtx_unlock(&ctl_softc->ctl_lock);
4489 		return (0);
4490 	}
4491 	lun->flags &= ~CTL_LUN_DISABLED;
4492 	mtx_unlock(&lun->lun_lock);
4493 
4494 	for (fe = STAILQ_FIRST(&ctl_softc->fe_list); fe != NULL; fe = nfe) {
4495 		nfe = STAILQ_NEXT(fe, links);
4496 
4497 		/*
4498 		 * Drop the lock while we call the FETD's enable routine.
4499 		 * This can lead to a callback into CTL (at least in the
4500 		 * case of the internal initiator frontend.
4501 		 */
4502 		mtx_unlock(&ctl_softc->ctl_lock);
4503 		retval = fe->lun_enable(fe->targ_lun_arg, lun->target,lun->lun);
4504 		mtx_lock(&ctl_softc->ctl_lock);
4505 		if (retval != 0) {
4506 			printf("%s: FETD %s port %d returned error "
4507 			       "%d for lun_enable on target %ju lun %jd\n",
4508 			       __func__, fe->port_name, fe->targ_port, retval,
4509 			       (uintmax_t)lun->target.id, (intmax_t)lun->lun);
4510 		}
4511 #if 0
4512 		 else {
4513             /* NOTE:  TODO:  why does lun enable affect port status? */
4514 			fe->status |= CTL_PORT_STATUS_LUN_ONLINE;
4515 		}
4516 #endif
4517 	}
4518 
4519 	mtx_unlock(&ctl_softc->ctl_lock);
4520 
4521 	return (0);
4522 }
4523 
4524 int
4525 ctl_disable_lun(struct ctl_be_lun *be_lun)
4526 {
4527 	struct ctl_softc *ctl_softc;
4528 	struct ctl_frontend *fe;
4529 	struct ctl_lun *lun;
4530 	int retval;
4531 
4532 	ctl_softc = control_softc;
4533 
4534 	lun = (struct ctl_lun *)be_lun->ctl_lun;
4535 
4536 	mtx_lock(&ctl_softc->ctl_lock);
4537 	mtx_lock(&lun->lun_lock);
4538 	if (lun->flags & CTL_LUN_DISABLED) {
4539 		mtx_unlock(&lun->lun_lock);
4540 		mtx_unlock(&ctl_softc->ctl_lock);
4541 		return (0);
4542 	}
4543 	lun->flags |= CTL_LUN_DISABLED;
4544 	mtx_unlock(&lun->lun_lock);
4545 
4546 	STAILQ_FOREACH(fe, &ctl_softc->fe_list, links) {
4547 		mtx_unlock(&ctl_softc->ctl_lock);
4548 		/*
4549 		 * Drop the lock before we call the frontend's disable
4550 		 * routine, to avoid lock order reversals.
4551 		 *
4552 		 * XXX KDM what happens if the frontend list changes while
4553 		 * we're traversing it?  It's unlikely, but should be handled.
4554 		 */
4555 		retval = fe->lun_disable(fe->targ_lun_arg, lun->target,
4556 					 lun->lun);
4557 		mtx_lock(&ctl_softc->ctl_lock);
4558 		if (retval != 0) {
4559 			printf("ctl_alloc_lun: FETD %s port %d returned error "
4560 			       "%d for lun_disable on target %ju lun %jd\n",
4561 			       fe->port_name, fe->targ_port, retval,
4562 			       (uintmax_t)lun->target.id, (intmax_t)lun->lun);
4563 		}
4564 	}
4565 
4566 	mtx_unlock(&ctl_softc->ctl_lock);
4567 
4568 	return (0);
4569 }
4570 
4571 int
4572 ctl_start_lun(struct ctl_be_lun *be_lun)
4573 {
4574 	struct ctl_softc *ctl_softc;
4575 	struct ctl_lun *lun;
4576 
4577 	ctl_softc = control_softc;
4578 
4579 	lun = (struct ctl_lun *)be_lun->ctl_lun;
4580 
4581 	mtx_lock(&lun->lun_lock);
4582 	lun->flags &= ~CTL_LUN_STOPPED;
4583 	mtx_unlock(&lun->lun_lock);
4584 
4585 	return (0);
4586 }
4587 
4588 int
4589 ctl_stop_lun(struct ctl_be_lun *be_lun)
4590 {
4591 	struct ctl_softc *ctl_softc;
4592 	struct ctl_lun *lun;
4593 
4594 	ctl_softc = control_softc;
4595 
4596 	lun = (struct ctl_lun *)be_lun->ctl_lun;
4597 
4598 	mtx_lock(&lun->lun_lock);
4599 	lun->flags |= CTL_LUN_STOPPED;
4600 	mtx_unlock(&lun->lun_lock);
4601 
4602 	return (0);
4603 }
4604 
4605 int
4606 ctl_lun_offline(struct ctl_be_lun *be_lun)
4607 {
4608 	struct ctl_softc *ctl_softc;
4609 	struct ctl_lun *lun;
4610 
4611 	ctl_softc = control_softc;
4612 
4613 	lun = (struct ctl_lun *)be_lun->ctl_lun;
4614 
4615 	mtx_lock(&lun->lun_lock);
4616 	lun->flags |= CTL_LUN_OFFLINE;
4617 	mtx_unlock(&lun->lun_lock);
4618 
4619 	return (0);
4620 }
4621 
4622 int
4623 ctl_lun_online(struct ctl_be_lun *be_lun)
4624 {
4625 	struct ctl_softc *ctl_softc;
4626 	struct ctl_lun *lun;
4627 
4628 	ctl_softc = control_softc;
4629 
4630 	lun = (struct ctl_lun *)be_lun->ctl_lun;
4631 
4632 	mtx_lock(&lun->lun_lock);
4633 	lun->flags &= ~CTL_LUN_OFFLINE;
4634 	mtx_unlock(&lun->lun_lock);
4635 
4636 	return (0);
4637 }
4638 
4639 int
4640 ctl_invalidate_lun(struct ctl_be_lun *be_lun)
4641 {
4642 	struct ctl_softc *ctl_softc;
4643 	struct ctl_lun *lun;
4644 
4645 	ctl_softc = control_softc;
4646 
4647 	lun = (struct ctl_lun *)be_lun->ctl_lun;
4648 
4649 	mtx_lock(&lun->lun_lock);
4650 
4651 	/*
4652 	 * The LUN needs to be disabled before it can be marked invalid.
4653 	 */
4654 	if ((lun->flags & CTL_LUN_DISABLED) == 0) {
4655 		mtx_unlock(&lun->lun_lock);
4656 		return (-1);
4657 	}
4658 	/*
4659 	 * Mark the LUN invalid.
4660 	 */
4661 	lun->flags |= CTL_LUN_INVALID;
4662 
4663 	/*
4664 	 * If there is nothing in the OOA queue, go ahead and free the LUN.
4665 	 * If we have something in the OOA queue, we'll free it when the
4666 	 * last I/O completes.
4667 	 */
4668 	if (TAILQ_EMPTY(&lun->ooa_queue)) {
4669 		mtx_unlock(&lun->lun_lock);
4670 		mtx_lock(&ctl_softc->ctl_lock);
4671 		ctl_free_lun(lun);
4672 		mtx_unlock(&ctl_softc->ctl_lock);
4673 	} else
4674 		mtx_unlock(&lun->lun_lock);
4675 
4676 	return (0);
4677 }
4678 
4679 int
4680 ctl_lun_inoperable(struct ctl_be_lun *be_lun)
4681 {
4682 	struct ctl_softc *ctl_softc;
4683 	struct ctl_lun *lun;
4684 
4685 	ctl_softc = control_softc;
4686 	lun = (struct ctl_lun *)be_lun->ctl_lun;
4687 
4688 	mtx_lock(&lun->lun_lock);
4689 	lun->flags |= CTL_LUN_INOPERABLE;
4690 	mtx_unlock(&lun->lun_lock);
4691 
4692 	return (0);
4693 }
4694 
4695 int
4696 ctl_lun_operable(struct ctl_be_lun *be_lun)
4697 {
4698 	struct ctl_softc *ctl_softc;
4699 	struct ctl_lun *lun;
4700 
4701 	ctl_softc = control_softc;
4702 	lun = (struct ctl_lun *)be_lun->ctl_lun;
4703 
4704 	mtx_lock(&lun->lun_lock);
4705 	lun->flags &= ~CTL_LUN_INOPERABLE;
4706 	mtx_unlock(&lun->lun_lock);
4707 
4708 	return (0);
4709 }
4710 
4711 int
4712 ctl_lun_power_lock(struct ctl_be_lun *be_lun, struct ctl_nexus *nexus,
4713 		   int lock)
4714 {
4715 	struct ctl_softc *softc;
4716 	struct ctl_lun *lun;
4717 	struct copan_aps_subpage *current_sp;
4718 	struct ctl_page_index *page_index;
4719 	int i;
4720 
4721 	softc = control_softc;
4722 
4723 	mtx_lock(&softc->ctl_lock);
4724 
4725 	lun = (struct ctl_lun *)be_lun->ctl_lun;
4726 	mtx_lock(&lun->lun_lock);
4727 
4728 	page_index = NULL;
4729 	for (i = 0; i < CTL_NUM_MODE_PAGES; i++) {
4730 		if ((lun->mode_pages.index[i].page_code & SMPH_PC_MASK) !=
4731 		     APS_PAGE_CODE)
4732 			continue;
4733 
4734 		if (lun->mode_pages.index[i].subpage != APS_SUBPAGE_CODE)
4735 			continue;
4736 		page_index = &lun->mode_pages.index[i];
4737 	}
4738 
4739 	if (page_index == NULL) {
4740 		mtx_unlock(&lun->lun_lock);
4741 		mtx_unlock(&softc->ctl_lock);
4742 		printf("%s: APS subpage not found for lun %ju!\n", __func__,
4743 		       (uintmax_t)lun->lun);
4744 		return (1);
4745 	}
4746 #if 0
4747 	if ((softc->aps_locked_lun != 0)
4748 	 && (softc->aps_locked_lun != lun->lun)) {
4749 		printf("%s: attempt to lock LUN %llu when %llu is already "
4750 		       "locked\n");
4751 		mtx_unlock(&lun->lun_lock);
4752 		mtx_unlock(&softc->ctl_lock);
4753 		return (1);
4754 	}
4755 #endif
4756 
4757 	current_sp = (struct copan_aps_subpage *)(page_index->page_data +
4758 		(page_index->page_len * CTL_PAGE_CURRENT));
4759 
4760 	if (lock != 0) {
4761 		current_sp->lock_active = APS_LOCK_ACTIVE;
4762 		softc->aps_locked_lun = lun->lun;
4763 	} else {
4764 		current_sp->lock_active = 0;
4765 		softc->aps_locked_lun = 0;
4766 	}
4767 
4768 
4769 	/*
4770 	 * If we're in HA mode, try to send the lock message to the other
4771 	 * side.
4772 	 */
4773 	if (ctl_is_single == 0) {
4774 		int isc_retval;
4775 		union ctl_ha_msg lock_msg;
4776 
4777 		lock_msg.hdr.nexus = *nexus;
4778 		lock_msg.hdr.msg_type = CTL_MSG_APS_LOCK;
4779 		if (lock != 0)
4780 			lock_msg.aps.lock_flag = 1;
4781 		else
4782 			lock_msg.aps.lock_flag = 0;
4783 		isc_retval = ctl_ha_msg_send(CTL_HA_CHAN_CTL, &lock_msg,
4784 					 sizeof(lock_msg), 0);
4785 		if (isc_retval > CTL_HA_STATUS_SUCCESS) {
4786 			printf("%s: APS (lock=%d) error returned from "
4787 			       "ctl_ha_msg_send: %d\n", __func__, lock, isc_retval);
4788 			mtx_unlock(&lun->lun_lock);
4789 			mtx_unlock(&softc->ctl_lock);
4790 			return (1);
4791 		}
4792 	}
4793 
4794 	mtx_unlock(&lun->lun_lock);
4795 	mtx_unlock(&softc->ctl_lock);
4796 
4797 	return (0);
4798 }
4799 
4800 void
4801 ctl_lun_capacity_changed(struct ctl_be_lun *be_lun)
4802 {
4803 	struct ctl_lun *lun;
4804 	struct ctl_softc *softc;
4805 	int i;
4806 
4807 	softc = control_softc;
4808 
4809 	lun = (struct ctl_lun *)be_lun->ctl_lun;
4810 
4811 	mtx_lock(&lun->lun_lock);
4812 
4813 	for (i = 0; i < CTL_MAX_INITIATORS; i++)
4814 		lun->pending_sense[i].ua_pending |= CTL_UA_CAPACITY_CHANGED;
4815 
4816 	mtx_unlock(&lun->lun_lock);
4817 }
4818 
4819 /*
4820  * Backend "memory move is complete" callback for requests that never
4821  * make it down to say RAIDCore's configuration code.
4822  */
4823 int
4824 ctl_config_move_done(union ctl_io *io)
4825 {
4826 	int retval;
4827 
4828 	retval = CTL_RETVAL_COMPLETE;
4829 
4830 
4831 	CTL_DEBUG_PRINT(("ctl_config_move_done\n"));
4832 	/*
4833 	 * XXX KDM this shouldn't happen, but what if it does?
4834 	 */
4835 	if (io->io_hdr.io_type != CTL_IO_SCSI)
4836 		panic("I/O type isn't CTL_IO_SCSI!");
4837 
4838 	if ((io->io_hdr.port_status == 0)
4839 	 && ((io->io_hdr.flags & CTL_FLAG_ABORT) == 0)
4840 	 && ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_STATUS_NONE))
4841 		io->io_hdr.status = CTL_SUCCESS;
4842 	else if ((io->io_hdr.port_status != 0)
4843 	      && ((io->io_hdr.flags & CTL_FLAG_ABORT) == 0)
4844 	      && ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_STATUS_NONE)){
4845 		/*
4846 		 * For hardware error sense keys, the sense key
4847 		 * specific value is defined to be a retry count,
4848 		 * but we use it to pass back an internal FETD
4849 		 * error code.  XXX KDM  Hopefully the FETD is only
4850 		 * using 16 bits for an error code, since that's
4851 		 * all the space we have in the sks field.
4852 		 */
4853 		ctl_set_internal_failure(&io->scsiio,
4854 					 /*sks_valid*/ 1,
4855 					 /*retry_count*/
4856 					 io->io_hdr.port_status);
4857 		if (io->io_hdr.flags & CTL_FLAG_ALLOCATED)
4858 			free(io->scsiio.kern_data_ptr, M_CTL);
4859 		ctl_done(io);
4860 		goto bailout;
4861 	}
4862 
4863 	if (((io->io_hdr.flags & CTL_FLAG_DATA_MASK) == CTL_FLAG_DATA_IN)
4864 	 || ((io->io_hdr.status & CTL_STATUS_MASK) != CTL_SUCCESS)
4865 	 || ((io->io_hdr.flags & CTL_FLAG_ABORT) != 0)) {
4866 		/*
4867 		 * XXX KDM just assuming a single pointer here, and not a
4868 		 * S/G list.  If we start using S/G lists for config data,
4869 		 * we'll need to know how to clean them up here as well.
4870 		 */
4871 		if (io->io_hdr.flags & CTL_FLAG_ALLOCATED)
4872 			free(io->scsiio.kern_data_ptr, M_CTL);
4873 		/* Hopefully the user has already set the status... */
4874 		ctl_done(io);
4875 	} else {
4876 		/*
4877 		 * XXX KDM now we need to continue data movement.  Some
4878 		 * options:
4879 		 * - call ctl_scsiio() again?  We don't do this for data
4880 		 *   writes, because for those at least we know ahead of
4881 		 *   time where the write will go and how long it is.  For
4882 		 *   config writes, though, that information is largely
4883 		 *   contained within the write itself, thus we need to
4884 		 *   parse out the data again.
4885 		 *
4886 		 * - Call some other function once the data is in?
4887 		 */
4888 
4889 		/*
4890 		 * XXX KDM call ctl_scsiio() again for now, and check flag
4891 		 * bits to see whether we're allocated or not.
4892 		 */
4893 		retval = ctl_scsiio(&io->scsiio);
4894 	}
4895 bailout:
4896 	return (retval);
4897 }
4898 
4899 /*
4900  * This gets called by a backend driver when it is done with a
4901  * data_submit method.
4902  */
4903 void
4904 ctl_data_submit_done(union ctl_io *io)
4905 {
4906 	/*
4907 	 * If the IO_CONT flag is set, we need to call the supplied
4908 	 * function to continue processing the I/O, instead of completing
4909 	 * the I/O just yet.
4910 	 *
4911 	 * If there is an error, though, we don't want to keep processing.
4912 	 * Instead, just send status back to the initiator.
4913 	 */
4914 	if ((io->io_hdr.flags & CTL_FLAG_IO_CONT) &&
4915 	    (io->io_hdr.flags & CTL_FLAG_ABORT) == 0 &&
4916 	    ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_STATUS_NONE ||
4917 	     (io->io_hdr.status & CTL_STATUS_MASK) == CTL_SUCCESS)) {
4918 		io->scsiio.io_cont(io);
4919 		return;
4920 	}
4921 	ctl_done(io);
4922 }
4923 
4924 /*
4925  * This gets called by a backend driver when it is done with a
4926  * configuration write.
4927  */
4928 void
4929 ctl_config_write_done(union ctl_io *io)
4930 {
4931 	/*
4932 	 * If the IO_CONT flag is set, we need to call the supplied
4933 	 * function to continue processing the I/O, instead of completing
4934 	 * the I/O just yet.
4935 	 *
4936 	 * If there is an error, though, we don't want to keep processing.
4937 	 * Instead, just send status back to the initiator.
4938 	 */
4939 	if ((io->io_hdr.flags & CTL_FLAG_IO_CONT)
4940 	 && (((io->io_hdr.status & CTL_STATUS_MASK) == CTL_STATUS_NONE)
4941 	  || ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_SUCCESS))) {
4942 		io->scsiio.io_cont(io);
4943 		return;
4944 	}
4945 	/*
4946 	 * Since a configuration write can be done for commands that actually
4947 	 * have data allocated, like write buffer, and commands that have
4948 	 * no data, like start/stop unit, we need to check here.
4949 	 */
4950 	if ((io->io_hdr.flags & CTL_FLAG_DATA_MASK) == CTL_FLAG_DATA_OUT)
4951 		free(io->scsiio.kern_data_ptr, M_CTL);
4952 	ctl_done(io);
4953 }
4954 
4955 /*
4956  * SCSI release command.
4957  */
4958 int
4959 ctl_scsi_release(struct ctl_scsiio *ctsio)
4960 {
4961 	int length, longid, thirdparty_id, resv_id;
4962 	struct ctl_softc *ctl_softc;
4963 	struct ctl_lun *lun;
4964 
4965 	length = 0;
4966 	resv_id = 0;
4967 
4968 	CTL_DEBUG_PRINT(("ctl_scsi_release\n"));
4969 
4970 	lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
4971 	ctl_softc = control_softc;
4972 
4973 	switch (ctsio->cdb[0]) {
4974 	case RELEASE: {
4975 		struct scsi_release *cdb;
4976 
4977 		cdb = (struct scsi_release *)ctsio->cdb;
4978 		if ((cdb->byte2 & 0x1f) != 0) {
4979 			ctl_set_invalid_field(ctsio,
4980 					      /*sks_valid*/ 1,
4981 					      /*command*/ 1,
4982 					      /*field*/ 1,
4983 					      /*bit_valid*/ 0,
4984 					      /*bit*/ 0);
4985 			ctl_done((union ctl_io *)ctsio);
4986 			return (CTL_RETVAL_COMPLETE);
4987 		}
4988 		break;
4989 	}
4990 	case RELEASE_10: {
4991 		struct scsi_release_10 *cdb;
4992 
4993 		cdb = (struct scsi_release_10 *)ctsio->cdb;
4994 
4995 		if ((cdb->byte2 & SR10_EXTENT) != 0) {
4996 			ctl_set_invalid_field(ctsio,
4997 					      /*sks_valid*/ 1,
4998 					      /*command*/ 1,
4999 					      /*field*/ 1,
5000 					      /*bit_valid*/ 1,
5001 					      /*bit*/ 0);
5002 			ctl_done((union ctl_io *)ctsio);
5003 			return (CTL_RETVAL_COMPLETE);
5004 
5005 		}
5006 
5007 		if ((cdb->byte2 & SR10_3RDPTY) != 0) {
5008 			ctl_set_invalid_field(ctsio,
5009 					      /*sks_valid*/ 1,
5010 					      /*command*/ 1,
5011 					      /*field*/ 1,
5012 					      /*bit_valid*/ 1,
5013 					      /*bit*/ 4);
5014 			ctl_done((union ctl_io *)ctsio);
5015 			return (CTL_RETVAL_COMPLETE);
5016 		}
5017 
5018 		if (cdb->byte2 & SR10_LONGID)
5019 			longid = 1;
5020 		else
5021 			thirdparty_id = cdb->thirdparty_id;
5022 
5023 		resv_id = cdb->resv_id;
5024 		length = scsi_2btoul(cdb->length);
5025 		break;
5026 	}
5027 	}
5028 
5029 
5030 	/*
5031 	 * XXX KDM right now, we only support LUN reservation.  We don't
5032 	 * support 3rd party reservations, or extent reservations, which
5033 	 * might actually need the parameter list.  If we've gotten this
5034 	 * far, we've got a LUN reservation.  Anything else got kicked out
5035 	 * above.  So, according to SPC, ignore the length.
5036 	 */
5037 	length = 0;
5038 
5039 	if (((ctsio->io_hdr.flags & CTL_FLAG_ALLOCATED) == 0)
5040 	 && (length > 0)) {
5041 		ctsio->kern_data_ptr = malloc(length, M_CTL, M_WAITOK);
5042 		ctsio->kern_data_len = length;
5043 		ctsio->kern_total_len = length;
5044 		ctsio->kern_data_resid = 0;
5045 		ctsio->kern_rel_offset = 0;
5046 		ctsio->kern_sg_entries = 0;
5047 		ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
5048 		ctsio->be_move_done = ctl_config_move_done;
5049 		ctl_datamove((union ctl_io *)ctsio);
5050 
5051 		return (CTL_RETVAL_COMPLETE);
5052 	}
5053 
5054 	if (length > 0)
5055 		thirdparty_id = scsi_8btou64(ctsio->kern_data_ptr);
5056 
5057 	mtx_lock(&lun->lun_lock);
5058 
5059 	/*
5060 	 * According to SPC, it is not an error for an intiator to attempt
5061 	 * to release a reservation on a LUN that isn't reserved, or that
5062 	 * is reserved by another initiator.  The reservation can only be
5063 	 * released, though, by the initiator who made it or by one of
5064 	 * several reset type events.
5065 	 */
5066 	if (lun->flags & CTL_LUN_RESERVED) {
5067 		if ((ctsio->io_hdr.nexus.initid.id == lun->rsv_nexus.initid.id)
5068 		 && (ctsio->io_hdr.nexus.targ_port == lun->rsv_nexus.targ_port)
5069 		 && (ctsio->io_hdr.nexus.targ_target.id ==
5070 		     lun->rsv_nexus.targ_target.id)) {
5071 			lun->flags &= ~CTL_LUN_RESERVED;
5072 		}
5073 	}
5074 
5075 	mtx_unlock(&lun->lun_lock);
5076 
5077 	ctsio->scsi_status = SCSI_STATUS_OK;
5078 	ctsio->io_hdr.status = CTL_SUCCESS;
5079 
5080 	if (ctsio->io_hdr.flags & CTL_FLAG_ALLOCATED) {
5081 		free(ctsio->kern_data_ptr, M_CTL);
5082 		ctsio->io_hdr.flags &= ~CTL_FLAG_ALLOCATED;
5083 	}
5084 
5085 	ctl_done((union ctl_io *)ctsio);
5086 	return (CTL_RETVAL_COMPLETE);
5087 }
5088 
5089 int
5090 ctl_scsi_reserve(struct ctl_scsiio *ctsio)
5091 {
5092 	int extent, thirdparty, longid;
5093 	int resv_id, length;
5094 	uint64_t thirdparty_id;
5095 	struct ctl_softc *ctl_softc;
5096 	struct ctl_lun *lun;
5097 
5098 	extent = 0;
5099 	thirdparty = 0;
5100 	longid = 0;
5101 	resv_id = 0;
5102 	length = 0;
5103 	thirdparty_id = 0;
5104 
5105 	CTL_DEBUG_PRINT(("ctl_reserve\n"));
5106 
5107 	lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
5108 	ctl_softc = control_softc;
5109 
5110 	switch (ctsio->cdb[0]) {
5111 	case RESERVE: {
5112 		struct scsi_reserve *cdb;
5113 
5114 		cdb = (struct scsi_reserve *)ctsio->cdb;
5115 		if ((cdb->byte2 & 0x1f) != 0) {
5116 			ctl_set_invalid_field(ctsio,
5117 					      /*sks_valid*/ 1,
5118 					      /*command*/ 1,
5119 					      /*field*/ 1,
5120 					      /*bit_valid*/ 0,
5121 					      /*bit*/ 0);
5122 			ctl_done((union ctl_io *)ctsio);
5123 			return (CTL_RETVAL_COMPLETE);
5124 		}
5125 		resv_id = cdb->resv_id;
5126 		length = scsi_2btoul(cdb->length);
5127 		break;
5128 	}
5129 	case RESERVE_10: {
5130 		struct scsi_reserve_10 *cdb;
5131 
5132 		cdb = (struct scsi_reserve_10 *)ctsio->cdb;
5133 
5134 		if ((cdb->byte2 & SR10_EXTENT) != 0) {
5135 			ctl_set_invalid_field(ctsio,
5136 					      /*sks_valid*/ 1,
5137 					      /*command*/ 1,
5138 					      /*field*/ 1,
5139 					      /*bit_valid*/ 1,
5140 					      /*bit*/ 0);
5141 			ctl_done((union ctl_io *)ctsio);
5142 			return (CTL_RETVAL_COMPLETE);
5143 		}
5144 		if ((cdb->byte2 & SR10_3RDPTY) != 0) {
5145 			ctl_set_invalid_field(ctsio,
5146 					      /*sks_valid*/ 1,
5147 					      /*command*/ 1,
5148 					      /*field*/ 1,
5149 					      /*bit_valid*/ 1,
5150 					      /*bit*/ 4);
5151 			ctl_done((union ctl_io *)ctsio);
5152 			return (CTL_RETVAL_COMPLETE);
5153 		}
5154 		if (cdb->byte2 & SR10_LONGID)
5155 			longid = 1;
5156 		else
5157 			thirdparty_id = cdb->thirdparty_id;
5158 
5159 		resv_id = cdb->resv_id;
5160 		length = scsi_2btoul(cdb->length);
5161 		break;
5162 	}
5163 	}
5164 
5165 	/*
5166 	 * XXX KDM right now, we only support LUN reservation.  We don't
5167 	 * support 3rd party reservations, or extent reservations, which
5168 	 * might actually need the parameter list.  If we've gotten this
5169 	 * far, we've got a LUN reservation.  Anything else got kicked out
5170 	 * above.  So, according to SPC, ignore the length.
5171 	 */
5172 	length = 0;
5173 
5174 	if (((ctsio->io_hdr.flags & CTL_FLAG_ALLOCATED) == 0)
5175 	 && (length > 0)) {
5176 		ctsio->kern_data_ptr = malloc(length, M_CTL, M_WAITOK);
5177 		ctsio->kern_data_len = length;
5178 		ctsio->kern_total_len = length;
5179 		ctsio->kern_data_resid = 0;
5180 		ctsio->kern_rel_offset = 0;
5181 		ctsio->kern_sg_entries = 0;
5182 		ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
5183 		ctsio->be_move_done = ctl_config_move_done;
5184 		ctl_datamove((union ctl_io *)ctsio);
5185 
5186 		return (CTL_RETVAL_COMPLETE);
5187 	}
5188 
5189 	if (length > 0)
5190 		thirdparty_id = scsi_8btou64(ctsio->kern_data_ptr);
5191 
5192 	mtx_lock(&lun->lun_lock);
5193 	if (lun->flags & CTL_LUN_RESERVED) {
5194 		if ((ctsio->io_hdr.nexus.initid.id != lun->rsv_nexus.initid.id)
5195 		 || (ctsio->io_hdr.nexus.targ_port != lun->rsv_nexus.targ_port)
5196 		 || (ctsio->io_hdr.nexus.targ_target.id !=
5197 		     lun->rsv_nexus.targ_target.id)) {
5198 			ctsio->scsi_status = SCSI_STATUS_RESERV_CONFLICT;
5199 			ctsio->io_hdr.status = CTL_SCSI_ERROR;
5200 			goto bailout;
5201 		}
5202 	}
5203 
5204 	lun->flags |= CTL_LUN_RESERVED;
5205 	lun->rsv_nexus = ctsio->io_hdr.nexus;
5206 
5207 	ctsio->scsi_status = SCSI_STATUS_OK;
5208 	ctsio->io_hdr.status = CTL_SUCCESS;
5209 
5210 bailout:
5211 	mtx_unlock(&lun->lun_lock);
5212 
5213 	if (ctsio->io_hdr.flags & CTL_FLAG_ALLOCATED) {
5214 		free(ctsio->kern_data_ptr, M_CTL);
5215 		ctsio->io_hdr.flags &= ~CTL_FLAG_ALLOCATED;
5216 	}
5217 
5218 	ctl_done((union ctl_io *)ctsio);
5219 	return (CTL_RETVAL_COMPLETE);
5220 }
5221 
5222 int
5223 ctl_start_stop(struct ctl_scsiio *ctsio)
5224 {
5225 	struct scsi_start_stop_unit *cdb;
5226 	struct ctl_lun *lun;
5227 	struct ctl_softc *ctl_softc;
5228 	int retval;
5229 
5230 	CTL_DEBUG_PRINT(("ctl_start_stop\n"));
5231 
5232 	lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
5233 	ctl_softc = control_softc;
5234 	retval = 0;
5235 
5236 	cdb = (struct scsi_start_stop_unit *)ctsio->cdb;
5237 
5238 	/*
5239 	 * XXX KDM
5240 	 * We don't support the immediate bit on a stop unit.  In order to
5241 	 * do that, we would need to code up a way to know that a stop is
5242 	 * pending, and hold off any new commands until it completes, one
5243 	 * way or another.  Then we could accept or reject those commands
5244 	 * depending on its status.  We would almost need to do the reverse
5245 	 * of what we do below for an immediate start -- return the copy of
5246 	 * the ctl_io to the FETD with status to send to the host (and to
5247 	 * free the copy!) and then free the original I/O once the stop
5248 	 * actually completes.  That way, the OOA queue mechanism can work
5249 	 * to block commands that shouldn't proceed.  Another alternative
5250 	 * would be to put the copy in the queue in place of the original,
5251 	 * and return the original back to the caller.  That could be
5252 	 * slightly safer..
5253 	 */
5254 	if ((cdb->byte2 & SSS_IMMED)
5255 	 && ((cdb->how & SSS_START) == 0)) {
5256 		ctl_set_invalid_field(ctsio,
5257 				      /*sks_valid*/ 1,
5258 				      /*command*/ 1,
5259 				      /*field*/ 1,
5260 				      /*bit_valid*/ 1,
5261 				      /*bit*/ 0);
5262 		ctl_done((union ctl_io *)ctsio);
5263 		return (CTL_RETVAL_COMPLETE);
5264 	}
5265 
5266 	/*
5267 	 * We don't support the power conditions field.  We need to check
5268 	 * this prior to checking the load/eject and start/stop bits.
5269 	 */
5270 	if ((cdb->how & SSS_PC_MASK) != SSS_PC_START_VALID) {
5271 		ctl_set_invalid_field(ctsio,
5272 				      /*sks_valid*/ 1,
5273 				      /*command*/ 1,
5274 				      /*field*/ 4,
5275 				      /*bit_valid*/ 1,
5276 				      /*bit*/ 4);
5277 		ctl_done((union ctl_io *)ctsio);
5278 		return (CTL_RETVAL_COMPLETE);
5279 	}
5280 
5281 	/*
5282 	 * Media isn't removable, so we can't load or eject it.
5283 	 */
5284 	if ((cdb->how & SSS_LOEJ) != 0) {
5285 		ctl_set_invalid_field(ctsio,
5286 				      /*sks_valid*/ 1,
5287 				      /*command*/ 1,
5288 				      /*field*/ 4,
5289 				      /*bit_valid*/ 1,
5290 				      /*bit*/ 1);
5291 		ctl_done((union ctl_io *)ctsio);
5292 		return (CTL_RETVAL_COMPLETE);
5293 	}
5294 
5295 	if ((lun->flags & CTL_LUN_PR_RESERVED)
5296 	 && ((cdb->how & SSS_START)==0)) {
5297 		uint32_t residx;
5298 
5299 		residx = ctl_get_resindex(&ctsio->io_hdr.nexus);
5300 		if (!lun->per_res[residx].registered
5301 		 || (lun->pr_res_idx!=residx && lun->res_type < 4)) {
5302 
5303 			ctl_set_reservation_conflict(ctsio);
5304 			ctl_done((union ctl_io *)ctsio);
5305 			return (CTL_RETVAL_COMPLETE);
5306 		}
5307 	}
5308 
5309 	/*
5310 	 * If there is no backend on this device, we can't start or stop
5311 	 * it.  In theory we shouldn't get any start/stop commands in the
5312 	 * first place at this level if the LUN doesn't have a backend.
5313 	 * That should get stopped by the command decode code.
5314 	 */
5315 	if (lun->backend == NULL) {
5316 		ctl_set_invalid_opcode(ctsio);
5317 		ctl_done((union ctl_io *)ctsio);
5318 		return (CTL_RETVAL_COMPLETE);
5319 	}
5320 
5321 	/*
5322 	 * XXX KDM Copan-specific offline behavior.
5323 	 * Figure out a reasonable way to port this?
5324 	 */
5325 #ifdef NEEDTOPORT
5326 	mtx_lock(&lun->lun_lock);
5327 
5328 	if (((cdb->byte2 & SSS_ONOFFLINE) == 0)
5329 	 && (lun->flags & CTL_LUN_OFFLINE)) {
5330 		/*
5331 		 * If the LUN is offline, and the on/offline bit isn't set,
5332 		 * reject the start or stop.  Otherwise, let it through.
5333 		 */
5334 		mtx_unlock(&lun->lun_lock);
5335 		ctl_set_lun_not_ready(ctsio);
5336 		ctl_done((union ctl_io *)ctsio);
5337 	} else {
5338 		mtx_unlock(&lun->lun_lock);
5339 #endif /* NEEDTOPORT */
5340 		/*
5341 		 * This could be a start or a stop when we're online,
5342 		 * or a stop/offline or start/online.  A start or stop when
5343 		 * we're offline is covered in the case above.
5344 		 */
5345 		/*
5346 		 * In the non-immediate case, we send the request to
5347 		 * the backend and return status to the user when
5348 		 * it is done.
5349 		 *
5350 		 * In the immediate case, we allocate a new ctl_io
5351 		 * to hold a copy of the request, and send that to
5352 		 * the backend.  We then set good status on the
5353 		 * user's request and return it immediately.
5354 		 */
5355 		if (cdb->byte2 & SSS_IMMED) {
5356 			union ctl_io *new_io;
5357 
5358 			new_io = ctl_alloc_io(ctsio->io_hdr.pool);
5359 			if (new_io == NULL) {
5360 				ctl_set_busy(ctsio);
5361 				ctl_done((union ctl_io *)ctsio);
5362 			} else {
5363 				ctl_copy_io((union ctl_io *)ctsio,
5364 					    new_io);
5365 				retval = lun->backend->config_write(new_io);
5366 				ctl_set_success(ctsio);
5367 				ctl_done((union ctl_io *)ctsio);
5368 			}
5369 		} else {
5370 			retval = lun->backend->config_write(
5371 				(union ctl_io *)ctsio);
5372 		}
5373 #ifdef NEEDTOPORT
5374 	}
5375 #endif
5376 	return (retval);
5377 }
5378 
5379 /*
5380  * We support the SYNCHRONIZE CACHE command (10 and 16 byte versions), but
5381  * we don't really do anything with the LBA and length fields if the user
5382  * passes them in.  Instead we'll just flush out the cache for the entire
5383  * LUN.
5384  */
5385 int
5386 ctl_sync_cache(struct ctl_scsiio *ctsio)
5387 {
5388 	struct ctl_lun *lun;
5389 	struct ctl_softc *ctl_softc;
5390 	uint64_t starting_lba;
5391 	uint32_t block_count;
5392 	int reladr, immed;
5393 	int retval;
5394 
5395 	CTL_DEBUG_PRINT(("ctl_sync_cache\n"));
5396 
5397 	lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
5398 	ctl_softc = control_softc;
5399 	retval = 0;
5400 	reladr = 0;
5401 	immed = 0;
5402 
5403 	switch (ctsio->cdb[0]) {
5404 	case SYNCHRONIZE_CACHE: {
5405 		struct scsi_sync_cache *cdb;
5406 		cdb = (struct scsi_sync_cache *)ctsio->cdb;
5407 
5408 		if (cdb->byte2 & SSC_RELADR)
5409 			reladr = 1;
5410 
5411 		if (cdb->byte2 & SSC_IMMED)
5412 			immed = 1;
5413 
5414 		starting_lba = scsi_4btoul(cdb->begin_lba);
5415 		block_count = scsi_2btoul(cdb->lb_count);
5416 		break;
5417 	}
5418 	case SYNCHRONIZE_CACHE_16: {
5419 		struct scsi_sync_cache_16 *cdb;
5420 		cdb = (struct scsi_sync_cache_16 *)ctsio->cdb;
5421 
5422 		if (cdb->byte2 & SSC_RELADR)
5423 			reladr = 1;
5424 
5425 		if (cdb->byte2 & SSC_IMMED)
5426 			immed = 1;
5427 
5428 		starting_lba = scsi_8btou64(cdb->begin_lba);
5429 		block_count = scsi_4btoul(cdb->lb_count);
5430 		break;
5431 	}
5432 	default:
5433 		ctl_set_invalid_opcode(ctsio);
5434 		ctl_done((union ctl_io *)ctsio);
5435 		goto bailout;
5436 		break; /* NOTREACHED */
5437 	}
5438 
5439 	if (immed) {
5440 		/*
5441 		 * We don't support the immediate bit.  Since it's in the
5442 		 * same place for the 10 and 16 byte SYNCHRONIZE CACHE
5443 		 * commands, we can just return the same error in either
5444 		 * case.
5445 		 */
5446 		ctl_set_invalid_field(ctsio,
5447 				      /*sks_valid*/ 1,
5448 				      /*command*/ 1,
5449 				      /*field*/ 1,
5450 				      /*bit_valid*/ 1,
5451 				      /*bit*/ 1);
5452 		ctl_done((union ctl_io *)ctsio);
5453 		goto bailout;
5454 	}
5455 
5456 	if (reladr) {
5457 		/*
5458 		 * We don't support the reladr bit either.  It can only be
5459 		 * used with linked commands, and we don't support linked
5460 		 * commands.  Since the bit is in the same place for the
5461 		 * 10 and 16 byte SYNCHRONIZE CACHE * commands, we can
5462 		 * just return the same error in either case.
5463 		 */
5464 		ctl_set_invalid_field(ctsio,
5465 				      /*sks_valid*/ 1,
5466 				      /*command*/ 1,
5467 				      /*field*/ 1,
5468 				      /*bit_valid*/ 1,
5469 				      /*bit*/ 0);
5470 		ctl_done((union ctl_io *)ctsio);
5471 		goto bailout;
5472 	}
5473 
5474 	/*
5475 	 * We check the LBA and length, but don't do anything with them.
5476 	 * A SYNCHRONIZE CACHE will cause the entire cache for this lun to
5477 	 * get flushed.  This check will just help satisfy anyone who wants
5478 	 * to see an error for an out of range LBA.
5479 	 */
5480 	if ((starting_lba + block_count) > (lun->be_lun->maxlba + 1)) {
5481 		ctl_set_lba_out_of_range(ctsio);
5482 		ctl_done((union ctl_io *)ctsio);
5483 		goto bailout;
5484 	}
5485 
5486 	/*
5487 	 * If this LUN has no backend, we can't flush the cache anyway.
5488 	 */
5489 	if (lun->backend == NULL) {
5490 		ctl_set_invalid_opcode(ctsio);
5491 		ctl_done((union ctl_io *)ctsio);
5492 		goto bailout;
5493 	}
5494 
5495 	/*
5496 	 * Check to see whether we're configured to send the SYNCHRONIZE
5497 	 * CACHE command directly to the back end.
5498 	 */
5499 	mtx_lock(&lun->lun_lock);
5500 	if ((ctl_softc->flags & CTL_FLAG_REAL_SYNC)
5501 	 && (++(lun->sync_count) >= lun->sync_interval)) {
5502 		lun->sync_count = 0;
5503 		mtx_unlock(&lun->lun_lock);
5504 		retval = lun->backend->config_write((union ctl_io *)ctsio);
5505 	} else {
5506 		mtx_unlock(&lun->lun_lock);
5507 		ctl_set_success(ctsio);
5508 		ctl_done((union ctl_io *)ctsio);
5509 	}
5510 
5511 bailout:
5512 
5513 	return (retval);
5514 }
5515 
5516 int
5517 ctl_format(struct ctl_scsiio *ctsio)
5518 {
5519 	struct scsi_format *cdb;
5520 	struct ctl_lun *lun;
5521 	struct ctl_softc *ctl_softc;
5522 	int length, defect_list_len;
5523 
5524 	CTL_DEBUG_PRINT(("ctl_format\n"));
5525 
5526 	lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
5527 	ctl_softc = control_softc;
5528 
5529 	cdb = (struct scsi_format *)ctsio->cdb;
5530 
5531 	length = 0;
5532 	if (cdb->byte2 & SF_FMTDATA) {
5533 		if (cdb->byte2 & SF_LONGLIST)
5534 			length = sizeof(struct scsi_format_header_long);
5535 		else
5536 			length = sizeof(struct scsi_format_header_short);
5537 	}
5538 
5539 	if (((ctsio->io_hdr.flags & CTL_FLAG_ALLOCATED) == 0)
5540 	 && (length > 0)) {
5541 		ctsio->kern_data_ptr = malloc(length, M_CTL, M_WAITOK);
5542 		ctsio->kern_data_len = length;
5543 		ctsio->kern_total_len = length;
5544 		ctsio->kern_data_resid = 0;
5545 		ctsio->kern_rel_offset = 0;
5546 		ctsio->kern_sg_entries = 0;
5547 		ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
5548 		ctsio->be_move_done = ctl_config_move_done;
5549 		ctl_datamove((union ctl_io *)ctsio);
5550 
5551 		return (CTL_RETVAL_COMPLETE);
5552 	}
5553 
5554 	defect_list_len = 0;
5555 
5556 	if (cdb->byte2 & SF_FMTDATA) {
5557 		if (cdb->byte2 & SF_LONGLIST) {
5558 			struct scsi_format_header_long *header;
5559 
5560 			header = (struct scsi_format_header_long *)
5561 				ctsio->kern_data_ptr;
5562 
5563 			defect_list_len = scsi_4btoul(header->defect_list_len);
5564 			if (defect_list_len != 0) {
5565 				ctl_set_invalid_field(ctsio,
5566 						      /*sks_valid*/ 1,
5567 						      /*command*/ 0,
5568 						      /*field*/ 2,
5569 						      /*bit_valid*/ 0,
5570 						      /*bit*/ 0);
5571 				goto bailout;
5572 			}
5573 		} else {
5574 			struct scsi_format_header_short *header;
5575 
5576 			header = (struct scsi_format_header_short *)
5577 				ctsio->kern_data_ptr;
5578 
5579 			defect_list_len = scsi_2btoul(header->defect_list_len);
5580 			if (defect_list_len != 0) {
5581 				ctl_set_invalid_field(ctsio,
5582 						      /*sks_valid*/ 1,
5583 						      /*command*/ 0,
5584 						      /*field*/ 2,
5585 						      /*bit_valid*/ 0,
5586 						      /*bit*/ 0);
5587 				goto bailout;
5588 			}
5589 		}
5590 	}
5591 
5592 	/*
5593 	 * The format command will clear out the "Medium format corrupted"
5594 	 * status if set by the configuration code.  That status is really
5595 	 * just a way to notify the host that we have lost the media, and
5596 	 * get them to issue a command that will basically make them think
5597 	 * they're blowing away the media.
5598 	 */
5599 	mtx_lock(&lun->lun_lock);
5600 	lun->flags &= ~CTL_LUN_INOPERABLE;
5601 	mtx_unlock(&lun->lun_lock);
5602 
5603 	ctsio->scsi_status = SCSI_STATUS_OK;
5604 	ctsio->io_hdr.status = CTL_SUCCESS;
5605 bailout:
5606 
5607 	if (ctsio->io_hdr.flags & CTL_FLAG_ALLOCATED) {
5608 		free(ctsio->kern_data_ptr, M_CTL);
5609 		ctsio->io_hdr.flags &= ~CTL_FLAG_ALLOCATED;
5610 	}
5611 
5612 	ctl_done((union ctl_io *)ctsio);
5613 	return (CTL_RETVAL_COMPLETE);
5614 }
5615 
5616 int
5617 ctl_read_buffer(struct ctl_scsiio *ctsio)
5618 {
5619 	struct scsi_read_buffer *cdb;
5620 	struct ctl_lun *lun;
5621 	int buffer_offset, len;
5622 	static uint8_t descr[4];
5623 	static uint8_t echo_descr[4] = { 0 };
5624 
5625 	CTL_DEBUG_PRINT(("ctl_read_buffer\n"));
5626 
5627 	lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
5628 	cdb = (struct scsi_read_buffer *)ctsio->cdb;
5629 
5630 	if (lun->flags & CTL_LUN_PR_RESERVED) {
5631 		uint32_t residx;
5632 
5633 		/*
5634 		 * XXX KDM need a lock here.
5635 		 */
5636 		residx = ctl_get_resindex(&ctsio->io_hdr.nexus);
5637 		if ((lun->res_type == SPR_TYPE_EX_AC
5638 		  && residx != lun->pr_res_idx)
5639 		 || ((lun->res_type == SPR_TYPE_EX_AC_RO
5640 		   || lun->res_type == SPR_TYPE_EX_AC_AR)
5641 		  && !lun->per_res[residx].registered)) {
5642 			ctl_set_reservation_conflict(ctsio);
5643 			ctl_done((union ctl_io *)ctsio);
5644 			return (CTL_RETVAL_COMPLETE);
5645 	        }
5646 	}
5647 
5648 	if ((cdb->byte2 & RWB_MODE) != RWB_MODE_DATA &&
5649 	    (cdb->byte2 & RWB_MODE) != RWB_MODE_ECHO_DESCR &&
5650 	    (cdb->byte2 & RWB_MODE) != RWB_MODE_DESCR) {
5651 		ctl_set_invalid_field(ctsio,
5652 				      /*sks_valid*/ 1,
5653 				      /*command*/ 1,
5654 				      /*field*/ 1,
5655 				      /*bit_valid*/ 1,
5656 				      /*bit*/ 4);
5657 		ctl_done((union ctl_io *)ctsio);
5658 		return (CTL_RETVAL_COMPLETE);
5659 	}
5660 	if (cdb->buffer_id != 0) {
5661 		ctl_set_invalid_field(ctsio,
5662 				      /*sks_valid*/ 1,
5663 				      /*command*/ 1,
5664 				      /*field*/ 2,
5665 				      /*bit_valid*/ 0,
5666 				      /*bit*/ 0);
5667 		ctl_done((union ctl_io *)ctsio);
5668 		return (CTL_RETVAL_COMPLETE);
5669 	}
5670 
5671 	len = scsi_3btoul(cdb->length);
5672 	buffer_offset = scsi_3btoul(cdb->offset);
5673 
5674 	if (buffer_offset + len > sizeof(lun->write_buffer)) {
5675 		ctl_set_invalid_field(ctsio,
5676 				      /*sks_valid*/ 1,
5677 				      /*command*/ 1,
5678 				      /*field*/ 6,
5679 				      /*bit_valid*/ 0,
5680 				      /*bit*/ 0);
5681 		ctl_done((union ctl_io *)ctsio);
5682 		return (CTL_RETVAL_COMPLETE);
5683 	}
5684 
5685 	if ((cdb->byte2 & RWB_MODE) == RWB_MODE_DESCR) {
5686 		descr[0] = 0;
5687 		scsi_ulto3b(sizeof(lun->write_buffer), &descr[1]);
5688 		ctsio->kern_data_ptr = descr;
5689 		len = min(len, sizeof(descr));
5690 	} else if ((cdb->byte2 & RWB_MODE) == RWB_MODE_ECHO_DESCR) {
5691 		ctsio->kern_data_ptr = echo_descr;
5692 		len = min(len, sizeof(echo_descr));
5693 	} else
5694 		ctsio->kern_data_ptr = lun->write_buffer + buffer_offset;
5695 	ctsio->kern_data_len = len;
5696 	ctsio->kern_total_len = len;
5697 	ctsio->kern_data_resid = 0;
5698 	ctsio->kern_rel_offset = 0;
5699 	ctsio->kern_sg_entries = 0;
5700 	ctsio->be_move_done = ctl_config_move_done;
5701 	ctl_datamove((union ctl_io *)ctsio);
5702 
5703 	return (CTL_RETVAL_COMPLETE);
5704 }
5705 
5706 int
5707 ctl_write_buffer(struct ctl_scsiio *ctsio)
5708 {
5709 	struct scsi_write_buffer *cdb;
5710 	struct ctl_lun *lun;
5711 	int buffer_offset, len;
5712 
5713 	CTL_DEBUG_PRINT(("ctl_write_buffer\n"));
5714 
5715 	lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
5716 	cdb = (struct scsi_write_buffer *)ctsio->cdb;
5717 
5718 	if ((cdb->byte2 & RWB_MODE) != RWB_MODE_DATA) {
5719 		ctl_set_invalid_field(ctsio,
5720 				      /*sks_valid*/ 1,
5721 				      /*command*/ 1,
5722 				      /*field*/ 1,
5723 				      /*bit_valid*/ 1,
5724 				      /*bit*/ 4);
5725 		ctl_done((union ctl_io *)ctsio);
5726 		return (CTL_RETVAL_COMPLETE);
5727 	}
5728 	if (cdb->buffer_id != 0) {
5729 		ctl_set_invalid_field(ctsio,
5730 				      /*sks_valid*/ 1,
5731 				      /*command*/ 1,
5732 				      /*field*/ 2,
5733 				      /*bit_valid*/ 0,
5734 				      /*bit*/ 0);
5735 		ctl_done((union ctl_io *)ctsio);
5736 		return (CTL_RETVAL_COMPLETE);
5737 	}
5738 
5739 	len = scsi_3btoul(cdb->length);
5740 	buffer_offset = scsi_3btoul(cdb->offset);
5741 
5742 	if (buffer_offset + len > sizeof(lun->write_buffer)) {
5743 		ctl_set_invalid_field(ctsio,
5744 				      /*sks_valid*/ 1,
5745 				      /*command*/ 1,
5746 				      /*field*/ 6,
5747 				      /*bit_valid*/ 0,
5748 				      /*bit*/ 0);
5749 		ctl_done((union ctl_io *)ctsio);
5750 		return (CTL_RETVAL_COMPLETE);
5751 	}
5752 
5753 	/*
5754 	 * If we've got a kernel request that hasn't been malloced yet,
5755 	 * malloc it and tell the caller the data buffer is here.
5756 	 */
5757 	if ((ctsio->io_hdr.flags & CTL_FLAG_ALLOCATED) == 0) {
5758 		ctsio->kern_data_ptr = lun->write_buffer + buffer_offset;
5759 		ctsio->kern_data_len = len;
5760 		ctsio->kern_total_len = len;
5761 		ctsio->kern_data_resid = 0;
5762 		ctsio->kern_rel_offset = 0;
5763 		ctsio->kern_sg_entries = 0;
5764 		ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
5765 		ctsio->be_move_done = ctl_config_move_done;
5766 		ctl_datamove((union ctl_io *)ctsio);
5767 
5768 		return (CTL_RETVAL_COMPLETE);
5769 	}
5770 
5771 	ctl_done((union ctl_io *)ctsio);
5772 
5773 	return (CTL_RETVAL_COMPLETE);
5774 }
5775 
5776 int
5777 ctl_write_same(struct ctl_scsiio *ctsio)
5778 {
5779 	struct ctl_lun *lun;
5780 	struct ctl_lba_len_flags *lbalen;
5781 	uint64_t lba;
5782 	uint32_t num_blocks;
5783 	int len, retval;
5784 	uint8_t byte2;
5785 
5786 	retval = CTL_RETVAL_COMPLETE;
5787 
5788 	CTL_DEBUG_PRINT(("ctl_write_same\n"));
5789 
5790 	lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
5791 
5792 	switch (ctsio->cdb[0]) {
5793 	case WRITE_SAME_10: {
5794 		struct scsi_write_same_10 *cdb;
5795 
5796 		cdb = (struct scsi_write_same_10 *)ctsio->cdb;
5797 
5798 		lba = scsi_4btoul(cdb->addr);
5799 		num_blocks = scsi_2btoul(cdb->length);
5800 		byte2 = cdb->byte2;
5801 		break;
5802 	}
5803 	case WRITE_SAME_16: {
5804 		struct scsi_write_same_16 *cdb;
5805 
5806 		cdb = (struct scsi_write_same_16 *)ctsio->cdb;
5807 
5808 		lba = scsi_8btou64(cdb->addr);
5809 		num_blocks = scsi_4btoul(cdb->length);
5810 		byte2 = cdb->byte2;
5811 		break;
5812 	}
5813 	default:
5814 		/*
5815 		 * We got a command we don't support.  This shouldn't
5816 		 * happen, commands should be filtered out above us.
5817 		 */
5818 		ctl_set_invalid_opcode(ctsio);
5819 		ctl_done((union ctl_io *)ctsio);
5820 
5821 		return (CTL_RETVAL_COMPLETE);
5822 		break; /* NOTREACHED */
5823 	}
5824 
5825 	/*
5826 	 * The first check is to make sure we're in bounds, the second
5827 	 * check is to catch wrap-around problems.  If the lba + num blocks
5828 	 * is less than the lba, then we've wrapped around and the block
5829 	 * range is invalid anyway.
5830 	 */
5831 	if (((lba + num_blocks) > (lun->be_lun->maxlba + 1))
5832 	 || ((lba + num_blocks) < lba)) {
5833 		ctl_set_lba_out_of_range(ctsio);
5834 		ctl_done((union ctl_io *)ctsio);
5835 		return (CTL_RETVAL_COMPLETE);
5836 	}
5837 
5838 	/* Zero number of blocks means "to the last logical block" */
5839 	if (num_blocks == 0) {
5840 		if ((lun->be_lun->maxlba + 1) - lba > UINT32_MAX) {
5841 			ctl_set_invalid_field(ctsio,
5842 					      /*sks_valid*/ 0,
5843 					      /*command*/ 1,
5844 					      /*field*/ 0,
5845 					      /*bit_valid*/ 0,
5846 					      /*bit*/ 0);
5847 			ctl_done((union ctl_io *)ctsio);
5848 			return (CTL_RETVAL_COMPLETE);
5849 		}
5850 		num_blocks = (lun->be_lun->maxlba + 1) - lba;
5851 	}
5852 
5853 	len = lun->be_lun->blocksize;
5854 
5855 	/*
5856 	 * If we've got a kernel request that hasn't been malloced yet,
5857 	 * malloc it and tell the caller the data buffer is here.
5858 	 */
5859 	if ((ctsio->io_hdr.flags & CTL_FLAG_ALLOCATED) == 0) {
5860 		ctsio->kern_data_ptr = malloc(len, M_CTL, M_WAITOK);;
5861 		ctsio->kern_data_len = len;
5862 		ctsio->kern_total_len = len;
5863 		ctsio->kern_data_resid = 0;
5864 		ctsio->kern_rel_offset = 0;
5865 		ctsio->kern_sg_entries = 0;
5866 		ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
5867 		ctsio->be_move_done = ctl_config_move_done;
5868 		ctl_datamove((union ctl_io *)ctsio);
5869 
5870 		return (CTL_RETVAL_COMPLETE);
5871 	}
5872 
5873 	lbalen = (struct ctl_lba_len_flags *)&ctsio->io_hdr.ctl_private[CTL_PRIV_LBA_LEN];
5874 	lbalen->lba = lba;
5875 	lbalen->len = num_blocks;
5876 	lbalen->flags = byte2;
5877 	retval = lun->backend->config_write((union ctl_io *)ctsio);
5878 
5879 	return (retval);
5880 }
5881 
5882 int
5883 ctl_unmap(struct ctl_scsiio *ctsio)
5884 {
5885 	struct ctl_lun *lun;
5886 	struct scsi_unmap *cdb;
5887 	struct ctl_ptr_len_flags *ptrlen;
5888 	struct scsi_unmap_header *hdr;
5889 	struct scsi_unmap_desc *buf, *end;
5890 	uint64_t lba;
5891 	uint32_t num_blocks;
5892 	int len, retval;
5893 	uint8_t byte2;
5894 
5895 	retval = CTL_RETVAL_COMPLETE;
5896 
5897 	CTL_DEBUG_PRINT(("ctl_unmap\n"));
5898 
5899 	lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
5900 	cdb = (struct scsi_unmap *)ctsio->cdb;
5901 
5902 	len = scsi_2btoul(cdb->length);
5903 	byte2 = cdb->byte2;
5904 
5905 	/*
5906 	 * If we've got a kernel request that hasn't been malloced yet,
5907 	 * malloc it and tell the caller the data buffer is here.
5908 	 */
5909 	if ((ctsio->io_hdr.flags & CTL_FLAG_ALLOCATED) == 0) {
5910 		ctsio->kern_data_ptr = malloc(len, M_CTL, M_WAITOK);;
5911 		ctsio->kern_data_len = len;
5912 		ctsio->kern_total_len = len;
5913 		ctsio->kern_data_resid = 0;
5914 		ctsio->kern_rel_offset = 0;
5915 		ctsio->kern_sg_entries = 0;
5916 		ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
5917 		ctsio->be_move_done = ctl_config_move_done;
5918 		ctl_datamove((union ctl_io *)ctsio);
5919 
5920 		return (CTL_RETVAL_COMPLETE);
5921 	}
5922 
5923 	len = ctsio->kern_total_len - ctsio->kern_data_resid;
5924 	hdr = (struct scsi_unmap_header *)ctsio->kern_data_ptr;
5925 	if (len < sizeof (*hdr) ||
5926 	    len < (scsi_2btoul(hdr->length) + sizeof(hdr->length)) ||
5927 	    len < (scsi_2btoul(hdr->desc_length) + sizeof (*hdr)) ||
5928 	    scsi_2btoul(hdr->desc_length) % sizeof(*buf) != 0) {
5929 		ctl_set_invalid_field(ctsio,
5930 				      /*sks_valid*/ 0,
5931 				      /*command*/ 0,
5932 				      /*field*/ 0,
5933 				      /*bit_valid*/ 0,
5934 				      /*bit*/ 0);
5935 		ctl_done((union ctl_io *)ctsio);
5936 		return (CTL_RETVAL_COMPLETE);
5937 	}
5938 	len = scsi_2btoul(hdr->desc_length);
5939 	buf = (struct scsi_unmap_desc *)(hdr + 1);
5940 	end = buf + len / sizeof(*buf);
5941 
5942 	ptrlen = (struct ctl_ptr_len_flags *)&ctsio->io_hdr.ctl_private[CTL_PRIV_LBA_LEN];
5943 	ptrlen->ptr = (void *)buf;
5944 	ptrlen->len = len;
5945 	ptrlen->flags = byte2;
5946 
5947 	for (; buf < end; buf++) {
5948 		lba = scsi_8btou64(buf->lba);
5949 		num_blocks = scsi_4btoul(buf->length);
5950 		if (((lba + num_blocks) > (lun->be_lun->maxlba + 1))
5951 		 || ((lba + num_blocks) < lba)) {
5952 			ctl_set_lba_out_of_range(ctsio);
5953 			ctl_done((union ctl_io *)ctsio);
5954 			return (CTL_RETVAL_COMPLETE);
5955 		}
5956 	}
5957 
5958 	retval = lun->backend->config_write((union ctl_io *)ctsio);
5959 
5960 	return (retval);
5961 }
5962 
5963 /*
5964  * Note that this function currently doesn't actually do anything inside
5965  * CTL to enforce things if the DQue bit is turned on.
5966  *
5967  * Also note that this function can't be used in the default case, because
5968  * the DQue bit isn't set in the changeable mask for the control mode page
5969  * anyway.  This is just here as an example for how to implement a page
5970  * handler, and a placeholder in case we want to allow the user to turn
5971  * tagged queueing on and off.
5972  *
5973  * The D_SENSE bit handling is functional, however, and will turn
5974  * descriptor sense on and off for a given LUN.
5975  */
5976 int
5977 ctl_control_page_handler(struct ctl_scsiio *ctsio,
5978 			 struct ctl_page_index *page_index, uint8_t *page_ptr)
5979 {
5980 	struct scsi_control_page *current_cp, *saved_cp, *user_cp;
5981 	struct ctl_lun *lun;
5982 	struct ctl_softc *softc;
5983 	int set_ua;
5984 	uint32_t initidx;
5985 
5986 	lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
5987 	initidx = ctl_get_initindex(&ctsio->io_hdr.nexus);
5988 	set_ua = 0;
5989 
5990 	user_cp = (struct scsi_control_page *)page_ptr;
5991 	current_cp = (struct scsi_control_page *)
5992 		(page_index->page_data + (page_index->page_len *
5993 		CTL_PAGE_CURRENT));
5994 	saved_cp = (struct scsi_control_page *)
5995 		(page_index->page_data + (page_index->page_len *
5996 		CTL_PAGE_SAVED));
5997 
5998 	softc = control_softc;
5999 
6000 	mtx_lock(&lun->lun_lock);
6001 	if (((current_cp->rlec & SCP_DSENSE) == 0)
6002 	 && ((user_cp->rlec & SCP_DSENSE) != 0)) {
6003 		/*
6004 		 * Descriptor sense is currently turned off and the user
6005 		 * wants to turn it on.
6006 		 */
6007 		current_cp->rlec |= SCP_DSENSE;
6008 		saved_cp->rlec |= SCP_DSENSE;
6009 		lun->flags |= CTL_LUN_SENSE_DESC;
6010 		set_ua = 1;
6011 	} else if (((current_cp->rlec & SCP_DSENSE) != 0)
6012 		&& ((user_cp->rlec & SCP_DSENSE) == 0)) {
6013 		/*
6014 		 * Descriptor sense is currently turned on, and the user
6015 		 * wants to turn it off.
6016 		 */
6017 		current_cp->rlec &= ~SCP_DSENSE;
6018 		saved_cp->rlec &= ~SCP_DSENSE;
6019 		lun->flags &= ~CTL_LUN_SENSE_DESC;
6020 		set_ua = 1;
6021 	}
6022 	if (current_cp->queue_flags & SCP_QUEUE_DQUE) {
6023 		if (user_cp->queue_flags & SCP_QUEUE_DQUE) {
6024 #ifdef NEEDTOPORT
6025 			csevent_log(CSC_CTL | CSC_SHELF_SW |
6026 				    CTL_UNTAG_TO_UNTAG,
6027 				    csevent_LogType_Trace,
6028 				    csevent_Severity_Information,
6029 				    csevent_AlertLevel_Green,
6030 				    csevent_FRU_Firmware,
6031 				    csevent_FRU_Unknown,
6032 				    "Received untagged to untagged transition");
6033 #endif /* NEEDTOPORT */
6034 		} else {
6035 #ifdef NEEDTOPORT
6036 			csevent_log(CSC_CTL | CSC_SHELF_SW |
6037 				    CTL_UNTAG_TO_TAG,
6038 				    csevent_LogType_ConfigChange,
6039 				    csevent_Severity_Information,
6040 				    csevent_AlertLevel_Green,
6041 				    csevent_FRU_Firmware,
6042 				    csevent_FRU_Unknown,
6043 				    "Received untagged to tagged "
6044 				    "queueing transition");
6045 #endif /* NEEDTOPORT */
6046 
6047 			current_cp->queue_flags &= ~SCP_QUEUE_DQUE;
6048 			saved_cp->queue_flags &= ~SCP_QUEUE_DQUE;
6049 			set_ua = 1;
6050 		}
6051 	} else {
6052 		if (user_cp->queue_flags & SCP_QUEUE_DQUE) {
6053 #ifdef NEEDTOPORT
6054 			csevent_log(CSC_CTL | CSC_SHELF_SW |
6055 				    CTL_TAG_TO_UNTAG,
6056 				    csevent_LogType_ConfigChange,
6057 				    csevent_Severity_Warning,
6058 				    csevent_AlertLevel_Yellow,
6059 				    csevent_FRU_Firmware,
6060 				    csevent_FRU_Unknown,
6061 				    "Received tagged queueing to untagged "
6062 				    "transition");
6063 #endif /* NEEDTOPORT */
6064 
6065 			current_cp->queue_flags |= SCP_QUEUE_DQUE;
6066 			saved_cp->queue_flags |= SCP_QUEUE_DQUE;
6067 			set_ua = 1;
6068 		} else {
6069 #ifdef NEEDTOPORT
6070 			csevent_log(CSC_CTL | CSC_SHELF_SW |
6071 				    CTL_TAG_TO_TAG,
6072 				    csevent_LogType_Trace,
6073 				    csevent_Severity_Information,
6074 				    csevent_AlertLevel_Green,
6075 				    csevent_FRU_Firmware,
6076 				    csevent_FRU_Unknown,
6077 				    "Received tagged queueing to tagged "
6078 				    "queueing transition");
6079 #endif /* NEEDTOPORT */
6080 		}
6081 	}
6082 	if (set_ua != 0) {
6083 		int i;
6084 		/*
6085 		 * Let other initiators know that the mode
6086 		 * parameters for this LUN have changed.
6087 		 */
6088 		for (i = 0; i < CTL_MAX_INITIATORS; i++) {
6089 			if (i == initidx)
6090 				continue;
6091 
6092 			lun->pending_sense[i].ua_pending |=
6093 				CTL_UA_MODE_CHANGE;
6094 		}
6095 	}
6096 	mtx_unlock(&lun->lun_lock);
6097 
6098 	return (0);
6099 }
6100 
6101 int
6102 ctl_power_sp_handler(struct ctl_scsiio *ctsio,
6103 		     struct ctl_page_index *page_index, uint8_t *page_ptr)
6104 {
6105 	return (0);
6106 }
6107 
6108 int
6109 ctl_power_sp_sense_handler(struct ctl_scsiio *ctsio,
6110 			   struct ctl_page_index *page_index, int pc)
6111 {
6112 	struct copan_power_subpage *page;
6113 
6114 	page = (struct copan_power_subpage *)page_index->page_data +
6115 		(page_index->page_len * pc);
6116 
6117 	switch (pc) {
6118 	case SMS_PAGE_CTRL_CHANGEABLE >> 6:
6119 		/*
6120 		 * We don't update the changable bits for this page.
6121 		 */
6122 		break;
6123 	case SMS_PAGE_CTRL_CURRENT >> 6:
6124 	case SMS_PAGE_CTRL_DEFAULT >> 6:
6125 	case SMS_PAGE_CTRL_SAVED >> 6:
6126 #ifdef NEEDTOPORT
6127 		ctl_update_power_subpage(page);
6128 #endif
6129 		break;
6130 	default:
6131 #ifdef NEEDTOPORT
6132 		EPRINT(0, "Invalid PC %d!!", pc);
6133 #endif
6134 		break;
6135 	}
6136 	return (0);
6137 }
6138 
6139 
6140 int
6141 ctl_aps_sp_handler(struct ctl_scsiio *ctsio,
6142 		   struct ctl_page_index *page_index, uint8_t *page_ptr)
6143 {
6144 	struct copan_aps_subpage *user_sp;
6145 	struct copan_aps_subpage *current_sp;
6146 	union ctl_modepage_info *modepage_info;
6147 	struct ctl_softc *softc;
6148 	struct ctl_lun *lun;
6149 	int retval;
6150 
6151 	retval = CTL_RETVAL_COMPLETE;
6152 	current_sp = (struct copan_aps_subpage *)(page_index->page_data +
6153 		     (page_index->page_len * CTL_PAGE_CURRENT));
6154 	softc = control_softc;
6155 	lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
6156 
6157 	user_sp = (struct copan_aps_subpage *)page_ptr;
6158 
6159 	modepage_info = (union ctl_modepage_info *)
6160 		ctsio->io_hdr.ctl_private[CTL_PRIV_MODEPAGE].bytes;
6161 
6162 	modepage_info->header.page_code = page_index->page_code & SMPH_PC_MASK;
6163 	modepage_info->header.subpage = page_index->subpage;
6164 	modepage_info->aps.lock_active = user_sp->lock_active;
6165 
6166 	mtx_lock(&softc->ctl_lock);
6167 
6168 	/*
6169 	 * If there is a request to lock the LUN and another LUN is locked
6170 	 * this is an error. If the requested LUN is already locked ignore
6171 	 * the request. If no LUN is locked attempt to lock it.
6172 	 * if there is a request to unlock the LUN and the LUN is currently
6173 	 * locked attempt to unlock it. Otherwise ignore the request. i.e.
6174 	 * if another LUN is locked or no LUN is locked.
6175 	 */
6176 	if (user_sp->lock_active & APS_LOCK_ACTIVE) {
6177 		if (softc->aps_locked_lun == lun->lun) {
6178 			/*
6179 			 * This LUN is already locked, so we're done.
6180 			 */
6181 			retval = CTL_RETVAL_COMPLETE;
6182 		} else if (softc->aps_locked_lun == 0) {
6183 			/*
6184 			 * No one has the lock, pass the request to the
6185 			 * backend.
6186 			 */
6187 			retval = lun->backend->config_write(
6188 				(union ctl_io *)ctsio);
6189 		} else {
6190 			/*
6191 			 * Someone else has the lock, throw out the request.
6192 			 */
6193 			ctl_set_already_locked(ctsio);
6194 			free(ctsio->kern_data_ptr, M_CTL);
6195 			ctl_done((union ctl_io *)ctsio);
6196 
6197 			/*
6198 			 * Set the return value so that ctl_do_mode_select()
6199 			 * won't try to complete the command.  We already
6200 			 * completed it here.
6201 			 */
6202 			retval = CTL_RETVAL_ERROR;
6203 		}
6204 	} else if (softc->aps_locked_lun == lun->lun) {
6205 		/*
6206 		 * This LUN is locked, so pass the unlock request to the
6207 		 * backend.
6208 		 */
6209 		retval = lun->backend->config_write((union ctl_io *)ctsio);
6210 	}
6211 	mtx_unlock(&softc->ctl_lock);
6212 
6213 	return (retval);
6214 }
6215 
6216 int
6217 ctl_debugconf_sp_select_handler(struct ctl_scsiio *ctsio,
6218 				struct ctl_page_index *page_index,
6219 				uint8_t *page_ptr)
6220 {
6221 	uint8_t *c;
6222 	int i;
6223 
6224 	c = ((struct copan_debugconf_subpage *)page_ptr)->ctl_time_io_secs;
6225 	ctl_time_io_secs =
6226 		(c[0] << 8) |
6227 		(c[1] << 0) |
6228 		0;
6229 	CTL_DEBUG_PRINT(("set ctl_time_io_secs to %d\n", ctl_time_io_secs));
6230 	printf("set ctl_time_io_secs to %d\n", ctl_time_io_secs);
6231 	printf("page data:");
6232 	for (i=0; i<8; i++)
6233 		printf(" %.2x",page_ptr[i]);
6234 	printf("\n");
6235 	return (0);
6236 }
6237 
6238 int
6239 ctl_debugconf_sp_sense_handler(struct ctl_scsiio *ctsio,
6240 			       struct ctl_page_index *page_index,
6241 			       int pc)
6242 {
6243 	struct copan_debugconf_subpage *page;
6244 
6245 	page = (struct copan_debugconf_subpage *)page_index->page_data +
6246 		(page_index->page_len * pc);
6247 
6248 	switch (pc) {
6249 	case SMS_PAGE_CTRL_CHANGEABLE >> 6:
6250 	case SMS_PAGE_CTRL_DEFAULT >> 6:
6251 	case SMS_PAGE_CTRL_SAVED >> 6:
6252 		/*
6253 		 * We don't update the changable or default bits for this page.
6254 		 */
6255 		break;
6256 	case SMS_PAGE_CTRL_CURRENT >> 6:
6257 		page->ctl_time_io_secs[0] = ctl_time_io_secs >> 8;
6258 		page->ctl_time_io_secs[1] = ctl_time_io_secs >> 0;
6259 		break;
6260 	default:
6261 #ifdef NEEDTOPORT
6262 		EPRINT(0, "Invalid PC %d!!", pc);
6263 #endif /* NEEDTOPORT */
6264 		break;
6265 	}
6266 	return (0);
6267 }
6268 
6269 
6270 static int
6271 ctl_do_mode_select(union ctl_io *io)
6272 {
6273 	struct scsi_mode_page_header *page_header;
6274 	struct ctl_page_index *page_index;
6275 	struct ctl_scsiio *ctsio;
6276 	int control_dev, page_len;
6277 	int page_len_offset, page_len_size;
6278 	union ctl_modepage_info *modepage_info;
6279 	struct ctl_lun *lun;
6280 	int *len_left, *len_used;
6281 	int retval, i;
6282 
6283 	ctsio = &io->scsiio;
6284 	page_index = NULL;
6285 	page_len = 0;
6286 	retval = CTL_RETVAL_COMPLETE;
6287 
6288 	lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
6289 
6290 	if (lun->be_lun->lun_type != T_DIRECT)
6291 		control_dev = 1;
6292 	else
6293 		control_dev = 0;
6294 
6295 	modepage_info = (union ctl_modepage_info *)
6296 		ctsio->io_hdr.ctl_private[CTL_PRIV_MODEPAGE].bytes;
6297 	len_left = &modepage_info->header.len_left;
6298 	len_used = &modepage_info->header.len_used;
6299 
6300 do_next_page:
6301 
6302 	page_header = (struct scsi_mode_page_header *)
6303 		(ctsio->kern_data_ptr + *len_used);
6304 
6305 	if (*len_left == 0) {
6306 		free(ctsio->kern_data_ptr, M_CTL);
6307 		ctl_set_success(ctsio);
6308 		ctl_done((union ctl_io *)ctsio);
6309 		return (CTL_RETVAL_COMPLETE);
6310 	} else if (*len_left < sizeof(struct scsi_mode_page_header)) {
6311 
6312 		free(ctsio->kern_data_ptr, M_CTL);
6313 		ctl_set_param_len_error(ctsio);
6314 		ctl_done((union ctl_io *)ctsio);
6315 		return (CTL_RETVAL_COMPLETE);
6316 
6317 	} else if ((page_header->page_code & SMPH_SPF)
6318 		&& (*len_left < sizeof(struct scsi_mode_page_header_sp))) {
6319 
6320 		free(ctsio->kern_data_ptr, M_CTL);
6321 		ctl_set_param_len_error(ctsio);
6322 		ctl_done((union ctl_io *)ctsio);
6323 		return (CTL_RETVAL_COMPLETE);
6324 	}
6325 
6326 
6327 	/*
6328 	 * XXX KDM should we do something with the block descriptor?
6329 	 */
6330 	for (i = 0; i < CTL_NUM_MODE_PAGES; i++) {
6331 
6332 		if ((control_dev != 0)
6333 		 && (lun->mode_pages.index[i].page_flags &
6334 		     CTL_PAGE_FLAG_DISK_ONLY))
6335 			continue;
6336 
6337 		if ((lun->mode_pages.index[i].page_code & SMPH_PC_MASK) !=
6338 		    (page_header->page_code & SMPH_PC_MASK))
6339 			continue;
6340 
6341 		/*
6342 		 * If neither page has a subpage code, then we've got a
6343 		 * match.
6344 		 */
6345 		if (((lun->mode_pages.index[i].page_code & SMPH_SPF) == 0)
6346 		 && ((page_header->page_code & SMPH_SPF) == 0)) {
6347 			page_index = &lun->mode_pages.index[i];
6348 			page_len = page_header->page_length;
6349 			break;
6350 		}
6351 
6352 		/*
6353 		 * If both pages have subpages, then the subpage numbers
6354 		 * have to match.
6355 		 */
6356 		if ((lun->mode_pages.index[i].page_code & SMPH_SPF)
6357 		  && (page_header->page_code & SMPH_SPF)) {
6358 			struct scsi_mode_page_header_sp *sph;
6359 
6360 			sph = (struct scsi_mode_page_header_sp *)page_header;
6361 
6362 			if (lun->mode_pages.index[i].subpage ==
6363 			    sph->subpage) {
6364 				page_index = &lun->mode_pages.index[i];
6365 				page_len = scsi_2btoul(sph->page_length);
6366 				break;
6367 			}
6368 		}
6369 	}
6370 
6371 	/*
6372 	 * If we couldn't find the page, or if we don't have a mode select
6373 	 * handler for it, send back an error to the user.
6374 	 */
6375 	if ((page_index == NULL)
6376 	 || (page_index->select_handler == NULL)) {
6377 		ctl_set_invalid_field(ctsio,
6378 				      /*sks_valid*/ 1,
6379 				      /*command*/ 0,
6380 				      /*field*/ *len_used,
6381 				      /*bit_valid*/ 0,
6382 				      /*bit*/ 0);
6383 		free(ctsio->kern_data_ptr, M_CTL);
6384 		ctl_done((union ctl_io *)ctsio);
6385 		return (CTL_RETVAL_COMPLETE);
6386 	}
6387 
6388 	if (page_index->page_code & SMPH_SPF) {
6389 		page_len_offset = 2;
6390 		page_len_size = 2;
6391 	} else {
6392 		page_len_size = 1;
6393 		page_len_offset = 1;
6394 	}
6395 
6396 	/*
6397 	 * If the length the initiator gives us isn't the one we specify in
6398 	 * the mode page header, or if they didn't specify enough data in
6399 	 * the CDB to avoid truncating this page, kick out the request.
6400 	 */
6401 	if ((page_len != (page_index->page_len - page_len_offset -
6402 			  page_len_size))
6403 	 || (*len_left < page_index->page_len)) {
6404 
6405 
6406 		ctl_set_invalid_field(ctsio,
6407 				      /*sks_valid*/ 1,
6408 				      /*command*/ 0,
6409 				      /*field*/ *len_used + page_len_offset,
6410 				      /*bit_valid*/ 0,
6411 				      /*bit*/ 0);
6412 		free(ctsio->kern_data_ptr, M_CTL);
6413 		ctl_done((union ctl_io *)ctsio);
6414 		return (CTL_RETVAL_COMPLETE);
6415 	}
6416 
6417 	/*
6418 	 * Run through the mode page, checking to make sure that the bits
6419 	 * the user changed are actually legal for him to change.
6420 	 */
6421 	for (i = 0; i < page_index->page_len; i++) {
6422 		uint8_t *user_byte, *change_mask, *current_byte;
6423 		int bad_bit;
6424 		int j;
6425 
6426 		user_byte = (uint8_t *)page_header + i;
6427 		change_mask = page_index->page_data +
6428 			      (page_index->page_len * CTL_PAGE_CHANGEABLE) + i;
6429 		current_byte = page_index->page_data +
6430 			       (page_index->page_len * CTL_PAGE_CURRENT) + i;
6431 
6432 		/*
6433 		 * Check to see whether the user set any bits in this byte
6434 		 * that he is not allowed to set.
6435 		 */
6436 		if ((*user_byte & ~(*change_mask)) ==
6437 		    (*current_byte & ~(*change_mask)))
6438 			continue;
6439 
6440 		/*
6441 		 * Go through bit by bit to determine which one is illegal.
6442 		 */
6443 		bad_bit = 0;
6444 		for (j = 7; j >= 0; j--) {
6445 			if ((((1 << i) & ~(*change_mask)) & *user_byte) !=
6446 			    (((1 << i) & ~(*change_mask)) & *current_byte)) {
6447 				bad_bit = i;
6448 				break;
6449 			}
6450 		}
6451 		ctl_set_invalid_field(ctsio,
6452 				      /*sks_valid*/ 1,
6453 				      /*command*/ 0,
6454 				      /*field*/ *len_used + i,
6455 				      /*bit_valid*/ 1,
6456 				      /*bit*/ bad_bit);
6457 		free(ctsio->kern_data_ptr, M_CTL);
6458 		ctl_done((union ctl_io *)ctsio);
6459 		return (CTL_RETVAL_COMPLETE);
6460 	}
6461 
6462 	/*
6463 	 * Decrement these before we call the page handler, since we may
6464 	 * end up getting called back one way or another before the handler
6465 	 * returns to this context.
6466 	 */
6467 	*len_left -= page_index->page_len;
6468 	*len_used += page_index->page_len;
6469 
6470 	retval = page_index->select_handler(ctsio, page_index,
6471 					    (uint8_t *)page_header);
6472 
6473 	/*
6474 	 * If the page handler returns CTL_RETVAL_QUEUED, then we need to
6475 	 * wait until this queued command completes to finish processing
6476 	 * the mode page.  If it returns anything other than
6477 	 * CTL_RETVAL_COMPLETE (e.g. CTL_RETVAL_ERROR), then it should have
6478 	 * already set the sense information, freed the data pointer, and
6479 	 * completed the io for us.
6480 	 */
6481 	if (retval != CTL_RETVAL_COMPLETE)
6482 		goto bailout_no_done;
6483 
6484 	/*
6485 	 * If the initiator sent us more than one page, parse the next one.
6486 	 */
6487 	if (*len_left > 0)
6488 		goto do_next_page;
6489 
6490 	ctl_set_success(ctsio);
6491 	free(ctsio->kern_data_ptr, M_CTL);
6492 	ctl_done((union ctl_io *)ctsio);
6493 
6494 bailout_no_done:
6495 
6496 	return (CTL_RETVAL_COMPLETE);
6497 
6498 }
6499 
6500 int
6501 ctl_mode_select(struct ctl_scsiio *ctsio)
6502 {
6503 	int param_len, pf, sp;
6504 	int header_size, bd_len;
6505 	int len_left, len_used;
6506 	struct ctl_page_index *page_index;
6507 	struct ctl_lun *lun;
6508 	int control_dev, page_len;
6509 	union ctl_modepage_info *modepage_info;
6510 	int retval;
6511 
6512 	pf = 0;
6513 	sp = 0;
6514 	page_len = 0;
6515 	len_used = 0;
6516 	len_left = 0;
6517 	retval = 0;
6518 	bd_len = 0;
6519 	page_index = NULL;
6520 
6521 	lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
6522 
6523 	if (lun->be_lun->lun_type != T_DIRECT)
6524 		control_dev = 1;
6525 	else
6526 		control_dev = 0;
6527 
6528 	switch (ctsio->cdb[0]) {
6529 	case MODE_SELECT_6: {
6530 		struct scsi_mode_select_6 *cdb;
6531 
6532 		cdb = (struct scsi_mode_select_6 *)ctsio->cdb;
6533 
6534 		pf = (cdb->byte2 & SMS_PF) ? 1 : 0;
6535 		sp = (cdb->byte2 & SMS_SP) ? 1 : 0;
6536 
6537 		param_len = cdb->length;
6538 		header_size = sizeof(struct scsi_mode_header_6);
6539 		break;
6540 	}
6541 	case MODE_SELECT_10: {
6542 		struct scsi_mode_select_10 *cdb;
6543 
6544 		cdb = (struct scsi_mode_select_10 *)ctsio->cdb;
6545 
6546 		pf = (cdb->byte2 & SMS_PF) ? 1 : 0;
6547 		sp = (cdb->byte2 & SMS_SP) ? 1 : 0;
6548 
6549 		param_len = scsi_2btoul(cdb->length);
6550 		header_size = sizeof(struct scsi_mode_header_10);
6551 		break;
6552 	}
6553 	default:
6554 		ctl_set_invalid_opcode(ctsio);
6555 		ctl_done((union ctl_io *)ctsio);
6556 		return (CTL_RETVAL_COMPLETE);
6557 		break; /* NOTREACHED */
6558 	}
6559 
6560 	/*
6561 	 * From SPC-3:
6562 	 * "A parameter list length of zero indicates that the Data-Out Buffer
6563 	 * shall be empty. This condition shall not be considered as an error."
6564 	 */
6565 	if (param_len == 0) {
6566 		ctl_set_success(ctsio);
6567 		ctl_done((union ctl_io *)ctsio);
6568 		return (CTL_RETVAL_COMPLETE);
6569 	}
6570 
6571 	/*
6572 	 * Since we'll hit this the first time through, prior to
6573 	 * allocation, we don't need to free a data buffer here.
6574 	 */
6575 	if (param_len < header_size) {
6576 		ctl_set_param_len_error(ctsio);
6577 		ctl_done((union ctl_io *)ctsio);
6578 		return (CTL_RETVAL_COMPLETE);
6579 	}
6580 
6581 	/*
6582 	 * Allocate the data buffer and grab the user's data.  In theory,
6583 	 * we shouldn't have to sanity check the parameter list length here
6584 	 * because the maximum size is 64K.  We should be able to malloc
6585 	 * that much without too many problems.
6586 	 */
6587 	if ((ctsio->io_hdr.flags & CTL_FLAG_ALLOCATED) == 0) {
6588 		ctsio->kern_data_ptr = malloc(param_len, M_CTL, M_WAITOK);
6589 		ctsio->kern_data_len = param_len;
6590 		ctsio->kern_total_len = param_len;
6591 		ctsio->kern_data_resid = 0;
6592 		ctsio->kern_rel_offset = 0;
6593 		ctsio->kern_sg_entries = 0;
6594 		ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
6595 		ctsio->be_move_done = ctl_config_move_done;
6596 		ctl_datamove((union ctl_io *)ctsio);
6597 
6598 		return (CTL_RETVAL_COMPLETE);
6599 	}
6600 
6601 	switch (ctsio->cdb[0]) {
6602 	case MODE_SELECT_6: {
6603 		struct scsi_mode_header_6 *mh6;
6604 
6605 		mh6 = (struct scsi_mode_header_6 *)ctsio->kern_data_ptr;
6606 		bd_len = mh6->blk_desc_len;
6607 		break;
6608 	}
6609 	case MODE_SELECT_10: {
6610 		struct scsi_mode_header_10 *mh10;
6611 
6612 		mh10 = (struct scsi_mode_header_10 *)ctsio->kern_data_ptr;
6613 		bd_len = scsi_2btoul(mh10->blk_desc_len);
6614 		break;
6615 	}
6616 	default:
6617 		panic("Invalid CDB type %#x", ctsio->cdb[0]);
6618 		break;
6619 	}
6620 
6621 	if (param_len < (header_size + bd_len)) {
6622 		free(ctsio->kern_data_ptr, M_CTL);
6623 		ctl_set_param_len_error(ctsio);
6624 		ctl_done((union ctl_io *)ctsio);
6625 		return (CTL_RETVAL_COMPLETE);
6626 	}
6627 
6628 	/*
6629 	 * Set the IO_CONT flag, so that if this I/O gets passed to
6630 	 * ctl_config_write_done(), it'll get passed back to
6631 	 * ctl_do_mode_select() for further processing, or completion if
6632 	 * we're all done.
6633 	 */
6634 	ctsio->io_hdr.flags |= CTL_FLAG_IO_CONT;
6635 	ctsio->io_cont = ctl_do_mode_select;
6636 
6637 	modepage_info = (union ctl_modepage_info *)
6638 		ctsio->io_hdr.ctl_private[CTL_PRIV_MODEPAGE].bytes;
6639 
6640 	memset(modepage_info, 0, sizeof(*modepage_info));
6641 
6642 	len_left = param_len - header_size - bd_len;
6643 	len_used = header_size + bd_len;
6644 
6645 	modepage_info->header.len_left = len_left;
6646 	modepage_info->header.len_used = len_used;
6647 
6648 	return (ctl_do_mode_select((union ctl_io *)ctsio));
6649 }
6650 
6651 int
6652 ctl_mode_sense(struct ctl_scsiio *ctsio)
6653 {
6654 	struct ctl_lun *lun;
6655 	int pc, page_code, dbd, llba, subpage;
6656 	int alloc_len, page_len, header_len, total_len;
6657 	struct scsi_mode_block_descr *block_desc;
6658 	struct ctl_page_index *page_index;
6659 	int control_dev;
6660 
6661 	dbd = 0;
6662 	llba = 0;
6663 	block_desc = NULL;
6664 	page_index = NULL;
6665 
6666 	CTL_DEBUG_PRINT(("ctl_mode_sense\n"));
6667 
6668 	lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
6669 
6670 	if (lun->be_lun->lun_type != T_DIRECT)
6671 		control_dev = 1;
6672 	else
6673 		control_dev = 0;
6674 
6675 	if (lun->flags & CTL_LUN_PR_RESERVED) {
6676 		uint32_t residx;
6677 
6678 		/*
6679 		 * XXX KDM need a lock here.
6680 		 */
6681 		residx = ctl_get_resindex(&ctsio->io_hdr.nexus);
6682 		if ((lun->res_type == SPR_TYPE_EX_AC
6683 		  && residx != lun->pr_res_idx)
6684 		 || ((lun->res_type == SPR_TYPE_EX_AC_RO
6685 		   || lun->res_type == SPR_TYPE_EX_AC_AR)
6686 		  && !lun->per_res[residx].registered)) {
6687 			ctl_set_reservation_conflict(ctsio);
6688 			ctl_done((union ctl_io *)ctsio);
6689 			return (CTL_RETVAL_COMPLETE);
6690 		}
6691 	}
6692 
6693 	switch (ctsio->cdb[0]) {
6694 	case MODE_SENSE_6: {
6695 		struct scsi_mode_sense_6 *cdb;
6696 
6697 		cdb = (struct scsi_mode_sense_6 *)ctsio->cdb;
6698 
6699 		header_len = sizeof(struct scsi_mode_hdr_6);
6700 		if (cdb->byte2 & SMS_DBD)
6701 			dbd = 1;
6702 		else
6703 			header_len += sizeof(struct scsi_mode_block_descr);
6704 
6705 		pc = (cdb->page & SMS_PAGE_CTRL_MASK) >> 6;
6706 		page_code = cdb->page & SMS_PAGE_CODE;
6707 		subpage = cdb->subpage;
6708 		alloc_len = cdb->length;
6709 		break;
6710 	}
6711 	case MODE_SENSE_10: {
6712 		struct scsi_mode_sense_10 *cdb;
6713 
6714 		cdb = (struct scsi_mode_sense_10 *)ctsio->cdb;
6715 
6716 		header_len = sizeof(struct scsi_mode_hdr_10);
6717 
6718 		if (cdb->byte2 & SMS_DBD)
6719 			dbd = 1;
6720 		else
6721 			header_len += sizeof(struct scsi_mode_block_descr);
6722 		if (cdb->byte2 & SMS10_LLBAA)
6723 			llba = 1;
6724 		pc = (cdb->page & SMS_PAGE_CTRL_MASK) >> 6;
6725 		page_code = cdb->page & SMS_PAGE_CODE;
6726 		subpage = cdb->subpage;
6727 		alloc_len = scsi_2btoul(cdb->length);
6728 		break;
6729 	}
6730 	default:
6731 		ctl_set_invalid_opcode(ctsio);
6732 		ctl_done((union ctl_io *)ctsio);
6733 		return (CTL_RETVAL_COMPLETE);
6734 		break; /* NOTREACHED */
6735 	}
6736 
6737 	/*
6738 	 * We have to make a first pass through to calculate the size of
6739 	 * the pages that match the user's query.  Then we allocate enough
6740 	 * memory to hold it, and actually copy the data into the buffer.
6741 	 */
6742 	switch (page_code) {
6743 	case SMS_ALL_PAGES_PAGE: {
6744 		int i;
6745 
6746 		page_len = 0;
6747 
6748 		/*
6749 		 * At the moment, values other than 0 and 0xff here are
6750 		 * reserved according to SPC-3.
6751 		 */
6752 		if ((subpage != SMS_SUBPAGE_PAGE_0)
6753 		 && (subpage != SMS_SUBPAGE_ALL)) {
6754 			ctl_set_invalid_field(ctsio,
6755 					      /*sks_valid*/ 1,
6756 					      /*command*/ 1,
6757 					      /*field*/ 3,
6758 					      /*bit_valid*/ 0,
6759 					      /*bit*/ 0);
6760 			ctl_done((union ctl_io *)ctsio);
6761 			return (CTL_RETVAL_COMPLETE);
6762 		}
6763 
6764 		for (i = 0; i < CTL_NUM_MODE_PAGES; i++) {
6765 			if ((control_dev != 0)
6766 			 && (lun->mode_pages.index[i].page_flags &
6767 			     CTL_PAGE_FLAG_DISK_ONLY))
6768 				continue;
6769 
6770 			/*
6771 			 * We don't use this subpage if the user didn't
6772 			 * request all subpages.
6773 			 */
6774 			if ((lun->mode_pages.index[i].subpage != 0)
6775 			 && (subpage == SMS_SUBPAGE_PAGE_0))
6776 				continue;
6777 
6778 #if 0
6779 			printf("found page %#x len %d\n",
6780 			       lun->mode_pages.index[i].page_code &
6781 			       SMPH_PC_MASK,
6782 			       lun->mode_pages.index[i].page_len);
6783 #endif
6784 			page_len += lun->mode_pages.index[i].page_len;
6785 		}
6786 		break;
6787 	}
6788 	default: {
6789 		int i;
6790 
6791 		page_len = 0;
6792 
6793 		for (i = 0; i < CTL_NUM_MODE_PAGES; i++) {
6794 			/* Look for the right page code */
6795 			if ((lun->mode_pages.index[i].page_code &
6796 			     SMPH_PC_MASK) != page_code)
6797 				continue;
6798 
6799 			/* Look for the right subpage or the subpage wildcard*/
6800 			if ((lun->mode_pages.index[i].subpage != subpage)
6801 			 && (subpage != SMS_SUBPAGE_ALL))
6802 				continue;
6803 
6804 			/* Make sure the page is supported for this dev type */
6805 			if ((control_dev != 0)
6806 			 && (lun->mode_pages.index[i].page_flags &
6807 			     CTL_PAGE_FLAG_DISK_ONLY))
6808 				continue;
6809 
6810 #if 0
6811 			printf("found page %#x len %d\n",
6812 			       lun->mode_pages.index[i].page_code &
6813 			       SMPH_PC_MASK,
6814 			       lun->mode_pages.index[i].page_len);
6815 #endif
6816 
6817 			page_len += lun->mode_pages.index[i].page_len;
6818 		}
6819 
6820 		if (page_len == 0) {
6821 			ctl_set_invalid_field(ctsio,
6822 					      /*sks_valid*/ 1,
6823 					      /*command*/ 1,
6824 					      /*field*/ 2,
6825 					      /*bit_valid*/ 1,
6826 					      /*bit*/ 5);
6827 			ctl_done((union ctl_io *)ctsio);
6828 			return (CTL_RETVAL_COMPLETE);
6829 		}
6830 		break;
6831 	}
6832 	}
6833 
6834 	total_len = header_len + page_len;
6835 #if 0
6836 	printf("header_len = %d, page_len = %d, total_len = %d\n",
6837 	       header_len, page_len, total_len);
6838 #endif
6839 
6840 	ctsio->kern_data_ptr = malloc(total_len, M_CTL, M_WAITOK | M_ZERO);
6841 	ctsio->kern_sg_entries = 0;
6842 	ctsio->kern_data_resid = 0;
6843 	ctsio->kern_rel_offset = 0;
6844 	if (total_len < alloc_len) {
6845 		ctsio->residual = alloc_len - total_len;
6846 		ctsio->kern_data_len = total_len;
6847 		ctsio->kern_total_len = total_len;
6848 	} else {
6849 		ctsio->residual = 0;
6850 		ctsio->kern_data_len = alloc_len;
6851 		ctsio->kern_total_len = alloc_len;
6852 	}
6853 
6854 	switch (ctsio->cdb[0]) {
6855 	case MODE_SENSE_6: {
6856 		struct scsi_mode_hdr_6 *header;
6857 
6858 		header = (struct scsi_mode_hdr_6 *)ctsio->kern_data_ptr;
6859 
6860 		header->datalen = ctl_min(total_len - 1, 254);
6861 
6862 		if (dbd)
6863 			header->block_descr_len = 0;
6864 		else
6865 			header->block_descr_len =
6866 				sizeof(struct scsi_mode_block_descr);
6867 		block_desc = (struct scsi_mode_block_descr *)&header[1];
6868 		break;
6869 	}
6870 	case MODE_SENSE_10: {
6871 		struct scsi_mode_hdr_10 *header;
6872 		int datalen;
6873 
6874 		header = (struct scsi_mode_hdr_10 *)ctsio->kern_data_ptr;
6875 
6876 		datalen = ctl_min(total_len - 2, 65533);
6877 		scsi_ulto2b(datalen, header->datalen);
6878 		if (dbd)
6879 			scsi_ulto2b(0, header->block_descr_len);
6880 		else
6881 			scsi_ulto2b(sizeof(struct scsi_mode_block_descr),
6882 				    header->block_descr_len);
6883 		block_desc = (struct scsi_mode_block_descr *)&header[1];
6884 		break;
6885 	}
6886 	default:
6887 		panic("invalid CDB type %#x", ctsio->cdb[0]);
6888 		break; /* NOTREACHED */
6889 	}
6890 
6891 	/*
6892 	 * If we've got a disk, use its blocksize in the block
6893 	 * descriptor.  Otherwise, just set it to 0.
6894 	 */
6895 	if (dbd == 0) {
6896 		if (control_dev != 0)
6897 			scsi_ulto3b(lun->be_lun->blocksize,
6898 				    block_desc->block_len);
6899 		else
6900 			scsi_ulto3b(0, block_desc->block_len);
6901 	}
6902 
6903 	switch (page_code) {
6904 	case SMS_ALL_PAGES_PAGE: {
6905 		int i, data_used;
6906 
6907 		data_used = header_len;
6908 		for (i = 0; i < CTL_NUM_MODE_PAGES; i++) {
6909 			struct ctl_page_index *page_index;
6910 
6911 			page_index = &lun->mode_pages.index[i];
6912 
6913 			if ((control_dev != 0)
6914 			 && (page_index->page_flags &
6915 			    CTL_PAGE_FLAG_DISK_ONLY))
6916 				continue;
6917 
6918 			/*
6919 			 * We don't use this subpage if the user didn't
6920 			 * request all subpages.  We already checked (above)
6921 			 * to make sure the user only specified a subpage
6922 			 * of 0 or 0xff in the SMS_ALL_PAGES_PAGE case.
6923 			 */
6924 			if ((page_index->subpage != 0)
6925 			 && (subpage == SMS_SUBPAGE_PAGE_0))
6926 				continue;
6927 
6928 			/*
6929 			 * Call the handler, if it exists, to update the
6930 			 * page to the latest values.
6931 			 */
6932 			if (page_index->sense_handler != NULL)
6933 				page_index->sense_handler(ctsio, page_index,pc);
6934 
6935 			memcpy(ctsio->kern_data_ptr + data_used,
6936 			       page_index->page_data +
6937 			       (page_index->page_len * pc),
6938 			       page_index->page_len);
6939 			data_used += page_index->page_len;
6940 		}
6941 		break;
6942 	}
6943 	default: {
6944 		int i, data_used;
6945 
6946 		data_used = header_len;
6947 
6948 		for (i = 0; i < CTL_NUM_MODE_PAGES; i++) {
6949 			struct ctl_page_index *page_index;
6950 
6951 			page_index = &lun->mode_pages.index[i];
6952 
6953 			/* Look for the right page code */
6954 			if ((page_index->page_code & SMPH_PC_MASK) != page_code)
6955 				continue;
6956 
6957 			/* Look for the right subpage or the subpage wildcard*/
6958 			if ((page_index->subpage != subpage)
6959 			 && (subpage != SMS_SUBPAGE_ALL))
6960 				continue;
6961 
6962 			/* Make sure the page is supported for this dev type */
6963 			if ((control_dev != 0)
6964 			 && (page_index->page_flags &
6965 			     CTL_PAGE_FLAG_DISK_ONLY))
6966 				continue;
6967 
6968 			/*
6969 			 * Call the handler, if it exists, to update the
6970 			 * page to the latest values.
6971 			 */
6972 			if (page_index->sense_handler != NULL)
6973 				page_index->sense_handler(ctsio, page_index,pc);
6974 
6975 			memcpy(ctsio->kern_data_ptr + data_used,
6976 			       page_index->page_data +
6977 			       (page_index->page_len * pc),
6978 			       page_index->page_len);
6979 			data_used += page_index->page_len;
6980 		}
6981 		break;
6982 	}
6983 	}
6984 
6985 	ctsio->scsi_status = SCSI_STATUS_OK;
6986 
6987 	ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
6988 	ctsio->be_move_done = ctl_config_move_done;
6989 	ctl_datamove((union ctl_io *)ctsio);
6990 
6991 	return (CTL_RETVAL_COMPLETE);
6992 }
6993 
6994 int
6995 ctl_read_capacity(struct ctl_scsiio *ctsio)
6996 {
6997 	struct scsi_read_capacity *cdb;
6998 	struct scsi_read_capacity_data *data;
6999 	struct ctl_lun *lun;
7000 	uint32_t lba;
7001 
7002 	CTL_DEBUG_PRINT(("ctl_read_capacity\n"));
7003 
7004 	cdb = (struct scsi_read_capacity *)ctsio->cdb;
7005 
7006 	lba = scsi_4btoul(cdb->addr);
7007 	if (((cdb->pmi & SRC_PMI) == 0)
7008 	 && (lba != 0)) {
7009 		ctl_set_invalid_field(/*ctsio*/ ctsio,
7010 				      /*sks_valid*/ 1,
7011 				      /*command*/ 1,
7012 				      /*field*/ 2,
7013 				      /*bit_valid*/ 0,
7014 				      /*bit*/ 0);
7015 		ctl_done((union ctl_io *)ctsio);
7016 		return (CTL_RETVAL_COMPLETE);
7017 	}
7018 
7019 	lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
7020 
7021 	ctsio->kern_data_ptr = malloc(sizeof(*data), M_CTL, M_WAITOK | M_ZERO);
7022 	data = (struct scsi_read_capacity_data *)ctsio->kern_data_ptr;
7023 	ctsio->residual = 0;
7024 	ctsio->kern_data_len = sizeof(*data);
7025 	ctsio->kern_total_len = sizeof(*data);
7026 	ctsio->kern_data_resid = 0;
7027 	ctsio->kern_rel_offset = 0;
7028 	ctsio->kern_sg_entries = 0;
7029 
7030 	/*
7031 	 * If the maximum LBA is greater than 0xfffffffe, the user must
7032 	 * issue a SERVICE ACTION IN (16) command, with the read capacity
7033 	 * serivce action set.
7034 	 */
7035 	if (lun->be_lun->maxlba > 0xfffffffe)
7036 		scsi_ulto4b(0xffffffff, data->addr);
7037 	else
7038 		scsi_ulto4b(lun->be_lun->maxlba, data->addr);
7039 
7040 	/*
7041 	 * XXX KDM this may not be 512 bytes...
7042 	 */
7043 	scsi_ulto4b(lun->be_lun->blocksize, data->length);
7044 
7045 	ctsio->scsi_status = SCSI_STATUS_OK;
7046 
7047 	ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
7048 	ctsio->be_move_done = ctl_config_move_done;
7049 	ctl_datamove((union ctl_io *)ctsio);
7050 
7051 	return (CTL_RETVAL_COMPLETE);
7052 }
7053 
7054 static int
7055 ctl_read_capacity_16(struct ctl_scsiio *ctsio)
7056 {
7057 	struct scsi_read_capacity_16 *cdb;
7058 	struct scsi_read_capacity_data_long *data;
7059 	struct ctl_lun *lun;
7060 	uint64_t lba;
7061 	uint32_t alloc_len;
7062 
7063 	CTL_DEBUG_PRINT(("ctl_read_capacity_16\n"));
7064 
7065 	cdb = (struct scsi_read_capacity_16 *)ctsio->cdb;
7066 
7067 	alloc_len = scsi_4btoul(cdb->alloc_len);
7068 	lba = scsi_8btou64(cdb->addr);
7069 
7070 	if ((cdb->reladr & SRC16_PMI)
7071 	 && (lba != 0)) {
7072 		ctl_set_invalid_field(/*ctsio*/ ctsio,
7073 				      /*sks_valid*/ 1,
7074 				      /*command*/ 1,
7075 				      /*field*/ 2,
7076 				      /*bit_valid*/ 0,
7077 				      /*bit*/ 0);
7078 		ctl_done((union ctl_io *)ctsio);
7079 		return (CTL_RETVAL_COMPLETE);
7080 	}
7081 
7082 	lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
7083 
7084 	ctsio->kern_data_ptr = malloc(sizeof(*data), M_CTL, M_WAITOK | M_ZERO);
7085 	data = (struct scsi_read_capacity_data_long *)ctsio->kern_data_ptr;
7086 
7087 	if (sizeof(*data) < alloc_len) {
7088 		ctsio->residual = alloc_len - sizeof(*data);
7089 		ctsio->kern_data_len = sizeof(*data);
7090 		ctsio->kern_total_len = sizeof(*data);
7091 	} else {
7092 		ctsio->residual = 0;
7093 		ctsio->kern_data_len = alloc_len;
7094 		ctsio->kern_total_len = alloc_len;
7095 	}
7096 	ctsio->kern_data_resid = 0;
7097 	ctsio->kern_rel_offset = 0;
7098 	ctsio->kern_sg_entries = 0;
7099 
7100 	scsi_u64to8b(lun->be_lun->maxlba, data->addr);
7101 	/* XXX KDM this may not be 512 bytes... */
7102 	scsi_ulto4b(lun->be_lun->blocksize, data->length);
7103 	data->prot_lbppbe = lun->be_lun->pblockexp & SRC16_LBPPBE;
7104 	scsi_ulto2b(lun->be_lun->pblockoff & SRC16_LALBA_A, data->lalba_lbp);
7105 	if (lun->be_lun->flags & CTL_LUN_FLAG_UNMAP)
7106 		data->lalba_lbp[0] |= SRC16_LBPME;
7107 
7108 	ctsio->scsi_status = SCSI_STATUS_OK;
7109 
7110 	ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
7111 	ctsio->be_move_done = ctl_config_move_done;
7112 	ctl_datamove((union ctl_io *)ctsio);
7113 
7114 	return (CTL_RETVAL_COMPLETE);
7115 }
7116 
7117 int
7118 ctl_service_action_in(struct ctl_scsiio *ctsio)
7119 {
7120 	struct scsi_service_action_in *cdb;
7121 	int retval;
7122 
7123 	CTL_DEBUG_PRINT(("ctl_service_action_in\n"));
7124 
7125 	cdb = (struct scsi_service_action_in *)ctsio->cdb;
7126 
7127 	retval = CTL_RETVAL_COMPLETE;
7128 
7129 	switch (cdb->service_action) {
7130 	case SRC16_SERVICE_ACTION:
7131 		retval = ctl_read_capacity_16(ctsio);
7132 		break;
7133 	default:
7134 		ctl_set_invalid_field(/*ctsio*/ ctsio,
7135 				      /*sks_valid*/ 1,
7136 				      /*command*/ 1,
7137 				      /*field*/ 1,
7138 				      /*bit_valid*/ 1,
7139 				      /*bit*/ 4);
7140 		ctl_done((union ctl_io *)ctsio);
7141 		break;
7142 	}
7143 
7144 	return (retval);
7145 }
7146 
7147 int
7148 ctl_maintenance_in(struct ctl_scsiio *ctsio)
7149 {
7150 	struct scsi_maintenance_in *cdb;
7151 	int retval;
7152 	int alloc_len, total_len = 0;
7153 	int num_target_port_groups, single;
7154 	struct ctl_lun *lun;
7155 	struct ctl_softc *softc;
7156 	struct scsi_target_group_data *rtg_ptr;
7157 	struct scsi_target_port_group_descriptor *tpg_desc_ptr1, *tpg_desc_ptr2;
7158 	struct scsi_target_port_descriptor  *tp_desc_ptr1_1, *tp_desc_ptr1_2,
7159 	                                    *tp_desc_ptr2_1, *tp_desc_ptr2_2;
7160 
7161 	CTL_DEBUG_PRINT(("ctl_maintenance_in\n"));
7162 
7163 	cdb = (struct scsi_maintenance_in *)ctsio->cdb;
7164 	softc = control_softc;
7165 	lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
7166 
7167 	retval = CTL_RETVAL_COMPLETE;
7168 
7169 	if ((cdb->byte2 & SERVICE_ACTION_MASK) != SA_RPRT_TRGT_GRP) {
7170 		ctl_set_invalid_field(/*ctsio*/ ctsio,
7171 				      /*sks_valid*/ 1,
7172 				      /*command*/ 1,
7173 				      /*field*/ 1,
7174 				      /*bit_valid*/ 1,
7175 				      /*bit*/ 4);
7176 		ctl_done((union ctl_io *)ctsio);
7177 		return(retval);
7178 	}
7179 
7180 	single = ctl_is_single;
7181 	if (single)
7182         	num_target_port_groups = NUM_TARGET_PORT_GROUPS - 1;
7183 	else
7184         	num_target_port_groups = NUM_TARGET_PORT_GROUPS;
7185 
7186 	total_len = sizeof(struct scsi_target_group_data) +
7187 		sizeof(struct scsi_target_port_group_descriptor) *
7188 		num_target_port_groups +
7189 		sizeof(struct scsi_target_port_descriptor) *
7190 		NUM_PORTS_PER_GRP * num_target_port_groups;
7191 
7192 	alloc_len = scsi_4btoul(cdb->length);
7193 
7194 	ctsio->kern_data_ptr = malloc(total_len, M_CTL, M_WAITOK | M_ZERO);
7195 
7196 	ctsio->kern_sg_entries = 0;
7197 
7198 	if (total_len < alloc_len) {
7199 		ctsio->residual = alloc_len - total_len;
7200 		ctsio->kern_data_len = total_len;
7201 		ctsio->kern_total_len = total_len;
7202 	} else {
7203 		ctsio->residual = 0;
7204 		ctsio->kern_data_len = alloc_len;
7205 		ctsio->kern_total_len = alloc_len;
7206 	}
7207 	ctsio->kern_data_resid = 0;
7208 	ctsio->kern_rel_offset = 0;
7209 
7210 	rtg_ptr = (struct scsi_target_group_data *)ctsio->kern_data_ptr;
7211 
7212 	tpg_desc_ptr1 = &rtg_ptr->groups[0];
7213 	tp_desc_ptr1_1 = &tpg_desc_ptr1->descriptors[0];
7214 	tp_desc_ptr1_2 = (struct scsi_target_port_descriptor *)
7215 	        &tp_desc_ptr1_1->desc_list[0];
7216 
7217 	if (single == 0) {
7218 		tpg_desc_ptr2 = (struct scsi_target_port_group_descriptor *)
7219 	                &tp_desc_ptr1_2->desc_list[0];
7220 		tp_desc_ptr2_1 = &tpg_desc_ptr2->descriptors[0];
7221 		tp_desc_ptr2_2 = (struct scsi_target_port_descriptor *)
7222 	        	&tp_desc_ptr2_1->desc_list[0];
7223         } else {
7224 		tpg_desc_ptr2 = NULL;
7225 		tp_desc_ptr2_1 = NULL;
7226 		tp_desc_ptr2_2 = NULL;
7227 	}
7228 
7229 	scsi_ulto4b(total_len - 4, rtg_ptr->length);
7230 	if (single == 0) {
7231         	if (ctsio->io_hdr.nexus.targ_port < CTL_MAX_PORTS) {
7232 			if (lun->flags & CTL_LUN_PRIMARY_SC) {
7233 				tpg_desc_ptr1->pref_state = TPG_PRIMARY;
7234 				tpg_desc_ptr2->pref_state =
7235 					TPG_ASYMMETRIC_ACCESS_NONOPTIMIZED;
7236 			} else {
7237 				tpg_desc_ptr1->pref_state =
7238 					TPG_ASYMMETRIC_ACCESS_NONOPTIMIZED;
7239 				tpg_desc_ptr2->pref_state = TPG_PRIMARY;
7240 			}
7241 		} else {
7242 			if (lun->flags & CTL_LUN_PRIMARY_SC) {
7243 				tpg_desc_ptr1->pref_state =
7244 					TPG_ASYMMETRIC_ACCESS_NONOPTIMIZED;
7245 				tpg_desc_ptr2->pref_state = TPG_PRIMARY;
7246 			} else {
7247 				tpg_desc_ptr1->pref_state = TPG_PRIMARY;
7248 				tpg_desc_ptr2->pref_state =
7249 					TPG_ASYMMETRIC_ACCESS_NONOPTIMIZED;
7250 			}
7251 		}
7252 	} else {
7253 		tpg_desc_ptr1->pref_state = TPG_PRIMARY;
7254 	}
7255 	tpg_desc_ptr1->support = 0;
7256 	tpg_desc_ptr1->target_port_group[1] = 1;
7257 	tpg_desc_ptr1->status = TPG_IMPLICIT;
7258 	tpg_desc_ptr1->target_port_count= NUM_PORTS_PER_GRP;
7259 
7260 	if (single == 0) {
7261 		tpg_desc_ptr2->support = 0;
7262 		tpg_desc_ptr2->target_port_group[1] = 2;
7263 		tpg_desc_ptr2->status = TPG_IMPLICIT;
7264 		tpg_desc_ptr2->target_port_count = NUM_PORTS_PER_GRP;
7265 
7266 		tp_desc_ptr1_1->relative_target_port_identifier[1] = 1;
7267 		tp_desc_ptr1_2->relative_target_port_identifier[1] = 2;
7268 
7269 		tp_desc_ptr2_1->relative_target_port_identifier[1] = 9;
7270 		tp_desc_ptr2_2->relative_target_port_identifier[1] = 10;
7271 	} else {
7272         	if (ctsio->io_hdr.nexus.targ_port < CTL_MAX_PORTS) {
7273 			tp_desc_ptr1_1->relative_target_port_identifier[1] = 1;
7274 			tp_desc_ptr1_2->relative_target_port_identifier[1] = 2;
7275 		} else {
7276 			tp_desc_ptr1_1->relative_target_port_identifier[1] = 9;
7277 			tp_desc_ptr1_2->relative_target_port_identifier[1] = 10;
7278 		}
7279 	}
7280 
7281 	ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
7282 	ctsio->be_move_done = ctl_config_move_done;
7283 
7284 	CTL_DEBUG_PRINT(("buf = %x %x %x %x %x %x %x %x\n",
7285 			 ctsio->kern_data_ptr[0], ctsio->kern_data_ptr[1],
7286 			 ctsio->kern_data_ptr[2], ctsio->kern_data_ptr[3],
7287 			 ctsio->kern_data_ptr[4], ctsio->kern_data_ptr[5],
7288 			 ctsio->kern_data_ptr[6], ctsio->kern_data_ptr[7]));
7289 
7290 	ctl_datamove((union ctl_io *)ctsio);
7291 	return(retval);
7292 }
7293 
7294 int
7295 ctl_persistent_reserve_in(struct ctl_scsiio *ctsio)
7296 {
7297 	struct scsi_per_res_in *cdb;
7298 	int alloc_len, total_len = 0;
7299 	/* struct scsi_per_res_in_rsrv in_data; */
7300 	struct ctl_lun *lun;
7301 	struct ctl_softc *softc;
7302 
7303 	CTL_DEBUG_PRINT(("ctl_persistent_reserve_in\n"));
7304 
7305 	softc = control_softc;
7306 
7307 	cdb = (struct scsi_per_res_in *)ctsio->cdb;
7308 
7309 	alloc_len = scsi_2btoul(cdb->length);
7310 
7311 	lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
7312 
7313 retry:
7314 	mtx_lock(&lun->lun_lock);
7315 	switch (cdb->action) {
7316 	case SPRI_RK: /* read keys */
7317 		total_len = sizeof(struct scsi_per_res_in_keys) +
7318 			lun->pr_key_count *
7319 			sizeof(struct scsi_per_res_key);
7320 		break;
7321 	case SPRI_RR: /* read reservation */
7322 		if (lun->flags & CTL_LUN_PR_RESERVED)
7323 			total_len = sizeof(struct scsi_per_res_in_rsrv);
7324 		else
7325 			total_len = sizeof(struct scsi_per_res_in_header);
7326 		break;
7327 	case SPRI_RC: /* report capabilities */
7328 		total_len = sizeof(struct scsi_per_res_cap);
7329 		break;
7330 	case SPRI_RS: /* read full status */
7331 	default:
7332 		mtx_unlock(&lun->lun_lock);
7333 		ctl_set_invalid_field(ctsio,
7334 				      /*sks_valid*/ 1,
7335 				      /*command*/ 1,
7336 				      /*field*/ 1,
7337 				      /*bit_valid*/ 1,
7338 				      /*bit*/ 0);
7339 		ctl_done((union ctl_io *)ctsio);
7340 		return (CTL_RETVAL_COMPLETE);
7341 		break; /* NOTREACHED */
7342 	}
7343 	mtx_unlock(&lun->lun_lock);
7344 
7345 	ctsio->kern_data_ptr = malloc(total_len, M_CTL, M_WAITOK | M_ZERO);
7346 
7347 	if (total_len < alloc_len) {
7348 		ctsio->residual = alloc_len - total_len;
7349 		ctsio->kern_data_len = total_len;
7350 		ctsio->kern_total_len = total_len;
7351 	} else {
7352 		ctsio->residual = 0;
7353 		ctsio->kern_data_len = alloc_len;
7354 		ctsio->kern_total_len = alloc_len;
7355 	}
7356 
7357 	ctsio->kern_data_resid = 0;
7358 	ctsio->kern_rel_offset = 0;
7359 	ctsio->kern_sg_entries = 0;
7360 
7361 	mtx_lock(&lun->lun_lock);
7362 	switch (cdb->action) {
7363 	case SPRI_RK: { // read keys
7364         struct scsi_per_res_in_keys *res_keys;
7365 		int i, key_count;
7366 
7367 		res_keys = (struct scsi_per_res_in_keys*)ctsio->kern_data_ptr;
7368 
7369 		/*
7370 		 * We had to drop the lock to allocate our buffer, which
7371 		 * leaves time for someone to come in with another
7372 		 * persistent reservation.  (That is unlikely, though,
7373 		 * since this should be the only persistent reservation
7374 		 * command active right now.)
7375 		 */
7376 		if (total_len != (sizeof(struct scsi_per_res_in_keys) +
7377 		    (lun->pr_key_count *
7378 		     sizeof(struct scsi_per_res_key)))){
7379 			mtx_unlock(&lun->lun_lock);
7380 			free(ctsio->kern_data_ptr, M_CTL);
7381 			printf("%s: reservation length changed, retrying\n",
7382 			       __func__);
7383 			goto retry;
7384 		}
7385 
7386 		scsi_ulto4b(lun->PRGeneration, res_keys->header.generation);
7387 
7388 		scsi_ulto4b(sizeof(struct scsi_per_res_key) *
7389 			     lun->pr_key_count, res_keys->header.length);
7390 
7391 		for (i = 0, key_count = 0; i < 2*CTL_MAX_INITIATORS; i++) {
7392 			if (!lun->per_res[i].registered)
7393 				continue;
7394 
7395 			/*
7396 			 * We used lun->pr_key_count to calculate the
7397 			 * size to allocate.  If it turns out the number of
7398 			 * initiators with the registered flag set is
7399 			 * larger than that (i.e. they haven't been kept in
7400 			 * sync), we've got a problem.
7401 			 */
7402 			if (key_count >= lun->pr_key_count) {
7403 #ifdef NEEDTOPORT
7404 				csevent_log(CSC_CTL | CSC_SHELF_SW |
7405 					    CTL_PR_ERROR,
7406 					    csevent_LogType_Fault,
7407 					    csevent_AlertLevel_Yellow,
7408 					    csevent_FRU_ShelfController,
7409 					    csevent_FRU_Firmware,
7410 				        csevent_FRU_Unknown,
7411 					    "registered keys %d >= key "
7412 					    "count %d", key_count,
7413 					    lun->pr_key_count);
7414 #endif
7415 				key_count++;
7416 				continue;
7417 			}
7418 			memcpy(res_keys->keys[key_count].key,
7419 			       lun->per_res[i].res_key.key,
7420 			       ctl_min(sizeof(res_keys->keys[key_count].key),
7421 			       sizeof(lun->per_res[i].res_key)));
7422 			key_count++;
7423 		}
7424 		break;
7425 	}
7426 	case SPRI_RR: { // read reservation
7427 		struct scsi_per_res_in_rsrv *res;
7428 		int tmp_len, header_only;
7429 
7430 		res = (struct scsi_per_res_in_rsrv *)ctsio->kern_data_ptr;
7431 
7432 		scsi_ulto4b(lun->PRGeneration, res->header.generation);
7433 
7434 		if (lun->flags & CTL_LUN_PR_RESERVED)
7435 		{
7436 			tmp_len = sizeof(struct scsi_per_res_in_rsrv);
7437 			scsi_ulto4b(sizeof(struct scsi_per_res_in_rsrv_data),
7438 				    res->header.length);
7439 			header_only = 0;
7440 		} else {
7441 			tmp_len = sizeof(struct scsi_per_res_in_header);
7442 			scsi_ulto4b(0, res->header.length);
7443 			header_only = 1;
7444 		}
7445 
7446 		/*
7447 		 * We had to drop the lock to allocate our buffer, which
7448 		 * leaves time for someone to come in with another
7449 		 * persistent reservation.  (That is unlikely, though,
7450 		 * since this should be the only persistent reservation
7451 		 * command active right now.)
7452 		 */
7453 		if (tmp_len != total_len) {
7454 			mtx_unlock(&lun->lun_lock);
7455 			free(ctsio->kern_data_ptr, M_CTL);
7456 			printf("%s: reservation status changed, retrying\n",
7457 			       __func__);
7458 			goto retry;
7459 		}
7460 
7461 		/*
7462 		 * No reservation held, so we're done.
7463 		 */
7464 		if (header_only != 0)
7465 			break;
7466 
7467 		/*
7468 		 * If the registration is an All Registrants type, the key
7469 		 * is 0, since it doesn't really matter.
7470 		 */
7471 		if (lun->pr_res_idx != CTL_PR_ALL_REGISTRANTS) {
7472 			memcpy(res->data.reservation,
7473 			       &lun->per_res[lun->pr_res_idx].res_key,
7474 			       sizeof(struct scsi_per_res_key));
7475 		}
7476 		res->data.scopetype = lun->res_type;
7477 		break;
7478 	}
7479 	case SPRI_RC:     //report capabilities
7480 	{
7481 		struct scsi_per_res_cap *res_cap;
7482 		uint16_t type_mask;
7483 
7484 		res_cap = (struct scsi_per_res_cap *)ctsio->kern_data_ptr;
7485 		scsi_ulto2b(sizeof(*res_cap), res_cap->length);
7486 		res_cap->flags2 |= SPRI_TMV | SPRI_ALLOW_3;
7487 		type_mask = SPRI_TM_WR_EX_AR |
7488 			    SPRI_TM_EX_AC_RO |
7489 			    SPRI_TM_WR_EX_RO |
7490 			    SPRI_TM_EX_AC |
7491 			    SPRI_TM_WR_EX |
7492 			    SPRI_TM_EX_AC_AR;
7493 		scsi_ulto2b(type_mask, res_cap->type_mask);
7494 		break;
7495 	}
7496 	case SPRI_RS: //read full status
7497 	default:
7498 		/*
7499 		 * This is a bug, because we just checked for this above,
7500 		 * and should have returned an error.
7501 		 */
7502 		panic("Invalid PR type %x", cdb->action);
7503 		break; /* NOTREACHED */
7504 	}
7505 	mtx_unlock(&lun->lun_lock);
7506 
7507 	ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
7508 	ctsio->be_move_done = ctl_config_move_done;
7509 
7510 	CTL_DEBUG_PRINT(("buf = %x %x %x %x %x %x %x %x\n",
7511 			 ctsio->kern_data_ptr[0], ctsio->kern_data_ptr[1],
7512 			 ctsio->kern_data_ptr[2], ctsio->kern_data_ptr[3],
7513 			 ctsio->kern_data_ptr[4], ctsio->kern_data_ptr[5],
7514 			 ctsio->kern_data_ptr[6], ctsio->kern_data_ptr[7]));
7515 
7516 	ctl_datamove((union ctl_io *)ctsio);
7517 
7518 	return (CTL_RETVAL_COMPLETE);
7519 }
7520 
7521 /*
7522  * Returns 0 if ctl_persistent_reserve_out() should continue, non-zero if
7523  * it should return.
7524  */
7525 static int
7526 ctl_pro_preempt(struct ctl_softc *softc, struct ctl_lun *lun, uint64_t res_key,
7527 		uint64_t sa_res_key, uint8_t type, uint32_t residx,
7528 		struct ctl_scsiio *ctsio, struct scsi_per_res_out *cdb,
7529 		struct scsi_per_res_out_parms* param)
7530 {
7531 	union ctl_ha_msg persis_io;
7532 	int retval, i;
7533 	int isc_retval;
7534 
7535 	retval = 0;
7536 
7537 	mtx_lock(&lun->lun_lock);
7538 	if (sa_res_key == 0) {
7539 		if (lun->pr_res_idx == CTL_PR_ALL_REGISTRANTS) {
7540 			/* validate scope and type */
7541 			if ((cdb->scope_type & SPR_SCOPE_MASK) !=
7542 			     SPR_LU_SCOPE) {
7543 				mtx_unlock(&lun->lun_lock);
7544 				ctl_set_invalid_field(/*ctsio*/ ctsio,
7545 						      /*sks_valid*/ 1,
7546 						      /*command*/ 1,
7547 						      /*field*/ 2,
7548 						      /*bit_valid*/ 1,
7549 						      /*bit*/ 4);
7550 				ctl_done((union ctl_io *)ctsio);
7551 				return (1);
7552 			}
7553 
7554 		        if (type>8 || type==2 || type==4 || type==0) {
7555 				mtx_unlock(&lun->lun_lock);
7556 				ctl_set_invalid_field(/*ctsio*/ ctsio,
7557        	           				      /*sks_valid*/ 1,
7558 						      /*command*/ 1,
7559 						      /*field*/ 2,
7560 						      /*bit_valid*/ 1,
7561 						      /*bit*/ 0);
7562 				ctl_done((union ctl_io *)ctsio);
7563 				return (1);
7564 		        }
7565 
7566 			/* temporarily unregister this nexus */
7567 			lun->per_res[residx].registered = 0;
7568 
7569 			/*
7570 			 * Unregister everybody else and build UA for
7571 			 * them
7572 			 */
7573 			for(i=0; i < 2*CTL_MAX_INITIATORS; i++) {
7574 				if (lun->per_res[i].registered == 0)
7575 					continue;
7576 
7577 				if (!persis_offset
7578 				 && i <CTL_MAX_INITIATORS)
7579 					lun->pending_sense[i].ua_pending |=
7580 						CTL_UA_REG_PREEMPT;
7581 				else if (persis_offset
7582 				      && i >= persis_offset)
7583 					lun->pending_sense[i-persis_offset
7584 						].ua_pending |=
7585 						CTL_UA_REG_PREEMPT;
7586 				lun->per_res[i].registered = 0;
7587 				memset(&lun->per_res[i].res_key, 0,
7588 				       sizeof(struct scsi_per_res_key));
7589 			}
7590 			lun->per_res[residx].registered = 1;
7591 			lun->pr_key_count = 1;
7592 			lun->res_type = type;
7593 			if (lun->res_type != SPR_TYPE_WR_EX_AR
7594 			 && lun->res_type != SPR_TYPE_EX_AC_AR)
7595 				lun->pr_res_idx = residx;
7596 
7597 			/* send msg to other side */
7598 			persis_io.hdr.nexus = ctsio->io_hdr.nexus;
7599 			persis_io.hdr.msg_type = CTL_MSG_PERS_ACTION;
7600 			persis_io.pr.pr_info.action = CTL_PR_PREEMPT;
7601 			persis_io.pr.pr_info.residx = lun->pr_res_idx;
7602 			persis_io.pr.pr_info.res_type = type;
7603 			memcpy(persis_io.pr.pr_info.sa_res_key,
7604 			       param->serv_act_res_key,
7605 			       sizeof(param->serv_act_res_key));
7606 			if ((isc_retval=ctl_ha_msg_send(CTL_HA_CHAN_CTL,
7607 			     &persis_io, sizeof(persis_io), 0)) >
7608 			     CTL_HA_STATUS_SUCCESS) {
7609 				printf("CTL:Persis Out error returned "
7610 				       "from ctl_ha_msg_send %d\n",
7611 				       isc_retval);
7612 			}
7613 		} else {
7614 			/* not all registrants */
7615 			mtx_unlock(&lun->lun_lock);
7616 			free(ctsio->kern_data_ptr, M_CTL);
7617 			ctl_set_invalid_field(ctsio,
7618 					      /*sks_valid*/ 1,
7619 					      /*command*/ 0,
7620 					      /*field*/ 8,
7621 					      /*bit_valid*/ 0,
7622 					      /*bit*/ 0);
7623 			ctl_done((union ctl_io *)ctsio);
7624 			return (1);
7625 		}
7626 	} else if (lun->pr_res_idx == CTL_PR_ALL_REGISTRANTS
7627 		|| !(lun->flags & CTL_LUN_PR_RESERVED)) {
7628 		int found = 0;
7629 
7630 		if (res_key == sa_res_key) {
7631 			/* special case */
7632 			/*
7633 			 * The spec implies this is not good but doesn't
7634 			 * say what to do. There are two choices either
7635 			 * generate a res conflict or check condition
7636 			 * with illegal field in parameter data. Since
7637 			 * that is what is done when the sa_res_key is
7638 			 * zero I'll take that approach since this has
7639 			 * to do with the sa_res_key.
7640 			 */
7641 			mtx_unlock(&lun->lun_lock);
7642 			free(ctsio->kern_data_ptr, M_CTL);
7643 			ctl_set_invalid_field(ctsio,
7644 					      /*sks_valid*/ 1,
7645 					      /*command*/ 0,
7646 					      /*field*/ 8,
7647 					      /*bit_valid*/ 0,
7648 					      /*bit*/ 0);
7649 			ctl_done((union ctl_io *)ctsio);
7650 			return (1);
7651 		}
7652 
7653 		for (i=0; i < 2*CTL_MAX_INITIATORS; i++) {
7654 			if (lun->per_res[i].registered
7655 			 && memcmp(param->serv_act_res_key,
7656 			    lun->per_res[i].res_key.key,
7657 			    sizeof(struct scsi_per_res_key)) != 0)
7658 				continue;
7659 
7660 			found = 1;
7661 			lun->per_res[i].registered = 0;
7662 			memset(&lun->per_res[i].res_key, 0,
7663 			       sizeof(struct scsi_per_res_key));
7664 			lun->pr_key_count--;
7665 
7666 			if (!persis_offset
7667 			 && i < CTL_MAX_INITIATORS)
7668 				lun->pending_sense[i].ua_pending |=
7669 					CTL_UA_REG_PREEMPT;
7670 			else if (persis_offset
7671 			      && i >= persis_offset)
7672 				lun->pending_sense[i-persis_offset].ua_pending|=
7673 					CTL_UA_REG_PREEMPT;
7674 		}
7675 		if (!found) {
7676 			mtx_unlock(&lun->lun_lock);
7677 			free(ctsio->kern_data_ptr, M_CTL);
7678 			ctl_set_reservation_conflict(ctsio);
7679 			ctl_done((union ctl_io *)ctsio);
7680 			return (CTL_RETVAL_COMPLETE);
7681 		}
7682 		/* send msg to other side */
7683 		persis_io.hdr.nexus = ctsio->io_hdr.nexus;
7684 		persis_io.hdr.msg_type = CTL_MSG_PERS_ACTION;
7685 		persis_io.pr.pr_info.action = CTL_PR_PREEMPT;
7686 		persis_io.pr.pr_info.residx = lun->pr_res_idx;
7687 		persis_io.pr.pr_info.res_type = type;
7688 		memcpy(persis_io.pr.pr_info.sa_res_key,
7689 		       param->serv_act_res_key,
7690 		       sizeof(param->serv_act_res_key));
7691 		if ((isc_retval=ctl_ha_msg_send(CTL_HA_CHAN_CTL,
7692 		     &persis_io, sizeof(persis_io), 0)) >
7693 		     CTL_HA_STATUS_SUCCESS) {
7694 			printf("CTL:Persis Out error returned from "
7695 			       "ctl_ha_msg_send %d\n", isc_retval);
7696 		}
7697 	} else {
7698 		/* Reserved but not all registrants */
7699 		/* sa_res_key is res holder */
7700 		if (memcmp(param->serv_act_res_key,
7701                    lun->per_res[lun->pr_res_idx].res_key.key,
7702                    sizeof(struct scsi_per_res_key)) == 0) {
7703 			/* validate scope and type */
7704 			if ((cdb->scope_type & SPR_SCOPE_MASK) !=
7705 			     SPR_LU_SCOPE) {
7706 				mtx_unlock(&lun->lun_lock);
7707 				ctl_set_invalid_field(/*ctsio*/ ctsio,
7708 						      /*sks_valid*/ 1,
7709 						      /*command*/ 1,
7710 						      /*field*/ 2,
7711 						      /*bit_valid*/ 1,
7712 						      /*bit*/ 4);
7713 				ctl_done((union ctl_io *)ctsio);
7714 				return (1);
7715 			}
7716 
7717 			if (type>8 || type==2 || type==4 || type==0) {
7718 				mtx_unlock(&lun->lun_lock);
7719 				ctl_set_invalid_field(/*ctsio*/ ctsio,
7720 						      /*sks_valid*/ 1,
7721 						      /*command*/ 1,
7722 						      /*field*/ 2,
7723 						      /*bit_valid*/ 1,
7724 						      /*bit*/ 0);
7725 				ctl_done((union ctl_io *)ctsio);
7726 				return (1);
7727 			}
7728 
7729 			/*
7730 			 * Do the following:
7731 			 * if sa_res_key != res_key remove all
7732 			 * registrants w/sa_res_key and generate UA
7733 			 * for these registrants(Registrations
7734 			 * Preempted) if it wasn't an exclusive
7735 			 * reservation generate UA(Reservations
7736 			 * Preempted) for all other registered nexuses
7737 			 * if the type has changed. Establish the new
7738 			 * reservation and holder. If res_key and
7739 			 * sa_res_key are the same do the above
7740 			 * except don't unregister the res holder.
7741 			 */
7742 
7743 			/*
7744 			 * Temporarily unregister so it won't get
7745 			 * removed or UA generated
7746 			 */
7747 			lun->per_res[residx].registered = 0;
7748 			for(i=0; i < 2*CTL_MAX_INITIATORS; i++) {
7749 				if (lun->per_res[i].registered == 0)
7750 					continue;
7751 
7752 				if (memcmp(param->serv_act_res_key,
7753 				    lun->per_res[i].res_key.key,
7754 				    sizeof(struct scsi_per_res_key)) == 0) {
7755 					lun->per_res[i].registered = 0;
7756 					memset(&lun->per_res[i].res_key,
7757 					       0,
7758 					       sizeof(struct scsi_per_res_key));
7759 					lun->pr_key_count--;
7760 
7761 					if (!persis_offset
7762 					 && i < CTL_MAX_INITIATORS)
7763 						lun->pending_sense[i
7764 							].ua_pending |=
7765 							CTL_UA_REG_PREEMPT;
7766 					else if (persis_offset
7767 					      && i >= persis_offset)
7768 						lun->pending_sense[
7769 						  i-persis_offset].ua_pending |=
7770 						  CTL_UA_REG_PREEMPT;
7771 				} else if (type != lun->res_type
7772 					&& (lun->res_type == SPR_TYPE_WR_EX_RO
7773 					 || lun->res_type ==SPR_TYPE_EX_AC_RO)){
7774 						if (!persis_offset
7775 						 && i < CTL_MAX_INITIATORS)
7776 							lun->pending_sense[i
7777 							].ua_pending |=
7778 							CTL_UA_RES_RELEASE;
7779 						else if (persis_offset
7780 						      && i >= persis_offset)
7781 							lun->pending_sense[
7782 							i-persis_offset
7783 							].ua_pending |=
7784 							CTL_UA_RES_RELEASE;
7785 				}
7786 			}
7787 			lun->per_res[residx].registered = 1;
7788 			lun->res_type = type;
7789 			if (lun->res_type != SPR_TYPE_WR_EX_AR
7790 			 && lun->res_type != SPR_TYPE_EX_AC_AR)
7791 				lun->pr_res_idx = residx;
7792 			else
7793 				lun->pr_res_idx =
7794 					CTL_PR_ALL_REGISTRANTS;
7795 
7796 			persis_io.hdr.nexus = ctsio->io_hdr.nexus;
7797 			persis_io.hdr.msg_type = CTL_MSG_PERS_ACTION;
7798 			persis_io.pr.pr_info.action = CTL_PR_PREEMPT;
7799 			persis_io.pr.pr_info.residx = lun->pr_res_idx;
7800 			persis_io.pr.pr_info.res_type = type;
7801 			memcpy(persis_io.pr.pr_info.sa_res_key,
7802 			       param->serv_act_res_key,
7803 			       sizeof(param->serv_act_res_key));
7804 			if ((isc_retval=ctl_ha_msg_send(CTL_HA_CHAN_CTL,
7805 			     &persis_io, sizeof(persis_io), 0)) >
7806 			     CTL_HA_STATUS_SUCCESS) {
7807 				printf("CTL:Persis Out error returned "
7808 				       "from ctl_ha_msg_send %d\n",
7809 				       isc_retval);
7810 			}
7811 		} else {
7812 			/*
7813 			 * sa_res_key is not the res holder just
7814 			 * remove registrants
7815 			 */
7816 			int found=0;
7817 
7818 			for (i=0; i < 2*CTL_MAX_INITIATORS; i++) {
7819 				if (memcmp(param->serv_act_res_key,
7820 				    lun->per_res[i].res_key.key,
7821 				    sizeof(struct scsi_per_res_key)) != 0)
7822 					continue;
7823 
7824 				found = 1;
7825 				lun->per_res[i].registered = 0;
7826 				memset(&lun->per_res[i].res_key, 0,
7827 				       sizeof(struct scsi_per_res_key));
7828 				lun->pr_key_count--;
7829 
7830 				if (!persis_offset
7831 				 && i < CTL_MAX_INITIATORS)
7832 					lun->pending_sense[i].ua_pending |=
7833 						CTL_UA_REG_PREEMPT;
7834 				else if (persis_offset
7835 				      && i >= persis_offset)
7836 					lun->pending_sense[
7837 						i-persis_offset].ua_pending |=
7838 						CTL_UA_REG_PREEMPT;
7839 			}
7840 
7841 			if (!found) {
7842 				mtx_unlock(&lun->lun_lock);
7843 				free(ctsio->kern_data_ptr, M_CTL);
7844 				ctl_set_reservation_conflict(ctsio);
7845 				ctl_done((union ctl_io *)ctsio);
7846 		        	return (1);
7847 			}
7848 			persis_io.hdr.nexus = ctsio->io_hdr.nexus;
7849 			persis_io.hdr.msg_type = CTL_MSG_PERS_ACTION;
7850 			persis_io.pr.pr_info.action = CTL_PR_PREEMPT;
7851 			persis_io.pr.pr_info.residx = lun->pr_res_idx;
7852 			persis_io.pr.pr_info.res_type = type;
7853 			memcpy(persis_io.pr.pr_info.sa_res_key,
7854 			       param->serv_act_res_key,
7855 			       sizeof(param->serv_act_res_key));
7856 			if ((isc_retval=ctl_ha_msg_send(CTL_HA_CHAN_CTL,
7857 			     &persis_io, sizeof(persis_io), 0)) >
7858 			     CTL_HA_STATUS_SUCCESS) {
7859 				printf("CTL:Persis Out error returned "
7860 				       "from ctl_ha_msg_send %d\n",
7861 				isc_retval);
7862 			}
7863 		}
7864 	}
7865 
7866 	lun->PRGeneration++;
7867 	mtx_unlock(&lun->lun_lock);
7868 
7869 	return (retval);
7870 }
7871 
7872 static void
7873 ctl_pro_preempt_other(struct ctl_lun *lun, union ctl_ha_msg *msg)
7874 {
7875 	int i;
7876 
7877 	if (lun->pr_res_idx == CTL_PR_ALL_REGISTRANTS
7878 	 || lun->pr_res_idx == CTL_PR_NO_RESERVATION
7879 	 || memcmp(&lun->per_res[lun->pr_res_idx].res_key,
7880 		   msg->pr.pr_info.sa_res_key,
7881 		   sizeof(struct scsi_per_res_key)) != 0) {
7882 		uint64_t sa_res_key;
7883 		sa_res_key = scsi_8btou64(msg->pr.pr_info.sa_res_key);
7884 
7885 		if (sa_res_key == 0) {
7886 			/* temporarily unregister this nexus */
7887 			lun->per_res[msg->pr.pr_info.residx].registered = 0;
7888 
7889 			/*
7890 			 * Unregister everybody else and build UA for
7891 			 * them
7892 			 */
7893 			for(i=0; i < 2*CTL_MAX_INITIATORS; i++) {
7894 				if (lun->per_res[i].registered == 0)
7895 					continue;
7896 
7897 				if (!persis_offset
7898 				 && i < CTL_MAX_INITIATORS)
7899 					lun->pending_sense[i].ua_pending |=
7900 						CTL_UA_REG_PREEMPT;
7901 				else if (persis_offset && i >= persis_offset)
7902 					lun->pending_sense[i -
7903 						persis_offset].ua_pending |=
7904 						CTL_UA_REG_PREEMPT;
7905 				lun->per_res[i].registered = 0;
7906 				memset(&lun->per_res[i].res_key, 0,
7907 				       sizeof(struct scsi_per_res_key));
7908 			}
7909 
7910 			lun->per_res[msg->pr.pr_info.residx].registered = 1;
7911 			lun->pr_key_count = 1;
7912 			lun->res_type = msg->pr.pr_info.res_type;
7913 			if (lun->res_type != SPR_TYPE_WR_EX_AR
7914 			 && lun->res_type != SPR_TYPE_EX_AC_AR)
7915 				lun->pr_res_idx = msg->pr.pr_info.residx;
7916 		} else {
7917 		        for (i=0; i < 2*CTL_MAX_INITIATORS; i++) {
7918 				if (memcmp(msg->pr.pr_info.sa_res_key,
7919 		                   lun->per_res[i].res_key.key,
7920 		                   sizeof(struct scsi_per_res_key)) != 0)
7921 					continue;
7922 
7923 				lun->per_res[i].registered = 0;
7924 				memset(&lun->per_res[i].res_key, 0,
7925 				       sizeof(struct scsi_per_res_key));
7926 				lun->pr_key_count--;
7927 
7928 				if (!persis_offset
7929 				 && i < persis_offset)
7930 					lun->pending_sense[i].ua_pending |=
7931 						CTL_UA_REG_PREEMPT;
7932 				else if (persis_offset
7933 				      && i >= persis_offset)
7934 					lun->pending_sense[i -
7935 						persis_offset].ua_pending |=
7936 						CTL_UA_REG_PREEMPT;
7937 			}
7938 		}
7939 	} else {
7940 		/*
7941 		 * Temporarily unregister so it won't get removed
7942 		 * or UA generated
7943 		 */
7944 		lun->per_res[msg->pr.pr_info.residx].registered = 0;
7945 		for (i=0; i < 2*CTL_MAX_INITIATORS; i++) {
7946 			if (lun->per_res[i].registered == 0)
7947 				continue;
7948 
7949 			if (memcmp(msg->pr.pr_info.sa_res_key,
7950 	                   lun->per_res[i].res_key.key,
7951 	                   sizeof(struct scsi_per_res_key)) == 0) {
7952 				lun->per_res[i].registered = 0;
7953 				memset(&lun->per_res[i].res_key, 0,
7954 				       sizeof(struct scsi_per_res_key));
7955 				lun->pr_key_count--;
7956 				if (!persis_offset
7957 				 && i < CTL_MAX_INITIATORS)
7958 					lun->pending_sense[i].ua_pending |=
7959 						CTL_UA_REG_PREEMPT;
7960 				else if (persis_offset
7961 				      && i >= persis_offset)
7962 					lun->pending_sense[i -
7963 						persis_offset].ua_pending |=
7964 						CTL_UA_REG_PREEMPT;
7965 			} else if (msg->pr.pr_info.res_type != lun->res_type
7966 				&& (lun->res_type == SPR_TYPE_WR_EX_RO
7967 				 || lun->res_type == SPR_TYPE_EX_AC_RO)) {
7968 					if (!persis_offset
7969 					 && i < persis_offset)
7970 						lun->pending_sense[i
7971 							].ua_pending |=
7972 							CTL_UA_RES_RELEASE;
7973 					else if (persis_offset
7974 					      && i >= persis_offset)
7975 					lun->pending_sense[i -
7976 						persis_offset].ua_pending |=
7977 						CTL_UA_RES_RELEASE;
7978 			}
7979 		}
7980 		lun->per_res[msg->pr.pr_info.residx].registered = 1;
7981 		lun->res_type = msg->pr.pr_info.res_type;
7982 		if (lun->res_type != SPR_TYPE_WR_EX_AR
7983 		 && lun->res_type != SPR_TYPE_EX_AC_AR)
7984 			lun->pr_res_idx = msg->pr.pr_info.residx;
7985 		else
7986 			lun->pr_res_idx = CTL_PR_ALL_REGISTRANTS;
7987 	}
7988 	lun->PRGeneration++;
7989 
7990 }
7991 
7992 
7993 int
7994 ctl_persistent_reserve_out(struct ctl_scsiio *ctsio)
7995 {
7996 	int retval;
7997 	int isc_retval;
7998 	u_int32_t param_len;
7999 	struct scsi_per_res_out *cdb;
8000 	struct ctl_lun *lun;
8001 	struct scsi_per_res_out_parms* param;
8002 	struct ctl_softc *softc;
8003 	uint32_t residx;
8004 	uint64_t res_key, sa_res_key;
8005 	uint8_t type;
8006 	union ctl_ha_msg persis_io;
8007 	int    i;
8008 
8009 	CTL_DEBUG_PRINT(("ctl_persistent_reserve_out\n"));
8010 
8011 	retval = CTL_RETVAL_COMPLETE;
8012 
8013 	softc = control_softc;
8014 
8015 	cdb = (struct scsi_per_res_out *)ctsio->cdb;
8016 	lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
8017 
8018 	/*
8019 	 * We only support whole-LUN scope.  The scope & type are ignored for
8020 	 * register, register and ignore existing key and clear.
8021 	 * We sometimes ignore scope and type on preempts too!!
8022 	 * Verify reservation type here as well.
8023 	 */
8024 	type = cdb->scope_type & SPR_TYPE_MASK;
8025 	if ((cdb->action == SPRO_RESERVE)
8026 	 || (cdb->action == SPRO_RELEASE)) {
8027 		if ((cdb->scope_type & SPR_SCOPE_MASK) != SPR_LU_SCOPE) {
8028 			ctl_set_invalid_field(/*ctsio*/ ctsio,
8029 					      /*sks_valid*/ 1,
8030 					      /*command*/ 1,
8031 					      /*field*/ 2,
8032 					      /*bit_valid*/ 1,
8033 					      /*bit*/ 4);
8034 			ctl_done((union ctl_io *)ctsio);
8035 			return (CTL_RETVAL_COMPLETE);
8036 		}
8037 
8038 		if (type>8 || type==2 || type==4 || type==0) {
8039 			ctl_set_invalid_field(/*ctsio*/ ctsio,
8040 					      /*sks_valid*/ 1,
8041 					      /*command*/ 1,
8042 					      /*field*/ 2,
8043 					      /*bit_valid*/ 1,
8044 					      /*bit*/ 0);
8045 			ctl_done((union ctl_io *)ctsio);
8046 			return (CTL_RETVAL_COMPLETE);
8047 		}
8048 	}
8049 
8050 	switch (cdb->action & SPRO_ACTION_MASK) {
8051 	case SPRO_REGISTER:
8052 	case SPRO_RESERVE:
8053 	case SPRO_RELEASE:
8054 	case SPRO_CLEAR:
8055 	case SPRO_PREEMPT:
8056 	case SPRO_REG_IGNO:
8057 		break;
8058 	case SPRO_REG_MOVE:
8059 	case SPRO_PRE_ABO:
8060 	default:
8061 		ctl_set_invalid_field(/*ctsio*/ ctsio,
8062 				      /*sks_valid*/ 1,
8063 				      /*command*/ 1,
8064 				      /*field*/ 1,
8065 				      /*bit_valid*/ 1,
8066 				      /*bit*/ 0);
8067 		ctl_done((union ctl_io *)ctsio);
8068 		return (CTL_RETVAL_COMPLETE);
8069 		break; /* NOTREACHED */
8070 	}
8071 
8072 	param_len = scsi_4btoul(cdb->length);
8073 
8074 	if ((ctsio->io_hdr.flags & CTL_FLAG_ALLOCATED) == 0) {
8075 		ctsio->kern_data_ptr = malloc(param_len, M_CTL, M_WAITOK);
8076 		ctsio->kern_data_len = param_len;
8077 		ctsio->kern_total_len = param_len;
8078 		ctsio->kern_data_resid = 0;
8079 		ctsio->kern_rel_offset = 0;
8080 		ctsio->kern_sg_entries = 0;
8081 		ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
8082 		ctsio->be_move_done = ctl_config_move_done;
8083 		ctl_datamove((union ctl_io *)ctsio);
8084 
8085 		return (CTL_RETVAL_COMPLETE);
8086 	}
8087 
8088 	param = (struct scsi_per_res_out_parms *)ctsio->kern_data_ptr;
8089 
8090 	residx = ctl_get_resindex(&ctsio->io_hdr.nexus);
8091 	res_key = scsi_8btou64(param->res_key.key);
8092 	sa_res_key = scsi_8btou64(param->serv_act_res_key);
8093 
8094 	/*
8095 	 * Validate the reservation key here except for SPRO_REG_IGNO
8096 	 * This must be done for all other service actions
8097 	 */
8098 	if ((cdb->action & SPRO_ACTION_MASK) != SPRO_REG_IGNO) {
8099 		mtx_lock(&lun->lun_lock);
8100 		if (lun->per_res[residx].registered) {
8101 		    if (memcmp(param->res_key.key,
8102 			       lun->per_res[residx].res_key.key,
8103 			       ctl_min(sizeof(param->res_key),
8104 			       sizeof(lun->per_res[residx].res_key))) != 0) {
8105 				/*
8106 				 * The current key passed in doesn't match
8107 				 * the one the initiator previously
8108 				 * registered.
8109 				 */
8110 				mtx_unlock(&lun->lun_lock);
8111 				free(ctsio->kern_data_ptr, M_CTL);
8112 				ctl_set_reservation_conflict(ctsio);
8113 				ctl_done((union ctl_io *)ctsio);
8114 				return (CTL_RETVAL_COMPLETE);
8115 			}
8116 		} else if ((cdb->action & SPRO_ACTION_MASK) != SPRO_REGISTER) {
8117 			/*
8118 			 * We are not registered
8119 			 */
8120 			mtx_unlock(&lun->lun_lock);
8121 			free(ctsio->kern_data_ptr, M_CTL);
8122 			ctl_set_reservation_conflict(ctsio);
8123 			ctl_done((union ctl_io *)ctsio);
8124 			return (CTL_RETVAL_COMPLETE);
8125 		} else if (res_key != 0) {
8126 			/*
8127 			 * We are not registered and trying to register but
8128 			 * the register key isn't zero.
8129 			 */
8130 			mtx_unlock(&lun->lun_lock);
8131 			free(ctsio->kern_data_ptr, M_CTL);
8132 			ctl_set_reservation_conflict(ctsio);
8133 			ctl_done((union ctl_io *)ctsio);
8134 			return (CTL_RETVAL_COMPLETE);
8135 		}
8136 		mtx_unlock(&lun->lun_lock);
8137 	}
8138 
8139 	switch (cdb->action & SPRO_ACTION_MASK) {
8140 	case SPRO_REGISTER:
8141 	case SPRO_REG_IGNO: {
8142 
8143 #if 0
8144 		printf("Registration received\n");
8145 #endif
8146 
8147 		/*
8148 		 * We don't support any of these options, as we report in
8149 		 * the read capabilities request (see
8150 		 * ctl_persistent_reserve_in(), above).
8151 		 */
8152 		if ((param->flags & SPR_SPEC_I_PT)
8153 		 || (param->flags & SPR_ALL_TG_PT)
8154 		 || (param->flags & SPR_APTPL)) {
8155 			int bit_ptr;
8156 
8157 			if (param->flags & SPR_APTPL)
8158 				bit_ptr = 0;
8159 			else if (param->flags & SPR_ALL_TG_PT)
8160 				bit_ptr = 2;
8161 			else /* SPR_SPEC_I_PT */
8162 				bit_ptr = 3;
8163 
8164 			free(ctsio->kern_data_ptr, M_CTL);
8165 			ctl_set_invalid_field(ctsio,
8166 					      /*sks_valid*/ 1,
8167 					      /*command*/ 0,
8168 					      /*field*/ 20,
8169 					      /*bit_valid*/ 1,
8170 					      /*bit*/ bit_ptr);
8171 			ctl_done((union ctl_io *)ctsio);
8172 			return (CTL_RETVAL_COMPLETE);
8173 		}
8174 
8175 		mtx_lock(&lun->lun_lock);
8176 
8177 		/*
8178 		 * The initiator wants to clear the
8179 		 * key/unregister.
8180 		 */
8181 		if (sa_res_key == 0) {
8182 			if ((res_key == 0
8183 			  && (cdb->action & SPRO_ACTION_MASK) == SPRO_REGISTER)
8184 			 || ((cdb->action & SPRO_ACTION_MASK) == SPRO_REG_IGNO
8185 			  && !lun->per_res[residx].registered)) {
8186 				mtx_unlock(&lun->lun_lock);
8187 				goto done;
8188 			}
8189 
8190 			lun->per_res[residx].registered = 0;
8191 			memset(&lun->per_res[residx].res_key,
8192 			       0, sizeof(lun->per_res[residx].res_key));
8193 			lun->pr_key_count--;
8194 
8195 			if (residx == lun->pr_res_idx) {
8196 				lun->flags &= ~CTL_LUN_PR_RESERVED;
8197 				lun->pr_res_idx = CTL_PR_NO_RESERVATION;
8198 
8199 				if ((lun->res_type == SPR_TYPE_WR_EX_RO
8200 				  || lun->res_type == SPR_TYPE_EX_AC_RO)
8201 				 && lun->pr_key_count) {
8202 					/*
8203 					 * If the reservation is a registrants
8204 					 * only type we need to generate a UA
8205 					 * for other registered inits.  The
8206 					 * sense code should be RESERVATIONS
8207 					 * RELEASED
8208 					 */
8209 
8210 					for (i = 0; i < CTL_MAX_INITIATORS;i++){
8211 						if (lun->per_res[
8212 						    i+persis_offset].registered
8213 						    == 0)
8214 							continue;
8215 						lun->pending_sense[i
8216 							].ua_pending |=
8217 							CTL_UA_RES_RELEASE;
8218 					}
8219 				}
8220 				lun->res_type = 0;
8221 			} else if (lun->pr_res_idx == CTL_PR_ALL_REGISTRANTS) {
8222 				if (lun->pr_key_count==0) {
8223 					lun->flags &= ~CTL_LUN_PR_RESERVED;
8224 					lun->res_type = 0;
8225 					lun->pr_res_idx = CTL_PR_NO_RESERVATION;
8226 				}
8227 			}
8228 			persis_io.hdr.nexus = ctsio->io_hdr.nexus;
8229 			persis_io.hdr.msg_type = CTL_MSG_PERS_ACTION;
8230 			persis_io.pr.pr_info.action = CTL_PR_UNREG_KEY;
8231 			persis_io.pr.pr_info.residx = residx;
8232 			if ((isc_retval = ctl_ha_msg_send(CTL_HA_CHAN_CTL,
8233 			     &persis_io, sizeof(persis_io), 0 )) >
8234 			     CTL_HA_STATUS_SUCCESS) {
8235 				printf("CTL:Persis Out error returned from "
8236 				       "ctl_ha_msg_send %d\n", isc_retval);
8237 			}
8238 		} else /* sa_res_key != 0 */ {
8239 
8240 			/*
8241 			 * If we aren't registered currently then increment
8242 			 * the key count and set the registered flag.
8243 			 */
8244 			if (!lun->per_res[residx].registered) {
8245 				lun->pr_key_count++;
8246 				lun->per_res[residx].registered = 1;
8247 			}
8248 
8249 			memcpy(&lun->per_res[residx].res_key,
8250 			       param->serv_act_res_key,
8251 			       ctl_min(sizeof(param->serv_act_res_key),
8252 			       sizeof(lun->per_res[residx].res_key)));
8253 
8254 			persis_io.hdr.nexus = ctsio->io_hdr.nexus;
8255 			persis_io.hdr.msg_type = CTL_MSG_PERS_ACTION;
8256 			persis_io.pr.pr_info.action = CTL_PR_REG_KEY;
8257 			persis_io.pr.pr_info.residx = residx;
8258 			memcpy(persis_io.pr.pr_info.sa_res_key,
8259 			       param->serv_act_res_key,
8260 			       sizeof(param->serv_act_res_key));
8261 			if ((isc_retval=ctl_ha_msg_send(CTL_HA_CHAN_CTL,
8262 			     &persis_io, sizeof(persis_io), 0)) >
8263 			     CTL_HA_STATUS_SUCCESS) {
8264 				printf("CTL:Persis Out error returned from "
8265 				       "ctl_ha_msg_send %d\n", isc_retval);
8266 			}
8267 		}
8268 		lun->PRGeneration++;
8269 		mtx_unlock(&lun->lun_lock);
8270 
8271 		break;
8272 	}
8273 	case SPRO_RESERVE:
8274 #if 0
8275                 printf("Reserve executed type %d\n", type);
8276 #endif
8277 		mtx_lock(&lun->lun_lock);
8278 		if (lun->flags & CTL_LUN_PR_RESERVED) {
8279 			/*
8280 			 * if this isn't the reservation holder and it's
8281 			 * not a "all registrants" type or if the type is
8282 			 * different then we have a conflict
8283 			 */
8284 			if ((lun->pr_res_idx != residx
8285 			  && lun->pr_res_idx != CTL_PR_ALL_REGISTRANTS)
8286 			 || lun->res_type != type) {
8287 				mtx_unlock(&lun->lun_lock);
8288 				free(ctsio->kern_data_ptr, M_CTL);
8289 				ctl_set_reservation_conflict(ctsio);
8290 				ctl_done((union ctl_io *)ctsio);
8291 				return (CTL_RETVAL_COMPLETE);
8292 			}
8293 			mtx_unlock(&lun->lun_lock);
8294 		} else /* create a reservation */ {
8295 			/*
8296 			 * If it's not an "all registrants" type record
8297 			 * reservation holder
8298 			 */
8299 			if (type != SPR_TYPE_WR_EX_AR
8300 			 && type != SPR_TYPE_EX_AC_AR)
8301 				lun->pr_res_idx = residx; /* Res holder */
8302 			else
8303 				lun->pr_res_idx = CTL_PR_ALL_REGISTRANTS;
8304 
8305 			lun->flags |= CTL_LUN_PR_RESERVED;
8306 			lun->res_type = type;
8307 
8308 			mtx_unlock(&lun->lun_lock);
8309 
8310 			/* send msg to other side */
8311 			persis_io.hdr.nexus = ctsio->io_hdr.nexus;
8312 			persis_io.hdr.msg_type = CTL_MSG_PERS_ACTION;
8313 			persis_io.pr.pr_info.action = CTL_PR_RESERVE;
8314 			persis_io.pr.pr_info.residx = lun->pr_res_idx;
8315 			persis_io.pr.pr_info.res_type = type;
8316 			if ((isc_retval=ctl_ha_msg_send(CTL_HA_CHAN_CTL,
8317 			     &persis_io, sizeof(persis_io), 0)) >
8318 			     CTL_HA_STATUS_SUCCESS) {
8319 				printf("CTL:Persis Out error returned from "
8320 				       "ctl_ha_msg_send %d\n", isc_retval);
8321 			}
8322 		}
8323 		break;
8324 
8325 	case SPRO_RELEASE:
8326 		mtx_lock(&lun->lun_lock);
8327 		if ((lun->flags & CTL_LUN_PR_RESERVED) == 0) {
8328 			/* No reservation exists return good status */
8329 			mtx_unlock(&lun->lun_lock);
8330 			goto done;
8331 		}
8332 		/*
8333 		 * Is this nexus a reservation holder?
8334 		 */
8335 		if (lun->pr_res_idx != residx
8336 		 && lun->pr_res_idx != CTL_PR_ALL_REGISTRANTS) {
8337 			/*
8338 			 * not a res holder return good status but
8339 			 * do nothing
8340 			 */
8341 			mtx_unlock(&lun->lun_lock);
8342 			goto done;
8343 		}
8344 
8345 		if (lun->res_type != type) {
8346 			mtx_unlock(&lun->lun_lock);
8347 			free(ctsio->kern_data_ptr, M_CTL);
8348 			ctl_set_illegal_pr_release(ctsio);
8349 			ctl_done((union ctl_io *)ctsio);
8350 			return (CTL_RETVAL_COMPLETE);
8351 		}
8352 
8353 		/* okay to release */
8354 		lun->flags &= ~CTL_LUN_PR_RESERVED;
8355 		lun->pr_res_idx = CTL_PR_NO_RESERVATION;
8356 		lun->res_type = 0;
8357 
8358 		/*
8359 		 * if this isn't an exclusive access
8360 		 * res generate UA for all other
8361 		 * registrants.
8362 		 */
8363 		if (type != SPR_TYPE_EX_AC
8364 		 && type != SPR_TYPE_WR_EX) {
8365 			/*
8366 			 * temporarily unregister so we don't generate UA
8367 			 */
8368 			lun->per_res[residx].registered = 0;
8369 
8370 			for (i = 0; i < CTL_MAX_INITIATORS; i++) {
8371 				if (lun->per_res[i+persis_offset].registered
8372 				    == 0)
8373 					continue;
8374 				lun->pending_sense[i].ua_pending |=
8375 					CTL_UA_RES_RELEASE;
8376 			}
8377 
8378 			lun->per_res[residx].registered = 1;
8379 		}
8380 		mtx_unlock(&lun->lun_lock);
8381 		/* Send msg to other side */
8382 		persis_io.hdr.nexus = ctsio->io_hdr.nexus;
8383 		persis_io.hdr.msg_type = CTL_MSG_PERS_ACTION;
8384 		persis_io.pr.pr_info.action = CTL_PR_RELEASE;
8385 		if ((isc_retval=ctl_ha_msg_send( CTL_HA_CHAN_CTL, &persis_io,
8386 		     sizeof(persis_io), 0)) > CTL_HA_STATUS_SUCCESS) {
8387 			printf("CTL:Persis Out error returned from "
8388 			       "ctl_ha_msg_send %d\n", isc_retval);
8389 		}
8390 		break;
8391 
8392 	case SPRO_CLEAR:
8393 		/* send msg to other side */
8394 
8395 		mtx_lock(&lun->lun_lock);
8396 		lun->flags &= ~CTL_LUN_PR_RESERVED;
8397 		lun->res_type = 0;
8398 		lun->pr_key_count = 0;
8399 		lun->pr_res_idx = CTL_PR_NO_RESERVATION;
8400 
8401 
8402 		memset(&lun->per_res[residx].res_key,
8403 		       0, sizeof(lun->per_res[residx].res_key));
8404 		lun->per_res[residx].registered = 0;
8405 
8406 		for (i=0; i < 2*CTL_MAX_INITIATORS; i++)
8407 			if (lun->per_res[i].registered) {
8408 				if (!persis_offset && i < CTL_MAX_INITIATORS)
8409 					lun->pending_sense[i].ua_pending |=
8410 						CTL_UA_RES_PREEMPT;
8411 				else if (persis_offset && i >= persis_offset)
8412 					lun->pending_sense[i-persis_offset
8413 					    ].ua_pending |= CTL_UA_RES_PREEMPT;
8414 
8415 				memset(&lun->per_res[i].res_key,
8416 				       0, sizeof(struct scsi_per_res_key));
8417 				lun->per_res[i].registered = 0;
8418 			}
8419 		lun->PRGeneration++;
8420 		mtx_unlock(&lun->lun_lock);
8421 		persis_io.hdr.nexus = ctsio->io_hdr.nexus;
8422 		persis_io.hdr.msg_type = CTL_MSG_PERS_ACTION;
8423 		persis_io.pr.pr_info.action = CTL_PR_CLEAR;
8424 		if ((isc_retval=ctl_ha_msg_send(CTL_HA_CHAN_CTL, &persis_io,
8425 		     sizeof(persis_io), 0)) > CTL_HA_STATUS_SUCCESS) {
8426 			printf("CTL:Persis Out error returned from "
8427 			       "ctl_ha_msg_send %d\n", isc_retval);
8428 		}
8429 		break;
8430 
8431 	case SPRO_PREEMPT: {
8432 		int nretval;
8433 
8434 		nretval = ctl_pro_preempt(softc, lun, res_key, sa_res_key, type,
8435 					  residx, ctsio, cdb, param);
8436 		if (nretval != 0)
8437 			return (CTL_RETVAL_COMPLETE);
8438 		break;
8439 	}
8440 	case SPRO_REG_MOVE:
8441 	case SPRO_PRE_ABO:
8442 	default:
8443 		free(ctsio->kern_data_ptr, M_CTL);
8444 		ctl_set_invalid_field(/*ctsio*/ ctsio,
8445 				      /*sks_valid*/ 1,
8446 				      /*command*/ 1,
8447 				      /*field*/ 1,
8448 				      /*bit_valid*/ 1,
8449 				      /*bit*/ 0);
8450 		ctl_done((union ctl_io *)ctsio);
8451 		return (CTL_RETVAL_COMPLETE);
8452 		break; /* NOTREACHED */
8453 	}
8454 
8455 done:
8456 	free(ctsio->kern_data_ptr, M_CTL);
8457 	ctl_set_success(ctsio);
8458 	ctl_done((union ctl_io *)ctsio);
8459 
8460 	return (retval);
8461 }
8462 
8463 /*
8464  * This routine is for handling a message from the other SC pertaining to
8465  * persistent reserve out. All the error checking will have been done
8466  * so only perorming the action need be done here to keep the two
8467  * in sync.
8468  */
8469 static void
8470 ctl_hndl_per_res_out_on_other_sc(union ctl_ha_msg *msg)
8471 {
8472 	struct ctl_lun *lun;
8473 	struct ctl_softc *softc;
8474 	int i;
8475 	uint32_t targ_lun;
8476 
8477 	softc = control_softc;
8478 
8479 	targ_lun = msg->hdr.nexus.targ_mapped_lun;
8480 	lun = softc->ctl_luns[targ_lun];
8481 	mtx_lock(&lun->lun_lock);
8482 	switch(msg->pr.pr_info.action) {
8483 	case CTL_PR_REG_KEY:
8484 		if (!lun->per_res[msg->pr.pr_info.residx].registered) {
8485 			lun->per_res[msg->pr.pr_info.residx].registered = 1;
8486 			lun->pr_key_count++;
8487 		}
8488 		lun->PRGeneration++;
8489 		memcpy(&lun->per_res[msg->pr.pr_info.residx].res_key,
8490 		       msg->pr.pr_info.sa_res_key,
8491 		       sizeof(struct scsi_per_res_key));
8492 		break;
8493 
8494 	case CTL_PR_UNREG_KEY:
8495 		lun->per_res[msg->pr.pr_info.residx].registered = 0;
8496 		memset(&lun->per_res[msg->pr.pr_info.residx].res_key,
8497 		       0, sizeof(struct scsi_per_res_key));
8498 		lun->pr_key_count--;
8499 
8500 		/* XXX Need to see if the reservation has been released */
8501 		/* if so do we need to generate UA? */
8502 		if (msg->pr.pr_info.residx == lun->pr_res_idx) {
8503 			lun->flags &= ~CTL_LUN_PR_RESERVED;
8504 			lun->pr_res_idx = CTL_PR_NO_RESERVATION;
8505 
8506 			if ((lun->res_type == SPR_TYPE_WR_EX_RO
8507 			  || lun->res_type == SPR_TYPE_EX_AC_RO)
8508 			 && lun->pr_key_count) {
8509 				/*
8510 				 * If the reservation is a registrants
8511 				 * only type we need to generate a UA
8512 				 * for other registered inits.  The
8513 				 * sense code should be RESERVATIONS
8514 				 * RELEASED
8515 				 */
8516 
8517 				for (i = 0; i < CTL_MAX_INITIATORS; i++) {
8518 					if (lun->per_res[i+
8519 					    persis_offset].registered == 0)
8520 						continue;
8521 
8522 					lun->pending_sense[i
8523 						].ua_pending |=
8524 						CTL_UA_RES_RELEASE;
8525 				}
8526 			}
8527 			lun->res_type = 0;
8528 		} else if (lun->pr_res_idx == CTL_PR_ALL_REGISTRANTS) {
8529 			if (lun->pr_key_count==0) {
8530 				lun->flags &= ~CTL_LUN_PR_RESERVED;
8531 				lun->res_type = 0;
8532 				lun->pr_res_idx = CTL_PR_NO_RESERVATION;
8533 			}
8534 		}
8535 		lun->PRGeneration++;
8536 		break;
8537 
8538 	case CTL_PR_RESERVE:
8539 		lun->flags |= CTL_LUN_PR_RESERVED;
8540 		lun->res_type = msg->pr.pr_info.res_type;
8541 		lun->pr_res_idx = msg->pr.pr_info.residx;
8542 
8543 		break;
8544 
8545 	case CTL_PR_RELEASE:
8546 		/*
8547 		 * if this isn't an exclusive access res generate UA for all
8548 		 * other registrants.
8549 		 */
8550 		if (lun->res_type != SPR_TYPE_EX_AC
8551 		 && lun->res_type != SPR_TYPE_WR_EX) {
8552 			for (i = 0; i < CTL_MAX_INITIATORS; i++)
8553 				if (lun->per_res[i+persis_offset].registered)
8554 					lun->pending_sense[i].ua_pending |=
8555 						CTL_UA_RES_RELEASE;
8556 		}
8557 
8558 		lun->flags &= ~CTL_LUN_PR_RESERVED;
8559 		lun->pr_res_idx = CTL_PR_NO_RESERVATION;
8560 		lun->res_type = 0;
8561 		break;
8562 
8563 	case CTL_PR_PREEMPT:
8564 		ctl_pro_preempt_other(lun, msg);
8565 		break;
8566 	case CTL_PR_CLEAR:
8567 		lun->flags &= ~CTL_LUN_PR_RESERVED;
8568 		lun->res_type = 0;
8569 		lun->pr_key_count = 0;
8570 		lun->pr_res_idx = CTL_PR_NO_RESERVATION;
8571 
8572 		for (i=0; i < 2*CTL_MAX_INITIATORS; i++) {
8573 			if (lun->per_res[i].registered == 0)
8574 				continue;
8575 			if (!persis_offset
8576 			 && i < CTL_MAX_INITIATORS)
8577 				lun->pending_sense[i].ua_pending |=
8578 					CTL_UA_RES_PREEMPT;
8579 			else if (persis_offset
8580 			      && i >= persis_offset)
8581    				lun->pending_sense[i-persis_offset].ua_pending|=
8582 					CTL_UA_RES_PREEMPT;
8583 			memset(&lun->per_res[i].res_key, 0,
8584 			       sizeof(struct scsi_per_res_key));
8585 			lun->per_res[i].registered = 0;
8586 		}
8587 		lun->PRGeneration++;
8588 		break;
8589 	}
8590 
8591 	mtx_unlock(&lun->lun_lock);
8592 }
8593 
8594 int
8595 ctl_read_write(struct ctl_scsiio *ctsio)
8596 {
8597 	struct ctl_lun *lun;
8598 	struct ctl_lba_len_flags *lbalen;
8599 	uint64_t lba;
8600 	uint32_t num_blocks;
8601 	int reladdr, fua, dpo, ebp;
8602 	int retval;
8603 	int isread;
8604 
8605 	lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
8606 
8607 	CTL_DEBUG_PRINT(("ctl_read_write: command: %#x\n", ctsio->cdb[0]));
8608 
8609 	reladdr = 0;
8610 	fua = 0;
8611 	dpo = 0;
8612 	ebp = 0;
8613 
8614 	retval = CTL_RETVAL_COMPLETE;
8615 
8616 	isread = ctsio->cdb[0] == READ_6  || ctsio->cdb[0] == READ_10
8617 	      || ctsio->cdb[0] == READ_12 || ctsio->cdb[0] == READ_16;
8618 	if (lun->flags & CTL_LUN_PR_RESERVED && isread) {
8619 		uint32_t residx;
8620 
8621 		/*
8622 		 * XXX KDM need a lock here.
8623 		 */
8624 		residx = ctl_get_resindex(&ctsio->io_hdr.nexus);
8625 		if ((lun->res_type == SPR_TYPE_EX_AC
8626 		  && residx != lun->pr_res_idx)
8627 		 || ((lun->res_type == SPR_TYPE_EX_AC_RO
8628 		   || lun->res_type == SPR_TYPE_EX_AC_AR)
8629 		  && !lun->per_res[residx].registered)) {
8630 			ctl_set_reservation_conflict(ctsio);
8631 			ctl_done((union ctl_io *)ctsio);
8632 			return (CTL_RETVAL_COMPLETE);
8633 	        }
8634 	}
8635 
8636 	switch (ctsio->cdb[0]) {
8637 	case READ_6:
8638 	case WRITE_6: {
8639 		struct scsi_rw_6 *cdb;
8640 
8641 		cdb = (struct scsi_rw_6 *)ctsio->cdb;
8642 
8643 		lba = scsi_3btoul(cdb->addr);
8644 		/* only 5 bits are valid in the most significant address byte */
8645 		lba &= 0x1fffff;
8646 		num_blocks = cdb->length;
8647 		/*
8648 		 * This is correct according to SBC-2.
8649 		 */
8650 		if (num_blocks == 0)
8651 			num_blocks = 256;
8652 		break;
8653 	}
8654 	case READ_10:
8655 	case WRITE_10: {
8656 		struct scsi_rw_10 *cdb;
8657 
8658 		cdb = (struct scsi_rw_10 *)ctsio->cdb;
8659 
8660 		if (cdb->byte2 & SRW10_RELADDR)
8661 			reladdr = 1;
8662 		if (cdb->byte2 & SRW10_FUA)
8663 			fua = 1;
8664 		if (cdb->byte2 & SRW10_DPO)
8665 			dpo = 1;
8666 
8667 		if ((cdb->opcode == WRITE_10)
8668 		 && (cdb->byte2 & SRW10_EBP))
8669 			ebp = 1;
8670 
8671 		lba = scsi_4btoul(cdb->addr);
8672 		num_blocks = scsi_2btoul(cdb->length);
8673 		break;
8674 	}
8675 	case WRITE_VERIFY_10: {
8676 		struct scsi_write_verify_10 *cdb;
8677 
8678 		cdb = (struct scsi_write_verify_10 *)ctsio->cdb;
8679 
8680 		/*
8681 		 * XXX KDM we should do actual write verify support at some
8682 		 * point.  This is obviously fake, we're just translating
8683 		 * things to a write.  So we don't even bother checking the
8684 		 * BYTCHK field, since we don't do any verification.  If
8685 		 * the user asks for it, we'll just pretend we did it.
8686 		 */
8687 		if (cdb->byte2 & SWV_DPO)
8688 			dpo = 1;
8689 
8690 		lba = scsi_4btoul(cdb->addr);
8691 		num_blocks = scsi_2btoul(cdb->length);
8692 		break;
8693 	}
8694 	case READ_12:
8695 	case WRITE_12: {
8696 		struct scsi_rw_12 *cdb;
8697 
8698 		cdb = (struct scsi_rw_12 *)ctsio->cdb;
8699 
8700 		if (cdb->byte2 & SRW12_RELADDR)
8701 			reladdr = 1;
8702 		if (cdb->byte2 & SRW12_FUA)
8703 			fua = 1;
8704 		if (cdb->byte2 & SRW12_DPO)
8705 			dpo = 1;
8706 		lba = scsi_4btoul(cdb->addr);
8707 		num_blocks = scsi_4btoul(cdb->length);
8708 		break;
8709 	}
8710 	case WRITE_VERIFY_12: {
8711 		struct scsi_write_verify_12 *cdb;
8712 
8713 		cdb = (struct scsi_write_verify_12 *)ctsio->cdb;
8714 
8715 		if (cdb->byte2 & SWV_DPO)
8716 			dpo = 1;
8717 
8718 		lba = scsi_4btoul(cdb->addr);
8719 		num_blocks = scsi_4btoul(cdb->length);
8720 
8721 		break;
8722 	}
8723 	case READ_16:
8724 	case WRITE_16: {
8725 		struct scsi_rw_16 *cdb;
8726 
8727 		cdb = (struct scsi_rw_16 *)ctsio->cdb;
8728 
8729 		if (cdb->byte2 & SRW12_RELADDR)
8730 			reladdr = 1;
8731 		if (cdb->byte2 & SRW12_FUA)
8732 			fua = 1;
8733 		if (cdb->byte2 & SRW12_DPO)
8734 			dpo = 1;
8735 
8736 		lba = scsi_8btou64(cdb->addr);
8737 		num_blocks = scsi_4btoul(cdb->length);
8738 		break;
8739 	}
8740 	case WRITE_VERIFY_16: {
8741 		struct scsi_write_verify_16 *cdb;
8742 
8743 		cdb = (struct scsi_write_verify_16 *)ctsio->cdb;
8744 
8745 		if (cdb->byte2 & SWV_DPO)
8746 			dpo = 1;
8747 
8748 		lba = scsi_8btou64(cdb->addr);
8749 		num_blocks = scsi_4btoul(cdb->length);
8750 		break;
8751 	}
8752 	default:
8753 		/*
8754 		 * We got a command we don't support.  This shouldn't
8755 		 * happen, commands should be filtered out above us.
8756 		 */
8757 		ctl_set_invalid_opcode(ctsio);
8758 		ctl_done((union ctl_io *)ctsio);
8759 
8760 		return (CTL_RETVAL_COMPLETE);
8761 		break; /* NOTREACHED */
8762 	}
8763 
8764 	/*
8765 	 * XXX KDM what do we do with the DPO and FUA bits?  FUA might be
8766 	 * interesting for us, but if RAIDCore is in write-back mode,
8767 	 * getting it to do write-through for a particular transaction may
8768 	 * not be possible.
8769 	 */
8770 	/*
8771 	 * We don't support relative addressing.  That also requires
8772 	 * supporting linked commands, which we don't do.
8773 	 */
8774 	if (reladdr != 0) {
8775 		ctl_set_invalid_field(ctsio,
8776 				      /*sks_valid*/ 1,
8777 				      /*command*/ 1,
8778 				      /*field*/ 1,
8779 				      /*bit_valid*/ 1,
8780 				      /*bit*/ 0);
8781 		ctl_done((union ctl_io *)ctsio);
8782 		return (CTL_RETVAL_COMPLETE);
8783 	}
8784 
8785 	/*
8786 	 * The first check is to make sure we're in bounds, the second
8787 	 * check is to catch wrap-around problems.  If the lba + num blocks
8788 	 * is less than the lba, then we've wrapped around and the block
8789 	 * range is invalid anyway.
8790 	 */
8791 	if (((lba + num_blocks) > (lun->be_lun->maxlba + 1))
8792 	 || ((lba + num_blocks) < lba)) {
8793 		ctl_set_lba_out_of_range(ctsio);
8794 		ctl_done((union ctl_io *)ctsio);
8795 		return (CTL_RETVAL_COMPLETE);
8796 	}
8797 
8798 	/*
8799 	 * According to SBC-3, a transfer length of 0 is not an error.
8800 	 * Note that this cannot happen with WRITE(6) or READ(6), since 0
8801 	 * translates to 256 blocks for those commands.
8802 	 */
8803 	if (num_blocks == 0) {
8804 		ctl_set_success(ctsio);
8805 		ctl_done((union ctl_io *)ctsio);
8806 		return (CTL_RETVAL_COMPLETE);
8807 	}
8808 
8809 	lbalen = (struct ctl_lba_len_flags *)
8810 	    &ctsio->io_hdr.ctl_private[CTL_PRIV_LBA_LEN];
8811 	lbalen->lba = lba;
8812 	lbalen->len = num_blocks;
8813 	lbalen->flags = isread ? CTL_LLF_READ : CTL_LLF_WRITE;
8814 
8815 	ctsio->kern_total_len = num_blocks * lun->be_lun->blocksize;
8816 	ctsio->kern_rel_offset = 0;
8817 
8818 	CTL_DEBUG_PRINT(("ctl_read_write: calling data_submit()\n"));
8819 
8820 	retval = lun->backend->data_submit((union ctl_io *)ctsio);
8821 
8822 	return (retval);
8823 }
8824 
8825 static int
8826 ctl_cnw_cont(union ctl_io *io)
8827 {
8828 	struct ctl_scsiio *ctsio;
8829 	struct ctl_lun *lun;
8830 	struct ctl_lba_len_flags *lbalen;
8831 	int retval;
8832 
8833 	ctsio = &io->scsiio;
8834 	ctsio->io_hdr.status = CTL_STATUS_NONE;
8835 	ctsio->io_hdr.flags &= ~CTL_FLAG_IO_CONT;
8836 	lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
8837 	lbalen = (struct ctl_lba_len_flags *)
8838 	    &ctsio->io_hdr.ctl_private[CTL_PRIV_LBA_LEN];
8839 	lbalen->flags = CTL_LLF_WRITE;
8840 
8841 	CTL_DEBUG_PRINT(("ctl_cnw_cont: calling data_submit()\n"));
8842 	retval = lun->backend->data_submit((union ctl_io *)ctsio);
8843 	return (retval);
8844 }
8845 
8846 int
8847 ctl_cnw(struct ctl_scsiio *ctsio)
8848 {
8849 	struct ctl_lun *lun;
8850 	struct ctl_lba_len_flags *lbalen;
8851 	uint64_t lba;
8852 	uint32_t num_blocks;
8853 	int fua, dpo;
8854 	int retval;
8855 
8856 	lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
8857 
8858 	CTL_DEBUG_PRINT(("ctl_cnw: command: %#x\n", ctsio->cdb[0]));
8859 
8860 	fua = 0;
8861 	dpo = 0;
8862 
8863 	retval = CTL_RETVAL_COMPLETE;
8864 
8865 	switch (ctsio->cdb[0]) {
8866 	case COMPARE_AND_WRITE: {
8867 		struct scsi_compare_and_write *cdb;
8868 
8869 		cdb = (struct scsi_compare_and_write *)ctsio->cdb;
8870 
8871 		if (cdb->byte2 & SRW10_FUA)
8872 			fua = 1;
8873 		if (cdb->byte2 & SRW10_DPO)
8874 			dpo = 1;
8875 		lba = scsi_8btou64(cdb->addr);
8876 		num_blocks = cdb->length;
8877 		break;
8878 	}
8879 	default:
8880 		/*
8881 		 * We got a command we don't support.  This shouldn't
8882 		 * happen, commands should be filtered out above us.
8883 		 */
8884 		ctl_set_invalid_opcode(ctsio);
8885 		ctl_done((union ctl_io *)ctsio);
8886 
8887 		return (CTL_RETVAL_COMPLETE);
8888 		break; /* NOTREACHED */
8889 	}
8890 
8891 	/*
8892 	 * XXX KDM what do we do with the DPO and FUA bits?  FUA might be
8893 	 * interesting for us, but if RAIDCore is in write-back mode,
8894 	 * getting it to do write-through for a particular transaction may
8895 	 * not be possible.
8896 	 */
8897 
8898 	/*
8899 	 * The first check is to make sure we're in bounds, the second
8900 	 * check is to catch wrap-around problems.  If the lba + num blocks
8901 	 * is less than the lba, then we've wrapped around and the block
8902 	 * range is invalid anyway.
8903 	 */
8904 	if (((lba + num_blocks) > (lun->be_lun->maxlba + 1))
8905 	 || ((lba + num_blocks) < lba)) {
8906 		ctl_set_lba_out_of_range(ctsio);
8907 		ctl_done((union ctl_io *)ctsio);
8908 		return (CTL_RETVAL_COMPLETE);
8909 	}
8910 
8911 	/*
8912 	 * According to SBC-3, a transfer length of 0 is not an error.
8913 	 */
8914 	if (num_blocks == 0) {
8915 		ctl_set_success(ctsio);
8916 		ctl_done((union ctl_io *)ctsio);
8917 		return (CTL_RETVAL_COMPLETE);
8918 	}
8919 
8920 	ctsio->kern_total_len = 2 * num_blocks * lun->be_lun->blocksize;
8921 	ctsio->kern_rel_offset = 0;
8922 
8923 	/*
8924 	 * Set the IO_CONT flag, so that if this I/O gets passed to
8925 	 * ctl_data_submit_done(), it'll get passed back to
8926 	 * ctl_ctl_cnw_cont() for further processing.
8927 	 */
8928 	ctsio->io_hdr.flags |= CTL_FLAG_IO_CONT;
8929 	ctsio->io_cont = ctl_cnw_cont;
8930 
8931 	lbalen = (struct ctl_lba_len_flags *)
8932 	    &ctsio->io_hdr.ctl_private[CTL_PRIV_LBA_LEN];
8933 	lbalen->lba = lba;
8934 	lbalen->len = num_blocks;
8935 	lbalen->flags = CTL_LLF_COMPARE;
8936 
8937 	CTL_DEBUG_PRINT(("ctl_cnw: calling data_submit()\n"));
8938 	retval = lun->backend->data_submit((union ctl_io *)ctsio);
8939 	return (retval);
8940 }
8941 
8942 int
8943 ctl_verify(struct ctl_scsiio *ctsio)
8944 {
8945 	struct ctl_lun *lun;
8946 	struct ctl_lba_len_flags *lbalen;
8947 	uint64_t lba;
8948 	uint32_t num_blocks;
8949 	int bytchk, dpo;
8950 	int retval;
8951 
8952 	lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
8953 
8954 	CTL_DEBUG_PRINT(("ctl_verify: command: %#x\n", ctsio->cdb[0]));
8955 
8956 	bytchk = 0;
8957 	dpo = 0;
8958 	retval = CTL_RETVAL_COMPLETE;
8959 
8960 	switch (ctsio->cdb[0]) {
8961 	case VERIFY_10: {
8962 		struct scsi_verify_10 *cdb;
8963 
8964 		cdb = (struct scsi_verify_10 *)ctsio->cdb;
8965 		if (cdb->byte2 & SVFY_BYTCHK)
8966 			bytchk = 1;
8967 		if (cdb->byte2 & SVFY_DPO)
8968 			dpo = 1;
8969 		lba = scsi_4btoul(cdb->addr);
8970 		num_blocks = scsi_2btoul(cdb->length);
8971 		break;
8972 	}
8973 	case VERIFY_12: {
8974 		struct scsi_verify_12 *cdb;
8975 
8976 		cdb = (struct scsi_verify_12 *)ctsio->cdb;
8977 		if (cdb->byte2 & SVFY_BYTCHK)
8978 			bytchk = 1;
8979 		if (cdb->byte2 & SVFY_DPO)
8980 			dpo = 1;
8981 		lba = scsi_4btoul(cdb->addr);
8982 		num_blocks = scsi_4btoul(cdb->length);
8983 		break;
8984 	}
8985 	case VERIFY_16: {
8986 		struct scsi_rw_16 *cdb;
8987 
8988 		cdb = (struct scsi_rw_16 *)ctsio->cdb;
8989 		if (cdb->byte2 & SVFY_BYTCHK)
8990 			bytchk = 1;
8991 		if (cdb->byte2 & SVFY_DPO)
8992 			dpo = 1;
8993 		lba = scsi_8btou64(cdb->addr);
8994 		num_blocks = scsi_4btoul(cdb->length);
8995 		break;
8996 	}
8997 	default:
8998 		/*
8999 		 * We got a command we don't support.  This shouldn't
9000 		 * happen, commands should be filtered out above us.
9001 		 */
9002 		ctl_set_invalid_opcode(ctsio);
9003 		ctl_done((union ctl_io *)ctsio);
9004 		return (CTL_RETVAL_COMPLETE);
9005 	}
9006 
9007 	/*
9008 	 * The first check is to make sure we're in bounds, the second
9009 	 * check is to catch wrap-around problems.  If the lba + num blocks
9010 	 * is less than the lba, then we've wrapped around and the block
9011 	 * range is invalid anyway.
9012 	 */
9013 	if (((lba + num_blocks) > (lun->be_lun->maxlba + 1))
9014 	 || ((lba + num_blocks) < lba)) {
9015 		ctl_set_lba_out_of_range(ctsio);
9016 		ctl_done((union ctl_io *)ctsio);
9017 		return (CTL_RETVAL_COMPLETE);
9018 	}
9019 
9020 	/*
9021 	 * According to SBC-3, a transfer length of 0 is not an error.
9022 	 */
9023 	if (num_blocks == 0) {
9024 		ctl_set_success(ctsio);
9025 		ctl_done((union ctl_io *)ctsio);
9026 		return (CTL_RETVAL_COMPLETE);
9027 	}
9028 
9029 	lbalen = (struct ctl_lba_len_flags *)
9030 	    &ctsio->io_hdr.ctl_private[CTL_PRIV_LBA_LEN];
9031 	lbalen->lba = lba;
9032 	lbalen->len = num_blocks;
9033 	if (bytchk) {
9034 		lbalen->flags = CTL_LLF_COMPARE;
9035 		ctsio->kern_total_len = num_blocks * lun->be_lun->blocksize;
9036 	} else {
9037 		lbalen->flags = CTL_LLF_VERIFY;
9038 		ctsio->kern_total_len = 0;
9039 	}
9040 	ctsio->kern_rel_offset = 0;
9041 
9042 	CTL_DEBUG_PRINT(("ctl_verify: calling data_submit()\n"));
9043 	retval = lun->backend->data_submit((union ctl_io *)ctsio);
9044 	return (retval);
9045 }
9046 
9047 int
9048 ctl_report_luns(struct ctl_scsiio *ctsio)
9049 {
9050 	struct scsi_report_luns *cdb;
9051 	struct scsi_report_luns_data *lun_data;
9052 	struct ctl_lun *lun, *request_lun;
9053 	int num_luns, retval;
9054 	uint32_t alloc_len, lun_datalen;
9055 	int num_filled, well_known;
9056 	uint32_t initidx, targ_lun_id, lun_id;
9057 
9058 	retval = CTL_RETVAL_COMPLETE;
9059 	well_known = 0;
9060 
9061 	cdb = (struct scsi_report_luns *)ctsio->cdb;
9062 
9063 	CTL_DEBUG_PRINT(("ctl_report_luns\n"));
9064 
9065 	mtx_lock(&control_softc->ctl_lock);
9066 	num_luns = control_softc->num_luns;
9067 	mtx_unlock(&control_softc->ctl_lock);
9068 
9069 	switch (cdb->select_report) {
9070 	case RPL_REPORT_DEFAULT:
9071 	case RPL_REPORT_ALL:
9072 		break;
9073 	case RPL_REPORT_WELLKNOWN:
9074 		well_known = 1;
9075 		num_luns = 0;
9076 		break;
9077 	default:
9078 		ctl_set_invalid_field(ctsio,
9079 				      /*sks_valid*/ 1,
9080 				      /*command*/ 1,
9081 				      /*field*/ 2,
9082 				      /*bit_valid*/ 0,
9083 				      /*bit*/ 0);
9084 		ctl_done((union ctl_io *)ctsio);
9085 		return (retval);
9086 		break; /* NOTREACHED */
9087 	}
9088 
9089 	alloc_len = scsi_4btoul(cdb->length);
9090 	/*
9091 	 * The initiator has to allocate at least 16 bytes for this request,
9092 	 * so he can at least get the header and the first LUN.  Otherwise
9093 	 * we reject the request (per SPC-3 rev 14, section 6.21).
9094 	 */
9095 	if (alloc_len < (sizeof(struct scsi_report_luns_data) +
9096 	    sizeof(struct scsi_report_luns_lundata))) {
9097 		ctl_set_invalid_field(ctsio,
9098 				      /*sks_valid*/ 1,
9099 				      /*command*/ 1,
9100 				      /*field*/ 6,
9101 				      /*bit_valid*/ 0,
9102 				      /*bit*/ 0);
9103 		ctl_done((union ctl_io *)ctsio);
9104 		return (retval);
9105 	}
9106 
9107 	request_lun = (struct ctl_lun *)
9108 		ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
9109 
9110 	lun_datalen = sizeof(*lun_data) +
9111 		(num_luns * sizeof(struct scsi_report_luns_lundata));
9112 
9113 	ctsio->kern_data_ptr = malloc(lun_datalen, M_CTL, M_WAITOK | M_ZERO);
9114 	lun_data = (struct scsi_report_luns_data *)ctsio->kern_data_ptr;
9115 	ctsio->kern_sg_entries = 0;
9116 
9117 	initidx = ctl_get_initindex(&ctsio->io_hdr.nexus);
9118 
9119 	mtx_lock(&control_softc->ctl_lock);
9120 	for (targ_lun_id = 0, num_filled = 0; targ_lun_id < CTL_MAX_LUNS && num_filled < num_luns; targ_lun_id++) {
9121 		lun_id = targ_lun_id;
9122 		if (ctsio->io_hdr.nexus.lun_map_fn != NULL)
9123 			lun_id = ctsio->io_hdr.nexus.lun_map_fn(ctsio->io_hdr.nexus.lun_map_arg, lun_id);
9124 		if (lun_id >= CTL_MAX_LUNS)
9125 			continue;
9126 		lun = control_softc->ctl_luns[lun_id];
9127 		if (lun == NULL)
9128 			continue;
9129 
9130 		if (targ_lun_id <= 0xff) {
9131 			/*
9132 			 * Peripheral addressing method, bus number 0.
9133 			 */
9134 			lun_data->luns[num_filled].lundata[0] =
9135 				RPL_LUNDATA_ATYP_PERIPH;
9136 			lun_data->luns[num_filled].lundata[1] = targ_lun_id;
9137 			num_filled++;
9138 		} else if (targ_lun_id <= 0x3fff) {
9139 			/*
9140 			 * Flat addressing method.
9141 			 */
9142 			lun_data->luns[num_filled].lundata[0] =
9143 				RPL_LUNDATA_ATYP_FLAT |
9144 				(targ_lun_id & RPL_LUNDATA_FLAT_LUN_MASK);
9145 #ifdef OLDCTLHEADERS
9146 				(SRLD_ADDR_FLAT << SRLD_ADDR_SHIFT) |
9147 				(targ_lun_id & SRLD_BUS_LUN_MASK);
9148 #endif
9149 			lun_data->luns[num_filled].lundata[1] =
9150 #ifdef OLDCTLHEADERS
9151 				targ_lun_id >> SRLD_BUS_LUN_BITS;
9152 #endif
9153 				targ_lun_id >> RPL_LUNDATA_FLAT_LUN_BITS;
9154 			num_filled++;
9155 		} else {
9156 			printf("ctl_report_luns: bogus LUN number %jd, "
9157 			       "skipping\n", (intmax_t)targ_lun_id);
9158 		}
9159 		/*
9160 		 * According to SPC-3, rev 14 section 6.21:
9161 		 *
9162 		 * "The execution of a REPORT LUNS command to any valid and
9163 		 * installed logical unit shall clear the REPORTED LUNS DATA
9164 		 * HAS CHANGED unit attention condition for all logical
9165 		 * units of that target with respect to the requesting
9166 		 * initiator. A valid and installed logical unit is one
9167 		 * having a PERIPHERAL QUALIFIER of 000b in the standard
9168 		 * INQUIRY data (see 6.4.2)."
9169 		 *
9170 		 * If request_lun is NULL, the LUN this report luns command
9171 		 * was issued to is either disabled or doesn't exist. In that
9172 		 * case, we shouldn't clear any pending lun change unit
9173 		 * attention.
9174 		 */
9175 		if (request_lun != NULL) {
9176 			mtx_lock(&lun->lun_lock);
9177 			lun->pending_sense[initidx].ua_pending &=
9178 				~CTL_UA_LUN_CHANGE;
9179 			mtx_unlock(&lun->lun_lock);
9180 		}
9181 	}
9182 	mtx_unlock(&control_softc->ctl_lock);
9183 
9184 	/*
9185 	 * It's quite possible that we've returned fewer LUNs than we allocated
9186 	 * space for.  Trim it.
9187 	 */
9188 	lun_datalen = sizeof(*lun_data) +
9189 		(num_filled * sizeof(struct scsi_report_luns_lundata));
9190 
9191 	if (lun_datalen < alloc_len) {
9192 		ctsio->residual = alloc_len - lun_datalen;
9193 		ctsio->kern_data_len = lun_datalen;
9194 		ctsio->kern_total_len = lun_datalen;
9195 	} else {
9196 		ctsio->residual = 0;
9197 		ctsio->kern_data_len = alloc_len;
9198 		ctsio->kern_total_len = alloc_len;
9199 	}
9200 	ctsio->kern_data_resid = 0;
9201 	ctsio->kern_rel_offset = 0;
9202 	ctsio->kern_sg_entries = 0;
9203 
9204 	/*
9205 	 * We set this to the actual data length, regardless of how much
9206 	 * space we actually have to return results.  If the user looks at
9207 	 * this value, he'll know whether or not he allocated enough space
9208 	 * and reissue the command if necessary.  We don't support well
9209 	 * known logical units, so if the user asks for that, return none.
9210 	 */
9211 	scsi_ulto4b(lun_datalen - 8, lun_data->length);
9212 
9213 	/*
9214 	 * We can only return SCSI_STATUS_CHECK_COND when we can't satisfy
9215 	 * this request.
9216 	 */
9217 	ctsio->scsi_status = SCSI_STATUS_OK;
9218 
9219 	ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
9220 	ctsio->be_move_done = ctl_config_move_done;
9221 	ctl_datamove((union ctl_io *)ctsio);
9222 
9223 	return (retval);
9224 }
9225 
9226 int
9227 ctl_request_sense(struct ctl_scsiio *ctsio)
9228 {
9229 	struct scsi_request_sense *cdb;
9230 	struct scsi_sense_data *sense_ptr;
9231 	struct ctl_lun *lun;
9232 	uint32_t initidx;
9233 	int have_error;
9234 	scsi_sense_data_type sense_format;
9235 
9236 	cdb = (struct scsi_request_sense *)ctsio->cdb;
9237 
9238 	lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
9239 
9240 	CTL_DEBUG_PRINT(("ctl_request_sense\n"));
9241 
9242 	/*
9243 	 * Determine which sense format the user wants.
9244 	 */
9245 	if (cdb->byte2 & SRS_DESC)
9246 		sense_format = SSD_TYPE_DESC;
9247 	else
9248 		sense_format = SSD_TYPE_FIXED;
9249 
9250 	ctsio->kern_data_ptr = malloc(sizeof(*sense_ptr), M_CTL, M_WAITOK);
9251 	sense_ptr = (struct scsi_sense_data *)ctsio->kern_data_ptr;
9252 	ctsio->kern_sg_entries = 0;
9253 
9254 	/*
9255 	 * struct scsi_sense_data, which is currently set to 256 bytes, is
9256 	 * larger than the largest allowed value for the length field in the
9257 	 * REQUEST SENSE CDB, which is 252 bytes as of SPC-4.
9258 	 */
9259 	ctsio->residual = 0;
9260 	ctsio->kern_data_len = cdb->length;
9261 	ctsio->kern_total_len = cdb->length;
9262 
9263 	ctsio->kern_data_resid = 0;
9264 	ctsio->kern_rel_offset = 0;
9265 	ctsio->kern_sg_entries = 0;
9266 
9267 	/*
9268 	 * If we don't have a LUN, we don't have any pending sense.
9269 	 */
9270 	if (lun == NULL)
9271 		goto no_sense;
9272 
9273 	have_error = 0;
9274 	initidx = ctl_get_initindex(&ctsio->io_hdr.nexus);
9275 	/*
9276 	 * Check for pending sense, and then for pending unit attentions.
9277 	 * Pending sense gets returned first, then pending unit attentions.
9278 	 */
9279 	mtx_lock(&lun->lun_lock);
9280 	if (ctl_is_set(lun->have_ca, initidx)) {
9281 		scsi_sense_data_type stored_format;
9282 
9283 		/*
9284 		 * Check to see which sense format was used for the stored
9285 		 * sense data.
9286 		 */
9287 		stored_format = scsi_sense_type(
9288 		    &lun->pending_sense[initidx].sense);
9289 
9290 		/*
9291 		 * If the user requested a different sense format than the
9292 		 * one we stored, then we need to convert it to the other
9293 		 * format.  If we're going from descriptor to fixed format
9294 		 * sense data, we may lose things in translation, depending
9295 		 * on what options were used.
9296 		 *
9297 		 * If the stored format is SSD_TYPE_NONE (i.e. invalid),
9298 		 * for some reason we'll just copy it out as-is.
9299 		 */
9300 		if ((stored_format == SSD_TYPE_FIXED)
9301 		 && (sense_format == SSD_TYPE_DESC))
9302 			ctl_sense_to_desc((struct scsi_sense_data_fixed *)
9303 			    &lun->pending_sense[initidx].sense,
9304 			    (struct scsi_sense_data_desc *)sense_ptr);
9305 		else if ((stored_format == SSD_TYPE_DESC)
9306 		      && (sense_format == SSD_TYPE_FIXED))
9307 			ctl_sense_to_fixed((struct scsi_sense_data_desc *)
9308 			    &lun->pending_sense[initidx].sense,
9309 			    (struct scsi_sense_data_fixed *)sense_ptr);
9310 		else
9311 			memcpy(sense_ptr, &lun->pending_sense[initidx].sense,
9312 			       ctl_min(sizeof(*sense_ptr),
9313 			       sizeof(lun->pending_sense[initidx].sense)));
9314 
9315 		ctl_clear_mask(lun->have_ca, initidx);
9316 		have_error = 1;
9317 	} else if (lun->pending_sense[initidx].ua_pending != CTL_UA_NONE) {
9318 		ctl_ua_type ua_type;
9319 
9320 		ua_type = ctl_build_ua(lun->pending_sense[initidx].ua_pending,
9321 				       sense_ptr, sense_format);
9322 		if (ua_type != CTL_UA_NONE) {
9323 			have_error = 1;
9324 			/* We're reporting this UA, so clear it */
9325 			lun->pending_sense[initidx].ua_pending &= ~ua_type;
9326 		}
9327 	}
9328 	mtx_unlock(&lun->lun_lock);
9329 
9330 	/*
9331 	 * We already have a pending error, return it.
9332 	 */
9333 	if (have_error != 0) {
9334 		/*
9335 		 * We report the SCSI status as OK, since the status of the
9336 		 * request sense command itself is OK.
9337 		 */
9338 		ctsio->scsi_status = SCSI_STATUS_OK;
9339 
9340 		/*
9341 		 * We report 0 for the sense length, because we aren't doing
9342 		 * autosense in this case.  We're reporting sense as
9343 		 * parameter data.
9344 		 */
9345 		ctsio->sense_len = 0;
9346 		ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
9347 		ctsio->be_move_done = ctl_config_move_done;
9348 		ctl_datamove((union ctl_io *)ctsio);
9349 
9350 		return (CTL_RETVAL_COMPLETE);
9351 	}
9352 
9353 no_sense:
9354 
9355 	/*
9356 	 * No sense information to report, so we report that everything is
9357 	 * okay.
9358 	 */
9359 	ctl_set_sense_data(sense_ptr,
9360 			   lun,
9361 			   sense_format,
9362 			   /*current_error*/ 1,
9363 			   /*sense_key*/ SSD_KEY_NO_SENSE,
9364 			   /*asc*/ 0x00,
9365 			   /*ascq*/ 0x00,
9366 			   SSD_ELEM_NONE);
9367 
9368 	ctsio->scsi_status = SCSI_STATUS_OK;
9369 
9370 	/*
9371 	 * We report 0 for the sense length, because we aren't doing
9372 	 * autosense in this case.  We're reporting sense as parameter data.
9373 	 */
9374 	ctsio->sense_len = 0;
9375 	ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
9376 	ctsio->be_move_done = ctl_config_move_done;
9377 	ctl_datamove((union ctl_io *)ctsio);
9378 
9379 	return (CTL_RETVAL_COMPLETE);
9380 }
9381 
9382 int
9383 ctl_tur(struct ctl_scsiio *ctsio)
9384 {
9385 	struct ctl_lun *lun;
9386 
9387 	lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
9388 
9389 	CTL_DEBUG_PRINT(("ctl_tur\n"));
9390 
9391 	if (lun == NULL)
9392 		return (-EINVAL);
9393 
9394 	ctsio->scsi_status = SCSI_STATUS_OK;
9395 	ctsio->io_hdr.status = CTL_SUCCESS;
9396 
9397 	ctl_done((union ctl_io *)ctsio);
9398 
9399 	return (CTL_RETVAL_COMPLETE);
9400 }
9401 
9402 #ifdef notyet
9403 static int
9404 ctl_cmddt_inquiry(struct ctl_scsiio *ctsio)
9405 {
9406 
9407 }
9408 #endif
9409 
9410 static int
9411 ctl_inquiry_evpd_supported(struct ctl_scsiio *ctsio, int alloc_len)
9412 {
9413 	struct scsi_vpd_supported_pages *pages;
9414 	int sup_page_size;
9415 	struct ctl_lun *lun;
9416 
9417 	lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
9418 
9419 	sup_page_size = sizeof(struct scsi_vpd_supported_pages) *
9420 	    SCSI_EVPD_NUM_SUPPORTED_PAGES;
9421 	ctsio->kern_data_ptr = malloc(sup_page_size, M_CTL, M_WAITOK | M_ZERO);
9422 	pages = (struct scsi_vpd_supported_pages *)ctsio->kern_data_ptr;
9423 	ctsio->kern_sg_entries = 0;
9424 
9425 	if (sup_page_size < alloc_len) {
9426 		ctsio->residual = alloc_len - sup_page_size;
9427 		ctsio->kern_data_len = sup_page_size;
9428 		ctsio->kern_total_len = sup_page_size;
9429 	} else {
9430 		ctsio->residual = 0;
9431 		ctsio->kern_data_len = alloc_len;
9432 		ctsio->kern_total_len = alloc_len;
9433 	}
9434 	ctsio->kern_data_resid = 0;
9435 	ctsio->kern_rel_offset = 0;
9436 	ctsio->kern_sg_entries = 0;
9437 
9438 	/*
9439 	 * The control device is always connected.  The disk device, on the
9440 	 * other hand, may not be online all the time.  Need to change this
9441 	 * to figure out whether the disk device is actually online or not.
9442 	 */
9443 	if (lun != NULL)
9444 		pages->device = (SID_QUAL_LU_CONNECTED << 5) |
9445 				lun->be_lun->lun_type;
9446 	else
9447 		pages->device = (SID_QUAL_LU_OFFLINE << 5) | T_DIRECT;
9448 
9449 	pages->length = SCSI_EVPD_NUM_SUPPORTED_PAGES;
9450 	/* Supported VPD pages */
9451 	pages->page_list[0] = SVPD_SUPPORTED_PAGES;
9452 	/* Serial Number */
9453 	pages->page_list[1] = SVPD_UNIT_SERIAL_NUMBER;
9454 	/* Device Identification */
9455 	pages->page_list[2] = SVPD_DEVICE_ID;
9456 	/* Block limits */
9457 	pages->page_list[3] = SVPD_BLOCK_LIMITS;
9458 	/* Logical Block Provisioning */
9459 	pages->page_list[4] = SVPD_LBP;
9460 
9461 	ctsio->scsi_status = SCSI_STATUS_OK;
9462 
9463 	ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
9464 	ctsio->be_move_done = ctl_config_move_done;
9465 	ctl_datamove((union ctl_io *)ctsio);
9466 
9467 	return (CTL_RETVAL_COMPLETE);
9468 }
9469 
9470 static int
9471 ctl_inquiry_evpd_serial(struct ctl_scsiio *ctsio, int alloc_len)
9472 {
9473 	struct scsi_vpd_unit_serial_number *sn_ptr;
9474 	struct ctl_lun *lun;
9475 
9476 	lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
9477 
9478 	ctsio->kern_data_ptr = malloc(sizeof(*sn_ptr), M_CTL, M_WAITOK | M_ZERO);
9479 	sn_ptr = (struct scsi_vpd_unit_serial_number *)ctsio->kern_data_ptr;
9480 	ctsio->kern_sg_entries = 0;
9481 
9482 	if (sizeof(*sn_ptr) < alloc_len) {
9483 		ctsio->residual = alloc_len - sizeof(*sn_ptr);
9484 		ctsio->kern_data_len = sizeof(*sn_ptr);
9485 		ctsio->kern_total_len = sizeof(*sn_ptr);
9486 	} else {
9487 		ctsio->residual = 0;
9488 		ctsio->kern_data_len = alloc_len;
9489 		ctsio->kern_total_len = alloc_len;
9490 	}
9491 	ctsio->kern_data_resid = 0;
9492 	ctsio->kern_rel_offset = 0;
9493 	ctsio->kern_sg_entries = 0;
9494 
9495 	/*
9496 	 * The control device is always connected.  The disk device, on the
9497 	 * other hand, may not be online all the time.  Need to change this
9498 	 * to figure out whether the disk device is actually online or not.
9499 	 */
9500 	if (lun != NULL)
9501 		sn_ptr->device = (SID_QUAL_LU_CONNECTED << 5) |
9502 				  lun->be_lun->lun_type;
9503 	else
9504 		sn_ptr->device = (SID_QUAL_LU_OFFLINE << 5) | T_DIRECT;
9505 
9506 	sn_ptr->page_code = SVPD_UNIT_SERIAL_NUMBER;
9507 	sn_ptr->length = ctl_min(sizeof(*sn_ptr) - 4, CTL_SN_LEN);
9508 	/*
9509 	 * If we don't have a LUN, we just leave the serial number as
9510 	 * all spaces.
9511 	 */
9512 	memset(sn_ptr->serial_num, 0x20, sizeof(sn_ptr->serial_num));
9513 	if (lun != NULL) {
9514 		strncpy((char *)sn_ptr->serial_num,
9515 			(char *)lun->be_lun->serial_num, CTL_SN_LEN);
9516 	}
9517 	ctsio->scsi_status = SCSI_STATUS_OK;
9518 
9519 	ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
9520 	ctsio->be_move_done = ctl_config_move_done;
9521 	ctl_datamove((union ctl_io *)ctsio);
9522 
9523 	return (CTL_RETVAL_COMPLETE);
9524 }
9525 
9526 
9527 static int
9528 ctl_inquiry_evpd_devid(struct ctl_scsiio *ctsio, int alloc_len)
9529 {
9530 	struct scsi_vpd_device_id *devid_ptr;
9531 	struct scsi_vpd_id_descriptor *desc, *desc1;
9532 	struct scsi_vpd_id_descriptor *desc2, *desc3; /* for types 4h and 5h */
9533 	struct scsi_vpd_id_t10 *t10id;
9534 	struct ctl_softc *ctl_softc;
9535 	struct ctl_lun *lun;
9536 	struct ctl_frontend *fe;
9537 	char *val;
9538 	int data_len, devid_len;
9539 
9540 	ctl_softc = control_softc;
9541 
9542 	fe = ctl_softc->ctl_ports[ctl_port_idx(ctsio->io_hdr.nexus.targ_port)];
9543 
9544 	if (fe->devid != NULL)
9545 		return ((fe->devid)(ctsio, alloc_len));
9546 
9547 	lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
9548 
9549 	if (lun == NULL) {
9550 		devid_len = CTL_DEVID_MIN_LEN;
9551 	} else {
9552 		devid_len = max(CTL_DEVID_MIN_LEN,
9553 		    strnlen(lun->be_lun->device_id, CTL_DEVID_LEN));
9554 	}
9555 
9556 	data_len = sizeof(struct scsi_vpd_device_id) +
9557 		sizeof(struct scsi_vpd_id_descriptor) +
9558 		sizeof(struct scsi_vpd_id_t10) + devid_len +
9559 		sizeof(struct scsi_vpd_id_descriptor) + CTL_WWPN_LEN +
9560 		sizeof(struct scsi_vpd_id_descriptor) +
9561 		sizeof(struct scsi_vpd_id_rel_trgt_port_id) +
9562 		sizeof(struct scsi_vpd_id_descriptor) +
9563 		sizeof(struct scsi_vpd_id_trgt_port_grp_id);
9564 
9565 	ctsio->kern_data_ptr = malloc(data_len, M_CTL, M_WAITOK | M_ZERO);
9566 	devid_ptr = (struct scsi_vpd_device_id *)ctsio->kern_data_ptr;
9567 	ctsio->kern_sg_entries = 0;
9568 
9569 	if (data_len < alloc_len) {
9570 		ctsio->residual = alloc_len - data_len;
9571 		ctsio->kern_data_len = data_len;
9572 		ctsio->kern_total_len = data_len;
9573 	} else {
9574 		ctsio->residual = 0;
9575 		ctsio->kern_data_len = alloc_len;
9576 		ctsio->kern_total_len = alloc_len;
9577 	}
9578 	ctsio->kern_data_resid = 0;
9579 	ctsio->kern_rel_offset = 0;
9580 	ctsio->kern_sg_entries = 0;
9581 
9582 	desc = (struct scsi_vpd_id_descriptor *)devid_ptr->desc_list;
9583 	t10id = (struct scsi_vpd_id_t10 *)&desc->identifier[0];
9584 	desc1 = (struct scsi_vpd_id_descriptor *)(&desc->identifier[0] +
9585 		sizeof(struct scsi_vpd_id_t10) + devid_len);
9586 	desc2 = (struct scsi_vpd_id_descriptor *)(&desc1->identifier[0] +
9587 	          CTL_WWPN_LEN);
9588 	desc3 = (struct scsi_vpd_id_descriptor *)(&desc2->identifier[0] +
9589 	         sizeof(struct scsi_vpd_id_rel_trgt_port_id));
9590 
9591 	/*
9592 	 * The control device is always connected.  The disk device, on the
9593 	 * other hand, may not be online all the time.
9594 	 */
9595 	if (lun != NULL)
9596 		devid_ptr->device = (SID_QUAL_LU_CONNECTED << 5) |
9597 				     lun->be_lun->lun_type;
9598 	else
9599 		devid_ptr->device = (SID_QUAL_LU_OFFLINE << 5) | T_DIRECT;
9600 
9601 	devid_ptr->page_code = SVPD_DEVICE_ID;
9602 
9603 	scsi_ulto2b(data_len - 4, devid_ptr->length);
9604 
9605 	/*
9606 	 * For Fibre channel,
9607 	 */
9608 	if (fe->port_type == CTL_PORT_FC)
9609 	{
9610 		desc->proto_codeset = (SCSI_PROTO_FC << 4) |
9611 				      SVPD_ID_CODESET_ASCII;
9612         	desc1->proto_codeset = (SCSI_PROTO_FC << 4) |
9613 		              SVPD_ID_CODESET_BINARY;
9614 	}
9615 	else
9616 	{
9617 		desc->proto_codeset = (SCSI_PROTO_SPI << 4) |
9618 				      SVPD_ID_CODESET_ASCII;
9619         	desc1->proto_codeset = (SCSI_PROTO_SPI << 4) |
9620 		              SVPD_ID_CODESET_BINARY;
9621 	}
9622 	desc2->proto_codeset = desc3->proto_codeset = desc1->proto_codeset;
9623 
9624 	/*
9625 	 * We're using a LUN association here.  i.e., this device ID is a
9626 	 * per-LUN identifier.
9627 	 */
9628 	desc->id_type = SVPD_ID_PIV | SVPD_ID_ASSOC_LUN | SVPD_ID_TYPE_T10;
9629 	desc->length = sizeof(*t10id) + devid_len;
9630 	if (lun == NULL || (val = ctl_get_opt(lun->be_lun, "vendor")) == NULL) {
9631 		strncpy((char *)t10id->vendor, CTL_VENDOR, sizeof(t10id->vendor));
9632 	} else {
9633 		memset(t10id->vendor, ' ', sizeof(t10id->vendor));
9634 		strncpy(t10id->vendor, val,
9635 		    min(sizeof(t10id->vendor), strlen(val)));
9636 	}
9637 
9638 	/*
9639 	 * desc1 is for the WWPN which is a port asscociation.
9640 	 */
9641 	desc1->id_type = SVPD_ID_PIV | SVPD_ID_ASSOC_PORT | SVPD_ID_TYPE_NAA;
9642 	desc1->length = CTL_WWPN_LEN;
9643 	/* XXX Call Reggie's get_WWNN func here then add port # to the end */
9644 	/* For testing just create the WWPN */
9645 #if 0
9646 	ddb_GetWWNN((char *)desc1->identifier);
9647 
9648 	/* NOTE: if the port is 0 or 8 we don't want to subtract 1 */
9649 	/* This is so Copancontrol will return something sane */
9650 	if (ctsio->io_hdr.nexus.targ_port!=0 &&
9651 	    ctsio->io_hdr.nexus.targ_port!=8)
9652 		desc1->identifier[7] += ctsio->io_hdr.nexus.targ_port-1;
9653 	else
9654 		desc1->identifier[7] += ctsio->io_hdr.nexus.targ_port;
9655 #endif
9656 
9657 	be64enc(desc1->identifier, fe->wwpn);
9658 
9659 	/*
9660 	 * desc2 is for the Relative Target Port(type 4h) identifier
9661 	 */
9662 	desc2->id_type = SVPD_ID_PIV | SVPD_ID_ASSOC_PORT
9663 	                 | SVPD_ID_TYPE_RELTARG;
9664 	desc2->length = 4;
9665 //#if 0
9666 	/* NOTE: if the port is 0 or 8 we don't want to subtract 1 */
9667 	/* This is so Copancontrol will return something sane */
9668 	if (ctsio->io_hdr.nexus.targ_port!=0 &&
9669 	    ctsio->io_hdr.nexus.targ_port!=8)
9670 		desc2->identifier[3] = ctsio->io_hdr.nexus.targ_port - 1;
9671 	else
9672 	        desc2->identifier[3] = ctsio->io_hdr.nexus.targ_port;
9673 //#endif
9674 
9675 	/*
9676 	 * desc3 is for the Target Port Group(type 5h) identifier
9677 	 */
9678 	desc3->id_type = SVPD_ID_PIV | SVPD_ID_ASSOC_PORT
9679 	                 | SVPD_ID_TYPE_TPORTGRP;
9680 	desc3->length = 4;
9681 	if (ctsio->io_hdr.nexus.targ_port < CTL_MAX_PORTS || ctl_is_single)
9682 		desc3->identifier[3] = 1;
9683 	else
9684 		desc3->identifier[3] = 2;
9685 
9686 	/*
9687 	 * If we've actually got a backend, copy the device id from the
9688 	 * per-LUN data.  Otherwise, set it to all spaces.
9689 	 */
9690 	if (lun != NULL) {
9691 		/*
9692 		 * Copy the backend's LUN ID.
9693 		 */
9694 		strncpy((char *)t10id->vendor_spec_id,
9695 			(char *)lun->be_lun->device_id, devid_len);
9696 	} else {
9697 		/*
9698 		 * No backend, set this to spaces.
9699 		 */
9700 		memset(t10id->vendor_spec_id, 0x20, devid_len);
9701 	}
9702 
9703 	ctsio->scsi_status = SCSI_STATUS_OK;
9704 
9705 	ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
9706 	ctsio->be_move_done = ctl_config_move_done;
9707 	ctl_datamove((union ctl_io *)ctsio);
9708 
9709 	return (CTL_RETVAL_COMPLETE);
9710 }
9711 
9712 static int
9713 ctl_inquiry_evpd_block_limits(struct ctl_scsiio *ctsio, int alloc_len)
9714 {
9715 	struct scsi_vpd_block_limits *bl_ptr;
9716 	struct ctl_lun *lun;
9717 	int bs;
9718 
9719 	lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
9720 	bs = lun->be_lun->blocksize;
9721 
9722 	ctsio->kern_data_ptr = malloc(sizeof(*bl_ptr), M_CTL, M_WAITOK | M_ZERO);
9723 	bl_ptr = (struct scsi_vpd_block_limits *)ctsio->kern_data_ptr;
9724 	ctsio->kern_sg_entries = 0;
9725 
9726 	if (sizeof(*bl_ptr) < alloc_len) {
9727 		ctsio->residual = alloc_len - sizeof(*bl_ptr);
9728 		ctsio->kern_data_len = sizeof(*bl_ptr);
9729 		ctsio->kern_total_len = sizeof(*bl_ptr);
9730 	} else {
9731 		ctsio->residual = 0;
9732 		ctsio->kern_data_len = alloc_len;
9733 		ctsio->kern_total_len = alloc_len;
9734 	}
9735 	ctsio->kern_data_resid = 0;
9736 	ctsio->kern_rel_offset = 0;
9737 	ctsio->kern_sg_entries = 0;
9738 
9739 	/*
9740 	 * The control device is always connected.  The disk device, on the
9741 	 * other hand, may not be online all the time.  Need to change this
9742 	 * to figure out whether the disk device is actually online or not.
9743 	 */
9744 	if (lun != NULL)
9745 		bl_ptr->device = (SID_QUAL_LU_CONNECTED << 5) |
9746 				  lun->be_lun->lun_type;
9747 	else
9748 		bl_ptr->device = (SID_QUAL_LU_OFFLINE << 5) | T_DIRECT;
9749 
9750 	bl_ptr->page_code = SVPD_BLOCK_LIMITS;
9751 	scsi_ulto2b(sizeof(*bl_ptr), bl_ptr->page_length);
9752 	bl_ptr->max_cmp_write_len = 0xff;
9753 	scsi_ulto4b(0xffffffff, bl_ptr->max_txfer_len);
9754 	scsi_ulto4b(MAXPHYS / bs, bl_ptr->opt_txfer_len);
9755 	if (lun->be_lun->flags & CTL_LUN_FLAG_UNMAP) {
9756 		scsi_ulto4b(0xffffffff, bl_ptr->max_unmap_lba_cnt);
9757 		scsi_ulto4b(0xffffffff, bl_ptr->max_unmap_blk_cnt);
9758 	}
9759 	scsi_u64to8b(UINT64_MAX, bl_ptr->max_write_same_length);
9760 
9761 	ctsio->scsi_status = SCSI_STATUS_OK;
9762 	ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
9763 	ctsio->be_move_done = ctl_config_move_done;
9764 	ctl_datamove((union ctl_io *)ctsio);
9765 
9766 	return (CTL_RETVAL_COMPLETE);
9767 }
9768 
9769 static int
9770 ctl_inquiry_evpd_lbp(struct ctl_scsiio *ctsio, int alloc_len)
9771 {
9772 	struct scsi_vpd_logical_block_prov *lbp_ptr;
9773 	struct ctl_lun *lun;
9774 	int bs;
9775 
9776 	lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
9777 	bs = lun->be_lun->blocksize;
9778 
9779 	ctsio->kern_data_ptr = malloc(sizeof(*lbp_ptr), M_CTL, M_WAITOK | M_ZERO);
9780 	lbp_ptr = (struct scsi_vpd_logical_block_prov *)ctsio->kern_data_ptr;
9781 	ctsio->kern_sg_entries = 0;
9782 
9783 	if (sizeof(*lbp_ptr) < alloc_len) {
9784 		ctsio->residual = alloc_len - sizeof(*lbp_ptr);
9785 		ctsio->kern_data_len = sizeof(*lbp_ptr);
9786 		ctsio->kern_total_len = sizeof(*lbp_ptr);
9787 	} else {
9788 		ctsio->residual = 0;
9789 		ctsio->kern_data_len = alloc_len;
9790 		ctsio->kern_total_len = alloc_len;
9791 	}
9792 	ctsio->kern_data_resid = 0;
9793 	ctsio->kern_rel_offset = 0;
9794 	ctsio->kern_sg_entries = 0;
9795 
9796 	/*
9797 	 * The control device is always connected.  The disk device, on the
9798 	 * other hand, may not be online all the time.  Need to change this
9799 	 * to figure out whether the disk device is actually online or not.
9800 	 */
9801 	if (lun != NULL)
9802 		lbp_ptr->device = (SID_QUAL_LU_CONNECTED << 5) |
9803 				  lun->be_lun->lun_type;
9804 	else
9805 		lbp_ptr->device = (SID_QUAL_LU_OFFLINE << 5) | T_DIRECT;
9806 
9807 	lbp_ptr->page_code = SVPD_LBP;
9808 	if (lun->be_lun->flags & CTL_LUN_FLAG_UNMAP)
9809 		lbp_ptr->flags = SVPD_LBP_UNMAP | SVPD_LBP_WS16 | SVPD_LBP_WS10;
9810 
9811 	ctsio->scsi_status = SCSI_STATUS_OK;
9812 	ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
9813 	ctsio->be_move_done = ctl_config_move_done;
9814 	ctl_datamove((union ctl_io *)ctsio);
9815 
9816 	return (CTL_RETVAL_COMPLETE);
9817 }
9818 
9819 static int
9820 ctl_inquiry_evpd(struct ctl_scsiio *ctsio)
9821 {
9822 	struct scsi_inquiry *cdb;
9823 	struct ctl_lun *lun;
9824 	int alloc_len, retval;
9825 
9826 	lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
9827 	cdb = (struct scsi_inquiry *)ctsio->cdb;
9828 
9829 	retval = CTL_RETVAL_COMPLETE;
9830 
9831 	alloc_len = scsi_2btoul(cdb->length);
9832 
9833 	switch (cdb->page_code) {
9834 	case SVPD_SUPPORTED_PAGES:
9835 		retval = ctl_inquiry_evpd_supported(ctsio, alloc_len);
9836 		break;
9837 	case SVPD_UNIT_SERIAL_NUMBER:
9838 		retval = ctl_inquiry_evpd_serial(ctsio, alloc_len);
9839 		break;
9840 	case SVPD_DEVICE_ID:
9841 		retval = ctl_inquiry_evpd_devid(ctsio, alloc_len);
9842 		break;
9843 	case SVPD_BLOCK_LIMITS:
9844 		retval = ctl_inquiry_evpd_block_limits(ctsio, alloc_len);
9845 		break;
9846 	case SVPD_LBP:
9847 		retval = ctl_inquiry_evpd_lbp(ctsio, alloc_len);
9848 		break;
9849 	default:
9850 		ctl_set_invalid_field(ctsio,
9851 				      /*sks_valid*/ 1,
9852 				      /*command*/ 1,
9853 				      /*field*/ 2,
9854 				      /*bit_valid*/ 0,
9855 				      /*bit*/ 0);
9856 		ctl_done((union ctl_io *)ctsio);
9857 		retval = CTL_RETVAL_COMPLETE;
9858 		break;
9859 	}
9860 
9861 	return (retval);
9862 }
9863 
9864 static int
9865 ctl_inquiry_std(struct ctl_scsiio *ctsio)
9866 {
9867 	struct scsi_inquiry_data *inq_ptr;
9868 	struct scsi_inquiry *cdb;
9869 	struct ctl_softc *ctl_softc;
9870 	struct ctl_lun *lun;
9871 	char *val;
9872 	uint32_t alloc_len;
9873 	int is_fc;
9874 
9875 	ctl_softc = control_softc;
9876 
9877 	/*
9878 	 * Figure out whether we're talking to a Fibre Channel port or not.
9879 	 * We treat the ioctl front end, and any SCSI adapters, as packetized
9880 	 * SCSI front ends.
9881 	 */
9882 	if (ctl_softc->ctl_ports[ctl_port_idx(ctsio->io_hdr.nexus.targ_port)]->port_type !=
9883 	    CTL_PORT_FC)
9884 		is_fc = 0;
9885 	else
9886 		is_fc = 1;
9887 
9888 	lun = ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
9889 	cdb = (struct scsi_inquiry *)ctsio->cdb;
9890 	alloc_len = scsi_2btoul(cdb->length);
9891 
9892 	/*
9893 	 * We malloc the full inquiry data size here and fill it
9894 	 * in.  If the user only asks for less, we'll give him
9895 	 * that much.
9896 	 */
9897 	ctsio->kern_data_ptr = malloc(sizeof(*inq_ptr), M_CTL, M_WAITOK | M_ZERO);
9898 	inq_ptr = (struct scsi_inquiry_data *)ctsio->kern_data_ptr;
9899 	ctsio->kern_sg_entries = 0;
9900 	ctsio->kern_data_resid = 0;
9901 	ctsio->kern_rel_offset = 0;
9902 
9903 	if (sizeof(*inq_ptr) < alloc_len) {
9904 		ctsio->residual = alloc_len - sizeof(*inq_ptr);
9905 		ctsio->kern_data_len = sizeof(*inq_ptr);
9906 		ctsio->kern_total_len = sizeof(*inq_ptr);
9907 	} else {
9908 		ctsio->residual = 0;
9909 		ctsio->kern_data_len = alloc_len;
9910 		ctsio->kern_total_len = alloc_len;
9911 	}
9912 
9913 	/*
9914 	 * If we have a LUN configured, report it as connected.  Otherwise,
9915 	 * report that it is offline or no device is supported, depending
9916 	 * on the value of inquiry_pq_no_lun.
9917 	 *
9918 	 * According to the spec (SPC-4 r34), the peripheral qualifier
9919 	 * SID_QUAL_LU_OFFLINE (001b) is used in the following scenario:
9920 	 *
9921 	 * "A peripheral device having the specified peripheral device type
9922 	 * is not connected to this logical unit. However, the device
9923 	 * server is capable of supporting the specified peripheral device
9924 	 * type on this logical unit."
9925 	 *
9926 	 * According to the same spec, the peripheral qualifier
9927 	 * SID_QUAL_BAD_LU (011b) is used in this scenario:
9928 	 *
9929 	 * "The device server is not capable of supporting a peripheral
9930 	 * device on this logical unit. For this peripheral qualifier the
9931 	 * peripheral device type shall be set to 1Fh. All other peripheral
9932 	 * device type values are reserved for this peripheral qualifier."
9933 	 *
9934 	 * Given the text, it would seem that we probably want to report that
9935 	 * the LUN is offline here.  There is no LUN connected, but we can
9936 	 * support a LUN at the given LUN number.
9937 	 *
9938 	 * In the real world, though, it sounds like things are a little
9939 	 * different:
9940 	 *
9941 	 * - Linux, when presented with a LUN with the offline peripheral
9942 	 *   qualifier, will create an sg driver instance for it.  So when
9943 	 *   you attach it to CTL, you wind up with a ton of sg driver
9944 	 *   instances.  (One for every LUN that Linux bothered to probe.)
9945 	 *   Linux does this despite the fact that it issues a REPORT LUNs
9946 	 *   to LUN 0 to get the inventory of supported LUNs.
9947 	 *
9948 	 * - There is other anecdotal evidence (from Emulex folks) about
9949 	 *   arrays that use the offline peripheral qualifier for LUNs that
9950 	 *   are on the "passive" path in an active/passive array.
9951 	 *
9952 	 * So the solution is provide a hopefully reasonable default
9953 	 * (return bad/no LUN) and allow the user to change the behavior
9954 	 * with a tunable/sysctl variable.
9955 	 */
9956 	if (lun != NULL)
9957 		inq_ptr->device = (SID_QUAL_LU_CONNECTED << 5) |
9958 				  lun->be_lun->lun_type;
9959 	else if (ctl_softc->inquiry_pq_no_lun == 0)
9960 		inq_ptr->device = (SID_QUAL_LU_OFFLINE << 5) | T_DIRECT;
9961 	else
9962 		inq_ptr->device = (SID_QUAL_BAD_LU << 5) | T_NODEVICE;
9963 
9964 	/* RMB in byte 2 is 0 */
9965 	inq_ptr->version = SCSI_REV_SPC3;
9966 
9967 	/*
9968 	 * According to SAM-3, even if a device only supports a single
9969 	 * level of LUN addressing, it should still set the HISUP bit:
9970 	 *
9971 	 * 4.9.1 Logical unit numbers overview
9972 	 *
9973 	 * All logical unit number formats described in this standard are
9974 	 * hierarchical in structure even when only a single level in that
9975 	 * hierarchy is used. The HISUP bit shall be set to one in the
9976 	 * standard INQUIRY data (see SPC-2) when any logical unit number
9977 	 * format described in this standard is used.  Non-hierarchical
9978 	 * formats are outside the scope of this standard.
9979 	 *
9980 	 * Therefore we set the HiSup bit here.
9981 	 *
9982 	 * The reponse format is 2, per SPC-3.
9983 	 */
9984 	inq_ptr->response_format = SID_HiSup | 2;
9985 
9986 	inq_ptr->additional_length = sizeof(*inq_ptr) - 4;
9987 	CTL_DEBUG_PRINT(("additional_length = %d\n",
9988 			 inq_ptr->additional_length));
9989 
9990 	inq_ptr->spc3_flags = SPC3_SID_TPGS_IMPLICIT;
9991 	/* 16 bit addressing */
9992 	if (is_fc == 0)
9993 		inq_ptr->spc2_flags = SPC2_SID_ADDR16;
9994 	/* XXX set the SID_MultiP bit here if we're actually going to
9995 	   respond on multiple ports */
9996 	inq_ptr->spc2_flags |= SPC2_SID_MultiP;
9997 
9998 	/* 16 bit data bus, synchronous transfers */
9999 	/* XXX these flags don't apply for FC */
10000 	if (is_fc == 0)
10001 		inq_ptr->flags = SID_WBus16 | SID_Sync;
10002 	/*
10003 	 * XXX KDM do we want to support tagged queueing on the control
10004 	 * device at all?
10005 	 */
10006 	if ((lun == NULL)
10007 	 || (lun->be_lun->lun_type != T_PROCESSOR))
10008 		inq_ptr->flags |= SID_CmdQue;
10009 	/*
10010 	 * Per SPC-3, unused bytes in ASCII strings are filled with spaces.
10011 	 * We have 8 bytes for the vendor name, and 16 bytes for the device
10012 	 * name and 4 bytes for the revision.
10013 	 */
10014 	if (lun == NULL || (val = ctl_get_opt(lun->be_lun, "vendor")) == NULL) {
10015 		strcpy(inq_ptr->vendor, CTL_VENDOR);
10016 	} else {
10017 		memset(inq_ptr->vendor, ' ', sizeof(inq_ptr->vendor));
10018 		strncpy(inq_ptr->vendor, val,
10019 		    min(sizeof(inq_ptr->vendor), strlen(val)));
10020 	}
10021 	if (lun == NULL) {
10022 		strcpy(inq_ptr->product, CTL_DIRECT_PRODUCT);
10023 	} else if ((val = ctl_get_opt(lun->be_lun, "product")) == NULL) {
10024 		switch (lun->be_lun->lun_type) {
10025 		case T_DIRECT:
10026 			strcpy(inq_ptr->product, CTL_DIRECT_PRODUCT);
10027 			break;
10028 		case T_PROCESSOR:
10029 			strcpy(inq_ptr->product, CTL_PROCESSOR_PRODUCT);
10030 			break;
10031 		default:
10032 			strcpy(inq_ptr->product, CTL_UNKNOWN_PRODUCT);
10033 			break;
10034 		}
10035 	} else {
10036 		memset(inq_ptr->product, ' ', sizeof(inq_ptr->product));
10037 		strncpy(inq_ptr->product, val,
10038 		    min(sizeof(inq_ptr->product), strlen(val)));
10039 	}
10040 
10041 	/*
10042 	 * XXX make this a macro somewhere so it automatically gets
10043 	 * incremented when we make changes.
10044 	 */
10045 	if (lun == NULL || (val = ctl_get_opt(lun->be_lun, "revision")) == NULL) {
10046 		strncpy(inq_ptr->revision, "0001", sizeof(inq_ptr->revision));
10047 	} else {
10048 		memset(inq_ptr->revision, ' ', sizeof(inq_ptr->revision));
10049 		strncpy(inq_ptr->revision, val,
10050 		    min(sizeof(inq_ptr->revision), strlen(val)));
10051 	}
10052 
10053 	/*
10054 	 * For parallel SCSI, we support double transition and single
10055 	 * transition clocking.  We also support QAS (Quick Arbitration
10056 	 * and Selection) and Information Unit transfers on both the
10057 	 * control and array devices.
10058 	 */
10059 	if (is_fc == 0)
10060 		inq_ptr->spi3data = SID_SPI_CLOCK_DT_ST | SID_SPI_QAS |
10061 				    SID_SPI_IUS;
10062 
10063 	/* SAM-3 */
10064 	scsi_ulto2b(0x0060, inq_ptr->version1);
10065 	/* SPC-3 (no version claimed) XXX should we claim a version? */
10066 	scsi_ulto2b(0x0300, inq_ptr->version2);
10067 	if (is_fc) {
10068 		/* FCP-2 ANSI INCITS.350:2003 */
10069 		scsi_ulto2b(0x0917, inq_ptr->version3);
10070 	} else {
10071 		/* SPI-4 ANSI INCITS.362:200x */
10072 		scsi_ulto2b(0x0B56, inq_ptr->version3);
10073 	}
10074 
10075 	if (lun == NULL) {
10076 		/* SBC-2 (no version claimed) XXX should we claim a version? */
10077 		scsi_ulto2b(0x0320, inq_ptr->version4);
10078 	} else {
10079 		switch (lun->be_lun->lun_type) {
10080 		case T_DIRECT:
10081 			/*
10082 			 * SBC-2 (no version claimed) XXX should we claim a
10083 			 * version?
10084 			 */
10085 			scsi_ulto2b(0x0320, inq_ptr->version4);
10086 			break;
10087 		case T_PROCESSOR:
10088 		default:
10089 			break;
10090 		}
10091 	}
10092 
10093 	ctsio->scsi_status = SCSI_STATUS_OK;
10094 	if (ctsio->kern_data_len > 0) {
10095 		ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
10096 		ctsio->be_move_done = ctl_config_move_done;
10097 		ctl_datamove((union ctl_io *)ctsio);
10098 	} else {
10099 		ctsio->io_hdr.status = CTL_SUCCESS;
10100 		ctl_done((union ctl_io *)ctsio);
10101 	}
10102 
10103 	return (CTL_RETVAL_COMPLETE);
10104 }
10105 
10106 int
10107 ctl_inquiry(struct ctl_scsiio *ctsio)
10108 {
10109 	struct scsi_inquiry *cdb;
10110 	int retval;
10111 
10112 	cdb = (struct scsi_inquiry *)ctsio->cdb;
10113 
10114 	retval = 0;
10115 
10116 	CTL_DEBUG_PRINT(("ctl_inquiry\n"));
10117 
10118 	/*
10119 	 * Right now, we don't support the CmdDt inquiry information.
10120 	 * This would be nice to support in the future.  When we do
10121 	 * support it, we should change this test so that it checks to make
10122 	 * sure SI_EVPD and SI_CMDDT aren't both set at the same time.
10123 	 */
10124 #ifdef notyet
10125 	if (((cdb->byte2 & SI_EVPD)
10126 	 && (cdb->byte2 & SI_CMDDT)))
10127 #endif
10128 	if (cdb->byte2 & SI_CMDDT) {
10129 		/*
10130 		 * Point to the SI_CMDDT bit.  We might change this
10131 		 * when we support SI_CMDDT, but since both bits would be
10132 		 * "wrong", this should probably just stay as-is then.
10133 		 */
10134 		ctl_set_invalid_field(ctsio,
10135 				      /*sks_valid*/ 1,
10136 				      /*command*/ 1,
10137 				      /*field*/ 1,
10138 				      /*bit_valid*/ 1,
10139 				      /*bit*/ 1);
10140 		ctl_done((union ctl_io *)ctsio);
10141 		return (CTL_RETVAL_COMPLETE);
10142 	}
10143 	if (cdb->byte2 & SI_EVPD)
10144 		retval = ctl_inquiry_evpd(ctsio);
10145 #ifdef notyet
10146 	else if (cdb->byte2 & SI_CMDDT)
10147 		retval = ctl_inquiry_cmddt(ctsio);
10148 #endif
10149 	else
10150 		retval = ctl_inquiry_std(ctsio);
10151 
10152 	return (retval);
10153 }
10154 
10155 /*
10156  * For known CDB types, parse the LBA and length.
10157  */
10158 static int
10159 ctl_get_lba_len(union ctl_io *io, uint64_t *lba, uint32_t *len)
10160 {
10161 	if (io->io_hdr.io_type != CTL_IO_SCSI)
10162 		return (1);
10163 
10164 	switch (io->scsiio.cdb[0]) {
10165 	case COMPARE_AND_WRITE: {
10166 		struct scsi_compare_and_write *cdb;
10167 
10168 		cdb = (struct scsi_compare_and_write *)io->scsiio.cdb;
10169 
10170 		*lba = scsi_8btou64(cdb->addr);
10171 		*len = cdb->length;
10172 		break;
10173 	}
10174 	case READ_6:
10175 	case WRITE_6: {
10176 		struct scsi_rw_6 *cdb;
10177 
10178 		cdb = (struct scsi_rw_6 *)io->scsiio.cdb;
10179 
10180 		*lba = scsi_3btoul(cdb->addr);
10181 		/* only 5 bits are valid in the most significant address byte */
10182 		*lba &= 0x1fffff;
10183 		*len = cdb->length;
10184 		break;
10185 	}
10186 	case READ_10:
10187 	case WRITE_10: {
10188 		struct scsi_rw_10 *cdb;
10189 
10190 		cdb = (struct scsi_rw_10 *)io->scsiio.cdb;
10191 
10192 		*lba = scsi_4btoul(cdb->addr);
10193 		*len = scsi_2btoul(cdb->length);
10194 		break;
10195 	}
10196 	case WRITE_VERIFY_10: {
10197 		struct scsi_write_verify_10 *cdb;
10198 
10199 		cdb = (struct scsi_write_verify_10 *)io->scsiio.cdb;
10200 
10201 		*lba = scsi_4btoul(cdb->addr);
10202 		*len = scsi_2btoul(cdb->length);
10203 		break;
10204 	}
10205 	case READ_12:
10206 	case WRITE_12: {
10207 		struct scsi_rw_12 *cdb;
10208 
10209 		cdb = (struct scsi_rw_12 *)io->scsiio.cdb;
10210 
10211 		*lba = scsi_4btoul(cdb->addr);
10212 		*len = scsi_4btoul(cdb->length);
10213 		break;
10214 	}
10215 	case WRITE_VERIFY_12: {
10216 		struct scsi_write_verify_12 *cdb;
10217 
10218 		cdb = (struct scsi_write_verify_12 *)io->scsiio.cdb;
10219 
10220 		*lba = scsi_4btoul(cdb->addr);
10221 		*len = scsi_4btoul(cdb->length);
10222 		break;
10223 	}
10224 	case READ_16:
10225 	case WRITE_16: {
10226 		struct scsi_rw_16 *cdb;
10227 
10228 		cdb = (struct scsi_rw_16 *)io->scsiio.cdb;
10229 
10230 		*lba = scsi_8btou64(cdb->addr);
10231 		*len = scsi_4btoul(cdb->length);
10232 		break;
10233 	}
10234 	case WRITE_VERIFY_16: {
10235 		struct scsi_write_verify_16 *cdb;
10236 
10237 		cdb = (struct scsi_write_verify_16 *)io->scsiio.cdb;
10238 
10239 
10240 		*lba = scsi_8btou64(cdb->addr);
10241 		*len = scsi_4btoul(cdb->length);
10242 		break;
10243 	}
10244 	case WRITE_SAME_10: {
10245 		struct scsi_write_same_10 *cdb;
10246 
10247 		cdb = (struct scsi_write_same_10 *)io->scsiio.cdb;
10248 
10249 		*lba = scsi_4btoul(cdb->addr);
10250 		*len = scsi_2btoul(cdb->length);
10251 		break;
10252 	}
10253 	case WRITE_SAME_16: {
10254 		struct scsi_write_same_16 *cdb;
10255 
10256 		cdb = (struct scsi_write_same_16 *)io->scsiio.cdb;
10257 
10258 		*lba = scsi_8btou64(cdb->addr);
10259 		*len = scsi_4btoul(cdb->length);
10260 		break;
10261 	}
10262 	case VERIFY_10: {
10263 		struct scsi_verify_10 *cdb;
10264 
10265 		cdb = (struct scsi_verify_10 *)io->scsiio.cdb;
10266 
10267 		*lba = scsi_4btoul(cdb->addr);
10268 		*len = scsi_2btoul(cdb->length);
10269 		break;
10270 	}
10271 	case VERIFY_12: {
10272 		struct scsi_verify_12 *cdb;
10273 
10274 		cdb = (struct scsi_verify_12 *)io->scsiio.cdb;
10275 
10276 		*lba = scsi_4btoul(cdb->addr);
10277 		*len = scsi_4btoul(cdb->length);
10278 		break;
10279 	}
10280 	case VERIFY_16: {
10281 		struct scsi_verify_16 *cdb;
10282 
10283 		cdb = (struct scsi_verify_16 *)io->scsiio.cdb;
10284 
10285 		*lba = scsi_8btou64(cdb->addr);
10286 		*len = scsi_4btoul(cdb->length);
10287 		break;
10288 	}
10289 	default:
10290 		return (1);
10291 		break; /* NOTREACHED */
10292 	}
10293 
10294 	return (0);
10295 }
10296 
10297 static ctl_action
10298 ctl_extent_check_lba(uint64_t lba1, uint32_t len1, uint64_t lba2, uint32_t len2)
10299 {
10300 	uint64_t endlba1, endlba2;
10301 
10302 	endlba1 = lba1 + len1 - 1;
10303 	endlba2 = lba2 + len2 - 1;
10304 
10305 	if ((endlba1 < lba2)
10306 	 || (endlba2 < lba1))
10307 		return (CTL_ACTION_PASS);
10308 	else
10309 		return (CTL_ACTION_BLOCK);
10310 }
10311 
10312 static ctl_action
10313 ctl_extent_check(union ctl_io *io1, union ctl_io *io2)
10314 {
10315 	uint64_t lba1, lba2;
10316 	uint32_t len1, len2;
10317 	int retval;
10318 
10319 	retval = ctl_get_lba_len(io1, &lba1, &len1);
10320 	if (retval != 0)
10321 		return (CTL_ACTION_ERROR);
10322 
10323 	retval = ctl_get_lba_len(io2, &lba2, &len2);
10324 	if (retval != 0)
10325 		return (CTL_ACTION_ERROR);
10326 
10327 	return (ctl_extent_check_lba(lba1, len1, lba2, len2));
10328 }
10329 
10330 static ctl_action
10331 ctl_check_for_blockage(union ctl_io *pending_io, union ctl_io *ooa_io)
10332 {
10333 	struct ctl_cmd_entry *pending_entry, *ooa_entry;
10334 	ctl_serialize_action *serialize_row;
10335 
10336 	/*
10337 	 * The initiator attempted multiple untagged commands at the same
10338 	 * time.  Can't do that.
10339 	 */
10340 	if ((pending_io->scsiio.tag_type == CTL_TAG_UNTAGGED)
10341 	 && (ooa_io->scsiio.tag_type == CTL_TAG_UNTAGGED)
10342 	 && ((pending_io->io_hdr.nexus.targ_port ==
10343 	      ooa_io->io_hdr.nexus.targ_port)
10344 	  && (pending_io->io_hdr.nexus.initid.id ==
10345 	      ooa_io->io_hdr.nexus.initid.id))
10346 	 && ((ooa_io->io_hdr.flags & CTL_FLAG_ABORT) == 0))
10347 		return (CTL_ACTION_OVERLAP);
10348 
10349 	/*
10350 	 * The initiator attempted to send multiple tagged commands with
10351 	 * the same ID.  (It's fine if different initiators have the same
10352 	 * tag ID.)
10353 	 *
10354 	 * Even if all of those conditions are true, we don't kill the I/O
10355 	 * if the command ahead of us has been aborted.  We won't end up
10356 	 * sending it to the FETD, and it's perfectly legal to resend a
10357 	 * command with the same tag number as long as the previous
10358 	 * instance of this tag number has been aborted somehow.
10359 	 */
10360 	if ((pending_io->scsiio.tag_type != CTL_TAG_UNTAGGED)
10361 	 && (ooa_io->scsiio.tag_type != CTL_TAG_UNTAGGED)
10362 	 && (pending_io->scsiio.tag_num == ooa_io->scsiio.tag_num)
10363 	 && ((pending_io->io_hdr.nexus.targ_port ==
10364 	      ooa_io->io_hdr.nexus.targ_port)
10365 	  && (pending_io->io_hdr.nexus.initid.id ==
10366 	      ooa_io->io_hdr.nexus.initid.id))
10367 	 && ((ooa_io->io_hdr.flags & CTL_FLAG_ABORT) == 0))
10368 		return (CTL_ACTION_OVERLAP_TAG);
10369 
10370 	/*
10371 	 * If we get a head of queue tag, SAM-3 says that we should
10372 	 * immediately execute it.
10373 	 *
10374 	 * What happens if this command would normally block for some other
10375 	 * reason?  e.g. a request sense with a head of queue tag
10376 	 * immediately after a write.  Normally that would block, but this
10377 	 * will result in its getting executed immediately...
10378 	 *
10379 	 * We currently return "pass" instead of "skip", so we'll end up
10380 	 * going through the rest of the queue to check for overlapped tags.
10381 	 *
10382 	 * XXX KDM check for other types of blockage first??
10383 	 */
10384 	if (pending_io->scsiio.tag_type == CTL_TAG_HEAD_OF_QUEUE)
10385 		return (CTL_ACTION_PASS);
10386 
10387 	/*
10388 	 * Ordered tags have to block until all items ahead of them
10389 	 * have completed.  If we get called with an ordered tag, we always
10390 	 * block, if something else is ahead of us in the queue.
10391 	 */
10392 	if (pending_io->scsiio.tag_type == CTL_TAG_ORDERED)
10393 		return (CTL_ACTION_BLOCK);
10394 
10395 	/*
10396 	 * Simple tags get blocked until all head of queue and ordered tags
10397 	 * ahead of them have completed.  I'm lumping untagged commands in
10398 	 * with simple tags here.  XXX KDM is that the right thing to do?
10399 	 */
10400 	if (((pending_io->scsiio.tag_type == CTL_TAG_UNTAGGED)
10401 	  || (pending_io->scsiio.tag_type == CTL_TAG_SIMPLE))
10402 	 && ((ooa_io->scsiio.tag_type == CTL_TAG_HEAD_OF_QUEUE)
10403 	  || (ooa_io->scsiio.tag_type == CTL_TAG_ORDERED)))
10404 		return (CTL_ACTION_BLOCK);
10405 
10406 	pending_entry = &ctl_cmd_table[pending_io->scsiio.cdb[0]];
10407 	ooa_entry = &ctl_cmd_table[ooa_io->scsiio.cdb[0]];
10408 
10409 	serialize_row = ctl_serialize_table[ooa_entry->seridx];
10410 
10411 	switch (serialize_row[pending_entry->seridx]) {
10412 	case CTL_SER_BLOCK:
10413 		return (CTL_ACTION_BLOCK);
10414 		break; /* NOTREACHED */
10415 	case CTL_SER_EXTENT:
10416 		return (ctl_extent_check(pending_io, ooa_io));
10417 		break; /* NOTREACHED */
10418 	case CTL_SER_PASS:
10419 		return (CTL_ACTION_PASS);
10420 		break; /* NOTREACHED */
10421 	case CTL_SER_SKIP:
10422 		return (CTL_ACTION_SKIP);
10423 		break;
10424 	default:
10425 		panic("invalid serialization value %d",
10426 		      serialize_row[pending_entry->seridx]);
10427 		break; /* NOTREACHED */
10428 	}
10429 
10430 	return (CTL_ACTION_ERROR);
10431 }
10432 
10433 /*
10434  * Check for blockage or overlaps against the OOA (Order Of Arrival) queue.
10435  * Assumptions:
10436  * - pending_io is generally either incoming, or on the blocked queue
10437  * - starting I/O is the I/O we want to start the check with.
10438  */
10439 static ctl_action
10440 ctl_check_ooa(struct ctl_lun *lun, union ctl_io *pending_io,
10441 	      union ctl_io *starting_io)
10442 {
10443 	union ctl_io *ooa_io;
10444 	ctl_action action;
10445 
10446 	mtx_assert(&lun->lun_lock, MA_OWNED);
10447 
10448 	/*
10449 	 * Run back along the OOA queue, starting with the current
10450 	 * blocked I/O and going through every I/O before it on the
10451 	 * queue.  If starting_io is NULL, we'll just end up returning
10452 	 * CTL_ACTION_PASS.
10453 	 */
10454 	for (ooa_io = starting_io; ooa_io != NULL;
10455 	     ooa_io = (union ctl_io *)TAILQ_PREV(&ooa_io->io_hdr, ctl_ooaq,
10456 	     ooa_links)){
10457 
10458 		/*
10459 		 * This routine just checks to see whether
10460 		 * cur_blocked is blocked by ooa_io, which is ahead
10461 		 * of it in the queue.  It doesn't queue/dequeue
10462 		 * cur_blocked.
10463 		 */
10464 		action = ctl_check_for_blockage(pending_io, ooa_io);
10465 		switch (action) {
10466 		case CTL_ACTION_BLOCK:
10467 		case CTL_ACTION_OVERLAP:
10468 		case CTL_ACTION_OVERLAP_TAG:
10469 		case CTL_ACTION_SKIP:
10470 		case CTL_ACTION_ERROR:
10471 			return (action);
10472 			break; /* NOTREACHED */
10473 		case CTL_ACTION_PASS:
10474 			break;
10475 		default:
10476 			panic("invalid action %d", action);
10477 			break;  /* NOTREACHED */
10478 		}
10479 	}
10480 
10481 	return (CTL_ACTION_PASS);
10482 }
10483 
10484 /*
10485  * Assumptions:
10486  * - An I/O has just completed, and has been removed from the per-LUN OOA
10487  *   queue, so some items on the blocked queue may now be unblocked.
10488  */
10489 static int
10490 ctl_check_blocked(struct ctl_lun *lun)
10491 {
10492 	union ctl_io *cur_blocked, *next_blocked;
10493 
10494 	mtx_assert(&lun->lun_lock, MA_OWNED);
10495 
10496 	/*
10497 	 * Run forward from the head of the blocked queue, checking each
10498 	 * entry against the I/Os prior to it on the OOA queue to see if
10499 	 * there is still any blockage.
10500 	 *
10501 	 * We cannot use the TAILQ_FOREACH() macro, because it can't deal
10502 	 * with our removing a variable on it while it is traversing the
10503 	 * list.
10504 	 */
10505 	for (cur_blocked = (union ctl_io *)TAILQ_FIRST(&lun->blocked_queue);
10506 	     cur_blocked != NULL; cur_blocked = next_blocked) {
10507 		union ctl_io *prev_ooa;
10508 		ctl_action action;
10509 
10510 		next_blocked = (union ctl_io *)TAILQ_NEXT(&cur_blocked->io_hdr,
10511 							  blocked_links);
10512 
10513 		prev_ooa = (union ctl_io *)TAILQ_PREV(&cur_blocked->io_hdr,
10514 						      ctl_ooaq, ooa_links);
10515 
10516 		/*
10517 		 * If cur_blocked happens to be the first item in the OOA
10518 		 * queue now, prev_ooa will be NULL, and the action
10519 		 * returned will just be CTL_ACTION_PASS.
10520 		 */
10521 		action = ctl_check_ooa(lun, cur_blocked, prev_ooa);
10522 
10523 		switch (action) {
10524 		case CTL_ACTION_BLOCK:
10525 			/* Nothing to do here, still blocked */
10526 			break;
10527 		case CTL_ACTION_OVERLAP:
10528 		case CTL_ACTION_OVERLAP_TAG:
10529 			/*
10530 			 * This shouldn't happen!  In theory we've already
10531 			 * checked this command for overlap...
10532 			 */
10533 			break;
10534 		case CTL_ACTION_PASS:
10535 		case CTL_ACTION_SKIP: {
10536 			struct ctl_softc *softc;
10537 			struct ctl_cmd_entry *entry;
10538 			uint32_t initidx;
10539 			uint8_t opcode;
10540 			int isc_retval;
10541 
10542 			/*
10543 			 * The skip case shouldn't happen, this transaction
10544 			 * should have never made it onto the blocked queue.
10545 			 */
10546 			/*
10547 			 * This I/O is no longer blocked, we can remove it
10548 			 * from the blocked queue.  Since this is a TAILQ
10549 			 * (doubly linked list), we can do O(1) removals
10550 			 * from any place on the list.
10551 			 */
10552 			TAILQ_REMOVE(&lun->blocked_queue, &cur_blocked->io_hdr,
10553 				     blocked_links);
10554 			cur_blocked->io_hdr.flags &= ~CTL_FLAG_BLOCKED;
10555 
10556 			if (cur_blocked->io_hdr.flags & CTL_FLAG_FROM_OTHER_SC){
10557 				/*
10558 				 * Need to send IO back to original side to
10559 				 * run
10560 				 */
10561 				union ctl_ha_msg msg_info;
10562 
10563 				msg_info.hdr.original_sc =
10564 					cur_blocked->io_hdr.original_sc;
10565 				msg_info.hdr.serializing_sc = cur_blocked;
10566 				msg_info.hdr.msg_type = CTL_MSG_R2R;
10567 				if ((isc_retval=ctl_ha_msg_send(CTL_HA_CHAN_CTL,
10568 				     &msg_info, sizeof(msg_info), 0)) >
10569 				     CTL_HA_STATUS_SUCCESS) {
10570 					printf("CTL:Check Blocked error from "
10571 					       "ctl_ha_msg_send %d\n",
10572 					       isc_retval);
10573 				}
10574 				break;
10575 			}
10576 			opcode = cur_blocked->scsiio.cdb[0];
10577 			entry = &ctl_cmd_table[opcode];
10578 			softc = control_softc;
10579 
10580 			initidx = ctl_get_initindex(&cur_blocked->io_hdr.nexus);
10581 
10582 			/*
10583 			 * Check this I/O for LUN state changes that may
10584 			 * have happened while this command was blocked.
10585 			 * The LUN state may have been changed by a command
10586 			 * ahead of us in the queue, so we need to re-check
10587 			 * for any states that can be caused by SCSI
10588 			 * commands.
10589 			 */
10590 			if (ctl_scsiio_lun_check(softc, lun, entry,
10591 						 &cur_blocked->scsiio) == 0) {
10592 				cur_blocked->io_hdr.flags |=
10593 				                      CTL_FLAG_IS_WAS_ON_RTR;
10594 				ctl_enqueue_rtr(cur_blocked);
10595 			} else
10596 				ctl_done(cur_blocked);
10597 			break;
10598 		}
10599 		default:
10600 			/*
10601 			 * This probably shouldn't happen -- we shouldn't
10602 			 * get CTL_ACTION_ERROR, or anything else.
10603 			 */
10604 			break;
10605 		}
10606 	}
10607 
10608 	return (CTL_RETVAL_COMPLETE);
10609 }
10610 
10611 /*
10612  * This routine (with one exception) checks LUN flags that can be set by
10613  * commands ahead of us in the OOA queue.  These flags have to be checked
10614  * when a command initially comes in, and when we pull a command off the
10615  * blocked queue and are preparing to execute it.  The reason we have to
10616  * check these flags for commands on the blocked queue is that the LUN
10617  * state may have been changed by a command ahead of us while we're on the
10618  * blocked queue.
10619  *
10620  * Ordering is somewhat important with these checks, so please pay
10621  * careful attention to the placement of any new checks.
10622  */
10623 static int
10624 ctl_scsiio_lun_check(struct ctl_softc *ctl_softc, struct ctl_lun *lun,
10625 		     struct ctl_cmd_entry *entry, struct ctl_scsiio *ctsio)
10626 {
10627 	int retval;
10628 
10629 	retval = 0;
10630 
10631 	mtx_assert(&lun->lun_lock, MA_OWNED);
10632 
10633 	/*
10634 	 * If this shelf is a secondary shelf controller, we have to reject
10635 	 * any media access commands.
10636 	 */
10637 #if 0
10638 	/* No longer needed for HA */
10639 	if (((ctl_softc->flags & CTL_FLAG_MASTER_SHELF) == 0)
10640 	 && ((entry->flags & CTL_CMD_FLAG_OK_ON_SECONDARY) == 0)) {
10641 		ctl_set_lun_standby(ctsio);
10642 		retval = 1;
10643 		goto bailout;
10644 	}
10645 #endif
10646 
10647 	/*
10648 	 * Check for a reservation conflict.  If this command isn't allowed
10649 	 * even on reserved LUNs, and if this initiator isn't the one who
10650 	 * reserved us, reject the command with a reservation conflict.
10651 	 */
10652 	if ((lun->flags & CTL_LUN_RESERVED)
10653 	 && ((entry->flags & CTL_CMD_FLAG_ALLOW_ON_RESV) == 0)) {
10654 		if ((ctsio->io_hdr.nexus.initid.id != lun->rsv_nexus.initid.id)
10655 		 || (ctsio->io_hdr.nexus.targ_port != lun->rsv_nexus.targ_port)
10656 		 || (ctsio->io_hdr.nexus.targ_target.id !=
10657 		     lun->rsv_nexus.targ_target.id)) {
10658 			ctsio->scsi_status = SCSI_STATUS_RESERV_CONFLICT;
10659 			ctsio->io_hdr.status = CTL_SCSI_ERROR;
10660 			retval = 1;
10661 			goto bailout;
10662 		}
10663 	}
10664 
10665 	if ( (lun->flags & CTL_LUN_PR_RESERVED)
10666 	 && ((entry->flags & CTL_CMD_FLAG_ALLOW_ON_PR_RESV) == 0)) {
10667 		uint32_t residx;
10668 
10669 		residx = ctl_get_resindex(&ctsio->io_hdr.nexus);
10670 		/*
10671 		 * if we aren't registered or it's a res holder type
10672 		 * reservation and this isn't the res holder then set a
10673 		 * conflict.
10674 		 * NOTE: Commands which might be allowed on write exclusive
10675 		 * type reservations are checked in the particular command
10676 		 * for a conflict. Read and SSU are the only ones.
10677 		 */
10678 		if (!lun->per_res[residx].registered
10679 		 || (residx != lun->pr_res_idx && lun->res_type < 4)) {
10680 			ctsio->scsi_status = SCSI_STATUS_RESERV_CONFLICT;
10681 			ctsio->io_hdr.status = CTL_SCSI_ERROR;
10682 			retval = 1;
10683 			goto bailout;
10684 		}
10685 
10686 	}
10687 
10688 	if ((lun->flags & CTL_LUN_OFFLINE)
10689 	 && ((entry->flags & CTL_CMD_FLAG_OK_ON_OFFLINE) == 0)) {
10690 		ctl_set_lun_not_ready(ctsio);
10691 		retval = 1;
10692 		goto bailout;
10693 	}
10694 
10695 	/*
10696 	 * If the LUN is stopped, see if this particular command is allowed
10697 	 * for a stopped lun.  Otherwise, reject it with 0x04,0x02.
10698 	 */
10699 	if ((lun->flags & CTL_LUN_STOPPED)
10700 	 && ((entry->flags & CTL_CMD_FLAG_OK_ON_STOPPED) == 0)) {
10701 		/* "Logical unit not ready, initializing cmd. required" */
10702 		ctl_set_lun_stopped(ctsio);
10703 		retval = 1;
10704 		goto bailout;
10705 	}
10706 
10707 	if ((lun->flags & CTL_LUN_INOPERABLE)
10708 	 && ((entry->flags & CTL_CMD_FLAG_OK_ON_INOPERABLE) == 0)) {
10709 		/* "Medium format corrupted" */
10710 		ctl_set_medium_format_corrupted(ctsio);
10711 		retval = 1;
10712 		goto bailout;
10713 	}
10714 
10715 bailout:
10716 	return (retval);
10717 
10718 }
10719 
10720 static void
10721 ctl_failover_io(union ctl_io *io, int have_lock)
10722 {
10723 	ctl_set_busy(&io->scsiio);
10724 	ctl_done(io);
10725 }
10726 
10727 static void
10728 ctl_failover(void)
10729 {
10730 	struct ctl_lun *lun;
10731 	struct ctl_softc *ctl_softc;
10732 	union ctl_io *next_io, *pending_io;
10733 	union ctl_io *io;
10734 	int lun_idx;
10735 	int i;
10736 
10737 	ctl_softc = control_softc;
10738 
10739 	mtx_lock(&ctl_softc->ctl_lock);
10740 	/*
10741 	 * Remove any cmds from the other SC from the rtr queue.  These
10742 	 * will obviously only be for LUNs for which we're the primary.
10743 	 * We can't send status or get/send data for these commands.
10744 	 * Since they haven't been executed yet, we can just remove them.
10745 	 * We'll either abort them or delete them below, depending on
10746 	 * which HA mode we're in.
10747 	 */
10748 #ifdef notyet
10749 	mtx_lock(&ctl_softc->queue_lock);
10750 	for (io = (union ctl_io *)STAILQ_FIRST(&ctl_softc->rtr_queue);
10751 	     io != NULL; io = next_io) {
10752 		next_io = (union ctl_io *)STAILQ_NEXT(&io->io_hdr, links);
10753 		if (io->io_hdr.flags & CTL_FLAG_FROM_OTHER_SC)
10754 			STAILQ_REMOVE(&ctl_softc->rtr_queue, &io->io_hdr,
10755 				      ctl_io_hdr, links);
10756 	}
10757 	mtx_unlock(&ctl_softc->queue_lock);
10758 #endif
10759 
10760 	for (lun_idx=0; lun_idx < ctl_softc->num_luns; lun_idx++) {
10761 		lun = ctl_softc->ctl_luns[lun_idx];
10762 		if (lun==NULL)
10763 			continue;
10764 
10765 		/*
10766 		 * Processor LUNs are primary on both sides.
10767 		 * XXX will this always be true?
10768 		 */
10769 		if (lun->be_lun->lun_type == T_PROCESSOR)
10770 			continue;
10771 
10772 		if ((lun->flags & CTL_LUN_PRIMARY_SC)
10773 		 && (ctl_softc->ha_mode == CTL_HA_MODE_SER_ONLY)) {
10774 			printf("FAILOVER: primary lun %d\n", lun_idx);
10775 		        /*
10776 			 * Remove all commands from the other SC. First from the
10777 			 * blocked queue then from the ooa queue. Once we have
10778 			 * removed them. Call ctl_check_blocked to see if there
10779 			 * is anything that can run.
10780 			 */
10781 			for (io = (union ctl_io *)TAILQ_FIRST(
10782 			     &lun->blocked_queue); io != NULL; io = next_io) {
10783 
10784 		        	next_io = (union ctl_io *)TAILQ_NEXT(
10785 				    &io->io_hdr, blocked_links);
10786 
10787 				if (io->io_hdr.flags & CTL_FLAG_FROM_OTHER_SC) {
10788 					TAILQ_REMOVE(&lun->blocked_queue,
10789 						     &io->io_hdr,blocked_links);
10790 					io->io_hdr.flags &= ~CTL_FLAG_BLOCKED;
10791 					TAILQ_REMOVE(&lun->ooa_queue,
10792 						     &io->io_hdr, ooa_links);
10793 
10794 					ctl_free_io(io);
10795 				}
10796 			}
10797 
10798 			for (io = (union ctl_io *)TAILQ_FIRST(&lun->ooa_queue);
10799 	     		     io != NULL; io = next_io) {
10800 
10801 		        	next_io = (union ctl_io *)TAILQ_NEXT(
10802 				    &io->io_hdr, ooa_links);
10803 
10804 				if (io->io_hdr.flags & CTL_FLAG_FROM_OTHER_SC) {
10805 
10806 					TAILQ_REMOVE(&lun->ooa_queue,
10807 						&io->io_hdr,
10808 					     	ooa_links);
10809 
10810 					ctl_free_io(io);
10811 				}
10812 			}
10813 			ctl_check_blocked(lun);
10814 		} else if ((lun->flags & CTL_LUN_PRIMARY_SC)
10815 			&& (ctl_softc->ha_mode == CTL_HA_MODE_XFER)) {
10816 
10817 			printf("FAILOVER: primary lun %d\n", lun_idx);
10818 			/*
10819 			 * Abort all commands from the other SC.  We can't
10820 			 * send status back for them now.  These should get
10821 			 * cleaned up when they are completed or come out
10822 			 * for a datamove operation.
10823 			 */
10824 			for (io = (union ctl_io *)TAILQ_FIRST(&lun->ooa_queue);
10825 	     		     io != NULL; io = next_io) {
10826 		        	next_io = (union ctl_io *)TAILQ_NEXT(
10827 					&io->io_hdr, ooa_links);
10828 
10829 				if (io->io_hdr.flags & CTL_FLAG_FROM_OTHER_SC)
10830 					io->io_hdr.flags |= CTL_FLAG_ABORT;
10831 			}
10832 		} else if (((lun->flags & CTL_LUN_PRIMARY_SC) == 0)
10833 			&& (ctl_softc->ha_mode == CTL_HA_MODE_XFER)) {
10834 
10835 			printf("FAILOVER: secondary lun %d\n", lun_idx);
10836 
10837 			lun->flags |= CTL_LUN_PRIMARY_SC;
10838 
10839 			/*
10840 			 * We send all I/O that was sent to this controller
10841 			 * and redirected to the other side back with
10842 			 * busy status, and have the initiator retry it.
10843 			 * Figuring out how much data has been transferred,
10844 			 * etc. and picking up where we left off would be
10845 			 * very tricky.
10846 			 *
10847 			 * XXX KDM need to remove I/O from the blocked
10848 			 * queue as well!
10849 			 */
10850 			for (pending_io = (union ctl_io *)TAILQ_FIRST(
10851 			     &lun->ooa_queue); pending_io != NULL;
10852 			     pending_io = next_io) {
10853 
10854 				next_io =  (union ctl_io *)TAILQ_NEXT(
10855 					&pending_io->io_hdr, ooa_links);
10856 
10857 				pending_io->io_hdr.flags &=
10858 					~CTL_FLAG_SENT_2OTHER_SC;
10859 
10860 				if (pending_io->io_hdr.flags &
10861 				    CTL_FLAG_IO_ACTIVE) {
10862 					pending_io->io_hdr.flags |=
10863 						CTL_FLAG_FAILOVER;
10864 				} else {
10865 					ctl_set_busy(&pending_io->scsiio);
10866 					ctl_done(pending_io);
10867 				}
10868 			}
10869 
10870 			/*
10871 			 * Build Unit Attention
10872 			 */
10873 			for (i = 0; i < CTL_MAX_INITIATORS; i++) {
10874 				lun->pending_sense[i].ua_pending |=
10875 				                     CTL_UA_ASYM_ACC_CHANGE;
10876 			}
10877 		} else if (((lun->flags & CTL_LUN_PRIMARY_SC) == 0)
10878 			&& (ctl_softc->ha_mode == CTL_HA_MODE_SER_ONLY)) {
10879 			printf("FAILOVER: secondary lun %d\n", lun_idx);
10880 			/*
10881 			 * if the first io on the OOA is not on the RtR queue
10882 			 * add it.
10883 			 */
10884 			lun->flags |= CTL_LUN_PRIMARY_SC;
10885 
10886 			pending_io = (union ctl_io *)TAILQ_FIRST(
10887 			    &lun->ooa_queue);
10888 			if (pending_io==NULL) {
10889 				printf("Nothing on OOA queue\n");
10890 				continue;
10891 			}
10892 
10893 			pending_io->io_hdr.flags &= ~CTL_FLAG_SENT_2OTHER_SC;
10894 			if ((pending_io->io_hdr.flags &
10895 			     CTL_FLAG_IS_WAS_ON_RTR) == 0) {
10896 				pending_io->io_hdr.flags |=
10897 				    CTL_FLAG_IS_WAS_ON_RTR;
10898 				ctl_enqueue_rtr(pending_io);
10899 			}
10900 #if 0
10901 			else
10902 			{
10903 				printf("Tag 0x%04x is running\n",
10904 				      pending_io->scsiio.tag_num);
10905 			}
10906 #endif
10907 
10908 			next_io = (union ctl_io *)TAILQ_NEXT(
10909 			    &pending_io->io_hdr, ooa_links);
10910 			for (pending_io=next_io; pending_io != NULL;
10911 			     pending_io = next_io) {
10912 				pending_io->io_hdr.flags &=
10913 				    ~CTL_FLAG_SENT_2OTHER_SC;
10914 				next_io = (union ctl_io *)TAILQ_NEXT(
10915 					&pending_io->io_hdr, ooa_links);
10916 				if (pending_io->io_hdr.flags &
10917 				    CTL_FLAG_IS_WAS_ON_RTR) {
10918 #if 0
10919 				        printf("Tag 0x%04x is running\n",
10920 				      		pending_io->scsiio.tag_num);
10921 #endif
10922 					continue;
10923 				}
10924 
10925 				switch (ctl_check_ooa(lun, pending_io,
10926 			            (union ctl_io *)TAILQ_PREV(
10927 				    &pending_io->io_hdr, ctl_ooaq,
10928 				    ooa_links))) {
10929 
10930 				case CTL_ACTION_BLOCK:
10931 					TAILQ_INSERT_TAIL(&lun->blocked_queue,
10932 							  &pending_io->io_hdr,
10933 							  blocked_links);
10934 					pending_io->io_hdr.flags |=
10935 					    CTL_FLAG_BLOCKED;
10936 					break;
10937 				case CTL_ACTION_PASS:
10938 				case CTL_ACTION_SKIP:
10939 					pending_io->io_hdr.flags |=
10940 					    CTL_FLAG_IS_WAS_ON_RTR;
10941 					ctl_enqueue_rtr(pending_io);
10942 					break;
10943 				case CTL_ACTION_OVERLAP:
10944 					ctl_set_overlapped_cmd(
10945 					    (struct ctl_scsiio *)pending_io);
10946 					ctl_done(pending_io);
10947 					break;
10948 				case CTL_ACTION_OVERLAP_TAG:
10949 					ctl_set_overlapped_tag(
10950 					    (struct ctl_scsiio *)pending_io,
10951 					    pending_io->scsiio.tag_num & 0xff);
10952 					ctl_done(pending_io);
10953 					break;
10954 				case CTL_ACTION_ERROR:
10955 				default:
10956 					ctl_set_internal_failure(
10957 						(struct ctl_scsiio *)pending_io,
10958 						0,  // sks_valid
10959 						0); //retry count
10960 					ctl_done(pending_io);
10961 					break;
10962 				}
10963 			}
10964 
10965 			/*
10966 			 * Build Unit Attention
10967 			 */
10968 			for (i = 0; i < CTL_MAX_INITIATORS; i++) {
10969 				lun->pending_sense[i].ua_pending |=
10970 				                     CTL_UA_ASYM_ACC_CHANGE;
10971 			}
10972 		} else {
10973 			panic("Unhandled HA mode failover, LUN flags = %#x, "
10974 			      "ha_mode = #%x", lun->flags, ctl_softc->ha_mode);
10975 		}
10976 	}
10977 	ctl_pause_rtr = 0;
10978 	mtx_unlock(&ctl_softc->ctl_lock);
10979 }
10980 
10981 static int
10982 ctl_scsiio_precheck(struct ctl_softc *ctl_softc, struct ctl_scsiio *ctsio)
10983 {
10984 	struct ctl_lun *lun;
10985 	struct ctl_cmd_entry *entry;
10986 	uint8_t opcode;
10987 	uint32_t initidx, targ_lun;
10988 	int retval;
10989 
10990 	retval = 0;
10991 
10992 	lun = NULL;
10993 
10994 	opcode = ctsio->cdb[0];
10995 
10996 	targ_lun = ctsio->io_hdr.nexus.targ_mapped_lun;
10997 	if ((targ_lun < CTL_MAX_LUNS)
10998 	 && (ctl_softc->ctl_luns[targ_lun] != NULL)) {
10999 		lun = ctl_softc->ctl_luns[targ_lun];
11000 		/*
11001 		 * If the LUN is invalid, pretend that it doesn't exist.
11002 		 * It will go away as soon as all pending I/O has been
11003 		 * completed.
11004 		 */
11005 		if (lun->flags & CTL_LUN_DISABLED) {
11006 			lun = NULL;
11007 		} else {
11008 			ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr = lun;
11009 			ctsio->io_hdr.ctl_private[CTL_PRIV_BACKEND_LUN].ptr =
11010 				lun->be_lun;
11011 			if (lun->be_lun->lun_type == T_PROCESSOR) {
11012 				ctsio->io_hdr.flags |= CTL_FLAG_CONTROL_DEV;
11013 			}
11014 		}
11015 	} else {
11016 		ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr = NULL;
11017 		ctsio->io_hdr.ctl_private[CTL_PRIV_BACKEND_LUN].ptr = NULL;
11018 	}
11019 
11020 	entry = &ctl_cmd_table[opcode];
11021 
11022 	ctsio->io_hdr.flags &= ~CTL_FLAG_DATA_MASK;
11023 	ctsio->io_hdr.flags |= entry->flags & CTL_FLAG_DATA_MASK;
11024 
11025 	/*
11026 	 * Check to see whether we can send this command to LUNs that don't
11027 	 * exist.  This should pretty much only be the case for inquiry
11028 	 * and request sense.  Further checks, below, really require having
11029 	 * a LUN, so we can't really check the command anymore.  Just put
11030 	 * it on the rtr queue.
11031 	 */
11032 	if (lun == NULL) {
11033 		if (entry->flags & CTL_CMD_FLAG_OK_ON_ALL_LUNS) {
11034 			ctsio->io_hdr.flags |= CTL_FLAG_IS_WAS_ON_RTR;
11035 			ctl_enqueue_rtr((union ctl_io *)ctsio);
11036 			return (retval);
11037 		}
11038 
11039 		ctl_set_unsupported_lun(ctsio);
11040 		ctl_done((union ctl_io *)ctsio);
11041 		CTL_DEBUG_PRINT(("ctl_scsiio_precheck: bailing out due to invalid LUN\n"));
11042 		return (retval);
11043 	} else {
11044 		mtx_lock(&lun->lun_lock);
11045 
11046 		/*
11047 		 * Every I/O goes into the OOA queue for a particular LUN, and
11048 		 * stays there until completion.
11049 		 */
11050 		TAILQ_INSERT_TAIL(&lun->ooa_queue, &ctsio->io_hdr, ooa_links);
11051 
11052 		/*
11053 		 * Make sure we support this particular command on this LUN.
11054 		 * e.g., we don't support writes to the control LUN.
11055 		 */
11056 		switch (lun->be_lun->lun_type) {
11057 		case T_PROCESSOR:
11058 		 	if (((entry->flags & CTL_CMD_FLAG_OK_ON_PROC) == 0)
11059 			 && ((entry->flags & CTL_CMD_FLAG_OK_ON_ALL_LUNS)
11060 			      == 0)) {
11061 				mtx_unlock(&lun->lun_lock);
11062 				ctl_set_invalid_opcode(ctsio);
11063 				ctl_done((union ctl_io *)ctsio);
11064 				return (retval);
11065 			}
11066 			break;
11067 		case T_DIRECT:
11068 			if (((entry->flags & CTL_CMD_FLAG_OK_ON_SLUN) == 0)
11069 			 && ((entry->flags & CTL_CMD_FLAG_OK_ON_ALL_LUNS)
11070 			      == 0)){
11071 				mtx_unlock(&lun->lun_lock);
11072 				ctl_set_invalid_opcode(ctsio);
11073 				ctl_done((union ctl_io *)ctsio);
11074 				return (retval);
11075 			}
11076 			break;
11077 		default:
11078 			mtx_unlock(&lun->lun_lock);
11079 			panic("Unsupported CTL LUN type %d\n",
11080 			      lun->be_lun->lun_type);
11081 		}
11082 	}
11083 
11084 	initidx = ctl_get_initindex(&ctsio->io_hdr.nexus);
11085 
11086 	/*
11087 	 * If we've got a request sense, it'll clear the contingent
11088 	 * allegiance condition.  Otherwise, if we have a CA condition for
11089 	 * this initiator, clear it, because it sent down a command other
11090 	 * than request sense.
11091 	 */
11092 	if ((opcode != REQUEST_SENSE)
11093 	 && (ctl_is_set(lun->have_ca, initidx)))
11094 		ctl_clear_mask(lun->have_ca, initidx);
11095 
11096 	/*
11097 	 * If the command has this flag set, it handles its own unit
11098 	 * attention reporting, we shouldn't do anything.  Otherwise we
11099 	 * check for any pending unit attentions, and send them back to the
11100 	 * initiator.  We only do this when a command initially comes in,
11101 	 * not when we pull it off the blocked queue.
11102 	 *
11103 	 * According to SAM-3, section 5.3.2, the order that things get
11104 	 * presented back to the host is basically unit attentions caused
11105 	 * by some sort of reset event, busy status, reservation conflicts
11106 	 * or task set full, and finally any other status.
11107 	 *
11108 	 * One issue here is that some of the unit attentions we report
11109 	 * don't fall into the "reset" category (e.g. "reported luns data
11110 	 * has changed").  So reporting it here, before the reservation
11111 	 * check, may be technically wrong.  I guess the only thing to do
11112 	 * would be to check for and report the reset events here, and then
11113 	 * check for the other unit attention types after we check for a
11114 	 * reservation conflict.
11115 	 *
11116 	 * XXX KDM need to fix this
11117 	 */
11118 	if ((entry->flags & CTL_CMD_FLAG_NO_SENSE) == 0) {
11119 		ctl_ua_type ua_type;
11120 
11121 		ua_type = lun->pending_sense[initidx].ua_pending;
11122 		if (ua_type != CTL_UA_NONE) {
11123 			scsi_sense_data_type sense_format;
11124 
11125 			if (lun != NULL)
11126 				sense_format = (lun->flags &
11127 				    CTL_LUN_SENSE_DESC) ? SSD_TYPE_DESC :
11128 				    SSD_TYPE_FIXED;
11129 			else
11130 				sense_format = SSD_TYPE_FIXED;
11131 
11132 			ua_type = ctl_build_ua(ua_type, &ctsio->sense_data,
11133 					       sense_format);
11134 			if (ua_type != CTL_UA_NONE) {
11135 				ctsio->scsi_status = SCSI_STATUS_CHECK_COND;
11136 				ctsio->io_hdr.status = CTL_SCSI_ERROR |
11137 						       CTL_AUTOSENSE;
11138 				ctsio->sense_len = SSD_FULL_SIZE;
11139 				lun->pending_sense[initidx].ua_pending &=
11140 					~ua_type;
11141 				mtx_unlock(&lun->lun_lock);
11142 				ctl_done((union ctl_io *)ctsio);
11143 				return (retval);
11144 			}
11145 		}
11146 	}
11147 
11148 
11149 	if (ctl_scsiio_lun_check(ctl_softc, lun, entry, ctsio) != 0) {
11150 		mtx_unlock(&lun->lun_lock);
11151 		ctl_done((union ctl_io *)ctsio);
11152 		return (retval);
11153 	}
11154 
11155 	/*
11156 	 * XXX CHD this is where we want to send IO to other side if
11157 	 * this LUN is secondary on this SC. We will need to make a copy
11158 	 * of the IO and flag the IO on this side as SENT_2OTHER and the flag
11159 	 * the copy we send as FROM_OTHER.
11160 	 * We also need to stuff the address of the original IO so we can
11161 	 * find it easily. Something similar will need be done on the other
11162 	 * side so when we are done we can find the copy.
11163 	 */
11164 	if ((lun->flags & CTL_LUN_PRIMARY_SC) == 0) {
11165 		union ctl_ha_msg msg_info;
11166 		int isc_retval;
11167 
11168 		ctsio->io_hdr.flags |= CTL_FLAG_SENT_2OTHER_SC;
11169 
11170 		msg_info.hdr.msg_type = CTL_MSG_SERIALIZE;
11171 		msg_info.hdr.original_sc = (union ctl_io *)ctsio;
11172 #if 0
11173 		printf("1. ctsio %p\n", ctsio);
11174 #endif
11175 		msg_info.hdr.serializing_sc = NULL;
11176 		msg_info.hdr.nexus = ctsio->io_hdr.nexus;
11177 		msg_info.scsi.tag_num = ctsio->tag_num;
11178 		msg_info.scsi.tag_type = ctsio->tag_type;
11179 		memcpy(msg_info.scsi.cdb, ctsio->cdb, CTL_MAX_CDBLEN);
11180 
11181 		ctsio->io_hdr.flags &= ~CTL_FLAG_IO_ACTIVE;
11182 
11183 		if ((isc_retval=ctl_ha_msg_send(CTL_HA_CHAN_CTL,
11184 		    (void *)&msg_info, sizeof(msg_info), 0)) >
11185 		    CTL_HA_STATUS_SUCCESS) {
11186 			printf("CTL:precheck, ctl_ha_msg_send returned %d\n",
11187 			       isc_retval);
11188 			printf("CTL:opcode is %x\n",opcode);
11189 		} else {
11190 #if 0
11191 			printf("CTL:Precheck sent msg, opcode is %x\n",opcode);
11192 #endif
11193 		}
11194 
11195 		/*
11196 		 * XXX KDM this I/O is off the incoming queue, but hasn't
11197 		 * been inserted on any other queue.  We may need to come
11198 		 * up with a holding queue while we wait for serialization
11199 		 * so that we have an idea of what we're waiting for from
11200 		 * the other side.
11201 		 */
11202 		mtx_unlock(&lun->lun_lock);
11203 		return (retval);
11204 	}
11205 
11206 	switch (ctl_check_ooa(lun, (union ctl_io *)ctsio,
11207 			      (union ctl_io *)TAILQ_PREV(&ctsio->io_hdr,
11208 			      ctl_ooaq, ooa_links))) {
11209 	case CTL_ACTION_BLOCK:
11210 		ctsio->io_hdr.flags |= CTL_FLAG_BLOCKED;
11211 		TAILQ_INSERT_TAIL(&lun->blocked_queue, &ctsio->io_hdr,
11212 				  blocked_links);
11213 		mtx_unlock(&lun->lun_lock);
11214 		return (retval);
11215 	case CTL_ACTION_PASS:
11216 	case CTL_ACTION_SKIP:
11217 		ctsio->io_hdr.flags |= CTL_FLAG_IS_WAS_ON_RTR;
11218 		mtx_unlock(&lun->lun_lock);
11219 		ctl_enqueue_rtr((union ctl_io *)ctsio);
11220 		break;
11221 	case CTL_ACTION_OVERLAP:
11222 		mtx_unlock(&lun->lun_lock);
11223 		ctl_set_overlapped_cmd(ctsio);
11224 		ctl_done((union ctl_io *)ctsio);
11225 		break;
11226 	case CTL_ACTION_OVERLAP_TAG:
11227 		mtx_unlock(&lun->lun_lock);
11228 		ctl_set_overlapped_tag(ctsio, ctsio->tag_num & 0xff);
11229 		ctl_done((union ctl_io *)ctsio);
11230 		break;
11231 	case CTL_ACTION_ERROR:
11232 	default:
11233 		mtx_unlock(&lun->lun_lock);
11234 		ctl_set_internal_failure(ctsio,
11235 					 /*sks_valid*/ 0,
11236 					 /*retry_count*/ 0);
11237 		ctl_done((union ctl_io *)ctsio);
11238 		break;
11239 	}
11240 	return (retval);
11241 }
11242 
11243 static int
11244 ctl_scsiio(struct ctl_scsiio *ctsio)
11245 {
11246 	int retval;
11247 	struct ctl_cmd_entry *entry;
11248 
11249 	retval = CTL_RETVAL_COMPLETE;
11250 
11251 	CTL_DEBUG_PRINT(("ctl_scsiio cdb[0]=%02X\n", ctsio->cdb[0]));
11252 
11253 	entry = &ctl_cmd_table[ctsio->cdb[0]];
11254 
11255 	/*
11256 	 * If this I/O has been aborted, just send it straight to
11257 	 * ctl_done() without executing it.
11258 	 */
11259 	if (ctsio->io_hdr.flags & CTL_FLAG_ABORT) {
11260 		ctl_done((union ctl_io *)ctsio);
11261 		goto bailout;
11262 	}
11263 
11264 	/*
11265 	 * All the checks should have been handled by ctl_scsiio_precheck().
11266 	 * We should be clear now to just execute the I/O.
11267 	 */
11268 	retval = entry->execute(ctsio);
11269 
11270 bailout:
11271 	return (retval);
11272 }
11273 
11274 /*
11275  * Since we only implement one target right now, a bus reset simply resets
11276  * our single target.
11277  */
11278 static int
11279 ctl_bus_reset(struct ctl_softc *ctl_softc, union ctl_io *io)
11280 {
11281 	return(ctl_target_reset(ctl_softc, io, CTL_UA_BUS_RESET));
11282 }
11283 
11284 static int
11285 ctl_target_reset(struct ctl_softc *ctl_softc, union ctl_io *io,
11286 		 ctl_ua_type ua_type)
11287 {
11288 	struct ctl_lun *lun;
11289 	int retval;
11290 
11291 	if (!(io->io_hdr.flags & CTL_FLAG_FROM_OTHER_SC)) {
11292 		union ctl_ha_msg msg_info;
11293 
11294 		io->io_hdr.flags |= CTL_FLAG_SENT_2OTHER_SC;
11295 		msg_info.hdr.nexus = io->io_hdr.nexus;
11296 		if (ua_type==CTL_UA_TARG_RESET)
11297 			msg_info.task.task_action = CTL_TASK_TARGET_RESET;
11298 		else
11299 			msg_info.task.task_action = CTL_TASK_BUS_RESET;
11300 		msg_info.hdr.msg_type = CTL_MSG_MANAGE_TASKS;
11301 		msg_info.hdr.original_sc = NULL;
11302 		msg_info.hdr.serializing_sc = NULL;
11303 		if (CTL_HA_STATUS_SUCCESS != ctl_ha_msg_send(CTL_HA_CHAN_CTL,
11304 		    (void *)&msg_info, sizeof(msg_info), 0)) {
11305 		}
11306 	}
11307 	retval = 0;
11308 
11309 	mtx_lock(&ctl_softc->ctl_lock);
11310 	STAILQ_FOREACH(lun, &ctl_softc->lun_list, links)
11311 		retval += ctl_lun_reset(lun, io, ua_type);
11312 	mtx_unlock(&ctl_softc->ctl_lock);
11313 
11314 	return (retval);
11315 }
11316 
11317 /*
11318  * The LUN should always be set.  The I/O is optional, and is used to
11319  * distinguish between I/Os sent by this initiator, and by other
11320  * initiators.  We set unit attention for initiators other than this one.
11321  * SAM-3 is vague on this point.  It does say that a unit attention should
11322  * be established for other initiators when a LUN is reset (see section
11323  * 5.7.3), but it doesn't specifically say that the unit attention should
11324  * be established for this particular initiator when a LUN is reset.  Here
11325  * is the relevant text, from SAM-3 rev 8:
11326  *
11327  * 5.7.2 When a SCSI initiator port aborts its own tasks
11328  *
11329  * When a SCSI initiator port causes its own task(s) to be aborted, no
11330  * notification that the task(s) have been aborted shall be returned to
11331  * the SCSI initiator port other than the completion response for the
11332  * command or task management function action that caused the task(s) to
11333  * be aborted and notification(s) associated with related effects of the
11334  * action (e.g., a reset unit attention condition).
11335  *
11336  * XXX KDM for now, we're setting unit attention for all initiators.
11337  */
11338 static int
11339 ctl_lun_reset(struct ctl_lun *lun, union ctl_io *io, ctl_ua_type ua_type)
11340 {
11341 	union ctl_io *xio;
11342 #if 0
11343 	uint32_t initindex;
11344 #endif
11345 	int i;
11346 
11347 	mtx_lock(&lun->lun_lock);
11348 	/*
11349 	 * Run through the OOA queue and abort each I/O.
11350 	 */
11351 #if 0
11352 	TAILQ_FOREACH((struct ctl_io_hdr *)xio, &lun->ooa_queue, ooa_links) {
11353 #endif
11354 	for (xio = (union ctl_io *)TAILQ_FIRST(&lun->ooa_queue); xio != NULL;
11355 	     xio = (union ctl_io *)TAILQ_NEXT(&xio->io_hdr, ooa_links)) {
11356 		xio->io_hdr.flags |= CTL_FLAG_ABORT;
11357 	}
11358 
11359 	/*
11360 	 * This version sets unit attention for every
11361 	 */
11362 #if 0
11363 	initindex = ctl_get_initindex(&io->io_hdr.nexus);
11364 	for (i = 0; i < CTL_MAX_INITIATORS; i++) {
11365 		if (initindex == i)
11366 			continue;
11367 		lun->pending_sense[i].ua_pending |= ua_type;
11368 	}
11369 #endif
11370 
11371 	/*
11372 	 * A reset (any kind, really) clears reservations established with
11373 	 * RESERVE/RELEASE.  It does not clear reservations established
11374 	 * with PERSISTENT RESERVE OUT, but we don't support that at the
11375 	 * moment anyway.  See SPC-2, section 5.6.  SPC-3 doesn't address
11376 	 * reservations made with the RESERVE/RELEASE commands, because
11377 	 * those commands are obsolete in SPC-3.
11378 	 */
11379 	lun->flags &= ~CTL_LUN_RESERVED;
11380 
11381 	for (i = 0; i < CTL_MAX_INITIATORS; i++) {
11382 		ctl_clear_mask(lun->have_ca, i);
11383 		lun->pending_sense[i].ua_pending |= ua_type;
11384 	}
11385 	mtx_lock(&lun->lun_lock);
11386 
11387 	return (0);
11388 }
11389 
11390 static int
11391 ctl_abort_task(union ctl_io *io)
11392 {
11393 	union ctl_io *xio;
11394 	struct ctl_lun *lun;
11395 	struct ctl_softc *ctl_softc;
11396 #if 0
11397 	struct sbuf sb;
11398 	char printbuf[128];
11399 #endif
11400 	int found;
11401 	uint32_t targ_lun;
11402 
11403 	ctl_softc = control_softc;
11404 	found = 0;
11405 
11406 	/*
11407 	 * Look up the LUN.
11408 	 */
11409 	targ_lun = io->io_hdr.nexus.targ_mapped_lun;
11410 	mtx_lock(&ctl_softc->ctl_lock);
11411 	if ((targ_lun < CTL_MAX_LUNS)
11412 	 && (ctl_softc->ctl_luns[targ_lun] != NULL))
11413 		lun = ctl_softc->ctl_luns[targ_lun];
11414 	else {
11415 		mtx_unlock(&ctl_softc->ctl_lock);
11416 		goto bailout;
11417 	}
11418 
11419 #if 0
11420 	printf("ctl_abort_task: called for lun %lld, tag %d type %d\n",
11421 	       lun->lun, io->taskio.tag_num, io->taskio.tag_type);
11422 #endif
11423 
11424 	mtx_lock(&lun->lun_lock);
11425 	mtx_unlock(&ctl_softc->ctl_lock);
11426 	/*
11427 	 * Run through the OOA queue and attempt to find the given I/O.
11428 	 * The target port, initiator ID, tag type and tag number have to
11429 	 * match the values that we got from the initiator.  If we have an
11430 	 * untagged command to abort, simply abort the first untagged command
11431 	 * we come to.  We only allow one untagged command at a time of course.
11432 	 */
11433 #if 0
11434 	TAILQ_FOREACH((struct ctl_io_hdr *)xio, &lun->ooa_queue, ooa_links) {
11435 #endif
11436 	for (xio = (union ctl_io *)TAILQ_FIRST(&lun->ooa_queue); xio != NULL;
11437 	     xio = (union ctl_io *)TAILQ_NEXT(&xio->io_hdr, ooa_links)) {
11438 #if 0
11439 		sbuf_new(&sb, printbuf, sizeof(printbuf), SBUF_FIXEDLEN);
11440 
11441 		sbuf_printf(&sb, "LUN %lld tag %d type %d%s%s%s%s: ",
11442 			    lun->lun, xio->scsiio.tag_num,
11443 			    xio->scsiio.tag_type,
11444 			    (xio->io_hdr.blocked_links.tqe_prev
11445 			    == NULL) ? "" : " BLOCKED",
11446 			    (xio->io_hdr.flags &
11447 			    CTL_FLAG_DMA_INPROG) ? " DMA" : "",
11448 			    (xio->io_hdr.flags &
11449 			    CTL_FLAG_ABORT) ? " ABORT" : "",
11450 			    (xio->io_hdr.flags &
11451 			    CTL_FLAG_IS_WAS_ON_RTR ? " RTR" : ""));
11452 		ctl_scsi_command_string(&xio->scsiio, NULL, &sb);
11453 		sbuf_finish(&sb);
11454 		printf("%s\n", sbuf_data(&sb));
11455 #endif
11456 
11457 		if ((xio->io_hdr.nexus.targ_port == io->io_hdr.nexus.targ_port)
11458 		 && (xio->io_hdr.nexus.initid.id ==
11459 		     io->io_hdr.nexus.initid.id)) {
11460 			/*
11461 			 * If the abort says that the task is untagged, the
11462 			 * task in the queue must be untagged.  Otherwise,
11463 			 * we just check to see whether the tag numbers
11464 			 * match.  This is because the QLogic firmware
11465 			 * doesn't pass back the tag type in an abort
11466 			 * request.
11467 			 */
11468 #if 0
11469 			if (((xio->scsiio.tag_type == CTL_TAG_UNTAGGED)
11470 			  && (io->taskio.tag_type == CTL_TAG_UNTAGGED))
11471 			 || (xio->scsiio.tag_num == io->taskio.tag_num)) {
11472 #endif
11473 			/*
11474 			 * XXX KDM we've got problems with FC, because it
11475 			 * doesn't send down a tag type with aborts.  So we
11476 			 * can only really go by the tag number...
11477 			 * This may cause problems with parallel SCSI.
11478 			 * Need to figure that out!!
11479 			 */
11480 			if (xio->scsiio.tag_num == io->taskio.tag_num) {
11481 				xio->io_hdr.flags |= CTL_FLAG_ABORT;
11482 				found = 1;
11483 				if ((io->io_hdr.flags &
11484 				     CTL_FLAG_FROM_OTHER_SC) == 0 &&
11485 				    !(lun->flags & CTL_LUN_PRIMARY_SC)) {
11486 					union ctl_ha_msg msg_info;
11487 
11488 					io->io_hdr.flags |=
11489 					                CTL_FLAG_SENT_2OTHER_SC;
11490 					msg_info.hdr.nexus = io->io_hdr.nexus;
11491 					msg_info.task.task_action =
11492 						CTL_TASK_ABORT_TASK;
11493 					msg_info.task.tag_num =
11494 						io->taskio.tag_num;
11495 					msg_info.task.tag_type =
11496 						io->taskio.tag_type;
11497 					msg_info.hdr.msg_type =
11498 						CTL_MSG_MANAGE_TASKS;
11499 					msg_info.hdr.original_sc = NULL;
11500 					msg_info.hdr.serializing_sc = NULL;
11501 #if 0
11502 					printf("Sent Abort to other side\n");
11503 #endif
11504 					if (CTL_HA_STATUS_SUCCESS !=
11505 					        ctl_ha_msg_send(CTL_HA_CHAN_CTL,
11506 		    				(void *)&msg_info,
11507 						sizeof(msg_info), 0)) {
11508 					}
11509 				}
11510 #if 0
11511 				printf("ctl_abort_task: found I/O to abort\n");
11512 #endif
11513 				break;
11514 			}
11515 		}
11516 	}
11517 	mtx_unlock(&lun->lun_lock);
11518 
11519 bailout:
11520 
11521 	if (found == 0) {
11522 		/*
11523 		 * This isn't really an error.  It's entirely possible for
11524 		 * the abort and command completion to cross on the wire.
11525 		 * This is more of an informative/diagnostic error.
11526 		 */
11527 #if 0
11528 		printf("ctl_abort_task: ABORT sent for nonexistent I/O: "
11529 		       "%d:%d:%d:%d tag %d type %d\n",
11530 		       io->io_hdr.nexus.initid.id,
11531 		       io->io_hdr.nexus.targ_port,
11532 		       io->io_hdr.nexus.targ_target.id,
11533 		       io->io_hdr.nexus.targ_lun, io->taskio.tag_num,
11534 		       io->taskio.tag_type);
11535 #endif
11536 		return (1);
11537 	} else
11538 		return (0);
11539 }
11540 
11541 /*
11542  * This routine cannot block!  It must be callable from an interrupt
11543  * handler as well as from the work thread.
11544  */
11545 static void
11546 ctl_run_task(union ctl_io *io)
11547 {
11548 	struct ctl_softc *ctl_softc;
11549 	int retval;
11550 	const char *task_desc;
11551 
11552 	CTL_DEBUG_PRINT(("ctl_run_task\n"));
11553 
11554 	ctl_softc = control_softc;
11555 	retval = 0;
11556 
11557 	KASSERT(io->io_hdr.io_type == CTL_IO_TASK,
11558 	    ("ctl_run_task: Unextected io_type %d\n",
11559 	     io->io_hdr.io_type));
11560 
11561 	task_desc = ctl_scsi_task_string(&io->taskio);
11562 	if (task_desc != NULL) {
11563 #ifdef NEEDTOPORT
11564 		csevent_log(CSC_CTL | CSC_SHELF_SW |
11565 			    CTL_TASK_REPORT,
11566 			    csevent_LogType_Trace,
11567 			    csevent_Severity_Information,
11568 			    csevent_AlertLevel_Green,
11569 			    csevent_FRU_Firmware,
11570 			    csevent_FRU_Unknown,
11571 			    "CTL: received task: %s",task_desc);
11572 #endif
11573 	} else {
11574 #ifdef NEEDTOPORT
11575 		csevent_log(CSC_CTL | CSC_SHELF_SW |
11576 			    CTL_TASK_REPORT,
11577 			    csevent_LogType_Trace,
11578 			    csevent_Severity_Information,
11579 			    csevent_AlertLevel_Green,
11580 			    csevent_FRU_Firmware,
11581 			    csevent_FRU_Unknown,
11582 			    "CTL: received unknown task "
11583 			    "type: %d (%#x)",
11584 			    io->taskio.task_action,
11585 			    io->taskio.task_action);
11586 #endif
11587 	}
11588 	switch (io->taskio.task_action) {
11589 	case CTL_TASK_ABORT_TASK:
11590 		retval = ctl_abort_task(io);
11591 		break;
11592 	case CTL_TASK_ABORT_TASK_SET:
11593 		break;
11594 	case CTL_TASK_CLEAR_ACA:
11595 		break;
11596 	case CTL_TASK_CLEAR_TASK_SET:
11597 		break;
11598 	case CTL_TASK_LUN_RESET: {
11599 		struct ctl_lun *lun;
11600 		uint32_t targ_lun;
11601 		int retval;
11602 
11603 		targ_lun = io->io_hdr.nexus.targ_mapped_lun;
11604 		mtx_lock(&ctl_softc->ctl_lock);
11605 		if ((targ_lun < CTL_MAX_LUNS)
11606 		 && (ctl_softc->ctl_luns[targ_lun] != NULL))
11607 			lun = ctl_softc->ctl_luns[targ_lun];
11608 		else {
11609 			mtx_unlock(&ctl_softc->ctl_lock);
11610 			retval = 1;
11611 			break;
11612 		}
11613 
11614 		if (!(io->io_hdr.flags &
11615 		    CTL_FLAG_FROM_OTHER_SC)) {
11616 			union ctl_ha_msg msg_info;
11617 
11618 			io->io_hdr.flags |=
11619 				CTL_FLAG_SENT_2OTHER_SC;
11620 			msg_info.hdr.msg_type =
11621 				CTL_MSG_MANAGE_TASKS;
11622 			msg_info.hdr.nexus = io->io_hdr.nexus;
11623 			msg_info.task.task_action =
11624 				CTL_TASK_LUN_RESET;
11625 			msg_info.hdr.original_sc = NULL;
11626 			msg_info.hdr.serializing_sc = NULL;
11627 			if (CTL_HA_STATUS_SUCCESS !=
11628 			    ctl_ha_msg_send(CTL_HA_CHAN_CTL,
11629 			    (void *)&msg_info,
11630 			    sizeof(msg_info), 0)) {
11631 			}
11632 		}
11633 
11634 		retval = ctl_lun_reset(lun, io,
11635 				       CTL_UA_LUN_RESET);
11636 		mtx_unlock(&ctl_softc->ctl_lock);
11637 		break;
11638 	}
11639 	case CTL_TASK_TARGET_RESET:
11640 		retval = ctl_target_reset(ctl_softc, io, CTL_UA_TARG_RESET);
11641 		break;
11642 	case CTL_TASK_BUS_RESET:
11643 		retval = ctl_bus_reset(ctl_softc, io);
11644 		break;
11645 	case CTL_TASK_PORT_LOGIN:
11646 		break;
11647 	case CTL_TASK_PORT_LOGOUT:
11648 		break;
11649 	default:
11650 		printf("ctl_run_task: got unknown task management event %d\n",
11651 		       io->taskio.task_action);
11652 		break;
11653 	}
11654 	if (retval == 0)
11655 		io->io_hdr.status = CTL_SUCCESS;
11656 	else
11657 		io->io_hdr.status = CTL_ERROR;
11658 
11659 	/*
11660 	 * This will queue this I/O to the done queue, but the
11661 	 * work thread won't be able to process it until we
11662 	 * return and the lock is released.
11663 	 */
11664 	ctl_done(io);
11665 }
11666 
11667 /*
11668  * For HA operation.  Handle commands that come in from the other
11669  * controller.
11670  */
11671 static void
11672 ctl_handle_isc(union ctl_io *io)
11673 {
11674 	int free_io;
11675 	struct ctl_lun *lun;
11676 	struct ctl_softc *ctl_softc;
11677 	uint32_t targ_lun;
11678 
11679 	ctl_softc = control_softc;
11680 
11681 	targ_lun = io->io_hdr.nexus.targ_mapped_lun;
11682 	lun = ctl_softc->ctl_luns[targ_lun];
11683 
11684 	switch (io->io_hdr.msg_type) {
11685 	case CTL_MSG_SERIALIZE:
11686 		free_io = ctl_serialize_other_sc_cmd(&io->scsiio);
11687 		break;
11688 	case CTL_MSG_R2R: {
11689 		uint8_t opcode;
11690 		struct ctl_cmd_entry *entry;
11691 
11692 		/*
11693 		 * This is only used in SER_ONLY mode.
11694 		 */
11695 		free_io = 0;
11696 		opcode = io->scsiio.cdb[0];
11697 		entry = &ctl_cmd_table[opcode];
11698 		mtx_lock(&lun->lun_lock);
11699 		if (ctl_scsiio_lun_check(ctl_softc, lun,
11700 		    entry, (struct ctl_scsiio *)io) != 0) {
11701 			mtx_unlock(&lun->lun_lock);
11702 			ctl_done(io);
11703 			break;
11704 		}
11705 		io->io_hdr.flags |= CTL_FLAG_IS_WAS_ON_RTR;
11706 		mtx_unlock(&lun->lun_lock);
11707 		ctl_enqueue_rtr(io);
11708 		break;
11709 	}
11710 	case CTL_MSG_FINISH_IO:
11711 		if (ctl_softc->ha_mode == CTL_HA_MODE_XFER) {
11712 			free_io = 0;
11713 			ctl_done(io);
11714 		} else {
11715 			free_io = 1;
11716 			mtx_lock(&lun->lun_lock);
11717 			TAILQ_REMOVE(&lun->ooa_queue, &io->io_hdr,
11718 				     ooa_links);
11719 			ctl_check_blocked(lun);
11720 			mtx_unlock(&lun->lun_lock);
11721 		}
11722 		break;
11723 	case CTL_MSG_PERS_ACTION:
11724 		ctl_hndl_per_res_out_on_other_sc(
11725 			(union ctl_ha_msg *)&io->presio.pr_msg);
11726 		free_io = 1;
11727 		break;
11728 	case CTL_MSG_BAD_JUJU:
11729 		free_io = 0;
11730 		ctl_done(io);
11731 		break;
11732 	case CTL_MSG_DATAMOVE:
11733 		/* Only used in XFER mode */
11734 		free_io = 0;
11735 		ctl_datamove_remote(io);
11736 		break;
11737 	case CTL_MSG_DATAMOVE_DONE:
11738 		/* Only used in XFER mode */
11739 		free_io = 0;
11740 		io->scsiio.be_move_done(io);
11741 		break;
11742 	default:
11743 		free_io = 1;
11744 		printf("%s: Invalid message type %d\n",
11745 		       __func__, io->io_hdr.msg_type);
11746 		break;
11747 	}
11748 	if (free_io)
11749 		ctl_free_io(io);
11750 
11751 }
11752 
11753 
11754 /*
11755  * Returns the match type in the case of a match, or CTL_LUN_PAT_NONE if
11756  * there is no match.
11757  */
11758 static ctl_lun_error_pattern
11759 ctl_cmd_pattern_match(struct ctl_scsiio *ctsio, struct ctl_error_desc *desc)
11760 {
11761 	struct ctl_cmd_entry *entry;
11762 	ctl_lun_error_pattern filtered_pattern, pattern;
11763 	uint8_t opcode;
11764 
11765 	pattern = desc->error_pattern;
11766 
11767 	/*
11768 	 * XXX KDM we need more data passed into this function to match a
11769 	 * custom pattern, and we actually need to implement custom pattern
11770 	 * matching.
11771 	 */
11772 	if (pattern & CTL_LUN_PAT_CMD)
11773 		return (CTL_LUN_PAT_CMD);
11774 
11775 	if ((pattern & CTL_LUN_PAT_MASK) == CTL_LUN_PAT_ANY)
11776 		return (CTL_LUN_PAT_ANY);
11777 
11778 	opcode = ctsio->cdb[0];
11779 	entry = &ctl_cmd_table[opcode];
11780 
11781 	filtered_pattern = entry->pattern & pattern;
11782 
11783 	/*
11784 	 * If the user requested specific flags in the pattern (e.g.
11785 	 * CTL_LUN_PAT_RANGE), make sure the command supports all of those
11786 	 * flags.
11787 	 *
11788 	 * If the user did not specify any flags, it doesn't matter whether
11789 	 * or not the command supports the flags.
11790 	 */
11791 	if ((filtered_pattern & ~CTL_LUN_PAT_MASK) !=
11792 	     (pattern & ~CTL_LUN_PAT_MASK))
11793 		return (CTL_LUN_PAT_NONE);
11794 
11795 	/*
11796 	 * If the user asked for a range check, see if the requested LBA
11797 	 * range overlaps with this command's LBA range.
11798 	 */
11799 	if (filtered_pattern & CTL_LUN_PAT_RANGE) {
11800 		uint64_t lba1;
11801 		uint32_t len1;
11802 		ctl_action action;
11803 		int retval;
11804 
11805 		retval = ctl_get_lba_len((union ctl_io *)ctsio, &lba1, &len1);
11806 		if (retval != 0)
11807 			return (CTL_LUN_PAT_NONE);
11808 
11809 		action = ctl_extent_check_lba(lba1, len1, desc->lba_range.lba,
11810 					      desc->lba_range.len);
11811 		/*
11812 		 * A "pass" means that the LBA ranges don't overlap, so
11813 		 * this doesn't match the user's range criteria.
11814 		 */
11815 		if (action == CTL_ACTION_PASS)
11816 			return (CTL_LUN_PAT_NONE);
11817 	}
11818 
11819 	return (filtered_pattern);
11820 }
11821 
11822 static void
11823 ctl_inject_error(struct ctl_lun *lun, union ctl_io *io)
11824 {
11825 	struct ctl_error_desc *desc, *desc2;
11826 
11827 	mtx_assert(&lun->lun_lock, MA_OWNED);
11828 
11829 	STAILQ_FOREACH_SAFE(desc, &lun->error_list, links, desc2) {
11830 		ctl_lun_error_pattern pattern;
11831 		/*
11832 		 * Check to see whether this particular command matches
11833 		 * the pattern in the descriptor.
11834 		 */
11835 		pattern = ctl_cmd_pattern_match(&io->scsiio, desc);
11836 		if ((pattern & CTL_LUN_PAT_MASK) == CTL_LUN_PAT_NONE)
11837 			continue;
11838 
11839 		switch (desc->lun_error & CTL_LUN_INJ_TYPE) {
11840 		case CTL_LUN_INJ_ABORTED:
11841 			ctl_set_aborted(&io->scsiio);
11842 			break;
11843 		case CTL_LUN_INJ_MEDIUM_ERR:
11844 			ctl_set_medium_error(&io->scsiio);
11845 			break;
11846 		case CTL_LUN_INJ_UA:
11847 			/* 29h/00h  POWER ON, RESET, OR BUS DEVICE RESET
11848 			 * OCCURRED */
11849 			ctl_set_ua(&io->scsiio, 0x29, 0x00);
11850 			break;
11851 		case CTL_LUN_INJ_CUSTOM:
11852 			/*
11853 			 * We're assuming the user knows what he is doing.
11854 			 * Just copy the sense information without doing
11855 			 * checks.
11856 			 */
11857 			bcopy(&desc->custom_sense, &io->scsiio.sense_data,
11858 			      ctl_min(sizeof(desc->custom_sense),
11859 				      sizeof(io->scsiio.sense_data)));
11860 			io->scsiio.scsi_status = SCSI_STATUS_CHECK_COND;
11861 			io->scsiio.sense_len = SSD_FULL_SIZE;
11862 			io->io_hdr.status = CTL_SCSI_ERROR | CTL_AUTOSENSE;
11863 			break;
11864 		case CTL_LUN_INJ_NONE:
11865 		default:
11866 			/*
11867 			 * If this is an error injection type we don't know
11868 			 * about, clear the continuous flag (if it is set)
11869 			 * so it will get deleted below.
11870 			 */
11871 			desc->lun_error &= ~CTL_LUN_INJ_CONTINUOUS;
11872 			break;
11873 		}
11874 		/*
11875 		 * By default, each error injection action is a one-shot
11876 		 */
11877 		if (desc->lun_error & CTL_LUN_INJ_CONTINUOUS)
11878 			continue;
11879 
11880 		STAILQ_REMOVE(&lun->error_list, desc, ctl_error_desc, links);
11881 
11882 		free(desc, M_CTL);
11883 	}
11884 }
11885 
11886 #ifdef CTL_IO_DELAY
11887 static void
11888 ctl_datamove_timer_wakeup(void *arg)
11889 {
11890 	union ctl_io *io;
11891 
11892 	io = (union ctl_io *)arg;
11893 
11894 	ctl_datamove(io);
11895 }
11896 #endif /* CTL_IO_DELAY */
11897 
11898 void
11899 ctl_datamove(union ctl_io *io)
11900 {
11901 	void (*fe_datamove)(union ctl_io *io);
11902 
11903 	mtx_assert(&control_softc->ctl_lock, MA_NOTOWNED);
11904 
11905 	CTL_DEBUG_PRINT(("ctl_datamove\n"));
11906 
11907 #ifdef CTL_TIME_IO
11908 	if ((time_uptime - io->io_hdr.start_time) > ctl_time_io_secs) {
11909 		char str[256];
11910 		char path_str[64];
11911 		struct sbuf sb;
11912 
11913 		ctl_scsi_path_string(io, path_str, sizeof(path_str));
11914 		sbuf_new(&sb, str, sizeof(str), SBUF_FIXEDLEN);
11915 
11916 		sbuf_cat(&sb, path_str);
11917 		switch (io->io_hdr.io_type) {
11918 		case CTL_IO_SCSI:
11919 			ctl_scsi_command_string(&io->scsiio, NULL, &sb);
11920 			sbuf_printf(&sb, "\n");
11921 			sbuf_cat(&sb, path_str);
11922 			sbuf_printf(&sb, "Tag: 0x%04x, type %d\n",
11923 				    io->scsiio.tag_num, io->scsiio.tag_type);
11924 			break;
11925 		case CTL_IO_TASK:
11926 			sbuf_printf(&sb, "Task I/O type: %d, Tag: 0x%04x, "
11927 				    "Tag Type: %d\n", io->taskio.task_action,
11928 				    io->taskio.tag_num, io->taskio.tag_type);
11929 			break;
11930 		default:
11931 			printf("Invalid CTL I/O type %d\n", io->io_hdr.io_type);
11932 			panic("Invalid CTL I/O type %d\n", io->io_hdr.io_type);
11933 			break;
11934 		}
11935 		sbuf_cat(&sb, path_str);
11936 		sbuf_printf(&sb, "ctl_datamove: %jd seconds\n",
11937 			    (intmax_t)time_uptime - io->io_hdr.start_time);
11938 		sbuf_finish(&sb);
11939 		printf("%s", sbuf_data(&sb));
11940 	}
11941 #endif /* CTL_TIME_IO */
11942 
11943 #ifdef CTL_IO_DELAY
11944 	if (io->io_hdr.flags & CTL_FLAG_DELAY_DONE) {
11945 		struct ctl_lun *lun;
11946 
11947 		lun =(struct ctl_lun *)io->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
11948 
11949 		io->io_hdr.flags &= ~CTL_FLAG_DELAY_DONE;
11950 	} else {
11951 		struct ctl_lun *lun;
11952 
11953 		lun =(struct ctl_lun *)io->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
11954 		if ((lun != NULL)
11955 		 && (lun->delay_info.datamove_delay > 0)) {
11956 			struct callout *callout;
11957 
11958 			callout = (struct callout *)&io->io_hdr.timer_bytes;
11959 			callout_init(callout, /*mpsafe*/ 1);
11960 			io->io_hdr.flags |= CTL_FLAG_DELAY_DONE;
11961 			callout_reset(callout,
11962 				      lun->delay_info.datamove_delay * hz,
11963 				      ctl_datamove_timer_wakeup, io);
11964 			if (lun->delay_info.datamove_type ==
11965 			    CTL_DELAY_TYPE_ONESHOT)
11966 				lun->delay_info.datamove_delay = 0;
11967 			return;
11968 		}
11969 	}
11970 #endif
11971 
11972 	/*
11973 	 * This command has been aborted.  Set the port status, so we fail
11974 	 * the data move.
11975 	 */
11976 	if (io->io_hdr.flags & CTL_FLAG_ABORT) {
11977 		printf("ctl_datamove: tag 0x%04x on (%ju:%d:%ju:%d) aborted\n",
11978 		       io->scsiio.tag_num,(uintmax_t)io->io_hdr.nexus.initid.id,
11979 		       io->io_hdr.nexus.targ_port,
11980 		       (uintmax_t)io->io_hdr.nexus.targ_target.id,
11981 		       io->io_hdr.nexus.targ_lun);
11982 		io->io_hdr.status = CTL_CMD_ABORTED;
11983 		io->io_hdr.port_status = 31337;
11984 		/*
11985 		 * Note that the backend, in this case, will get the
11986 		 * callback in its context.  In other cases it may get
11987 		 * called in the frontend's interrupt thread context.
11988 		 */
11989 		io->scsiio.be_move_done(io);
11990 		return;
11991 	}
11992 
11993 	/*
11994 	 * If we're in XFER mode and this I/O is from the other shelf
11995 	 * controller, we need to send the DMA to the other side to
11996 	 * actually transfer the data to/from the host.  In serialize only
11997 	 * mode the transfer happens below CTL and ctl_datamove() is only
11998 	 * called on the machine that originally received the I/O.
11999 	 */
12000 	if ((control_softc->ha_mode == CTL_HA_MODE_XFER)
12001 	 && (io->io_hdr.flags & CTL_FLAG_FROM_OTHER_SC)) {
12002 		union ctl_ha_msg msg;
12003 		uint32_t sg_entries_sent;
12004 		int do_sg_copy;
12005 		int i;
12006 
12007 		memset(&msg, 0, sizeof(msg));
12008 		msg.hdr.msg_type = CTL_MSG_DATAMOVE;
12009 		msg.hdr.original_sc = io->io_hdr.original_sc;
12010 		msg.hdr.serializing_sc = io;
12011 		msg.hdr.nexus = io->io_hdr.nexus;
12012 		msg.dt.flags = io->io_hdr.flags;
12013 		/*
12014 		 * We convert everything into a S/G list here.  We can't
12015 		 * pass by reference, only by value between controllers.
12016 		 * So we can't pass a pointer to the S/G list, only as many
12017 		 * S/G entries as we can fit in here.  If it's possible for
12018 		 * us to get more than CTL_HA_MAX_SG_ENTRIES S/G entries,
12019 		 * then we need to break this up into multiple transfers.
12020 		 */
12021 		if (io->scsiio.kern_sg_entries == 0) {
12022 			msg.dt.kern_sg_entries = 1;
12023 			/*
12024 			 * If this is in cached memory, flush the cache
12025 			 * before we send the DMA request to the other
12026 			 * controller.  We want to do this in either the
12027 			 * read or the write case.  The read case is
12028 			 * straightforward.  In the write case, we want to
12029 			 * make sure nothing is in the local cache that
12030 			 * could overwrite the DMAed data.
12031 			 */
12032 			if ((io->io_hdr.flags & CTL_FLAG_NO_DATASYNC) == 0) {
12033 				/*
12034 				 * XXX KDM use bus_dmamap_sync() here.
12035 				 */
12036 			}
12037 
12038 			/*
12039 			 * Convert to a physical address if this is a
12040 			 * virtual address.
12041 			 */
12042 			if (io->io_hdr.flags & CTL_FLAG_BUS_ADDR) {
12043 				msg.dt.sg_list[0].addr =
12044 					io->scsiio.kern_data_ptr;
12045 			} else {
12046 				/*
12047 				 * XXX KDM use busdma here!
12048 				 */
12049 #if 0
12050 				msg.dt.sg_list[0].addr = (void *)
12051 					vtophys(io->scsiio.kern_data_ptr);
12052 #endif
12053 			}
12054 
12055 			msg.dt.sg_list[0].len = io->scsiio.kern_data_len;
12056 			do_sg_copy = 0;
12057 		} else {
12058 			struct ctl_sg_entry *sgl;
12059 
12060 			do_sg_copy = 1;
12061 			msg.dt.kern_sg_entries = io->scsiio.kern_sg_entries;
12062 			sgl = (struct ctl_sg_entry *)io->scsiio.kern_data_ptr;
12063 			if ((io->io_hdr.flags & CTL_FLAG_NO_DATASYNC) == 0) {
12064 				/*
12065 				 * XXX KDM use bus_dmamap_sync() here.
12066 				 */
12067 			}
12068 		}
12069 
12070 		msg.dt.kern_data_len = io->scsiio.kern_data_len;
12071 		msg.dt.kern_total_len = io->scsiio.kern_total_len;
12072 		msg.dt.kern_data_resid = io->scsiio.kern_data_resid;
12073 		msg.dt.kern_rel_offset = io->scsiio.kern_rel_offset;
12074 		msg.dt.sg_sequence = 0;
12075 
12076 		/*
12077 		 * Loop until we've sent all of the S/G entries.  On the
12078 		 * other end, we'll recompose these S/G entries into one
12079 		 * contiguous list before passing it to the
12080 		 */
12081 		for (sg_entries_sent = 0; sg_entries_sent <
12082 		     msg.dt.kern_sg_entries; msg.dt.sg_sequence++) {
12083 			msg.dt.cur_sg_entries = ctl_min((sizeof(msg.dt.sg_list)/
12084 				sizeof(msg.dt.sg_list[0])),
12085 				msg.dt.kern_sg_entries - sg_entries_sent);
12086 
12087 			if (do_sg_copy != 0) {
12088 				struct ctl_sg_entry *sgl;
12089 				int j;
12090 
12091 				sgl = (struct ctl_sg_entry *)
12092 					io->scsiio.kern_data_ptr;
12093 				/*
12094 				 * If this is in cached memory, flush the cache
12095 				 * before we send the DMA request to the other
12096 				 * controller.  We want to do this in either
12097 				 * the * read or the write case.  The read
12098 				 * case is straightforward.  In the write
12099 				 * case, we want to make sure nothing is
12100 				 * in the local cache that could overwrite
12101 				 * the DMAed data.
12102 				 */
12103 
12104 				for (i = sg_entries_sent, j = 0;
12105 				     i < msg.dt.cur_sg_entries; i++, j++) {
12106 					if ((io->io_hdr.flags &
12107 					     CTL_FLAG_NO_DATASYNC) == 0) {
12108 						/*
12109 						 * XXX KDM use bus_dmamap_sync()
12110 						 */
12111 					}
12112 					if ((io->io_hdr.flags &
12113 					     CTL_FLAG_BUS_ADDR) == 0) {
12114 						/*
12115 						 * XXX KDM use busdma.
12116 						 */
12117 #if 0
12118 						msg.dt.sg_list[j].addr =(void *)
12119 						       vtophys(sgl[i].addr);
12120 #endif
12121 					} else {
12122 						msg.dt.sg_list[j].addr =
12123 							sgl[i].addr;
12124 					}
12125 					msg.dt.sg_list[j].len = sgl[i].len;
12126 				}
12127 			}
12128 
12129 			sg_entries_sent += msg.dt.cur_sg_entries;
12130 			if (sg_entries_sent >= msg.dt.kern_sg_entries)
12131 				msg.dt.sg_last = 1;
12132 			else
12133 				msg.dt.sg_last = 0;
12134 
12135 			/*
12136 			 * XXX KDM drop and reacquire the lock here?
12137 			 */
12138 			if (ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg,
12139 			    sizeof(msg), 0) > CTL_HA_STATUS_SUCCESS) {
12140 				/*
12141 				 * XXX do something here.
12142 				 */
12143 			}
12144 
12145 			msg.dt.sent_sg_entries = sg_entries_sent;
12146 		}
12147 		io->io_hdr.flags &= ~CTL_FLAG_IO_ACTIVE;
12148 		if (io->io_hdr.flags & CTL_FLAG_FAILOVER)
12149 			ctl_failover_io(io, /*have_lock*/ 0);
12150 
12151 	} else {
12152 
12153 		/*
12154 		 * Lookup the fe_datamove() function for this particular
12155 		 * front end.
12156 		 */
12157 		fe_datamove =
12158 		    control_softc->ctl_ports[ctl_port_idx(io->io_hdr.nexus.targ_port)]->fe_datamove;
12159 
12160 		fe_datamove(io);
12161 	}
12162 }
12163 
12164 static void
12165 ctl_send_datamove_done(union ctl_io *io, int have_lock)
12166 {
12167 	union ctl_ha_msg msg;
12168 	int isc_status;
12169 
12170 	memset(&msg, 0, sizeof(msg));
12171 
12172 	msg.hdr.msg_type = CTL_MSG_DATAMOVE_DONE;
12173 	msg.hdr.original_sc = io;
12174 	msg.hdr.serializing_sc = io->io_hdr.serializing_sc;
12175 	msg.hdr.nexus = io->io_hdr.nexus;
12176 	msg.hdr.status = io->io_hdr.status;
12177 	msg.scsi.tag_num = io->scsiio.tag_num;
12178 	msg.scsi.tag_type = io->scsiio.tag_type;
12179 	msg.scsi.scsi_status = io->scsiio.scsi_status;
12180 	memcpy(&msg.scsi.sense_data, &io->scsiio.sense_data,
12181 	       sizeof(io->scsiio.sense_data));
12182 	msg.scsi.sense_len = io->scsiio.sense_len;
12183 	msg.scsi.sense_residual = io->scsiio.sense_residual;
12184 	msg.scsi.fetd_status = io->io_hdr.port_status;
12185 	msg.scsi.residual = io->scsiio.residual;
12186 	io->io_hdr.flags &= ~CTL_FLAG_IO_ACTIVE;
12187 
12188 	if (io->io_hdr.flags & CTL_FLAG_FAILOVER) {
12189 		ctl_failover_io(io, /*have_lock*/ have_lock);
12190 		return;
12191 	}
12192 
12193 	isc_status = ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg, sizeof(msg), 0);
12194 	if (isc_status > CTL_HA_STATUS_SUCCESS) {
12195 		/* XXX do something if this fails */
12196 	}
12197 
12198 }
12199 
12200 /*
12201  * The DMA to the remote side is done, now we need to tell the other side
12202  * we're done so it can continue with its data movement.
12203  */
12204 static void
12205 ctl_datamove_remote_write_cb(struct ctl_ha_dt_req *rq)
12206 {
12207 	union ctl_io *io;
12208 
12209 	io = rq->context;
12210 
12211 	if (rq->ret != CTL_HA_STATUS_SUCCESS) {
12212 		printf("%s: ISC DMA write failed with error %d", __func__,
12213 		       rq->ret);
12214 		ctl_set_internal_failure(&io->scsiio,
12215 					 /*sks_valid*/ 1,
12216 					 /*retry_count*/ rq->ret);
12217 	}
12218 
12219 	ctl_dt_req_free(rq);
12220 
12221 	/*
12222 	 * In this case, we had to malloc the memory locally.  Free it.
12223 	 */
12224 	if ((io->io_hdr.flags & CTL_FLAG_AUTO_MIRROR) == 0) {
12225 		int i;
12226 		for (i = 0; i < io->scsiio.kern_sg_entries; i++)
12227 			free(io->io_hdr.local_sglist[i].addr, M_CTL);
12228 	}
12229 	/*
12230 	 * The data is in local and remote memory, so now we need to send
12231 	 * status (good or back) back to the other side.
12232 	 */
12233 	ctl_send_datamove_done(io, /*have_lock*/ 0);
12234 }
12235 
12236 /*
12237  * We've moved the data from the host/controller into local memory.  Now we
12238  * need to push it over to the remote controller's memory.
12239  */
12240 static int
12241 ctl_datamove_remote_dm_write_cb(union ctl_io *io)
12242 {
12243 	int retval;
12244 
12245 	retval = 0;
12246 
12247 	retval = ctl_datamove_remote_xfer(io, CTL_HA_DT_CMD_WRITE,
12248 					  ctl_datamove_remote_write_cb);
12249 
12250 	return (retval);
12251 }
12252 
12253 static void
12254 ctl_datamove_remote_write(union ctl_io *io)
12255 {
12256 	int retval;
12257 	void (*fe_datamove)(union ctl_io *io);
12258 
12259 	/*
12260 	 * - Get the data from the host/HBA into local memory.
12261 	 * - DMA memory from the local controller to the remote controller.
12262 	 * - Send status back to the remote controller.
12263 	 */
12264 
12265 	retval = ctl_datamove_remote_sgl_setup(io);
12266 	if (retval != 0)
12267 		return;
12268 
12269 	/* Switch the pointer over so the FETD knows what to do */
12270 	io->scsiio.kern_data_ptr = (uint8_t *)io->io_hdr.local_sglist;
12271 
12272 	/*
12273 	 * Use a custom move done callback, since we need to send completion
12274 	 * back to the other controller, not to the backend on this side.
12275 	 */
12276 	io->scsiio.be_move_done = ctl_datamove_remote_dm_write_cb;
12277 
12278 	fe_datamove = control_softc->ctl_ports[ctl_port_idx(io->io_hdr.nexus.targ_port)]->fe_datamove;
12279 
12280 	fe_datamove(io);
12281 
12282 	return;
12283 
12284 }
12285 
12286 static int
12287 ctl_datamove_remote_dm_read_cb(union ctl_io *io)
12288 {
12289 #if 0
12290 	char str[256];
12291 	char path_str[64];
12292 	struct sbuf sb;
12293 #endif
12294 
12295 	/*
12296 	 * In this case, we had to malloc the memory locally.  Free it.
12297 	 */
12298 	if ((io->io_hdr.flags & CTL_FLAG_AUTO_MIRROR) == 0) {
12299 		int i;
12300 		for (i = 0; i < io->scsiio.kern_sg_entries; i++)
12301 			free(io->io_hdr.local_sglist[i].addr, M_CTL);
12302 	}
12303 
12304 #if 0
12305 	scsi_path_string(io, path_str, sizeof(path_str));
12306 	sbuf_new(&sb, str, sizeof(str), SBUF_FIXEDLEN);
12307 	sbuf_cat(&sb, path_str);
12308 	scsi_command_string(&io->scsiio, NULL, &sb);
12309 	sbuf_printf(&sb, "\n");
12310 	sbuf_cat(&sb, path_str);
12311 	sbuf_printf(&sb, "Tag: 0x%04x, type %d\n",
12312 		    io->scsiio.tag_num, io->scsiio.tag_type);
12313 	sbuf_cat(&sb, path_str);
12314 	sbuf_printf(&sb, "%s: flags %#x, status %#x\n", __func__,
12315 		    io->io_hdr.flags, io->io_hdr.status);
12316 	sbuf_finish(&sb);
12317 	printk("%s", sbuf_data(&sb));
12318 #endif
12319 
12320 
12321 	/*
12322 	 * The read is done, now we need to send status (good or bad) back
12323 	 * to the other side.
12324 	 */
12325 	ctl_send_datamove_done(io, /*have_lock*/ 0);
12326 
12327 	return (0);
12328 }
12329 
12330 static void
12331 ctl_datamove_remote_read_cb(struct ctl_ha_dt_req *rq)
12332 {
12333 	union ctl_io *io;
12334 	void (*fe_datamove)(union ctl_io *io);
12335 
12336 	io = rq->context;
12337 
12338 	if (rq->ret != CTL_HA_STATUS_SUCCESS) {
12339 		printf("%s: ISC DMA read failed with error %d", __func__,
12340 		       rq->ret);
12341 		ctl_set_internal_failure(&io->scsiio,
12342 					 /*sks_valid*/ 1,
12343 					 /*retry_count*/ rq->ret);
12344 	}
12345 
12346 	ctl_dt_req_free(rq);
12347 
12348 	/* Switch the pointer over so the FETD knows what to do */
12349 	io->scsiio.kern_data_ptr = (uint8_t *)io->io_hdr.local_sglist;
12350 
12351 	/*
12352 	 * Use a custom move done callback, since we need to send completion
12353 	 * back to the other controller, not to the backend on this side.
12354 	 */
12355 	io->scsiio.be_move_done = ctl_datamove_remote_dm_read_cb;
12356 
12357 	/* XXX KDM add checks like the ones in ctl_datamove? */
12358 
12359 	fe_datamove = control_softc->ctl_ports[ctl_port_idx(io->io_hdr.nexus.targ_port)]->fe_datamove;
12360 
12361 	fe_datamove(io);
12362 }
12363 
12364 static int
12365 ctl_datamove_remote_sgl_setup(union ctl_io *io)
12366 {
12367 	struct ctl_sg_entry *local_sglist, *remote_sglist;
12368 	struct ctl_sg_entry *local_dma_sglist, *remote_dma_sglist;
12369 	struct ctl_softc *softc;
12370 	int retval;
12371 	int i;
12372 
12373 	retval = 0;
12374 	softc = control_softc;
12375 
12376 	local_sglist = io->io_hdr.local_sglist;
12377 	local_dma_sglist = io->io_hdr.local_dma_sglist;
12378 	remote_sglist = io->io_hdr.remote_sglist;
12379 	remote_dma_sglist = io->io_hdr.remote_dma_sglist;
12380 
12381 	if (io->io_hdr.flags & CTL_FLAG_AUTO_MIRROR) {
12382 		for (i = 0; i < io->scsiio.kern_sg_entries; i++) {
12383 			local_sglist[i].len = remote_sglist[i].len;
12384 
12385 			/*
12386 			 * XXX Detect the situation where the RS-level I/O
12387 			 * redirector on the other side has already read the
12388 			 * data off of the AOR RS on this side, and
12389 			 * transferred it to remote (mirror) memory on the
12390 			 * other side.  Since we already have the data in
12391 			 * memory here, we just need to use it.
12392 			 *
12393 			 * XXX KDM this can probably be removed once we
12394 			 * get the cache device code in and take the
12395 			 * current AOR implementation out.
12396 			 */
12397 #ifdef NEEDTOPORT
12398 			if ((remote_sglist[i].addr >=
12399 			     (void *)vtophys(softc->mirr->addr))
12400 			 && (remote_sglist[i].addr <
12401 			     ((void *)vtophys(softc->mirr->addr) +
12402 			     CacheMirrorOffset))) {
12403 				local_sglist[i].addr = remote_sglist[i].addr -
12404 					CacheMirrorOffset;
12405 				if ((io->io_hdr.flags & CTL_FLAG_DATA_MASK) ==
12406 				     CTL_FLAG_DATA_IN)
12407 					io->io_hdr.flags |= CTL_FLAG_REDIR_DONE;
12408 			} else {
12409 				local_sglist[i].addr = remote_sglist[i].addr +
12410 					CacheMirrorOffset;
12411 			}
12412 #endif
12413 #if 0
12414 			printf("%s: local %p, remote %p, len %d\n",
12415 			       __func__, local_sglist[i].addr,
12416 			       remote_sglist[i].addr, local_sglist[i].len);
12417 #endif
12418 		}
12419 	} else {
12420 		uint32_t len_to_go;
12421 
12422 		/*
12423 		 * In this case, we don't have automatically allocated
12424 		 * memory for this I/O on this controller.  This typically
12425 		 * happens with internal CTL I/O -- e.g. inquiry, mode
12426 		 * sense, etc.  Anything coming from RAIDCore will have
12427 		 * a mirror area available.
12428 		 */
12429 		len_to_go = io->scsiio.kern_data_len;
12430 
12431 		/*
12432 		 * Clear the no datasync flag, we have to use malloced
12433 		 * buffers.
12434 		 */
12435 		io->io_hdr.flags &= ~CTL_FLAG_NO_DATASYNC;
12436 
12437 		/*
12438 		 * The difficult thing here is that the size of the various
12439 		 * S/G segments may be different than the size from the
12440 		 * remote controller.  That'll make it harder when DMAing
12441 		 * the data back to the other side.
12442 		 */
12443 		for (i = 0; (i < sizeof(io->io_hdr.remote_sglist) /
12444 		     sizeof(io->io_hdr.remote_sglist[0])) &&
12445 		     (len_to_go > 0); i++) {
12446 			local_sglist[i].len = ctl_min(len_to_go, 131072);
12447 			CTL_SIZE_8B(local_dma_sglist[i].len,
12448 				    local_sglist[i].len);
12449 			local_sglist[i].addr =
12450 				malloc(local_dma_sglist[i].len, M_CTL,M_WAITOK);
12451 
12452 			local_dma_sglist[i].addr = local_sglist[i].addr;
12453 
12454 			if (local_sglist[i].addr == NULL) {
12455 				int j;
12456 
12457 				printf("malloc failed for %zd bytes!",
12458 				       local_dma_sglist[i].len);
12459 				for (j = 0; j < i; j++) {
12460 					free(local_sglist[j].addr, M_CTL);
12461 				}
12462 				ctl_set_internal_failure(&io->scsiio,
12463 							 /*sks_valid*/ 1,
12464 							 /*retry_count*/ 4857);
12465 				retval = 1;
12466 				goto bailout_error;
12467 
12468 			}
12469 			/* XXX KDM do we need a sync here? */
12470 
12471 			len_to_go -= local_sglist[i].len;
12472 		}
12473 		/*
12474 		 * Reset the number of S/G entries accordingly.  The
12475 		 * original number of S/G entries is available in
12476 		 * rem_sg_entries.
12477 		 */
12478 		io->scsiio.kern_sg_entries = i;
12479 
12480 #if 0
12481 		printf("%s: kern_sg_entries = %d\n", __func__,
12482 		       io->scsiio.kern_sg_entries);
12483 		for (i = 0; i < io->scsiio.kern_sg_entries; i++)
12484 			printf("%s: sg[%d] = %p, %d (DMA: %d)\n", __func__, i,
12485 			       local_sglist[i].addr, local_sglist[i].len,
12486 			       local_dma_sglist[i].len);
12487 #endif
12488 	}
12489 
12490 
12491 	return (retval);
12492 
12493 bailout_error:
12494 
12495 	ctl_send_datamove_done(io, /*have_lock*/ 0);
12496 
12497 	return (retval);
12498 }
12499 
12500 static int
12501 ctl_datamove_remote_xfer(union ctl_io *io, unsigned command,
12502 			 ctl_ha_dt_cb callback)
12503 {
12504 	struct ctl_ha_dt_req *rq;
12505 	struct ctl_sg_entry *remote_sglist, *local_sglist;
12506 	struct ctl_sg_entry *remote_dma_sglist, *local_dma_sglist;
12507 	uint32_t local_used, remote_used, total_used;
12508 	int retval;
12509 	int i, j;
12510 
12511 	retval = 0;
12512 
12513 	rq = ctl_dt_req_alloc();
12514 
12515 	/*
12516 	 * If we failed to allocate the request, and if the DMA didn't fail
12517 	 * anyway, set busy status.  This is just a resource allocation
12518 	 * failure.
12519 	 */
12520 	if ((rq == NULL)
12521 	 && ((io->io_hdr.status & CTL_STATUS_MASK) != CTL_STATUS_NONE))
12522 		ctl_set_busy(&io->scsiio);
12523 
12524 	if ((io->io_hdr.status & CTL_STATUS_MASK) != CTL_STATUS_NONE) {
12525 
12526 		if (rq != NULL)
12527 			ctl_dt_req_free(rq);
12528 
12529 		/*
12530 		 * The data move failed.  We need to return status back
12531 		 * to the other controller.  No point in trying to DMA
12532 		 * data to the remote controller.
12533 		 */
12534 
12535 		ctl_send_datamove_done(io, /*have_lock*/ 0);
12536 
12537 		retval = 1;
12538 
12539 		goto bailout;
12540 	}
12541 
12542 	local_sglist = io->io_hdr.local_sglist;
12543 	local_dma_sglist = io->io_hdr.local_dma_sglist;
12544 	remote_sglist = io->io_hdr.remote_sglist;
12545 	remote_dma_sglist = io->io_hdr.remote_dma_sglist;
12546 	local_used = 0;
12547 	remote_used = 0;
12548 	total_used = 0;
12549 
12550 	if (io->io_hdr.flags & CTL_FLAG_REDIR_DONE) {
12551 		rq->ret = CTL_HA_STATUS_SUCCESS;
12552 		rq->context = io;
12553 		callback(rq);
12554 		goto bailout;
12555 	}
12556 
12557 	/*
12558 	 * Pull/push the data over the wire from/to the other controller.
12559 	 * This takes into account the possibility that the local and
12560 	 * remote sglists may not be identical in terms of the size of
12561 	 * the elements and the number of elements.
12562 	 *
12563 	 * One fundamental assumption here is that the length allocated for
12564 	 * both the local and remote sglists is identical.  Otherwise, we've
12565 	 * essentially got a coding error of some sort.
12566 	 */
12567 	for (i = 0, j = 0; total_used < io->scsiio.kern_data_len; ) {
12568 		int isc_ret;
12569 		uint32_t cur_len, dma_length;
12570 		uint8_t *tmp_ptr;
12571 
12572 		rq->id = CTL_HA_DATA_CTL;
12573 		rq->command = command;
12574 		rq->context = io;
12575 
12576 		/*
12577 		 * Both pointers should be aligned.  But it is possible
12578 		 * that the allocation length is not.  They should both
12579 		 * also have enough slack left over at the end, though,
12580 		 * to round up to the next 8 byte boundary.
12581 		 */
12582 		cur_len = ctl_min(local_sglist[i].len - local_used,
12583 				  remote_sglist[j].len - remote_used);
12584 
12585 		/*
12586 		 * In this case, we have a size issue and need to decrease
12587 		 * the size, except in the case where we actually have less
12588 		 * than 8 bytes left.  In that case, we need to increase
12589 		 * the DMA length to get the last bit.
12590 		 */
12591 		if ((cur_len & 0x7) != 0) {
12592 			if (cur_len > 0x7) {
12593 				cur_len = cur_len - (cur_len & 0x7);
12594 				dma_length = cur_len;
12595 			} else {
12596 				CTL_SIZE_8B(dma_length, cur_len);
12597 			}
12598 
12599 		} else
12600 			dma_length = cur_len;
12601 
12602 		/*
12603 		 * If we had to allocate memory for this I/O, instead of using
12604 		 * the non-cached mirror memory, we'll need to flush the cache
12605 		 * before trying to DMA to the other controller.
12606 		 *
12607 		 * We could end up doing this multiple times for the same
12608 		 * segment if we have a larger local segment than remote
12609 		 * segment.  That shouldn't be an issue.
12610 		 */
12611 		if ((io->io_hdr.flags & CTL_FLAG_NO_DATASYNC) == 0) {
12612 			/*
12613 			 * XXX KDM use bus_dmamap_sync() here.
12614 			 */
12615 		}
12616 
12617 		rq->size = dma_length;
12618 
12619 		tmp_ptr = (uint8_t *)local_sglist[i].addr;
12620 		tmp_ptr += local_used;
12621 
12622 		/* Use physical addresses when talking to ISC hardware */
12623 		if ((io->io_hdr.flags & CTL_FLAG_BUS_ADDR) == 0) {
12624 			/* XXX KDM use busdma */
12625 #if 0
12626 			rq->local = vtophys(tmp_ptr);
12627 #endif
12628 		} else
12629 			rq->local = tmp_ptr;
12630 
12631 		tmp_ptr = (uint8_t *)remote_sglist[j].addr;
12632 		tmp_ptr += remote_used;
12633 		rq->remote = tmp_ptr;
12634 
12635 		rq->callback = NULL;
12636 
12637 		local_used += cur_len;
12638 		if (local_used >= local_sglist[i].len) {
12639 			i++;
12640 			local_used = 0;
12641 		}
12642 
12643 		remote_used += cur_len;
12644 		if (remote_used >= remote_sglist[j].len) {
12645 			j++;
12646 			remote_used = 0;
12647 		}
12648 		total_used += cur_len;
12649 
12650 		if (total_used >= io->scsiio.kern_data_len)
12651 			rq->callback = callback;
12652 
12653 		if ((rq->size & 0x7) != 0) {
12654 			printf("%s: warning: size %d is not on 8b boundary\n",
12655 			       __func__, rq->size);
12656 		}
12657 		if (((uintptr_t)rq->local & 0x7) != 0) {
12658 			printf("%s: warning: local %p not on 8b boundary\n",
12659 			       __func__, rq->local);
12660 		}
12661 		if (((uintptr_t)rq->remote & 0x7) != 0) {
12662 			printf("%s: warning: remote %p not on 8b boundary\n",
12663 			       __func__, rq->local);
12664 		}
12665 #if 0
12666 		printf("%s: %s: local %#x remote %#x size %d\n", __func__,
12667 		       (command == CTL_HA_DT_CMD_WRITE) ? "WRITE" : "READ",
12668 		       rq->local, rq->remote, rq->size);
12669 #endif
12670 
12671 		isc_ret = ctl_dt_single(rq);
12672 		if (isc_ret == CTL_HA_STATUS_WAIT)
12673 			continue;
12674 
12675 		if (isc_ret == CTL_HA_STATUS_DISCONNECT) {
12676 			rq->ret = CTL_HA_STATUS_SUCCESS;
12677 		} else {
12678 			rq->ret = isc_ret;
12679 		}
12680 		callback(rq);
12681 		goto bailout;
12682 	}
12683 
12684 bailout:
12685 	return (retval);
12686 
12687 }
12688 
12689 static void
12690 ctl_datamove_remote_read(union ctl_io *io)
12691 {
12692 	int retval;
12693 	int i;
12694 
12695 	/*
12696 	 * This will send an error to the other controller in the case of a
12697 	 * failure.
12698 	 */
12699 	retval = ctl_datamove_remote_sgl_setup(io);
12700 	if (retval != 0)
12701 		return;
12702 
12703 	retval = ctl_datamove_remote_xfer(io, CTL_HA_DT_CMD_READ,
12704 					  ctl_datamove_remote_read_cb);
12705 	if ((retval != 0)
12706 	 && ((io->io_hdr.flags & CTL_FLAG_AUTO_MIRROR) == 0)) {
12707 		/*
12708 		 * Make sure we free memory if there was an error..  The
12709 		 * ctl_datamove_remote_xfer() function will send the
12710 		 * datamove done message, or call the callback with an
12711 		 * error if there is a problem.
12712 		 */
12713 		for (i = 0; i < io->scsiio.kern_sg_entries; i++)
12714 			free(io->io_hdr.local_sglist[i].addr, M_CTL);
12715 	}
12716 
12717 	return;
12718 }
12719 
12720 /*
12721  * Process a datamove request from the other controller.  This is used for
12722  * XFER mode only, not SER_ONLY mode.  For writes, we DMA into local memory
12723  * first.  Once that is complete, the data gets DMAed into the remote
12724  * controller's memory.  For reads, we DMA from the remote controller's
12725  * memory into our memory first, and then move it out to the FETD.
12726  */
12727 static void
12728 ctl_datamove_remote(union ctl_io *io)
12729 {
12730 	struct ctl_softc *softc;
12731 
12732 	softc = control_softc;
12733 
12734 	mtx_assert(&softc->ctl_lock, MA_NOTOWNED);
12735 
12736 	/*
12737 	 * Note that we look for an aborted I/O here, but don't do some of
12738 	 * the other checks that ctl_datamove() normally does.  We don't
12739 	 * need to run the task queue, because this I/O is on the ISC
12740 	 * queue, which is executed by the work thread after the task queue.
12741 	 * We don't need to run the datamove delay code, since that should
12742 	 * have been done if need be on the other controller.
12743 	 */
12744 	if (io->io_hdr.flags & CTL_FLAG_ABORT) {
12745 
12746 		printf("%s: tag 0x%04x on (%d:%d:%d:%d) aborted\n", __func__,
12747 		       io->scsiio.tag_num, io->io_hdr.nexus.initid.id,
12748 		       io->io_hdr.nexus.targ_port,
12749 		       io->io_hdr.nexus.targ_target.id,
12750 		       io->io_hdr.nexus.targ_lun);
12751 		io->io_hdr.status = CTL_CMD_ABORTED;
12752 		io->io_hdr.port_status = 31338;
12753 
12754 		ctl_send_datamove_done(io, /*have_lock*/ 0);
12755 
12756 		return;
12757 	}
12758 
12759 	if ((io->io_hdr.flags & CTL_FLAG_DATA_MASK) == CTL_FLAG_DATA_OUT) {
12760 		ctl_datamove_remote_write(io);
12761 	} else if ((io->io_hdr.flags & CTL_FLAG_DATA_MASK) == CTL_FLAG_DATA_IN){
12762 		ctl_datamove_remote_read(io);
12763 	} else {
12764 		union ctl_ha_msg msg;
12765 		struct scsi_sense_data *sense;
12766 		uint8_t sks[3];
12767 		int retry_count;
12768 
12769 		memset(&msg, 0, sizeof(msg));
12770 
12771 		msg.hdr.msg_type = CTL_MSG_BAD_JUJU;
12772 		msg.hdr.status = CTL_SCSI_ERROR;
12773 		msg.scsi.scsi_status = SCSI_STATUS_CHECK_COND;
12774 
12775 		retry_count = 4243;
12776 
12777 		sense = &msg.scsi.sense_data;
12778 		sks[0] = SSD_SCS_VALID;
12779 		sks[1] = (retry_count >> 8) & 0xff;
12780 		sks[2] = retry_count & 0xff;
12781 
12782 		/* "Internal target failure" */
12783 		scsi_set_sense_data(sense,
12784 				    /*sense_format*/ SSD_TYPE_NONE,
12785 				    /*current_error*/ 1,
12786 				    /*sense_key*/ SSD_KEY_HARDWARE_ERROR,
12787 				    /*asc*/ 0x44,
12788 				    /*ascq*/ 0x00,
12789 				    /*type*/ SSD_ELEM_SKS,
12790 				    /*size*/ sizeof(sks),
12791 				    /*data*/ sks,
12792 				    SSD_ELEM_NONE);
12793 
12794 		io->io_hdr.flags &= ~CTL_FLAG_IO_ACTIVE;
12795 		if (io->io_hdr.flags & CTL_FLAG_FAILOVER) {
12796 			ctl_failover_io(io, /*have_lock*/ 1);
12797 			return;
12798 		}
12799 
12800 		if (ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg, sizeof(msg), 0) >
12801 		    CTL_HA_STATUS_SUCCESS) {
12802 			/* XXX KDM what to do if this fails? */
12803 		}
12804 		return;
12805 	}
12806 
12807 }
12808 
12809 static int
12810 ctl_process_done(union ctl_io *io)
12811 {
12812 	struct ctl_lun *lun;
12813 	struct ctl_softc *ctl_softc;
12814 	void (*fe_done)(union ctl_io *io);
12815 	uint32_t targ_port = ctl_port_idx(io->io_hdr.nexus.targ_port);
12816 
12817 	CTL_DEBUG_PRINT(("ctl_process_done\n"));
12818 
12819 	fe_done =
12820 	    control_softc->ctl_ports[targ_port]->fe_done;
12821 
12822 #ifdef CTL_TIME_IO
12823 	if ((time_uptime - io->io_hdr.start_time) > ctl_time_io_secs) {
12824 		char str[256];
12825 		char path_str[64];
12826 		struct sbuf sb;
12827 
12828 		ctl_scsi_path_string(io, path_str, sizeof(path_str));
12829 		sbuf_new(&sb, str, sizeof(str), SBUF_FIXEDLEN);
12830 
12831 		sbuf_cat(&sb, path_str);
12832 		switch (io->io_hdr.io_type) {
12833 		case CTL_IO_SCSI:
12834 			ctl_scsi_command_string(&io->scsiio, NULL, &sb);
12835 			sbuf_printf(&sb, "\n");
12836 			sbuf_cat(&sb, path_str);
12837 			sbuf_printf(&sb, "Tag: 0x%04x, type %d\n",
12838 				    io->scsiio.tag_num, io->scsiio.tag_type);
12839 			break;
12840 		case CTL_IO_TASK:
12841 			sbuf_printf(&sb, "Task I/O type: %d, Tag: 0x%04x, "
12842 				    "Tag Type: %d\n", io->taskio.task_action,
12843 				    io->taskio.tag_num, io->taskio.tag_type);
12844 			break;
12845 		default:
12846 			printf("Invalid CTL I/O type %d\n", io->io_hdr.io_type);
12847 			panic("Invalid CTL I/O type %d\n", io->io_hdr.io_type);
12848 			break;
12849 		}
12850 		sbuf_cat(&sb, path_str);
12851 		sbuf_printf(&sb, "ctl_process_done: %jd seconds\n",
12852 			    (intmax_t)time_uptime - io->io_hdr.start_time);
12853 		sbuf_finish(&sb);
12854 		printf("%s", sbuf_data(&sb));
12855 	}
12856 #endif /* CTL_TIME_IO */
12857 
12858 	switch (io->io_hdr.io_type) {
12859 	case CTL_IO_SCSI:
12860 		break;
12861 	case CTL_IO_TASK:
12862 		if (bootverbose || verbose > 0)
12863 			ctl_io_error_print(io, NULL);
12864 		if (io->io_hdr.flags & CTL_FLAG_FROM_OTHER_SC)
12865 			ctl_free_io(io);
12866 		else
12867 			fe_done(io);
12868 		return (CTL_RETVAL_COMPLETE);
12869 		break;
12870 	default:
12871 		printf("ctl_process_done: invalid io type %d\n",
12872 		       io->io_hdr.io_type);
12873 		panic("ctl_process_done: invalid io type %d\n",
12874 		      io->io_hdr.io_type);
12875 		break; /* NOTREACHED */
12876 	}
12877 
12878 	lun = (struct ctl_lun *)io->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
12879 	if (lun == NULL) {
12880 		CTL_DEBUG_PRINT(("NULL LUN for lun %d\n",
12881 				 io->io_hdr.nexus.targ_mapped_lun));
12882 		fe_done(io);
12883 		goto bailout;
12884 	}
12885 	ctl_softc = lun->ctl_softc;
12886 
12887 	mtx_lock(&lun->lun_lock);
12888 
12889 	/*
12890 	 * Check to see if we have any errors to inject here.  We only
12891 	 * inject errors for commands that don't already have errors set.
12892 	 */
12893 	if ((STAILQ_FIRST(&lun->error_list) != NULL)
12894 	 && ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_SUCCESS))
12895 		ctl_inject_error(lun, io);
12896 
12897 	/*
12898 	 * XXX KDM how do we treat commands that aren't completed
12899 	 * successfully?
12900 	 *
12901 	 * XXX KDM should we also track I/O latency?
12902 	 */
12903 	if ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_SUCCESS) {
12904 		uint32_t blocksize;
12905 #ifdef CTL_TIME_IO
12906 		struct bintime cur_bt;
12907 #endif
12908 
12909 		if ((lun->be_lun != NULL)
12910 		 && (lun->be_lun->blocksize != 0))
12911 			blocksize = lun->be_lun->blocksize;
12912 		else
12913 			blocksize = 512;
12914 
12915 		switch (io->io_hdr.io_type) {
12916 		case CTL_IO_SCSI: {
12917 			int isread;
12918 			struct ctl_lba_len_flags *lbalen;
12919 
12920 			isread = 0;
12921 			switch (io->scsiio.cdb[0]) {
12922 			case READ_6:
12923 			case READ_10:
12924 			case READ_12:
12925 			case READ_16:
12926 				isread = 1;
12927 				/* FALLTHROUGH */
12928 			case WRITE_6:
12929 			case WRITE_10:
12930 			case WRITE_12:
12931 			case WRITE_16:
12932 			case WRITE_VERIFY_10:
12933 			case WRITE_VERIFY_12:
12934 			case WRITE_VERIFY_16:
12935 				lbalen = (struct ctl_lba_len_flags *)
12936 				    &io->io_hdr.ctl_private[CTL_PRIV_LBA_LEN];
12937 
12938 				if (isread) {
12939 					lun->stats.ports[targ_port].bytes[CTL_STATS_READ] +=
12940 					    lbalen->len * blocksize;
12941 					lun->stats.ports[targ_port].operations[CTL_STATS_READ]++;
12942 
12943 #ifdef CTL_TIME_IO
12944 					bintime_add(
12945 					   &lun->stats.ports[targ_port].dma_time[CTL_STATS_READ],
12946 					   &io->io_hdr.dma_bt);
12947 					lun->stats.ports[targ_port].num_dmas[CTL_STATS_READ] +=
12948 						io->io_hdr.num_dmas;
12949 					getbintime(&cur_bt);
12950 					bintime_sub(&cur_bt,
12951 						    &io->io_hdr.start_bt);
12952 
12953 					bintime_add(
12954 					    &lun->stats.ports[targ_port].time[CTL_STATS_READ],
12955 					    &cur_bt);
12956 
12957 #if 0
12958 					cs_prof_gettime(&cur_ticks);
12959 					lun->stats.time[CTL_STATS_READ] +=
12960 						cur_ticks -
12961 						io->io_hdr.start_ticks;
12962 #endif
12963 #if 0
12964 					lun->stats.time[CTL_STATS_READ] +=
12965 						jiffies - io->io_hdr.start_time;
12966 #endif
12967 #endif /* CTL_TIME_IO */
12968 				} else {
12969 					lun->stats.ports[targ_port].bytes[CTL_STATS_WRITE] +=
12970 					    lbalen->len * blocksize;
12971 					lun->stats.ports[targ_port].operations[
12972 						CTL_STATS_WRITE]++;
12973 
12974 #ifdef CTL_TIME_IO
12975 					bintime_add(
12976 					  &lun->stats.ports[targ_port].dma_time[CTL_STATS_WRITE],
12977 					  &io->io_hdr.dma_bt);
12978 					lun->stats.ports[targ_port].num_dmas[CTL_STATS_WRITE] +=
12979 						io->io_hdr.num_dmas;
12980 					getbintime(&cur_bt);
12981 					bintime_sub(&cur_bt,
12982 						    &io->io_hdr.start_bt);
12983 
12984 					bintime_add(
12985 					    &lun->stats.ports[targ_port].time[CTL_STATS_WRITE],
12986 					    &cur_bt);
12987 #if 0
12988 					cs_prof_gettime(&cur_ticks);
12989 					lun->stats.ports[targ_port].time[CTL_STATS_WRITE] +=
12990 						cur_ticks -
12991 						io->io_hdr.start_ticks;
12992 					lun->stats.ports[targ_port].time[CTL_STATS_WRITE] +=
12993 						jiffies - io->io_hdr.start_time;
12994 #endif
12995 #endif /* CTL_TIME_IO */
12996 				}
12997 				break;
12998 			default:
12999 				lun->stats.ports[targ_port].operations[CTL_STATS_NO_IO]++;
13000 
13001 #ifdef CTL_TIME_IO
13002 				bintime_add(
13003 				  &lun->stats.ports[targ_port].dma_time[CTL_STATS_NO_IO],
13004 				  &io->io_hdr.dma_bt);
13005 				lun->stats.ports[targ_port].num_dmas[CTL_STATS_NO_IO] +=
13006 					io->io_hdr.num_dmas;
13007 				getbintime(&cur_bt);
13008 				bintime_sub(&cur_bt, &io->io_hdr.start_bt);
13009 
13010 				bintime_add(&lun->stats.ports[targ_port].time[CTL_STATS_NO_IO],
13011 					    &cur_bt);
13012 
13013 #if 0
13014 				cs_prof_gettime(&cur_ticks);
13015 				lun->stats.ports[targ_port].time[CTL_STATS_NO_IO] +=
13016 					cur_ticks -
13017 					io->io_hdr.start_ticks;
13018 				lun->stats.ports[targ_port].time[CTL_STATS_NO_IO] +=
13019 					jiffies - io->io_hdr.start_time;
13020 #endif
13021 #endif /* CTL_TIME_IO */
13022 				break;
13023 			}
13024 			break;
13025 		}
13026 		default:
13027 			break;
13028 		}
13029 	}
13030 
13031 	/*
13032 	 * Remove this from the OOA queue.
13033 	 */
13034 	TAILQ_REMOVE(&lun->ooa_queue, &io->io_hdr, ooa_links);
13035 
13036 	/*
13037 	 * Run through the blocked queue on this LUN and see if anything
13038 	 * has become unblocked, now that this transaction is done.
13039 	 */
13040 	ctl_check_blocked(lun);
13041 
13042 	/*
13043 	 * If the LUN has been invalidated, free it if there is nothing
13044 	 * left on its OOA queue.
13045 	 */
13046 	if ((lun->flags & CTL_LUN_INVALID)
13047 	 && TAILQ_EMPTY(&lun->ooa_queue)) {
13048 		mtx_unlock(&lun->lun_lock);
13049 		mtx_lock(&ctl_softc->ctl_lock);
13050 		ctl_free_lun(lun);
13051 		mtx_unlock(&ctl_softc->ctl_lock);
13052 	} else
13053 		mtx_unlock(&lun->lun_lock);
13054 
13055 	/*
13056 	 * If this command has been aborted, make sure we set the status
13057 	 * properly.  The FETD is responsible for freeing the I/O and doing
13058 	 * whatever it needs to do to clean up its state.
13059 	 */
13060 	if (io->io_hdr.flags & CTL_FLAG_ABORT)
13061 		io->io_hdr.status = CTL_CMD_ABORTED;
13062 
13063 	/*
13064 	 * We print out status for every task management command.  For SCSI
13065 	 * commands, we filter out any unit attention errors; they happen
13066 	 * on every boot, and would clutter up the log.  Note:  task
13067 	 * management commands aren't printed here, they are printed above,
13068 	 * since they should never even make it down here.
13069 	 */
13070 	switch (io->io_hdr.io_type) {
13071 	case CTL_IO_SCSI: {
13072 		int error_code, sense_key, asc, ascq;
13073 
13074 		sense_key = 0;
13075 
13076 		if (((io->io_hdr.status & CTL_STATUS_MASK) == CTL_SCSI_ERROR)
13077 		 && (io->scsiio.scsi_status == SCSI_STATUS_CHECK_COND)) {
13078 			/*
13079 			 * Since this is just for printing, no need to
13080 			 * show errors here.
13081 			 */
13082 			scsi_extract_sense_len(&io->scsiio.sense_data,
13083 					       io->scsiio.sense_len,
13084 					       &error_code,
13085 					       &sense_key,
13086 					       &asc,
13087 					       &ascq,
13088 					       /*show_errors*/ 0);
13089 		}
13090 
13091 		if (((io->io_hdr.status & CTL_STATUS_MASK) != CTL_SUCCESS)
13092 		 && (((io->io_hdr.status & CTL_STATUS_MASK) != CTL_SCSI_ERROR)
13093 		  || (io->scsiio.scsi_status != SCSI_STATUS_CHECK_COND)
13094 		  || (sense_key != SSD_KEY_UNIT_ATTENTION))) {
13095 
13096 			if ((time_uptime - ctl_softc->last_print_jiffies) <= 0){
13097 				ctl_softc->skipped_prints++;
13098 			} else {
13099 				uint32_t skipped_prints;
13100 
13101 				skipped_prints = ctl_softc->skipped_prints;
13102 
13103 				ctl_softc->skipped_prints = 0;
13104 				ctl_softc->last_print_jiffies = time_uptime;
13105 
13106 				if (skipped_prints > 0) {
13107 #ifdef NEEDTOPORT
13108 					csevent_log(CSC_CTL | CSC_SHELF_SW |
13109 					    CTL_ERROR_REPORT,
13110 					    csevent_LogType_Trace,
13111 					    csevent_Severity_Information,
13112 					    csevent_AlertLevel_Green,
13113 					    csevent_FRU_Firmware,
13114 					    csevent_FRU_Unknown,
13115 					    "High CTL error volume, %d prints "
13116 					    "skipped", skipped_prints);
13117 #endif
13118 				}
13119 				if (bootverbose || verbose > 0)
13120 					ctl_io_error_print(io, NULL);
13121 			}
13122 		}
13123 		break;
13124 	}
13125 	case CTL_IO_TASK:
13126 		if (bootverbose || verbose > 0)
13127 			ctl_io_error_print(io, NULL);
13128 		break;
13129 	default:
13130 		break;
13131 	}
13132 
13133 	/*
13134 	 * Tell the FETD or the other shelf controller we're done with this
13135 	 * command.  Note that only SCSI commands get to this point.  Task
13136 	 * management commands are completed above.
13137 	 *
13138 	 * We only send status to the other controller if we're in XFER
13139 	 * mode.  In SER_ONLY mode, the I/O is done on the controller that
13140 	 * received the I/O (from CTL's perspective), and so the status is
13141 	 * generated there.
13142 	 *
13143 	 * XXX KDM if we hold the lock here, we could cause a deadlock
13144 	 * if the frontend comes back in in this context to queue
13145 	 * something.
13146 	 */
13147 	if ((ctl_softc->ha_mode == CTL_HA_MODE_XFER)
13148 	 && (io->io_hdr.flags & CTL_FLAG_FROM_OTHER_SC)) {
13149 		union ctl_ha_msg msg;
13150 
13151 		memset(&msg, 0, sizeof(msg));
13152 		msg.hdr.msg_type = CTL_MSG_FINISH_IO;
13153 		msg.hdr.original_sc = io->io_hdr.original_sc;
13154 		msg.hdr.nexus = io->io_hdr.nexus;
13155 		msg.hdr.status = io->io_hdr.status;
13156 		msg.scsi.scsi_status = io->scsiio.scsi_status;
13157 		msg.scsi.tag_num = io->scsiio.tag_num;
13158 		msg.scsi.tag_type = io->scsiio.tag_type;
13159 		msg.scsi.sense_len = io->scsiio.sense_len;
13160 		msg.scsi.sense_residual = io->scsiio.sense_residual;
13161 		msg.scsi.residual = io->scsiio.residual;
13162 		memcpy(&msg.scsi.sense_data, &io->scsiio.sense_data,
13163 		       sizeof(io->scsiio.sense_data));
13164 		/*
13165 		 * We copy this whether or not this is an I/O-related
13166 		 * command.  Otherwise, we'd have to go and check to see
13167 		 * whether it's a read/write command, and it really isn't
13168 		 * worth it.
13169 		 */
13170 		memcpy(&msg.scsi.lbalen,
13171 		       &io->io_hdr.ctl_private[CTL_PRIV_LBA_LEN].bytes,
13172 		       sizeof(msg.scsi.lbalen));
13173 
13174 		if (ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg,
13175 				sizeof(msg), 0) > CTL_HA_STATUS_SUCCESS) {
13176 			/* XXX do something here */
13177 		}
13178 
13179 		ctl_free_io(io);
13180 	} else
13181 		fe_done(io);
13182 
13183 bailout:
13184 
13185 	return (CTL_RETVAL_COMPLETE);
13186 }
13187 
13188 /*
13189  * Front end should call this if it doesn't do autosense.  When the request
13190  * sense comes back in from the initiator, we'll dequeue this and send it.
13191  */
13192 int
13193 ctl_queue_sense(union ctl_io *io)
13194 {
13195 	struct ctl_lun *lun;
13196 	struct ctl_softc *ctl_softc;
13197 	uint32_t initidx, targ_lun;
13198 
13199 	ctl_softc = control_softc;
13200 
13201 	CTL_DEBUG_PRINT(("ctl_queue_sense\n"));
13202 
13203 	/*
13204 	 * LUN lookup will likely move to the ctl_work_thread() once we
13205 	 * have our new queueing infrastructure (that doesn't put things on
13206 	 * a per-LUN queue initially).  That is so that we can handle
13207 	 * things like an INQUIRY to a LUN that we don't have enabled.  We
13208 	 * can't deal with that right now.
13209 	 */
13210 	mtx_lock(&ctl_softc->ctl_lock);
13211 
13212 	/*
13213 	 * If we don't have a LUN for this, just toss the sense
13214 	 * information.
13215 	 */
13216 	targ_lun = io->io_hdr.nexus.targ_lun;
13217 	if (io->io_hdr.nexus.lun_map_fn != NULL)
13218 		targ_lun = io->io_hdr.nexus.lun_map_fn(io->io_hdr.nexus.lun_map_arg, targ_lun);
13219 	if ((targ_lun < CTL_MAX_LUNS)
13220 	 && (ctl_softc->ctl_luns[targ_lun] != NULL))
13221 		lun = ctl_softc->ctl_luns[targ_lun];
13222 	else
13223 		goto bailout;
13224 
13225 	initidx = ctl_get_initindex(&io->io_hdr.nexus);
13226 
13227 	mtx_lock(&lun->lun_lock);
13228 	/*
13229 	 * Already have CA set for this LUN...toss the sense information.
13230 	 */
13231 	if (ctl_is_set(lun->have_ca, initidx)) {
13232 		mtx_unlock(&lun->lun_lock);
13233 		goto bailout;
13234 	}
13235 
13236 	memcpy(&lun->pending_sense[initidx].sense, &io->scsiio.sense_data,
13237 	       ctl_min(sizeof(lun->pending_sense[initidx].sense),
13238 	       sizeof(io->scsiio.sense_data)));
13239 	ctl_set_mask(lun->have_ca, initidx);
13240 	mtx_unlock(&lun->lun_lock);
13241 
13242 bailout:
13243 	mtx_unlock(&ctl_softc->ctl_lock);
13244 
13245 	ctl_free_io(io);
13246 
13247 	return (CTL_RETVAL_COMPLETE);
13248 }
13249 
13250 /*
13251  * Primary command inlet from frontend ports.  All SCSI and task I/O
13252  * requests must go through this function.
13253  */
13254 int
13255 ctl_queue(union ctl_io *io)
13256 {
13257 	struct ctl_softc *ctl_softc;
13258 
13259 	CTL_DEBUG_PRINT(("ctl_queue cdb[0]=%02X\n", io->scsiio.cdb[0]));
13260 
13261 	ctl_softc = control_softc;
13262 
13263 #ifdef CTL_TIME_IO
13264 	io->io_hdr.start_time = time_uptime;
13265 	getbintime(&io->io_hdr.start_bt);
13266 #endif /* CTL_TIME_IO */
13267 
13268 	/* Map FE-specific LUN ID into global one. */
13269 	if (io->io_hdr.nexus.lun_map_fn != NULL)
13270 		io->io_hdr.nexus.targ_mapped_lun = io->io_hdr.nexus.lun_map_fn(
13271 		    io->io_hdr.nexus.lun_map_arg, io->io_hdr.nexus.targ_lun);
13272 	else
13273 		io->io_hdr.nexus.targ_mapped_lun = io->io_hdr.nexus.targ_lun;
13274 
13275 	switch (io->io_hdr.io_type) {
13276 	case CTL_IO_SCSI:
13277 		ctl_enqueue_incoming(io);
13278 		break;
13279 	case CTL_IO_TASK:
13280 		ctl_run_task(io);
13281 		break;
13282 	default:
13283 		printf("ctl_queue: unknown I/O type %d\n", io->io_hdr.io_type);
13284 		return (-EINVAL);
13285 	}
13286 
13287 	return (CTL_RETVAL_COMPLETE);
13288 }
13289 
13290 #ifdef CTL_IO_DELAY
13291 static void
13292 ctl_done_timer_wakeup(void *arg)
13293 {
13294 	union ctl_io *io;
13295 
13296 	io = (union ctl_io *)arg;
13297 	ctl_done(io);
13298 }
13299 #endif /* CTL_IO_DELAY */
13300 
13301 void
13302 ctl_done(union ctl_io *io)
13303 {
13304 	struct ctl_softc *ctl_softc;
13305 
13306 	ctl_softc = control_softc;
13307 
13308 	/*
13309 	 * Enable this to catch duplicate completion issues.
13310 	 */
13311 #if 0
13312 	if (io->io_hdr.flags & CTL_FLAG_ALREADY_DONE) {
13313 		printf("%s: type %d msg %d cdb %x iptl: "
13314 		       "%d:%d:%d:%d tag 0x%04x "
13315 		       "flag %#x status %x\n",
13316 			__func__,
13317 			io->io_hdr.io_type,
13318 			io->io_hdr.msg_type,
13319 			io->scsiio.cdb[0],
13320 			io->io_hdr.nexus.initid.id,
13321 			io->io_hdr.nexus.targ_port,
13322 			io->io_hdr.nexus.targ_target.id,
13323 			io->io_hdr.nexus.targ_lun,
13324 			(io->io_hdr.io_type ==
13325 			CTL_IO_TASK) ?
13326 			io->taskio.tag_num :
13327 			io->scsiio.tag_num,
13328 		        io->io_hdr.flags,
13329 			io->io_hdr.status);
13330 	} else
13331 		io->io_hdr.flags |= CTL_FLAG_ALREADY_DONE;
13332 #endif
13333 
13334 	/*
13335 	 * This is an internal copy of an I/O, and should not go through
13336 	 * the normal done processing logic.
13337 	 */
13338 	if (io->io_hdr.flags & CTL_FLAG_INT_COPY)
13339 		return;
13340 
13341 	/*
13342 	 * We need to send a msg to the serializing shelf to finish the IO
13343 	 * as well.  We don't send a finish message to the other shelf if
13344 	 * this is a task management command.  Task management commands
13345 	 * aren't serialized in the OOA queue, but rather just executed on
13346 	 * both shelf controllers for commands that originated on that
13347 	 * controller.
13348 	 */
13349 	if ((io->io_hdr.flags & CTL_FLAG_SENT_2OTHER_SC)
13350 	 && (io->io_hdr.io_type != CTL_IO_TASK)) {
13351 		union ctl_ha_msg msg_io;
13352 
13353 		msg_io.hdr.msg_type = CTL_MSG_FINISH_IO;
13354 		msg_io.hdr.serializing_sc = io->io_hdr.serializing_sc;
13355 		if (ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg_io,
13356 		    sizeof(msg_io), 0 ) != CTL_HA_STATUS_SUCCESS) {
13357 		}
13358 		/* continue on to finish IO */
13359 	}
13360 #ifdef CTL_IO_DELAY
13361 	if (io->io_hdr.flags & CTL_FLAG_DELAY_DONE) {
13362 		struct ctl_lun *lun;
13363 
13364 		lun =(struct ctl_lun *)io->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
13365 
13366 		io->io_hdr.flags &= ~CTL_FLAG_DELAY_DONE;
13367 	} else {
13368 		struct ctl_lun *lun;
13369 
13370 		lun =(struct ctl_lun *)io->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
13371 
13372 		if ((lun != NULL)
13373 		 && (lun->delay_info.done_delay > 0)) {
13374 			struct callout *callout;
13375 
13376 			callout = (struct callout *)&io->io_hdr.timer_bytes;
13377 			callout_init(callout, /*mpsafe*/ 1);
13378 			io->io_hdr.flags |= CTL_FLAG_DELAY_DONE;
13379 			callout_reset(callout,
13380 				      lun->delay_info.done_delay * hz,
13381 				      ctl_done_timer_wakeup, io);
13382 			if (lun->delay_info.done_type == CTL_DELAY_TYPE_ONESHOT)
13383 				lun->delay_info.done_delay = 0;
13384 			return;
13385 		}
13386 	}
13387 #endif /* CTL_IO_DELAY */
13388 
13389 	ctl_enqueue_done(io);
13390 }
13391 
13392 int
13393 ctl_isc(struct ctl_scsiio *ctsio)
13394 {
13395 	struct ctl_lun *lun;
13396 	int retval;
13397 
13398 	lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
13399 
13400 	CTL_DEBUG_PRINT(("ctl_isc: command: %02x\n", ctsio->cdb[0]));
13401 
13402 	CTL_DEBUG_PRINT(("ctl_isc: calling data_submit()\n"));
13403 
13404 	retval = lun->backend->data_submit((union ctl_io *)ctsio);
13405 
13406 	return (retval);
13407 }
13408 
13409 
13410 static void
13411 ctl_work_thread(void *arg)
13412 {
13413 	struct ctl_thread *thr = (struct ctl_thread *)arg;
13414 	struct ctl_softc *softc = thr->ctl_softc;
13415 	union ctl_io *io;
13416 	int retval;
13417 
13418 	CTL_DEBUG_PRINT(("ctl_work_thread starting\n"));
13419 
13420 	for (;;) {
13421 		retval = 0;
13422 
13423 		/*
13424 		 * We handle the queues in this order:
13425 		 * - ISC
13426 		 * - done queue (to free up resources, unblock other commands)
13427 		 * - RtR queue
13428 		 * - incoming queue
13429 		 *
13430 		 * If those queues are empty, we break out of the loop and
13431 		 * go to sleep.
13432 		 */
13433 		mtx_lock(&thr->queue_lock);
13434 		io = (union ctl_io *)STAILQ_FIRST(&thr->isc_queue);
13435 		if (io != NULL) {
13436 			STAILQ_REMOVE_HEAD(&thr->isc_queue, links);
13437 			mtx_unlock(&thr->queue_lock);
13438 			ctl_handle_isc(io);
13439 			continue;
13440 		}
13441 		io = (union ctl_io *)STAILQ_FIRST(&thr->done_queue);
13442 		if (io != NULL) {
13443 			STAILQ_REMOVE_HEAD(&thr->done_queue, links);
13444 			/* clear any blocked commands, call fe_done */
13445 			mtx_unlock(&thr->queue_lock);
13446 			retval = ctl_process_done(io);
13447 			continue;
13448 		}
13449 		if (!ctl_pause_rtr) {
13450 			io = (union ctl_io *)STAILQ_FIRST(&thr->rtr_queue);
13451 			if (io != NULL) {
13452 				STAILQ_REMOVE_HEAD(&thr->rtr_queue, links);
13453 				mtx_unlock(&thr->queue_lock);
13454 				retval = ctl_scsiio(&io->scsiio);
13455 				if (retval != CTL_RETVAL_COMPLETE)
13456 					CTL_DEBUG_PRINT(("ctl_scsiio failed\n"));
13457 				continue;
13458 			}
13459 		}
13460 		io = (union ctl_io *)STAILQ_FIRST(&thr->incoming_queue);
13461 		if (io != NULL) {
13462 			STAILQ_REMOVE_HEAD(&thr->incoming_queue, links);
13463 			mtx_unlock(&thr->queue_lock);
13464 			ctl_scsiio_precheck(softc, &io->scsiio);
13465 			continue;
13466 		}
13467 
13468 		/* Sleep until we have something to do. */
13469 		mtx_sleep(thr, &thr->queue_lock, PDROP | PRIBIO, "-", 0);
13470 	}
13471 }
13472 
13473 static void
13474 ctl_lun_thread(void *arg)
13475 {
13476 	struct ctl_softc *softc = (struct ctl_softc *)arg;
13477 	struct ctl_be_lun *be_lun;
13478 	int retval;
13479 
13480 	CTL_DEBUG_PRINT(("ctl_lun_thread starting\n"));
13481 
13482 	for (;;) {
13483 		retval = 0;
13484 		mtx_lock(&softc->ctl_lock);
13485 		be_lun = STAILQ_FIRST(&softc->pending_lun_queue);
13486 		if (be_lun != NULL) {
13487 			STAILQ_REMOVE_HEAD(&softc->pending_lun_queue, links);
13488 			mtx_unlock(&softc->ctl_lock);
13489 			ctl_create_lun(be_lun);
13490 			continue;
13491 		}
13492 
13493 		/* Sleep until we have something to do. */
13494 		mtx_sleep(&softc->pending_lun_queue, &softc->ctl_lock,
13495 		    PDROP | PRIBIO, "-", 0);
13496 	}
13497 }
13498 
13499 static void
13500 ctl_enqueue_incoming(union ctl_io *io)
13501 {
13502 	struct ctl_softc *softc = control_softc;
13503 	struct ctl_thread *thr;
13504 
13505 	thr = &softc->threads[io->io_hdr.nexus.targ_mapped_lun % worker_threads];
13506 	mtx_lock(&thr->queue_lock);
13507 	STAILQ_INSERT_TAIL(&thr->incoming_queue, &io->io_hdr, links);
13508 	mtx_unlock(&thr->queue_lock);
13509 	wakeup(thr);
13510 }
13511 
13512 static void
13513 ctl_enqueue_rtr(union ctl_io *io)
13514 {
13515 	struct ctl_softc *softc = control_softc;
13516 	struct ctl_thread *thr;
13517 
13518 	thr = &softc->threads[io->io_hdr.nexus.targ_mapped_lun % worker_threads];
13519 	mtx_lock(&thr->queue_lock);
13520 	STAILQ_INSERT_TAIL(&thr->rtr_queue, &io->io_hdr, links);
13521 	mtx_unlock(&thr->queue_lock);
13522 	wakeup(thr);
13523 }
13524 
13525 static void
13526 ctl_enqueue_done(union ctl_io *io)
13527 {
13528 	struct ctl_softc *softc = control_softc;
13529 	struct ctl_thread *thr;
13530 
13531 	thr = &softc->threads[io->io_hdr.nexus.targ_mapped_lun % worker_threads];
13532 	mtx_lock(&thr->queue_lock);
13533 	STAILQ_INSERT_TAIL(&thr->done_queue, &io->io_hdr, links);
13534 	mtx_unlock(&thr->queue_lock);
13535 	wakeup(thr);
13536 }
13537 
13538 static void
13539 ctl_enqueue_isc(union ctl_io *io)
13540 {
13541 	struct ctl_softc *softc = control_softc;
13542 	struct ctl_thread *thr;
13543 
13544 	thr = &softc->threads[io->io_hdr.nexus.targ_mapped_lun % worker_threads];
13545 	mtx_lock(&thr->queue_lock);
13546 	STAILQ_INSERT_TAIL(&thr->isc_queue, &io->io_hdr, links);
13547 	mtx_unlock(&thr->queue_lock);
13548 	wakeup(thr);
13549 }
13550 
13551 /* Initialization and failover */
13552 
13553 void
13554 ctl_init_isc_msg(void)
13555 {
13556 	printf("CTL: Still calling this thing\n");
13557 }
13558 
13559 /*
13560  * Init component
13561  * 	Initializes component into configuration defined by bootMode
13562  *	(see hasc-sv.c)
13563  *  	returns hasc_Status:
13564  * 		OK
13565  *		ERROR - fatal error
13566  */
13567 static ctl_ha_comp_status
13568 ctl_isc_init(struct ctl_ha_component *c)
13569 {
13570 	ctl_ha_comp_status ret = CTL_HA_COMP_STATUS_OK;
13571 
13572 	c->status = ret;
13573 	return ret;
13574 }
13575 
13576 /* Start component
13577  * 	Starts component in state requested. If component starts successfully,
13578  *	it must set its own state to the requestrd state
13579  *	When requested state is HASC_STATE_HA, the component may refine it
13580  * 	by adding _SLAVE or _MASTER flags.
13581  *	Currently allowed state transitions are:
13582  *	UNKNOWN->HA		- initial startup
13583  *	UNKNOWN->SINGLE - initial startup when no parter detected
13584  *	HA->SINGLE		- failover
13585  * returns ctl_ha_comp_status:
13586  * 		OK	- component successfully started in requested state
13587  *		FAILED  - could not start the requested state, failover may
13588  * 			  be possible
13589  *		ERROR	- fatal error detected, no future startup possible
13590  */
13591 static ctl_ha_comp_status
13592 ctl_isc_start(struct ctl_ha_component *c, ctl_ha_state state)
13593 {
13594 	ctl_ha_comp_status ret = CTL_HA_COMP_STATUS_OK;
13595 
13596 	printf("%s: go\n", __func__);
13597 
13598 	// UNKNOWN->HA or UNKNOWN->SINGLE (bootstrap)
13599 	if (c->state == CTL_HA_STATE_UNKNOWN ) {
13600 		ctl_is_single = 0;
13601 		if (ctl_ha_msg_create(CTL_HA_CHAN_CTL, ctl_isc_event_handler)
13602 		    != CTL_HA_STATUS_SUCCESS) {
13603 			printf("ctl_isc_start: ctl_ha_msg_create failed.\n");
13604 			ret = CTL_HA_COMP_STATUS_ERROR;
13605 		}
13606 	} else if (CTL_HA_STATE_IS_HA(c->state)
13607 		&& CTL_HA_STATE_IS_SINGLE(state)){
13608 		// HA->SINGLE transition
13609 	        ctl_failover();
13610 		ctl_is_single = 1;
13611 	} else {
13612 		printf("ctl_isc_start:Invalid state transition %X->%X\n",
13613 		       c->state, state);
13614 		ret = CTL_HA_COMP_STATUS_ERROR;
13615 	}
13616 	if (CTL_HA_STATE_IS_SINGLE(state))
13617 		ctl_is_single = 1;
13618 
13619 	c->state = state;
13620 	c->status = ret;
13621 	return ret;
13622 }
13623 
13624 /*
13625  * Quiesce component
13626  * The component must clear any error conditions (set status to OK) and
13627  * prepare itself to another Start call
13628  * returns ctl_ha_comp_status:
13629  * 	OK
13630  *	ERROR
13631  */
13632 static ctl_ha_comp_status
13633 ctl_isc_quiesce(struct ctl_ha_component *c)
13634 {
13635 	int ret = CTL_HA_COMP_STATUS_OK;
13636 
13637 	ctl_pause_rtr = 1;
13638 	c->status = ret;
13639 	return ret;
13640 }
13641 
13642 struct ctl_ha_component ctl_ha_component_ctlisc =
13643 {
13644 	.name = "CTL ISC",
13645 	.state = CTL_HA_STATE_UNKNOWN,
13646 	.init = ctl_isc_init,
13647 	.start = ctl_isc_start,
13648 	.quiesce = ctl_isc_quiesce
13649 };
13650 
13651 /*
13652  *  vim: ts=8
13653  */
13654