xref: /freebsd/sys/cam/ctl/ctl.c (revision c36e54bb328697af1e6113812caecbd3bac89fe0)
1 /*-
2  * Copyright (c) 2003-2009 Silicon Graphics International Corp.
3  * Copyright (c) 2012 The FreeBSD Foundation
4  * All rights reserved.
5  *
6  * Portions of this software were developed by Edward Tomasz Napierala
7  * under sponsorship from the FreeBSD Foundation.
8  *
9  * Redistribution and use in source and binary forms, with or without
10  * modification, are permitted provided that the following conditions
11  * are met:
12  * 1. Redistributions of source code must retain the above copyright
13  *    notice, this list of conditions, and the following disclaimer,
14  *    without modification.
15  * 2. Redistributions in binary form must reproduce at minimum a disclaimer
16  *    substantially similar to the "NO WARRANTY" disclaimer below
17  *    ("Disclaimer") and any redistribution must be conditioned upon
18  *    including a substantially similar Disclaimer requirement for further
19  *    binary redistribution.
20  *
21  * NO WARRANTY
22  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
23  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
24  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTIBILITY AND FITNESS FOR
25  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
26  * HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
27  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
28  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
29  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
30  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
31  * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
32  * POSSIBILITY OF SUCH DAMAGES.
33  *
34  * $Id$
35  */
36 /*
37  * CAM Target Layer, a SCSI device emulation subsystem.
38  *
39  * Author: Ken Merry <ken@FreeBSD.org>
40  */
41 
42 #define _CTL_C
43 
44 #include <sys/cdefs.h>
45 __FBSDID("$FreeBSD$");
46 
47 #include <sys/param.h>
48 #include <sys/systm.h>
49 #include <sys/ctype.h>
50 #include <sys/kernel.h>
51 #include <sys/types.h>
52 #include <sys/kthread.h>
53 #include <sys/bio.h>
54 #include <sys/fcntl.h>
55 #include <sys/lock.h>
56 #include <sys/module.h>
57 #include <sys/mutex.h>
58 #include <sys/condvar.h>
59 #include <sys/malloc.h>
60 #include <sys/conf.h>
61 #include <sys/ioccom.h>
62 #include <sys/queue.h>
63 #include <sys/sbuf.h>
64 #include <sys/smp.h>
65 #include <sys/endian.h>
66 #include <sys/sysctl.h>
67 #include <vm/uma.h>
68 
69 #include <cam/cam.h>
70 #include <cam/scsi/scsi_all.h>
71 #include <cam/scsi/scsi_da.h>
72 #include <cam/ctl/ctl_io.h>
73 #include <cam/ctl/ctl.h>
74 #include <cam/ctl/ctl_frontend.h>
75 #include <cam/ctl/ctl_frontend_internal.h>
76 #include <cam/ctl/ctl_util.h>
77 #include <cam/ctl/ctl_backend.h>
78 #include <cam/ctl/ctl_ioctl.h>
79 #include <cam/ctl/ctl_ha.h>
80 #include <cam/ctl/ctl_private.h>
81 #include <cam/ctl/ctl_debug.h>
82 #include <cam/ctl/ctl_scsi_all.h>
83 #include <cam/ctl/ctl_error.h>
84 
85 struct ctl_softc *control_softc = NULL;
86 
87 /*
88  * Size and alignment macros needed for Copan-specific HA hardware.  These
89  * can go away when the HA code is re-written, and uses busdma for any
90  * hardware.
91  */
92 #define	CTL_ALIGN_8B(target, source, type)				\
93 	if (((uint32_t)source & 0x7) != 0)				\
94 		target = (type)(source + (0x8 - ((uint32_t)source & 0x7)));\
95 	else								\
96 		target = (type)source;
97 
98 #define	CTL_SIZE_8B(target, size)					\
99 	if ((size & 0x7) != 0)						\
100 		target = size + (0x8 - (size & 0x7));			\
101 	else								\
102 		target = size;
103 
104 #define CTL_ALIGN_8B_MARGIN	16
105 
106 /*
107  * Template mode pages.
108  */
109 
110 /*
111  * Note that these are default values only.  The actual values will be
112  * filled in when the user does a mode sense.
113  */
114 const static struct copan_debugconf_subpage debugconf_page_default = {
115 	DBGCNF_PAGE_CODE | SMPH_SPF,	/* page_code */
116 	DBGCNF_SUBPAGE_CODE,		/* subpage */
117 	{(sizeof(struct copan_debugconf_subpage) - 4) >> 8,
118 	 (sizeof(struct copan_debugconf_subpage) - 4) >> 0}, /* page_length */
119 	DBGCNF_VERSION,			/* page_version */
120 	{CTL_TIME_IO_DEFAULT_SECS>>8,
121 	 CTL_TIME_IO_DEFAULT_SECS>>0},	/* ctl_time_io_secs */
122 };
123 
124 const static struct copan_debugconf_subpage debugconf_page_changeable = {
125 	DBGCNF_PAGE_CODE | SMPH_SPF,	/* page_code */
126 	DBGCNF_SUBPAGE_CODE,		/* subpage */
127 	{(sizeof(struct copan_debugconf_subpage) - 4) >> 8,
128 	 (sizeof(struct copan_debugconf_subpage) - 4) >> 0}, /* page_length */
129 	0,				/* page_version */
130 	{0xff,0xff},			/* ctl_time_io_secs */
131 };
132 
133 const static struct scsi_da_rw_recovery_page rw_er_page_default = {
134 	/*page_code*/SMS_RW_ERROR_RECOVERY_PAGE,
135 	/*page_length*/sizeof(struct scsi_da_rw_recovery_page) - 2,
136 	/*byte3*/SMS_RWER_AWRE|SMS_RWER_ARRE,
137 	/*read_retry_count*/0,
138 	/*correction_span*/0,
139 	/*head_offset_count*/0,
140 	/*data_strobe_offset_cnt*/0,
141 	/*byte8*/SMS_RWER_LBPERE,
142 	/*write_retry_count*/0,
143 	/*reserved2*/0,
144 	/*recovery_time_limit*/{0, 0},
145 };
146 
147 const static struct scsi_da_rw_recovery_page rw_er_page_changeable = {
148 	/*page_code*/SMS_RW_ERROR_RECOVERY_PAGE,
149 	/*page_length*/sizeof(struct scsi_da_rw_recovery_page) - 2,
150 	/*byte3*/0,
151 	/*read_retry_count*/0,
152 	/*correction_span*/0,
153 	/*head_offset_count*/0,
154 	/*data_strobe_offset_cnt*/0,
155 	/*byte8*/0,
156 	/*write_retry_count*/0,
157 	/*reserved2*/0,
158 	/*recovery_time_limit*/{0, 0},
159 };
160 
161 const static struct scsi_format_page format_page_default = {
162 	/*page_code*/SMS_FORMAT_DEVICE_PAGE,
163 	/*page_length*/sizeof(struct scsi_format_page) - 2,
164 	/*tracks_per_zone*/ {0, 0},
165 	/*alt_sectors_per_zone*/ {0, 0},
166 	/*alt_tracks_per_zone*/ {0, 0},
167 	/*alt_tracks_per_lun*/ {0, 0},
168 	/*sectors_per_track*/ {(CTL_DEFAULT_SECTORS_PER_TRACK >> 8) & 0xff,
169 			        CTL_DEFAULT_SECTORS_PER_TRACK & 0xff},
170 	/*bytes_per_sector*/ {0, 0},
171 	/*interleave*/ {0, 0},
172 	/*track_skew*/ {0, 0},
173 	/*cylinder_skew*/ {0, 0},
174 	/*flags*/ SFP_HSEC,
175 	/*reserved*/ {0, 0, 0}
176 };
177 
178 const static struct scsi_format_page format_page_changeable = {
179 	/*page_code*/SMS_FORMAT_DEVICE_PAGE,
180 	/*page_length*/sizeof(struct scsi_format_page) - 2,
181 	/*tracks_per_zone*/ {0, 0},
182 	/*alt_sectors_per_zone*/ {0, 0},
183 	/*alt_tracks_per_zone*/ {0, 0},
184 	/*alt_tracks_per_lun*/ {0, 0},
185 	/*sectors_per_track*/ {0, 0},
186 	/*bytes_per_sector*/ {0, 0},
187 	/*interleave*/ {0, 0},
188 	/*track_skew*/ {0, 0},
189 	/*cylinder_skew*/ {0, 0},
190 	/*flags*/ 0,
191 	/*reserved*/ {0, 0, 0}
192 };
193 
194 const static struct scsi_rigid_disk_page rigid_disk_page_default = {
195 	/*page_code*/SMS_RIGID_DISK_PAGE,
196 	/*page_length*/sizeof(struct scsi_rigid_disk_page) - 2,
197 	/*cylinders*/ {0, 0, 0},
198 	/*heads*/ CTL_DEFAULT_HEADS,
199 	/*start_write_precomp*/ {0, 0, 0},
200 	/*start_reduced_current*/ {0, 0, 0},
201 	/*step_rate*/ {0, 0},
202 	/*landing_zone_cylinder*/ {0, 0, 0},
203 	/*rpl*/ SRDP_RPL_DISABLED,
204 	/*rotational_offset*/ 0,
205 	/*reserved1*/ 0,
206 	/*rotation_rate*/ {(CTL_DEFAULT_ROTATION_RATE >> 8) & 0xff,
207 			   CTL_DEFAULT_ROTATION_RATE & 0xff},
208 	/*reserved2*/ {0, 0}
209 };
210 
211 const static struct scsi_rigid_disk_page rigid_disk_page_changeable = {
212 	/*page_code*/SMS_RIGID_DISK_PAGE,
213 	/*page_length*/sizeof(struct scsi_rigid_disk_page) - 2,
214 	/*cylinders*/ {0, 0, 0},
215 	/*heads*/ 0,
216 	/*start_write_precomp*/ {0, 0, 0},
217 	/*start_reduced_current*/ {0, 0, 0},
218 	/*step_rate*/ {0, 0},
219 	/*landing_zone_cylinder*/ {0, 0, 0},
220 	/*rpl*/ 0,
221 	/*rotational_offset*/ 0,
222 	/*reserved1*/ 0,
223 	/*rotation_rate*/ {0, 0},
224 	/*reserved2*/ {0, 0}
225 };
226 
227 const static struct scsi_caching_page caching_page_default = {
228 	/*page_code*/SMS_CACHING_PAGE,
229 	/*page_length*/sizeof(struct scsi_caching_page) - 2,
230 	/*flags1*/ SCP_DISC | SCP_WCE,
231 	/*ret_priority*/ 0,
232 	/*disable_pf_transfer_len*/ {0xff, 0xff},
233 	/*min_prefetch*/ {0, 0},
234 	/*max_prefetch*/ {0xff, 0xff},
235 	/*max_pf_ceiling*/ {0xff, 0xff},
236 	/*flags2*/ 0,
237 	/*cache_segments*/ 0,
238 	/*cache_seg_size*/ {0, 0},
239 	/*reserved*/ 0,
240 	/*non_cache_seg_size*/ {0, 0, 0}
241 };
242 
243 const static struct scsi_caching_page caching_page_changeable = {
244 	/*page_code*/SMS_CACHING_PAGE,
245 	/*page_length*/sizeof(struct scsi_caching_page) - 2,
246 	/*flags1*/ SCP_WCE | SCP_RCD,
247 	/*ret_priority*/ 0,
248 	/*disable_pf_transfer_len*/ {0, 0},
249 	/*min_prefetch*/ {0, 0},
250 	/*max_prefetch*/ {0, 0},
251 	/*max_pf_ceiling*/ {0, 0},
252 	/*flags2*/ 0,
253 	/*cache_segments*/ 0,
254 	/*cache_seg_size*/ {0, 0},
255 	/*reserved*/ 0,
256 	/*non_cache_seg_size*/ {0, 0, 0}
257 };
258 
259 const static struct scsi_control_page control_page_default = {
260 	/*page_code*/SMS_CONTROL_MODE_PAGE,
261 	/*page_length*/sizeof(struct scsi_control_page) - 2,
262 	/*rlec*/0,
263 	/*queue_flags*/SCP_QUEUE_ALG_RESTRICTED,
264 	/*eca_and_aen*/0,
265 	/*flags4*/SCP_TAS,
266 	/*aen_holdoff_period*/{0, 0},
267 	/*busy_timeout_period*/{0, 0},
268 	/*extended_selftest_completion_time*/{0, 0}
269 };
270 
271 const static struct scsi_control_page control_page_changeable = {
272 	/*page_code*/SMS_CONTROL_MODE_PAGE,
273 	/*page_length*/sizeof(struct scsi_control_page) - 2,
274 	/*rlec*/SCP_DSENSE,
275 	/*queue_flags*/SCP_QUEUE_ALG_MASK,
276 	/*eca_and_aen*/SCP_SWP,
277 	/*flags4*/0,
278 	/*aen_holdoff_period*/{0, 0},
279 	/*busy_timeout_period*/{0, 0},
280 	/*extended_selftest_completion_time*/{0, 0}
281 };
282 
283 const static struct scsi_info_exceptions_page ie_page_default = {
284 	/*page_code*/SMS_INFO_EXCEPTIONS_PAGE,
285 	/*page_length*/sizeof(struct scsi_info_exceptions_page) - 2,
286 	/*info_flags*/SIEP_FLAGS_DEXCPT,
287 	/*mrie*/0,
288 	/*interval_timer*/{0, 0, 0, 0},
289 	/*report_count*/{0, 0, 0, 0}
290 };
291 
292 const static struct scsi_info_exceptions_page ie_page_changeable = {
293 	/*page_code*/SMS_INFO_EXCEPTIONS_PAGE,
294 	/*page_length*/sizeof(struct scsi_info_exceptions_page) - 2,
295 	/*info_flags*/0,
296 	/*mrie*/0,
297 	/*interval_timer*/{0, 0, 0, 0},
298 	/*report_count*/{0, 0, 0, 0}
299 };
300 
301 #define CTL_LBPM_LEN	(sizeof(struct ctl_logical_block_provisioning_page) - 4)
302 
303 const static struct ctl_logical_block_provisioning_page lbp_page_default = {{
304 	/*page_code*/SMS_INFO_EXCEPTIONS_PAGE | SMPH_SPF,
305 	/*subpage_code*/0x02,
306 	/*page_length*/{CTL_LBPM_LEN >> 8, CTL_LBPM_LEN},
307 	/*flags*/0,
308 	/*reserved*/{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
309 	/*descr*/{}},
310 	{{/*flags*/0,
311 	  /*resource*/0x01,
312 	  /*reserved*/{0, 0},
313 	  /*count*/{0, 0, 0, 0}},
314 	 {/*flags*/0,
315 	  /*resource*/0x02,
316 	  /*reserved*/{0, 0},
317 	  /*count*/{0, 0, 0, 0}},
318 	 {/*flags*/0,
319 	  /*resource*/0xf1,
320 	  /*reserved*/{0, 0},
321 	  /*count*/{0, 0, 0, 0}},
322 	 {/*flags*/0,
323 	  /*resource*/0xf2,
324 	  /*reserved*/{0, 0},
325 	  /*count*/{0, 0, 0, 0}}
326 	}
327 };
328 
329 const static struct ctl_logical_block_provisioning_page lbp_page_changeable = {{
330 	/*page_code*/SMS_INFO_EXCEPTIONS_PAGE | SMPH_SPF,
331 	/*subpage_code*/0x02,
332 	/*page_length*/{CTL_LBPM_LEN >> 8, CTL_LBPM_LEN},
333 	/*flags*/0,
334 	/*reserved*/{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
335 	/*descr*/{}},
336 	{{/*flags*/0,
337 	  /*resource*/0,
338 	  /*reserved*/{0, 0},
339 	  /*count*/{0, 0, 0, 0}},
340 	 {/*flags*/0,
341 	  /*resource*/0,
342 	  /*reserved*/{0, 0},
343 	  /*count*/{0, 0, 0, 0}},
344 	 {/*flags*/0,
345 	  /*resource*/0,
346 	  /*reserved*/{0, 0},
347 	  /*count*/{0, 0, 0, 0}},
348 	 {/*flags*/0,
349 	  /*resource*/0,
350 	  /*reserved*/{0, 0},
351 	  /*count*/{0, 0, 0, 0}}
352 	}
353 };
354 
355 /*
356  * XXX KDM move these into the softc.
357  */
358 static int rcv_sync_msg;
359 static uint8_t ctl_pause_rtr;
360 
361 SYSCTL_NODE(_kern_cam, OID_AUTO, ctl, CTLFLAG_RD, 0, "CAM Target Layer");
362 static int worker_threads = -1;
363 SYSCTL_INT(_kern_cam_ctl, OID_AUTO, worker_threads, CTLFLAG_RDTUN,
364     &worker_threads, 1, "Number of worker threads");
365 static int ctl_debug = CTL_DEBUG_NONE;
366 SYSCTL_INT(_kern_cam_ctl, OID_AUTO, debug, CTLFLAG_RWTUN,
367     &ctl_debug, 0, "Enabled debug flags");
368 
369 /*
370  * Supported pages (0x00), Serial number (0x80), Device ID (0x83),
371  * Extended INQUIRY Data (0x86), Mode Page Policy (0x87),
372  * SCSI Ports (0x88), Third-party Copy (0x8F), Block limits (0xB0),
373  * Block Device Characteristics (0xB1) and Logical Block Provisioning (0xB2)
374  */
375 #define SCSI_EVPD_NUM_SUPPORTED_PAGES	10
376 
377 static void ctl_isc_event_handler(ctl_ha_channel chanel, ctl_ha_event event,
378 				  int param);
379 static void ctl_copy_sense_data(union ctl_ha_msg *src, union ctl_io *dest);
380 static int ctl_init(void);
381 void ctl_shutdown(void);
382 static int ctl_open(struct cdev *dev, int flags, int fmt, struct thread *td);
383 static int ctl_close(struct cdev *dev, int flags, int fmt, struct thread *td);
384 static void ctl_ioctl_online(void *arg);
385 static void ctl_ioctl_offline(void *arg);
386 static int ctl_ioctl_lun_enable(void *arg, int lun_id);
387 static int ctl_ioctl_lun_disable(void *arg, int lun_id);
388 static int ctl_ioctl_do_datamove(struct ctl_scsiio *ctsio);
389 static int ctl_serialize_other_sc_cmd(struct ctl_scsiio *ctsio);
390 static int ctl_ioctl_submit_wait(union ctl_io *io);
391 static void ctl_ioctl_datamove(union ctl_io *io);
392 static void ctl_ioctl_done(union ctl_io *io);
393 static void ctl_ioctl_hard_startstop_callback(void *arg,
394 					      struct cfi_metatask *metatask);
395 static void ctl_ioctl_bbrread_callback(void *arg,struct cfi_metatask *metatask);
396 static int ctl_ioctl_fill_ooa(struct ctl_lun *lun, uint32_t *cur_fill_num,
397 			      struct ctl_ooa *ooa_hdr,
398 			      struct ctl_ooa_entry *kern_entries);
399 static int ctl_ioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flag,
400 		     struct thread *td);
401 static int ctl_alloc_lun(struct ctl_softc *ctl_softc, struct ctl_lun *lun,
402 			 struct ctl_be_lun *be_lun);
403 static int ctl_free_lun(struct ctl_lun *lun);
404 static void ctl_create_lun(struct ctl_be_lun *be_lun);
405 static struct ctl_port * ctl_io_port(struct ctl_io_hdr *io_hdr);
406 /**
407 static void ctl_failover_change_pages(struct ctl_softc *softc,
408 				      struct ctl_scsiio *ctsio, int master);
409 **/
410 
411 static int ctl_do_mode_select(union ctl_io *io);
412 static int ctl_pro_preempt(struct ctl_softc *softc, struct ctl_lun *lun,
413 			   uint64_t res_key, uint64_t sa_res_key,
414 			   uint8_t type, uint32_t residx,
415 			   struct ctl_scsiio *ctsio,
416 			   struct scsi_per_res_out *cdb,
417 			   struct scsi_per_res_out_parms* param);
418 static void ctl_pro_preempt_other(struct ctl_lun *lun,
419 				  union ctl_ha_msg *msg);
420 static void ctl_hndl_per_res_out_on_other_sc(union ctl_ha_msg *msg);
421 static int ctl_inquiry_evpd_supported(struct ctl_scsiio *ctsio, int alloc_len);
422 static int ctl_inquiry_evpd_serial(struct ctl_scsiio *ctsio, int alloc_len);
423 static int ctl_inquiry_evpd_devid(struct ctl_scsiio *ctsio, int alloc_len);
424 static int ctl_inquiry_evpd_eid(struct ctl_scsiio *ctsio, int alloc_len);
425 static int ctl_inquiry_evpd_mpp(struct ctl_scsiio *ctsio, int alloc_len);
426 static int ctl_inquiry_evpd_scsi_ports(struct ctl_scsiio *ctsio,
427 					 int alloc_len);
428 static int ctl_inquiry_evpd_block_limits(struct ctl_scsiio *ctsio,
429 					 int alloc_len);
430 static int ctl_inquiry_evpd_bdc(struct ctl_scsiio *ctsio, int alloc_len);
431 static int ctl_inquiry_evpd_lbp(struct ctl_scsiio *ctsio, int alloc_len);
432 static int ctl_inquiry_evpd(struct ctl_scsiio *ctsio);
433 static int ctl_inquiry_std(struct ctl_scsiio *ctsio);
434 static int ctl_get_lba_len(union ctl_io *io, uint64_t *lba, uint64_t *len);
435 static ctl_action ctl_extent_check(union ctl_io *io1, union ctl_io *io2,
436     bool seq);
437 static ctl_action ctl_extent_check_seq(union ctl_io *io1, union ctl_io *io2);
438 static ctl_action ctl_check_for_blockage(struct ctl_lun *lun,
439     union ctl_io *pending_io, union ctl_io *ooa_io);
440 static ctl_action ctl_check_ooa(struct ctl_lun *lun, union ctl_io *pending_io,
441 				union ctl_io *starting_io);
442 static int ctl_check_blocked(struct ctl_lun *lun);
443 static int ctl_scsiio_lun_check(struct ctl_lun *lun,
444 				const struct ctl_cmd_entry *entry,
445 				struct ctl_scsiio *ctsio);
446 //static int ctl_check_rtr(union ctl_io *pending_io, struct ctl_softc *softc);
447 static void ctl_failover(void);
448 static void ctl_clear_ua(struct ctl_softc *ctl_softc, uint32_t initidx,
449 			 ctl_ua_type ua_type);
450 static int ctl_scsiio_precheck(struct ctl_softc *ctl_softc,
451 			       struct ctl_scsiio *ctsio);
452 static int ctl_scsiio(struct ctl_scsiio *ctsio);
453 
454 static int ctl_bus_reset(struct ctl_softc *ctl_softc, union ctl_io *io);
455 static int ctl_target_reset(struct ctl_softc *ctl_softc, union ctl_io *io,
456 			    ctl_ua_type ua_type);
457 static int ctl_lun_reset(struct ctl_lun *lun, union ctl_io *io,
458 			 ctl_ua_type ua_type);
459 static int ctl_abort_task(union ctl_io *io);
460 static int ctl_abort_task_set(union ctl_io *io);
461 static int ctl_i_t_nexus_reset(union ctl_io *io);
462 static void ctl_run_task(union ctl_io *io);
463 #ifdef CTL_IO_DELAY
464 static void ctl_datamove_timer_wakeup(void *arg);
465 static void ctl_done_timer_wakeup(void *arg);
466 #endif /* CTL_IO_DELAY */
467 
468 static void ctl_send_datamove_done(union ctl_io *io, int have_lock);
469 static void ctl_datamove_remote_write_cb(struct ctl_ha_dt_req *rq);
470 static int ctl_datamove_remote_dm_write_cb(union ctl_io *io);
471 static void ctl_datamove_remote_write(union ctl_io *io);
472 static int ctl_datamove_remote_dm_read_cb(union ctl_io *io);
473 static void ctl_datamove_remote_read_cb(struct ctl_ha_dt_req *rq);
474 static int ctl_datamove_remote_sgl_setup(union ctl_io *io);
475 static int ctl_datamove_remote_xfer(union ctl_io *io, unsigned command,
476 				    ctl_ha_dt_cb callback);
477 static void ctl_datamove_remote_read(union ctl_io *io);
478 static void ctl_datamove_remote(union ctl_io *io);
479 static int ctl_process_done(union ctl_io *io);
480 static void ctl_lun_thread(void *arg);
481 static void ctl_thresh_thread(void *arg);
482 static void ctl_work_thread(void *arg);
483 static void ctl_enqueue_incoming(union ctl_io *io);
484 static void ctl_enqueue_rtr(union ctl_io *io);
485 static void ctl_enqueue_done(union ctl_io *io);
486 static void ctl_enqueue_isc(union ctl_io *io);
487 static const struct ctl_cmd_entry *
488     ctl_get_cmd_entry(struct ctl_scsiio *ctsio, int *sa);
489 static const struct ctl_cmd_entry *
490     ctl_validate_command(struct ctl_scsiio *ctsio);
491 static int ctl_cmd_applicable(uint8_t lun_type,
492     const struct ctl_cmd_entry *entry);
493 
494 /*
495  * Load the serialization table.  This isn't very pretty, but is probably
496  * the easiest way to do it.
497  */
498 #include "ctl_ser_table.c"
499 
500 /*
501  * We only need to define open, close and ioctl routines for this driver.
502  */
503 static struct cdevsw ctl_cdevsw = {
504 	.d_version =	D_VERSION,
505 	.d_flags =	0,
506 	.d_open =	ctl_open,
507 	.d_close =	ctl_close,
508 	.d_ioctl =	ctl_ioctl,
509 	.d_name =	"ctl",
510 };
511 
512 
513 MALLOC_DEFINE(M_CTL, "ctlmem", "Memory used for CTL");
514 MALLOC_DEFINE(M_CTLIO, "ctlio", "Memory used for CTL requests");
515 
516 static int ctl_module_event_handler(module_t, int /*modeventtype_t*/, void *);
517 
518 static moduledata_t ctl_moduledata = {
519 	"ctl",
520 	ctl_module_event_handler,
521 	NULL
522 };
523 
524 DECLARE_MODULE(ctl, ctl_moduledata, SI_SUB_CONFIGURE, SI_ORDER_THIRD);
525 MODULE_VERSION(ctl, 1);
526 
527 static struct ctl_frontend ioctl_frontend =
528 {
529 	.name = "ioctl",
530 };
531 
532 static void
533 ctl_isc_handler_finish_xfer(struct ctl_softc *ctl_softc,
534 			    union ctl_ha_msg *msg_info)
535 {
536 	struct ctl_scsiio *ctsio;
537 
538 	if (msg_info->hdr.original_sc == NULL) {
539 		printf("%s: original_sc == NULL!\n", __func__);
540 		/* XXX KDM now what? */
541 		return;
542 	}
543 
544 	ctsio = &msg_info->hdr.original_sc->scsiio;
545 	ctsio->io_hdr.flags |= CTL_FLAG_IO_ACTIVE;
546 	ctsio->io_hdr.msg_type = CTL_MSG_FINISH_IO;
547 	ctsio->io_hdr.status = msg_info->hdr.status;
548 	ctsio->scsi_status = msg_info->scsi.scsi_status;
549 	ctsio->sense_len = msg_info->scsi.sense_len;
550 	ctsio->sense_residual = msg_info->scsi.sense_residual;
551 	ctsio->residual = msg_info->scsi.residual;
552 	memcpy(&ctsio->sense_data, &msg_info->scsi.sense_data,
553 	       sizeof(ctsio->sense_data));
554 	memcpy(&ctsio->io_hdr.ctl_private[CTL_PRIV_LBA_LEN].bytes,
555 	       &msg_info->scsi.lbalen, sizeof(msg_info->scsi.lbalen));
556 	ctl_enqueue_isc((union ctl_io *)ctsio);
557 }
558 
559 static void
560 ctl_isc_handler_finish_ser_only(struct ctl_softc *ctl_softc,
561 				union ctl_ha_msg *msg_info)
562 {
563 	struct ctl_scsiio *ctsio;
564 
565 	if (msg_info->hdr.serializing_sc == NULL) {
566 		printf("%s: serializing_sc == NULL!\n", __func__);
567 		/* XXX KDM now what? */
568 		return;
569 	}
570 
571 	ctsio = &msg_info->hdr.serializing_sc->scsiio;
572 #if 0
573 	/*
574 	 * Attempt to catch the situation where an I/O has
575 	 * been freed, and we're using it again.
576 	 */
577 	if (ctsio->io_hdr.io_type == 0xff) {
578 		union ctl_io *tmp_io;
579 		tmp_io = (union ctl_io *)ctsio;
580 		printf("%s: %p use after free!\n", __func__,
581 		       ctsio);
582 		printf("%s: type %d msg %d cdb %x iptl: "
583 		       "%d:%d:%d:%d tag 0x%04x "
584 		       "flag %#x status %x\n",
585 			__func__,
586 			tmp_io->io_hdr.io_type,
587 			tmp_io->io_hdr.msg_type,
588 			tmp_io->scsiio.cdb[0],
589 			tmp_io->io_hdr.nexus.initid.id,
590 			tmp_io->io_hdr.nexus.targ_port,
591 			tmp_io->io_hdr.nexus.targ_target.id,
592 			tmp_io->io_hdr.nexus.targ_lun,
593 			(tmp_io->io_hdr.io_type ==
594 			CTL_IO_TASK) ?
595 			tmp_io->taskio.tag_num :
596 			tmp_io->scsiio.tag_num,
597 		        tmp_io->io_hdr.flags,
598 			tmp_io->io_hdr.status);
599 	}
600 #endif
601 	ctsio->io_hdr.msg_type = CTL_MSG_FINISH_IO;
602 	ctl_enqueue_isc((union ctl_io *)ctsio);
603 }
604 
605 /*
606  * ISC (Inter Shelf Communication) event handler.  Events from the HA
607  * subsystem come in here.
608  */
609 static void
610 ctl_isc_event_handler(ctl_ha_channel channel, ctl_ha_event event, int param)
611 {
612 	struct ctl_softc *softc;
613 	union ctl_io *io;
614 	struct ctl_prio *presio;
615 	ctl_ha_status isc_status;
616 
617 	softc = control_softc;
618 	io = NULL;
619 
620 
621 #if 0
622 	printf("CTL: Isc Msg event %d\n", event);
623 #endif
624 	if (event == CTL_HA_EVT_MSG_RECV) {
625 		union ctl_ha_msg msg_info;
626 
627 		isc_status = ctl_ha_msg_recv(CTL_HA_CHAN_CTL, &msg_info,
628 					     sizeof(msg_info), /*wait*/ 0);
629 #if 0
630 		printf("CTL: msg_type %d\n", msg_info.msg_type);
631 #endif
632 		if (isc_status != 0) {
633 			printf("Error receiving message, status = %d\n",
634 			       isc_status);
635 			return;
636 		}
637 
638 		switch (msg_info.hdr.msg_type) {
639 		case CTL_MSG_SERIALIZE:
640 #if 0
641 			printf("Serialize\n");
642 #endif
643 			io = ctl_alloc_io_nowait(softc->othersc_pool);
644 			if (io == NULL) {
645 				printf("ctl_isc_event_handler: can't allocate "
646 				       "ctl_io!\n");
647 				/* Bad Juju */
648 				/* Need to set busy and send msg back */
649 				msg_info.hdr.msg_type = CTL_MSG_BAD_JUJU;
650 				msg_info.hdr.status = CTL_SCSI_ERROR;
651 				msg_info.scsi.scsi_status = SCSI_STATUS_BUSY;
652 				msg_info.scsi.sense_len = 0;
653 			        if (ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg_info,
654 				    sizeof(msg_info), 0) > CTL_HA_STATUS_SUCCESS){
655 				}
656 				goto bailout;
657 			}
658 			ctl_zero_io(io);
659 			// populate ctsio from msg_info
660 			io->io_hdr.io_type = CTL_IO_SCSI;
661 			io->io_hdr.msg_type = CTL_MSG_SERIALIZE;
662 			io->io_hdr.original_sc = msg_info.hdr.original_sc;
663 #if 0
664 			printf("pOrig %x\n", (int)msg_info.original_sc);
665 #endif
666 			io->io_hdr.flags |= CTL_FLAG_FROM_OTHER_SC |
667 					    CTL_FLAG_IO_ACTIVE;
668 			/*
669 			 * If we're in serialization-only mode, we don't
670 			 * want to go through full done processing.  Thus
671 			 * the COPY flag.
672 			 *
673 			 * XXX KDM add another flag that is more specific.
674 			 */
675 			if (softc->ha_mode == CTL_HA_MODE_SER_ONLY)
676 				io->io_hdr.flags |= CTL_FLAG_INT_COPY;
677 			io->io_hdr.nexus = msg_info.hdr.nexus;
678 #if 0
679 			printf("targ %d, port %d, iid %d, lun %d\n",
680 			       io->io_hdr.nexus.targ_target.id,
681 			       io->io_hdr.nexus.targ_port,
682 			       io->io_hdr.nexus.initid.id,
683 			       io->io_hdr.nexus.targ_lun);
684 #endif
685 			io->scsiio.tag_num = msg_info.scsi.tag_num;
686 			io->scsiio.tag_type = msg_info.scsi.tag_type;
687 			memcpy(io->scsiio.cdb, msg_info.scsi.cdb,
688 			       CTL_MAX_CDBLEN);
689 			if (softc->ha_mode == CTL_HA_MODE_XFER) {
690 				const struct ctl_cmd_entry *entry;
691 
692 				entry = ctl_get_cmd_entry(&io->scsiio, NULL);
693 				io->io_hdr.flags &= ~CTL_FLAG_DATA_MASK;
694 				io->io_hdr.flags |=
695 					entry->flags & CTL_FLAG_DATA_MASK;
696 			}
697 			ctl_enqueue_isc(io);
698 			break;
699 
700 		/* Performed on the Originating SC, XFER mode only */
701 		case CTL_MSG_DATAMOVE: {
702 			struct ctl_sg_entry *sgl;
703 			int i, j;
704 
705 			io = msg_info.hdr.original_sc;
706 			if (io == NULL) {
707 				printf("%s: original_sc == NULL!\n", __func__);
708 				/* XXX KDM do something here */
709 				break;
710 			}
711 			io->io_hdr.msg_type = CTL_MSG_DATAMOVE;
712 			io->io_hdr.flags |= CTL_FLAG_IO_ACTIVE;
713 			/*
714 			 * Keep track of this, we need to send it back over
715 			 * when the datamove is complete.
716 			 */
717 			io->io_hdr.serializing_sc = msg_info.hdr.serializing_sc;
718 
719 			if (msg_info.dt.sg_sequence == 0) {
720 				/*
721 				 * XXX KDM we use the preallocated S/G list
722 				 * here, but we'll need to change this to
723 				 * dynamic allocation if we need larger S/G
724 				 * lists.
725 				 */
726 				if (msg_info.dt.kern_sg_entries >
727 				    sizeof(io->io_hdr.remote_sglist) /
728 				    sizeof(io->io_hdr.remote_sglist[0])) {
729 					printf("%s: number of S/G entries "
730 					    "needed %u > allocated num %zd\n",
731 					    __func__,
732 					    msg_info.dt.kern_sg_entries,
733 					    sizeof(io->io_hdr.remote_sglist)/
734 					    sizeof(io->io_hdr.remote_sglist[0]));
735 
736 					/*
737 					 * XXX KDM send a message back to
738 					 * the other side to shut down the
739 					 * DMA.  The error will come back
740 					 * through via the normal channel.
741 					 */
742 					break;
743 				}
744 				sgl = io->io_hdr.remote_sglist;
745 				memset(sgl, 0,
746 				       sizeof(io->io_hdr.remote_sglist));
747 
748 				io->scsiio.kern_data_ptr = (uint8_t *)sgl;
749 
750 				io->scsiio.kern_sg_entries =
751 					msg_info.dt.kern_sg_entries;
752 				io->scsiio.rem_sg_entries =
753 					msg_info.dt.kern_sg_entries;
754 				io->scsiio.kern_data_len =
755 					msg_info.dt.kern_data_len;
756 				io->scsiio.kern_total_len =
757 					msg_info.dt.kern_total_len;
758 				io->scsiio.kern_data_resid =
759 					msg_info.dt.kern_data_resid;
760 				io->scsiio.kern_rel_offset =
761 					msg_info.dt.kern_rel_offset;
762 				/*
763 				 * Clear out per-DMA flags.
764 				 */
765 				io->io_hdr.flags &= ~CTL_FLAG_RDMA_MASK;
766 				/*
767 				 * Add per-DMA flags that are set for this
768 				 * particular DMA request.
769 				 */
770 				io->io_hdr.flags |= msg_info.dt.flags &
771 						    CTL_FLAG_RDMA_MASK;
772 			} else
773 				sgl = (struct ctl_sg_entry *)
774 					io->scsiio.kern_data_ptr;
775 
776 			for (i = msg_info.dt.sent_sg_entries, j = 0;
777 			     i < (msg_info.dt.sent_sg_entries +
778 			     msg_info.dt.cur_sg_entries); i++, j++) {
779 				sgl[i].addr = msg_info.dt.sg_list[j].addr;
780 				sgl[i].len = msg_info.dt.sg_list[j].len;
781 
782 #if 0
783 				printf("%s: L: %p,%d -> %p,%d j=%d, i=%d\n",
784 				       __func__,
785 				       msg_info.dt.sg_list[j].addr,
786 				       msg_info.dt.sg_list[j].len,
787 				       sgl[i].addr, sgl[i].len, j, i);
788 #endif
789 			}
790 #if 0
791 			memcpy(&sgl[msg_info.dt.sent_sg_entries],
792 			       msg_info.dt.sg_list,
793 			       sizeof(*sgl) * msg_info.dt.cur_sg_entries);
794 #endif
795 
796 			/*
797 			 * If this is the last piece of the I/O, we've got
798 			 * the full S/G list.  Queue processing in the thread.
799 			 * Otherwise wait for the next piece.
800 			 */
801 			if (msg_info.dt.sg_last != 0)
802 				ctl_enqueue_isc(io);
803 			break;
804 		}
805 		/* Performed on the Serializing (primary) SC, XFER mode only */
806 		case CTL_MSG_DATAMOVE_DONE: {
807 			if (msg_info.hdr.serializing_sc == NULL) {
808 				printf("%s: serializing_sc == NULL!\n",
809 				       __func__);
810 				/* XXX KDM now what? */
811 				break;
812 			}
813 			/*
814 			 * We grab the sense information here in case
815 			 * there was a failure, so we can return status
816 			 * back to the initiator.
817 			 */
818 			io = msg_info.hdr.serializing_sc;
819 			io->io_hdr.msg_type = CTL_MSG_DATAMOVE_DONE;
820 			io->io_hdr.status = msg_info.hdr.status;
821 			io->scsiio.scsi_status = msg_info.scsi.scsi_status;
822 			io->scsiio.sense_len = msg_info.scsi.sense_len;
823 			io->scsiio.sense_residual =msg_info.scsi.sense_residual;
824 			io->io_hdr.port_status = msg_info.scsi.fetd_status;
825 			io->scsiio.residual = msg_info.scsi.residual;
826 			memcpy(&io->scsiio.sense_data,&msg_info.scsi.sense_data,
827 			       sizeof(io->scsiio.sense_data));
828 			ctl_enqueue_isc(io);
829 			break;
830 		}
831 
832 		/* Preformed on Originating SC, SER_ONLY mode */
833 		case CTL_MSG_R2R:
834 			io = msg_info.hdr.original_sc;
835 			if (io == NULL) {
836 				printf("%s: Major Bummer\n", __func__);
837 				return;
838 			} else {
839 #if 0
840 				printf("pOrig %x\n",(int) ctsio);
841 #endif
842 			}
843 			io->io_hdr.msg_type = CTL_MSG_R2R;
844 			io->io_hdr.serializing_sc = msg_info.hdr.serializing_sc;
845 			ctl_enqueue_isc(io);
846 			break;
847 
848 		/*
849 		 * Performed on Serializing(i.e. primary SC) SC in SER_ONLY
850 		 * mode.
851 		 * Performed on the Originating (i.e. secondary) SC in XFER
852 		 * mode
853 		 */
854 		case CTL_MSG_FINISH_IO:
855 			if (softc->ha_mode == CTL_HA_MODE_XFER)
856 				ctl_isc_handler_finish_xfer(softc,
857 							    &msg_info);
858 			else
859 				ctl_isc_handler_finish_ser_only(softc,
860 								&msg_info);
861 			break;
862 
863 		/* Preformed on Originating SC */
864 		case CTL_MSG_BAD_JUJU:
865 			io = msg_info.hdr.original_sc;
866 			if (io == NULL) {
867 				printf("%s: Bad JUJU!, original_sc is NULL!\n",
868 				       __func__);
869 				break;
870 			}
871 			ctl_copy_sense_data(&msg_info, io);
872 			/*
873 			 * IO should have already been cleaned up on other
874 			 * SC so clear this flag so we won't send a message
875 			 * back to finish the IO there.
876 			 */
877 			io->io_hdr.flags &= ~CTL_FLAG_SENT_2OTHER_SC;
878 			io->io_hdr.flags |= CTL_FLAG_IO_ACTIVE;
879 
880 			/* io = msg_info.hdr.serializing_sc; */
881 			io->io_hdr.msg_type = CTL_MSG_BAD_JUJU;
882 			ctl_enqueue_isc(io);
883 			break;
884 
885 		/* Handle resets sent from the other side */
886 		case CTL_MSG_MANAGE_TASKS: {
887 			struct ctl_taskio *taskio;
888 			taskio = (struct ctl_taskio *)ctl_alloc_io_nowait(
889 			    softc->othersc_pool);
890 			if (taskio == NULL) {
891 				printf("ctl_isc_event_handler: can't allocate "
892 				       "ctl_io!\n");
893 				/* Bad Juju */
894 				/* should I just call the proper reset func
895 				   here??? */
896 				goto bailout;
897 			}
898 			ctl_zero_io((union ctl_io *)taskio);
899 			taskio->io_hdr.io_type = CTL_IO_TASK;
900 			taskio->io_hdr.flags |= CTL_FLAG_FROM_OTHER_SC;
901 			taskio->io_hdr.nexus = msg_info.hdr.nexus;
902 			taskio->task_action = msg_info.task.task_action;
903 			taskio->tag_num = msg_info.task.tag_num;
904 			taskio->tag_type = msg_info.task.tag_type;
905 #ifdef CTL_TIME_IO
906 			taskio->io_hdr.start_time = time_uptime;
907 			getbintime(&taskio->io_hdr.start_bt);
908 #if 0
909 			cs_prof_gettime(&taskio->io_hdr.start_ticks);
910 #endif
911 #endif /* CTL_TIME_IO */
912 			ctl_run_task((union ctl_io *)taskio);
913 			break;
914 		}
915 		/* Persistent Reserve action which needs attention */
916 		case CTL_MSG_PERS_ACTION:
917 			presio = (struct ctl_prio *)ctl_alloc_io_nowait(
918 			    softc->othersc_pool);
919 			if (presio == NULL) {
920 				printf("ctl_isc_event_handler: can't allocate "
921 				       "ctl_io!\n");
922 				/* Bad Juju */
923 				/* Need to set busy and send msg back */
924 				goto bailout;
925 			}
926 			ctl_zero_io((union ctl_io *)presio);
927 			presio->io_hdr.msg_type = CTL_MSG_PERS_ACTION;
928 			presio->pr_msg = msg_info.pr;
929 			ctl_enqueue_isc((union ctl_io *)presio);
930 			break;
931 		case CTL_MSG_SYNC_FE:
932 			rcv_sync_msg = 1;
933 			break;
934 		default:
935 		        printf("How did I get here?\n");
936 		}
937 	} else if (event == CTL_HA_EVT_MSG_SENT) {
938 		if (param != CTL_HA_STATUS_SUCCESS) {
939 			printf("Bad status from ctl_ha_msg_send status %d\n",
940 			       param);
941 		}
942 		return;
943 	} else if (event == CTL_HA_EVT_DISCONNECT) {
944 		printf("CTL: Got a disconnect from Isc\n");
945 		return;
946 	} else {
947 		printf("ctl_isc_event_handler: Unknown event %d\n", event);
948 		return;
949 	}
950 
951 bailout:
952 	return;
953 }
954 
955 static void
956 ctl_copy_sense_data(union ctl_ha_msg *src, union ctl_io *dest)
957 {
958 	struct scsi_sense_data *sense;
959 
960 	sense = &dest->scsiio.sense_data;
961 	bcopy(&src->scsi.sense_data, sense, sizeof(*sense));
962 	dest->scsiio.scsi_status = src->scsi.scsi_status;
963 	dest->scsiio.sense_len = src->scsi.sense_len;
964 	dest->io_hdr.status = src->hdr.status;
965 }
966 
967 static void
968 ctl_est_ua(struct ctl_lun *lun, uint32_t initidx, ctl_ua_type ua)
969 {
970 	ctl_ua_type *pu;
971 
972 	mtx_assert(&lun->lun_lock, MA_OWNED);
973 	pu = lun->pending_ua[initidx / CTL_MAX_INIT_PER_PORT];
974 	if (pu == NULL)
975 		return;
976 	pu[initidx % CTL_MAX_INIT_PER_PORT] |= ua;
977 }
978 
979 static void
980 ctl_est_ua_all(struct ctl_lun *lun, uint32_t except, ctl_ua_type ua)
981 {
982 	int i, j;
983 
984 	mtx_assert(&lun->lun_lock, MA_OWNED);
985 	for (i = 0; i < CTL_MAX_PORTS; i++) {
986 		if (lun->pending_ua[i] == NULL)
987 			continue;
988 		for (j = 0; j < CTL_MAX_INIT_PER_PORT; j++) {
989 			if (i * CTL_MAX_INIT_PER_PORT + j == except)
990 				continue;
991 			lun->pending_ua[i][j] |= ua;
992 		}
993 	}
994 }
995 
996 static void
997 ctl_clr_ua(struct ctl_lun *lun, uint32_t initidx, ctl_ua_type ua)
998 {
999 	ctl_ua_type *pu;
1000 
1001 	mtx_assert(&lun->lun_lock, MA_OWNED);
1002 	pu = lun->pending_ua[initidx / CTL_MAX_INIT_PER_PORT];
1003 	if (pu == NULL)
1004 		return;
1005 	pu[initidx % CTL_MAX_INIT_PER_PORT] &= ~ua;
1006 }
1007 
1008 static void
1009 ctl_clr_ua_all(struct ctl_lun *lun, uint32_t except, ctl_ua_type ua)
1010 {
1011 	int i, j;
1012 
1013 	mtx_assert(&lun->lun_lock, MA_OWNED);
1014 	for (i = 0; i < CTL_MAX_PORTS; i++) {
1015 		if (lun->pending_ua[i] == NULL)
1016 			continue;
1017 		for (j = 0; j < CTL_MAX_INIT_PER_PORT; j++) {
1018 			if (i * CTL_MAX_INIT_PER_PORT + j == except)
1019 				continue;
1020 			lun->pending_ua[i][j] &= ~ua;
1021 		}
1022 	}
1023 }
1024 
1025 static int
1026 ctl_ha_state_sysctl(SYSCTL_HANDLER_ARGS)
1027 {
1028 	struct ctl_softc *softc = (struct ctl_softc *)arg1;
1029 	struct ctl_lun *lun;
1030 	int error, value;
1031 
1032 	if (softc->flags & CTL_FLAG_ACTIVE_SHELF)
1033 		value = 0;
1034 	else
1035 		value = 1;
1036 
1037 	error = sysctl_handle_int(oidp, &value, 0, req);
1038 	if ((error != 0) || (req->newptr == NULL))
1039 		return (error);
1040 
1041 	mtx_lock(&softc->ctl_lock);
1042 	if (value == 0)
1043 		softc->flags |= CTL_FLAG_ACTIVE_SHELF;
1044 	else
1045 		softc->flags &= ~CTL_FLAG_ACTIVE_SHELF;
1046 	STAILQ_FOREACH(lun, &softc->lun_list, links) {
1047 		mtx_lock(&lun->lun_lock);
1048 		ctl_est_ua_all(lun, -1, CTL_UA_ASYM_ACC_CHANGE);
1049 		mtx_unlock(&lun->lun_lock);
1050 	}
1051 	mtx_unlock(&softc->ctl_lock);
1052 	return (0);
1053 }
1054 
1055 static int
1056 ctl_init(void)
1057 {
1058 	struct ctl_softc *softc;
1059 	void *other_pool;
1060 	struct ctl_port *port;
1061 	int i, error, retval;
1062 	//int isc_retval;
1063 
1064 	retval = 0;
1065 	ctl_pause_rtr = 0;
1066         rcv_sync_msg = 0;
1067 
1068 	control_softc = malloc(sizeof(*control_softc), M_DEVBUF,
1069 			       M_WAITOK | M_ZERO);
1070 	softc = control_softc;
1071 
1072 	softc->dev = make_dev(&ctl_cdevsw, 0, UID_ROOT, GID_OPERATOR, 0600,
1073 			      "cam/ctl");
1074 
1075 	softc->dev->si_drv1 = softc;
1076 
1077 	/*
1078 	 * By default, return a "bad LUN" peripheral qualifier for unknown
1079 	 * LUNs.  The user can override this default using the tunable or
1080 	 * sysctl.  See the comment in ctl_inquiry_std() for more details.
1081 	 */
1082 	softc->inquiry_pq_no_lun = 1;
1083 	TUNABLE_INT_FETCH("kern.cam.ctl.inquiry_pq_no_lun",
1084 			  &softc->inquiry_pq_no_lun);
1085 	sysctl_ctx_init(&softc->sysctl_ctx);
1086 	softc->sysctl_tree = SYSCTL_ADD_NODE(&softc->sysctl_ctx,
1087 		SYSCTL_STATIC_CHILDREN(_kern_cam), OID_AUTO, "ctl",
1088 		CTLFLAG_RD, 0, "CAM Target Layer");
1089 
1090 	if (softc->sysctl_tree == NULL) {
1091 		printf("%s: unable to allocate sysctl tree\n", __func__);
1092 		destroy_dev(softc->dev);
1093 		free(control_softc, M_DEVBUF);
1094 		control_softc = NULL;
1095 		return (ENOMEM);
1096 	}
1097 
1098 	SYSCTL_ADD_INT(&softc->sysctl_ctx,
1099 		       SYSCTL_CHILDREN(softc->sysctl_tree), OID_AUTO,
1100 		       "inquiry_pq_no_lun", CTLFLAG_RW,
1101 		       &softc->inquiry_pq_no_lun, 0,
1102 		       "Report no lun possible for invalid LUNs");
1103 
1104 	mtx_init(&softc->ctl_lock, "CTL mutex", NULL, MTX_DEF);
1105 	softc->io_zone = uma_zcreate("CTL IO", sizeof(union ctl_io),
1106 	    NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0);
1107 	softc->open_count = 0;
1108 
1109 	/*
1110 	 * Default to actually sending a SYNCHRONIZE CACHE command down to
1111 	 * the drive.
1112 	 */
1113 	softc->flags = CTL_FLAG_REAL_SYNC;
1114 
1115 	/*
1116 	 * In Copan's HA scheme, the "master" and "slave" roles are
1117 	 * figured out through the slot the controller is in.  Although it
1118 	 * is an active/active system, someone has to be in charge.
1119 	 */
1120 	SYSCTL_ADD_INT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1121 	    OID_AUTO, "ha_id", CTLFLAG_RDTUN, &softc->ha_id, 0,
1122 	    "HA head ID (0 - no HA)");
1123 	if (softc->ha_id == 0) {
1124 		softc->flags |= CTL_FLAG_ACTIVE_SHELF;
1125 		softc->is_single = 1;
1126 		softc->port_offset = 0;
1127 	} else
1128 		softc->port_offset = (softc->ha_id - 1) * CTL_MAX_PORTS;
1129 	softc->persis_offset = softc->port_offset * CTL_MAX_INIT_PER_PORT;
1130 
1131 	STAILQ_INIT(&softc->lun_list);
1132 	STAILQ_INIT(&softc->pending_lun_queue);
1133 	STAILQ_INIT(&softc->fe_list);
1134 	STAILQ_INIT(&softc->port_list);
1135 	STAILQ_INIT(&softc->be_list);
1136 	ctl_tpc_init(softc);
1137 
1138 	if (ctl_pool_create(softc, "othersc", CTL_POOL_ENTRIES_OTHER_SC,
1139 	                    &other_pool) != 0)
1140 	{
1141 		printf("ctl: can't allocate %d entry other SC pool, "
1142 		       "exiting\n", CTL_POOL_ENTRIES_OTHER_SC);
1143 		return (ENOMEM);
1144 	}
1145 	softc->othersc_pool = other_pool;
1146 
1147 	if (worker_threads <= 0)
1148 		worker_threads = max(1, mp_ncpus / 4);
1149 	if (worker_threads > CTL_MAX_THREADS)
1150 		worker_threads = CTL_MAX_THREADS;
1151 
1152 	for (i = 0; i < worker_threads; i++) {
1153 		struct ctl_thread *thr = &softc->threads[i];
1154 
1155 		mtx_init(&thr->queue_lock, "CTL queue mutex", NULL, MTX_DEF);
1156 		thr->ctl_softc = softc;
1157 		STAILQ_INIT(&thr->incoming_queue);
1158 		STAILQ_INIT(&thr->rtr_queue);
1159 		STAILQ_INIT(&thr->done_queue);
1160 		STAILQ_INIT(&thr->isc_queue);
1161 
1162 		error = kproc_kthread_add(ctl_work_thread, thr,
1163 		    &softc->ctl_proc, &thr->thread, 0, 0, "ctl", "work%d", i);
1164 		if (error != 0) {
1165 			printf("error creating CTL work thread!\n");
1166 			ctl_pool_free(other_pool);
1167 			return (error);
1168 		}
1169 	}
1170 	error = kproc_kthread_add(ctl_lun_thread, softc,
1171 	    &softc->ctl_proc, NULL, 0, 0, "ctl", "lun");
1172 	if (error != 0) {
1173 		printf("error creating CTL lun thread!\n");
1174 		ctl_pool_free(other_pool);
1175 		return (error);
1176 	}
1177 	error = kproc_kthread_add(ctl_thresh_thread, softc,
1178 	    &softc->ctl_proc, NULL, 0, 0, "ctl", "thresh");
1179 	if (error != 0) {
1180 		printf("error creating CTL threshold thread!\n");
1181 		ctl_pool_free(other_pool);
1182 		return (error);
1183 	}
1184 	if (bootverbose)
1185 		printf("ctl: CAM Target Layer loaded\n");
1186 
1187 	/*
1188 	 * Initialize the ioctl front end.
1189 	 */
1190 	ctl_frontend_register(&ioctl_frontend);
1191 	port = &softc->ioctl_info.port;
1192 	port->frontend = &ioctl_frontend;
1193 	sprintf(softc->ioctl_info.port_name, "ioctl");
1194 	port->port_type = CTL_PORT_IOCTL;
1195 	port->num_requested_ctl_io = 100;
1196 	port->port_name = softc->ioctl_info.port_name;
1197 	port->port_online = ctl_ioctl_online;
1198 	port->port_offline = ctl_ioctl_offline;
1199 	port->onoff_arg = &softc->ioctl_info;
1200 	port->lun_enable = ctl_ioctl_lun_enable;
1201 	port->lun_disable = ctl_ioctl_lun_disable;
1202 	port->targ_lun_arg = &softc->ioctl_info;
1203 	port->fe_datamove = ctl_ioctl_datamove;
1204 	port->fe_done = ctl_ioctl_done;
1205 	port->max_targets = 15;
1206 	port->max_target_id = 15;
1207 
1208 	if (ctl_port_register(&softc->ioctl_info.port) != 0) {
1209 		printf("ctl: ioctl front end registration failed, will "
1210 		       "continue anyway\n");
1211 	}
1212 
1213 	SYSCTL_ADD_PROC(&softc->sysctl_ctx,SYSCTL_CHILDREN(softc->sysctl_tree),
1214 	    OID_AUTO, "ha_state", CTLTYPE_INT | CTLFLAG_RWTUN,
1215 	    softc, 0, ctl_ha_state_sysctl, "I", "HA state for this head");
1216 
1217 #ifdef CTL_IO_DELAY
1218 	if (sizeof(struct callout) > CTL_TIMER_BYTES) {
1219 		printf("sizeof(struct callout) %zd > CTL_TIMER_BYTES %zd\n",
1220 		       sizeof(struct callout), CTL_TIMER_BYTES);
1221 		return (EINVAL);
1222 	}
1223 #endif /* CTL_IO_DELAY */
1224 
1225 	return (0);
1226 }
1227 
1228 void
1229 ctl_shutdown(void)
1230 {
1231 	struct ctl_softc *softc;
1232 	struct ctl_lun *lun, *next_lun;
1233 
1234 	softc = (struct ctl_softc *)control_softc;
1235 
1236 	if (ctl_port_deregister(&softc->ioctl_info.port) != 0)
1237 		printf("ctl: ioctl front end deregistration failed\n");
1238 
1239 	mtx_lock(&softc->ctl_lock);
1240 
1241 	/*
1242 	 * Free up each LUN.
1243 	 */
1244 	for (lun = STAILQ_FIRST(&softc->lun_list); lun != NULL; lun = next_lun){
1245 		next_lun = STAILQ_NEXT(lun, links);
1246 		ctl_free_lun(lun);
1247 	}
1248 
1249 	mtx_unlock(&softc->ctl_lock);
1250 
1251 	ctl_frontend_deregister(&ioctl_frontend);
1252 
1253 #if 0
1254 	ctl_shutdown_thread(softc->work_thread);
1255 	mtx_destroy(&softc->queue_lock);
1256 #endif
1257 
1258 	ctl_tpc_shutdown(softc);
1259 	uma_zdestroy(softc->io_zone);
1260 	mtx_destroy(&softc->ctl_lock);
1261 
1262 	destroy_dev(softc->dev);
1263 
1264 	sysctl_ctx_free(&softc->sysctl_ctx);
1265 
1266 	free(control_softc, M_DEVBUF);
1267 	control_softc = NULL;
1268 
1269 	if (bootverbose)
1270 		printf("ctl: CAM Target Layer unloaded\n");
1271 }
1272 
1273 static int
1274 ctl_module_event_handler(module_t mod, int what, void *arg)
1275 {
1276 
1277 	switch (what) {
1278 	case MOD_LOAD:
1279 		return (ctl_init());
1280 	case MOD_UNLOAD:
1281 		return (EBUSY);
1282 	default:
1283 		return (EOPNOTSUPP);
1284 	}
1285 }
1286 
1287 /*
1288  * XXX KDM should we do some access checks here?  Bump a reference count to
1289  * prevent a CTL module from being unloaded while someone has it open?
1290  */
1291 static int
1292 ctl_open(struct cdev *dev, int flags, int fmt, struct thread *td)
1293 {
1294 	return (0);
1295 }
1296 
1297 static int
1298 ctl_close(struct cdev *dev, int flags, int fmt, struct thread *td)
1299 {
1300 	return (0);
1301 }
1302 
1303 int
1304 ctl_port_enable(ctl_port_type port_type)
1305 {
1306 	struct ctl_softc *softc = control_softc;
1307 	struct ctl_port *port;
1308 
1309 	if (softc->is_single == 0) {
1310 		union ctl_ha_msg msg_info;
1311 		int isc_retval;
1312 
1313 #if 0
1314 		printf("%s: HA mode, synchronizing frontend enable\n",
1315 		        __func__);
1316 #endif
1317 		msg_info.hdr.msg_type = CTL_MSG_SYNC_FE;
1318 	        if ((isc_retval=ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg_info,
1319 		        sizeof(msg_info), 1 )) > CTL_HA_STATUS_SUCCESS) {
1320 			printf("Sync msg send error retval %d\n", isc_retval);
1321 		}
1322 		if (!rcv_sync_msg) {
1323 			isc_retval=ctl_ha_msg_recv(CTL_HA_CHAN_CTL, &msg_info,
1324 			        sizeof(msg_info), 1);
1325 		}
1326 #if 0
1327         	printf("CTL:Frontend Enable\n");
1328 	} else {
1329 		printf("%s: single mode, skipping frontend synchronization\n",
1330 		        __func__);
1331 #endif
1332 	}
1333 
1334 	STAILQ_FOREACH(port, &softc->port_list, links) {
1335 		if (port_type & port->port_type)
1336 		{
1337 #if 0
1338 			printf("port %d\n", port->targ_port);
1339 #endif
1340 			ctl_port_online(port);
1341 		}
1342 	}
1343 
1344 	return (0);
1345 }
1346 
1347 int
1348 ctl_port_disable(ctl_port_type port_type)
1349 {
1350 	struct ctl_softc *softc;
1351 	struct ctl_port *port;
1352 
1353 	softc = control_softc;
1354 
1355 	STAILQ_FOREACH(port, &softc->port_list, links) {
1356 		if (port_type & port->port_type)
1357 			ctl_port_offline(port);
1358 	}
1359 
1360 	return (0);
1361 }
1362 
1363 /*
1364  * Returns 0 for success, 1 for failure.
1365  * Currently the only failure mode is if there aren't enough entries
1366  * allocated.  So, in case of a failure, look at num_entries_dropped,
1367  * reallocate and try again.
1368  */
1369 int
1370 ctl_port_list(struct ctl_port_entry *entries, int num_entries_alloced,
1371 	      int *num_entries_filled, int *num_entries_dropped,
1372 	      ctl_port_type port_type, int no_virtual)
1373 {
1374 	struct ctl_softc *softc;
1375 	struct ctl_port *port;
1376 	int entries_dropped, entries_filled;
1377 	int retval;
1378 	int i;
1379 
1380 	softc = control_softc;
1381 
1382 	retval = 0;
1383 	entries_filled = 0;
1384 	entries_dropped = 0;
1385 
1386 	i = 0;
1387 	mtx_lock(&softc->ctl_lock);
1388 	STAILQ_FOREACH(port, &softc->port_list, links) {
1389 		struct ctl_port_entry *entry;
1390 
1391 		if ((port->port_type & port_type) == 0)
1392 			continue;
1393 
1394 		if ((no_virtual != 0)
1395 		 && (port->virtual_port != 0))
1396 			continue;
1397 
1398 		if (entries_filled >= num_entries_alloced) {
1399 			entries_dropped++;
1400 			continue;
1401 		}
1402 		entry = &entries[i];
1403 
1404 		entry->port_type = port->port_type;
1405 		strlcpy(entry->port_name, port->port_name,
1406 			sizeof(entry->port_name));
1407 		entry->physical_port = port->physical_port;
1408 		entry->virtual_port = port->virtual_port;
1409 		entry->wwnn = port->wwnn;
1410 		entry->wwpn = port->wwpn;
1411 
1412 		i++;
1413 		entries_filled++;
1414 	}
1415 
1416 	mtx_unlock(&softc->ctl_lock);
1417 
1418 	if (entries_dropped > 0)
1419 		retval = 1;
1420 
1421 	*num_entries_dropped = entries_dropped;
1422 	*num_entries_filled = entries_filled;
1423 
1424 	return (retval);
1425 }
1426 
1427 static void
1428 ctl_ioctl_online(void *arg)
1429 {
1430 	struct ctl_ioctl_info *ioctl_info;
1431 
1432 	ioctl_info = (struct ctl_ioctl_info *)arg;
1433 
1434 	ioctl_info->flags |= CTL_IOCTL_FLAG_ENABLED;
1435 }
1436 
1437 static void
1438 ctl_ioctl_offline(void *arg)
1439 {
1440 	struct ctl_ioctl_info *ioctl_info;
1441 
1442 	ioctl_info = (struct ctl_ioctl_info *)arg;
1443 
1444 	ioctl_info->flags &= ~CTL_IOCTL_FLAG_ENABLED;
1445 }
1446 
1447 /*
1448  * Remove an initiator by port number and initiator ID.
1449  * Returns 0 for success, -1 for failure.
1450  */
1451 int
1452 ctl_remove_initiator(struct ctl_port *port, int iid)
1453 {
1454 	struct ctl_softc *softc = control_softc;
1455 
1456 	mtx_assert(&softc->ctl_lock, MA_NOTOWNED);
1457 
1458 	if (iid > CTL_MAX_INIT_PER_PORT) {
1459 		printf("%s: initiator ID %u > maximun %u!\n",
1460 		       __func__, iid, CTL_MAX_INIT_PER_PORT);
1461 		return (-1);
1462 	}
1463 
1464 	mtx_lock(&softc->ctl_lock);
1465 	port->wwpn_iid[iid].in_use--;
1466 	port->wwpn_iid[iid].last_use = time_uptime;
1467 	mtx_unlock(&softc->ctl_lock);
1468 
1469 	return (0);
1470 }
1471 
1472 /*
1473  * Add an initiator to the initiator map.
1474  * Returns iid for success, < 0 for failure.
1475  */
1476 int
1477 ctl_add_initiator(struct ctl_port *port, int iid, uint64_t wwpn, char *name)
1478 {
1479 	struct ctl_softc *softc = control_softc;
1480 	time_t best_time;
1481 	int i, best;
1482 
1483 	mtx_assert(&softc->ctl_lock, MA_NOTOWNED);
1484 
1485 	if (iid >= CTL_MAX_INIT_PER_PORT) {
1486 		printf("%s: WWPN %#jx initiator ID %u > maximum %u!\n",
1487 		       __func__, wwpn, iid, CTL_MAX_INIT_PER_PORT);
1488 		free(name, M_CTL);
1489 		return (-1);
1490 	}
1491 
1492 	mtx_lock(&softc->ctl_lock);
1493 
1494 	if (iid < 0 && (wwpn != 0 || name != NULL)) {
1495 		for (i = 0; i < CTL_MAX_INIT_PER_PORT; i++) {
1496 			if (wwpn != 0 && wwpn == port->wwpn_iid[i].wwpn) {
1497 				iid = i;
1498 				break;
1499 			}
1500 			if (name != NULL && port->wwpn_iid[i].name != NULL &&
1501 			    strcmp(name, port->wwpn_iid[i].name) == 0) {
1502 				iid = i;
1503 				break;
1504 			}
1505 		}
1506 	}
1507 
1508 	if (iid < 0) {
1509 		for (i = 0; i < CTL_MAX_INIT_PER_PORT; i++) {
1510 			if (port->wwpn_iid[i].in_use == 0 &&
1511 			    port->wwpn_iid[i].wwpn == 0 &&
1512 			    port->wwpn_iid[i].name == NULL) {
1513 				iid = i;
1514 				break;
1515 			}
1516 		}
1517 	}
1518 
1519 	if (iid < 0) {
1520 		best = -1;
1521 		best_time = INT32_MAX;
1522 		for (i = 0; i < CTL_MAX_INIT_PER_PORT; i++) {
1523 			if (port->wwpn_iid[i].in_use == 0) {
1524 				if (port->wwpn_iid[i].last_use < best_time) {
1525 					best = i;
1526 					best_time = port->wwpn_iid[i].last_use;
1527 				}
1528 			}
1529 		}
1530 		iid = best;
1531 	}
1532 
1533 	if (iid < 0) {
1534 		mtx_unlock(&softc->ctl_lock);
1535 		free(name, M_CTL);
1536 		return (-2);
1537 	}
1538 
1539 	if (port->wwpn_iid[iid].in_use > 0 && (wwpn != 0 || name != NULL)) {
1540 		/*
1541 		 * This is not an error yet.
1542 		 */
1543 		if (wwpn != 0 && wwpn == port->wwpn_iid[iid].wwpn) {
1544 #if 0
1545 			printf("%s: port %d iid %u WWPN %#jx arrived"
1546 			    " again\n", __func__, port->targ_port,
1547 			    iid, (uintmax_t)wwpn);
1548 #endif
1549 			goto take;
1550 		}
1551 		if (name != NULL && port->wwpn_iid[iid].name != NULL &&
1552 		    strcmp(name, port->wwpn_iid[iid].name) == 0) {
1553 #if 0
1554 			printf("%s: port %d iid %u name '%s' arrived"
1555 			    " again\n", __func__, port->targ_port,
1556 			    iid, name);
1557 #endif
1558 			goto take;
1559 		}
1560 
1561 		/*
1562 		 * This is an error, but what do we do about it?  The
1563 		 * driver is telling us we have a new WWPN for this
1564 		 * initiator ID, so we pretty much need to use it.
1565 		 */
1566 		printf("%s: port %d iid %u WWPN %#jx '%s' arrived,"
1567 		    " but WWPN %#jx '%s' is still at that address\n",
1568 		    __func__, port->targ_port, iid, wwpn, name,
1569 		    (uintmax_t)port->wwpn_iid[iid].wwpn,
1570 		    port->wwpn_iid[iid].name);
1571 
1572 		/*
1573 		 * XXX KDM clear have_ca and ua_pending on each LUN for
1574 		 * this initiator.
1575 		 */
1576 	}
1577 take:
1578 	free(port->wwpn_iid[iid].name, M_CTL);
1579 	port->wwpn_iid[iid].name = name;
1580 	port->wwpn_iid[iid].wwpn = wwpn;
1581 	port->wwpn_iid[iid].in_use++;
1582 	mtx_unlock(&softc->ctl_lock);
1583 
1584 	return (iid);
1585 }
1586 
1587 static int
1588 ctl_create_iid(struct ctl_port *port, int iid, uint8_t *buf)
1589 {
1590 	int len;
1591 
1592 	switch (port->port_type) {
1593 	case CTL_PORT_FC:
1594 	{
1595 		struct scsi_transportid_fcp *id =
1596 		    (struct scsi_transportid_fcp *)buf;
1597 		if (port->wwpn_iid[iid].wwpn == 0)
1598 			return (0);
1599 		memset(id, 0, sizeof(*id));
1600 		id->format_protocol = SCSI_PROTO_FC;
1601 		scsi_u64to8b(port->wwpn_iid[iid].wwpn, id->n_port_name);
1602 		return (sizeof(*id));
1603 	}
1604 	case CTL_PORT_ISCSI:
1605 	{
1606 		struct scsi_transportid_iscsi_port *id =
1607 		    (struct scsi_transportid_iscsi_port *)buf;
1608 		if (port->wwpn_iid[iid].name == NULL)
1609 			return (0);
1610 		memset(id, 0, 256);
1611 		id->format_protocol = SCSI_TRN_ISCSI_FORMAT_PORT |
1612 		    SCSI_PROTO_ISCSI;
1613 		len = strlcpy(id->iscsi_name, port->wwpn_iid[iid].name, 252) + 1;
1614 		len = roundup2(min(len, 252), 4);
1615 		scsi_ulto2b(len, id->additional_length);
1616 		return (sizeof(*id) + len);
1617 	}
1618 	case CTL_PORT_SAS:
1619 	{
1620 		struct scsi_transportid_sas *id =
1621 		    (struct scsi_transportid_sas *)buf;
1622 		if (port->wwpn_iid[iid].wwpn == 0)
1623 			return (0);
1624 		memset(id, 0, sizeof(*id));
1625 		id->format_protocol = SCSI_PROTO_SAS;
1626 		scsi_u64to8b(port->wwpn_iid[iid].wwpn, id->sas_address);
1627 		return (sizeof(*id));
1628 	}
1629 	default:
1630 	{
1631 		struct scsi_transportid_spi *id =
1632 		    (struct scsi_transportid_spi *)buf;
1633 		memset(id, 0, sizeof(*id));
1634 		id->format_protocol = SCSI_PROTO_SPI;
1635 		scsi_ulto2b(iid, id->scsi_addr);
1636 		scsi_ulto2b(port->targ_port, id->rel_trgt_port_id);
1637 		return (sizeof(*id));
1638 	}
1639 	}
1640 }
1641 
1642 static int
1643 ctl_ioctl_lun_enable(void *arg, int lun_id)
1644 {
1645 	return (0);
1646 }
1647 
1648 static int
1649 ctl_ioctl_lun_disable(void *arg, int lun_id)
1650 {
1651 	return (0);
1652 }
1653 
1654 /*
1655  * Data movement routine for the CTL ioctl frontend port.
1656  */
1657 static int
1658 ctl_ioctl_do_datamove(struct ctl_scsiio *ctsio)
1659 {
1660 	struct ctl_sg_entry *ext_sglist, *kern_sglist;
1661 	struct ctl_sg_entry ext_entry, kern_entry;
1662 	int ext_sglen, ext_sg_entries, kern_sg_entries;
1663 	int ext_sg_start, ext_offset;
1664 	int len_to_copy, len_copied;
1665 	int kern_watermark, ext_watermark;
1666 	int ext_sglist_malloced;
1667 	int i, j;
1668 
1669 	ext_sglist_malloced = 0;
1670 	ext_sg_start = 0;
1671 	ext_offset = 0;
1672 
1673 	CTL_DEBUG_PRINT(("ctl_ioctl_do_datamove\n"));
1674 
1675 	/*
1676 	 * If this flag is set, fake the data transfer.
1677 	 */
1678 	if (ctsio->io_hdr.flags & CTL_FLAG_NO_DATAMOVE) {
1679 		ctsio->ext_data_filled = ctsio->ext_data_len;
1680 		goto bailout;
1681 	}
1682 
1683 	/*
1684 	 * To simplify things here, if we have a single buffer, stick it in
1685 	 * a S/G entry and just make it a single entry S/G list.
1686 	 */
1687 	if (ctsio->io_hdr.flags & CTL_FLAG_EDPTR_SGLIST) {
1688 		int len_seen;
1689 
1690 		ext_sglen = ctsio->ext_sg_entries * sizeof(*ext_sglist);
1691 
1692 		ext_sglist = (struct ctl_sg_entry *)malloc(ext_sglen, M_CTL,
1693 							   M_WAITOK);
1694 		ext_sglist_malloced = 1;
1695 		if (copyin(ctsio->ext_data_ptr, ext_sglist,
1696 				   ext_sglen) != 0) {
1697 			ctl_set_internal_failure(ctsio,
1698 						 /*sks_valid*/ 0,
1699 						 /*retry_count*/ 0);
1700 			goto bailout;
1701 		}
1702 		ext_sg_entries = ctsio->ext_sg_entries;
1703 		len_seen = 0;
1704 		for (i = 0; i < ext_sg_entries; i++) {
1705 			if ((len_seen + ext_sglist[i].len) >=
1706 			     ctsio->ext_data_filled) {
1707 				ext_sg_start = i;
1708 				ext_offset = ctsio->ext_data_filled - len_seen;
1709 				break;
1710 			}
1711 			len_seen += ext_sglist[i].len;
1712 		}
1713 	} else {
1714 		ext_sglist = &ext_entry;
1715 		ext_sglist->addr = ctsio->ext_data_ptr;
1716 		ext_sglist->len = ctsio->ext_data_len;
1717 		ext_sg_entries = 1;
1718 		ext_sg_start = 0;
1719 		ext_offset = ctsio->ext_data_filled;
1720 	}
1721 
1722 	if (ctsio->kern_sg_entries > 0) {
1723 		kern_sglist = (struct ctl_sg_entry *)ctsio->kern_data_ptr;
1724 		kern_sg_entries = ctsio->kern_sg_entries;
1725 	} else {
1726 		kern_sglist = &kern_entry;
1727 		kern_sglist->addr = ctsio->kern_data_ptr;
1728 		kern_sglist->len = ctsio->kern_data_len;
1729 		kern_sg_entries = 1;
1730 	}
1731 
1732 
1733 	kern_watermark = 0;
1734 	ext_watermark = ext_offset;
1735 	len_copied = 0;
1736 	for (i = ext_sg_start, j = 0;
1737 	     i < ext_sg_entries && j < kern_sg_entries;) {
1738 		uint8_t *ext_ptr, *kern_ptr;
1739 
1740 		len_to_copy = MIN(ext_sglist[i].len - ext_watermark,
1741 				  kern_sglist[j].len - kern_watermark);
1742 
1743 		ext_ptr = (uint8_t *)ext_sglist[i].addr;
1744 		ext_ptr = ext_ptr + ext_watermark;
1745 		if (ctsio->io_hdr.flags & CTL_FLAG_BUS_ADDR) {
1746 			/*
1747 			 * XXX KDM fix this!
1748 			 */
1749 			panic("need to implement bus address support");
1750 #if 0
1751 			kern_ptr = bus_to_virt(kern_sglist[j].addr);
1752 #endif
1753 		} else
1754 			kern_ptr = (uint8_t *)kern_sglist[j].addr;
1755 		kern_ptr = kern_ptr + kern_watermark;
1756 
1757 		kern_watermark += len_to_copy;
1758 		ext_watermark += len_to_copy;
1759 
1760 		if ((ctsio->io_hdr.flags & CTL_FLAG_DATA_MASK) ==
1761 		     CTL_FLAG_DATA_IN) {
1762 			CTL_DEBUG_PRINT(("ctl_ioctl_do_datamove: copying %d "
1763 					 "bytes to user\n", len_to_copy));
1764 			CTL_DEBUG_PRINT(("ctl_ioctl_do_datamove: from %p "
1765 					 "to %p\n", kern_ptr, ext_ptr));
1766 			if (copyout(kern_ptr, ext_ptr, len_to_copy) != 0) {
1767 				ctl_set_internal_failure(ctsio,
1768 							 /*sks_valid*/ 0,
1769 							 /*retry_count*/ 0);
1770 				goto bailout;
1771 			}
1772 		} else {
1773 			CTL_DEBUG_PRINT(("ctl_ioctl_do_datamove: copying %d "
1774 					 "bytes from user\n", len_to_copy));
1775 			CTL_DEBUG_PRINT(("ctl_ioctl_do_datamove: from %p "
1776 					 "to %p\n", ext_ptr, kern_ptr));
1777 			if (copyin(ext_ptr, kern_ptr, len_to_copy)!= 0){
1778 				ctl_set_internal_failure(ctsio,
1779 							 /*sks_valid*/ 0,
1780 							 /*retry_count*/0);
1781 				goto bailout;
1782 			}
1783 		}
1784 
1785 		len_copied += len_to_copy;
1786 
1787 		if (ext_sglist[i].len == ext_watermark) {
1788 			i++;
1789 			ext_watermark = 0;
1790 		}
1791 
1792 		if (kern_sglist[j].len == kern_watermark) {
1793 			j++;
1794 			kern_watermark = 0;
1795 		}
1796 	}
1797 
1798 	ctsio->ext_data_filled += len_copied;
1799 
1800 	CTL_DEBUG_PRINT(("ctl_ioctl_do_datamove: ext_sg_entries: %d, "
1801 			 "kern_sg_entries: %d\n", ext_sg_entries,
1802 			 kern_sg_entries));
1803 	CTL_DEBUG_PRINT(("ctl_ioctl_do_datamove: ext_data_len = %d, "
1804 			 "kern_data_len = %d\n", ctsio->ext_data_len,
1805 			 ctsio->kern_data_len));
1806 
1807 
1808 	/* XXX KDM set residual?? */
1809 bailout:
1810 
1811 	if (ext_sglist_malloced != 0)
1812 		free(ext_sglist, M_CTL);
1813 
1814 	return (CTL_RETVAL_COMPLETE);
1815 }
1816 
1817 /*
1818  * Serialize a command that went down the "wrong" side, and so was sent to
1819  * this controller for execution.  The logic is a little different than the
1820  * standard case in ctl_scsiio_precheck().  Errors in this case need to get
1821  * sent back to the other side, but in the success case, we execute the
1822  * command on this side (XFER mode) or tell the other side to execute it
1823  * (SER_ONLY mode).
1824  */
1825 static int
1826 ctl_serialize_other_sc_cmd(struct ctl_scsiio *ctsio)
1827 {
1828 	struct ctl_softc *softc;
1829 	union ctl_ha_msg msg_info;
1830 	struct ctl_lun *lun;
1831 	int retval = 0;
1832 	uint32_t targ_lun;
1833 
1834 	softc = control_softc;
1835 
1836 	targ_lun = ctsio->io_hdr.nexus.targ_mapped_lun;
1837 	lun = softc->ctl_luns[targ_lun];
1838 	if (lun==NULL)
1839 	{
1840 		/*
1841 		 * Why isn't LUN defined? The other side wouldn't
1842 		 * send a cmd if the LUN is undefined.
1843 		 */
1844 		printf("%s: Bad JUJU!, LUN is NULL!\n", __func__);
1845 
1846 		/* "Logical unit not supported" */
1847 		ctl_set_sense_data(&msg_info.scsi.sense_data,
1848 				   lun,
1849 				   /*sense_format*/SSD_TYPE_NONE,
1850 				   /*current_error*/ 1,
1851 				   /*sense_key*/ SSD_KEY_ILLEGAL_REQUEST,
1852 				   /*asc*/ 0x25,
1853 				   /*ascq*/ 0x00,
1854 				   SSD_ELEM_NONE);
1855 
1856 		msg_info.scsi.sense_len = SSD_FULL_SIZE;
1857 		msg_info.scsi.scsi_status = SCSI_STATUS_CHECK_COND;
1858 		msg_info.hdr.status = CTL_SCSI_ERROR | CTL_AUTOSENSE;
1859 		msg_info.hdr.original_sc = ctsio->io_hdr.original_sc;
1860 		msg_info.hdr.serializing_sc = NULL;
1861 		msg_info.hdr.msg_type = CTL_MSG_BAD_JUJU;
1862 	        if (ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg_info,
1863 				sizeof(msg_info), 0 ) > CTL_HA_STATUS_SUCCESS) {
1864 		}
1865 		return(1);
1866 
1867 	}
1868 
1869 	mtx_lock(&lun->lun_lock);
1870     	TAILQ_INSERT_TAIL(&lun->ooa_queue, &ctsio->io_hdr, ooa_links);
1871 
1872 	switch (ctl_check_ooa(lun, (union ctl_io *)ctsio,
1873 		(union ctl_io *)TAILQ_PREV(&ctsio->io_hdr, ctl_ooaq,
1874 		 ooa_links))) {
1875 	case CTL_ACTION_BLOCK:
1876 		ctsio->io_hdr.flags |= CTL_FLAG_BLOCKED;
1877 		TAILQ_INSERT_TAIL(&lun->blocked_queue, &ctsio->io_hdr,
1878 				  blocked_links);
1879 		break;
1880 	case CTL_ACTION_PASS:
1881 	case CTL_ACTION_SKIP:
1882 		if (softc->ha_mode == CTL_HA_MODE_XFER) {
1883 			ctsio->io_hdr.flags |= CTL_FLAG_IS_WAS_ON_RTR;
1884 			ctl_enqueue_rtr((union ctl_io *)ctsio);
1885 		} else {
1886 
1887 			/* send msg back to other side */
1888 			msg_info.hdr.original_sc = ctsio->io_hdr.original_sc;
1889 			msg_info.hdr.serializing_sc = (union ctl_io *)ctsio;
1890 			msg_info.hdr.msg_type = CTL_MSG_R2R;
1891 #if 0
1892 			printf("2. pOrig %x\n", (int)msg_info.hdr.original_sc);
1893 #endif
1894 		        if (ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg_info,
1895 			    sizeof(msg_info), 0 ) > CTL_HA_STATUS_SUCCESS) {
1896 			}
1897 		}
1898 		break;
1899 	case CTL_ACTION_OVERLAP:
1900 		/* OVERLAPPED COMMANDS ATTEMPTED */
1901 		ctl_set_sense_data(&msg_info.scsi.sense_data,
1902 				   lun,
1903 				   /*sense_format*/SSD_TYPE_NONE,
1904 				   /*current_error*/ 1,
1905 				   /*sense_key*/ SSD_KEY_ILLEGAL_REQUEST,
1906 				   /*asc*/ 0x4E,
1907 				   /*ascq*/ 0x00,
1908 				   SSD_ELEM_NONE);
1909 
1910 		msg_info.scsi.sense_len = SSD_FULL_SIZE;
1911 		msg_info.scsi.scsi_status = SCSI_STATUS_CHECK_COND;
1912 		msg_info.hdr.status = CTL_SCSI_ERROR | CTL_AUTOSENSE;
1913 		msg_info.hdr.original_sc = ctsio->io_hdr.original_sc;
1914 		msg_info.hdr.serializing_sc = NULL;
1915 		msg_info.hdr.msg_type = CTL_MSG_BAD_JUJU;
1916 #if 0
1917 		printf("BAD JUJU:Major Bummer Overlap\n");
1918 #endif
1919 		TAILQ_REMOVE(&lun->ooa_queue, &ctsio->io_hdr, ooa_links);
1920 		retval = 1;
1921 		if (ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg_info,
1922 		    sizeof(msg_info), 0 ) > CTL_HA_STATUS_SUCCESS) {
1923 		}
1924 		break;
1925 	case CTL_ACTION_OVERLAP_TAG:
1926 		/* TAGGED OVERLAPPED COMMANDS (NN = QUEUE TAG) */
1927 		ctl_set_sense_data(&msg_info.scsi.sense_data,
1928 				   lun,
1929 				   /*sense_format*/SSD_TYPE_NONE,
1930 				   /*current_error*/ 1,
1931 				   /*sense_key*/ SSD_KEY_ILLEGAL_REQUEST,
1932 				   /*asc*/ 0x4D,
1933 				   /*ascq*/ ctsio->tag_num & 0xff,
1934 				   SSD_ELEM_NONE);
1935 
1936 		msg_info.scsi.sense_len = SSD_FULL_SIZE;
1937 		msg_info.scsi.scsi_status = SCSI_STATUS_CHECK_COND;
1938 		msg_info.hdr.status = CTL_SCSI_ERROR | CTL_AUTOSENSE;
1939 		msg_info.hdr.original_sc = ctsio->io_hdr.original_sc;
1940 		msg_info.hdr.serializing_sc = NULL;
1941 		msg_info.hdr.msg_type = CTL_MSG_BAD_JUJU;
1942 #if 0
1943 		printf("BAD JUJU:Major Bummer Overlap Tag\n");
1944 #endif
1945 		TAILQ_REMOVE(&lun->ooa_queue, &ctsio->io_hdr, ooa_links);
1946 		retval = 1;
1947 		if (ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg_info,
1948 		    sizeof(msg_info), 0 ) > CTL_HA_STATUS_SUCCESS) {
1949 		}
1950 		break;
1951 	case CTL_ACTION_ERROR:
1952 	default:
1953 		/* "Internal target failure" */
1954 		ctl_set_sense_data(&msg_info.scsi.sense_data,
1955 				   lun,
1956 				   /*sense_format*/SSD_TYPE_NONE,
1957 				   /*current_error*/ 1,
1958 				   /*sense_key*/ SSD_KEY_HARDWARE_ERROR,
1959 				   /*asc*/ 0x44,
1960 				   /*ascq*/ 0x00,
1961 				   SSD_ELEM_NONE);
1962 
1963 		msg_info.scsi.sense_len = SSD_FULL_SIZE;
1964 		msg_info.scsi.scsi_status = SCSI_STATUS_CHECK_COND;
1965 		msg_info.hdr.status = CTL_SCSI_ERROR | CTL_AUTOSENSE;
1966 		msg_info.hdr.original_sc = ctsio->io_hdr.original_sc;
1967 		msg_info.hdr.serializing_sc = NULL;
1968 		msg_info.hdr.msg_type = CTL_MSG_BAD_JUJU;
1969 #if 0
1970 		printf("BAD JUJU:Major Bummer HW Error\n");
1971 #endif
1972 		TAILQ_REMOVE(&lun->ooa_queue, &ctsio->io_hdr, ooa_links);
1973 		retval = 1;
1974 		if (ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg_info,
1975 		    sizeof(msg_info), 0 ) > CTL_HA_STATUS_SUCCESS) {
1976 		}
1977 		break;
1978 	}
1979 	mtx_unlock(&lun->lun_lock);
1980 	return (retval);
1981 }
1982 
1983 static int
1984 ctl_ioctl_submit_wait(union ctl_io *io)
1985 {
1986 	struct ctl_fe_ioctl_params params;
1987 	ctl_fe_ioctl_state last_state;
1988 	int done, retval;
1989 
1990 	retval = 0;
1991 
1992 	bzero(&params, sizeof(params));
1993 
1994 	mtx_init(&params.ioctl_mtx, "ctliocmtx", NULL, MTX_DEF);
1995 	cv_init(&params.sem, "ctlioccv");
1996 	params.state = CTL_IOCTL_INPROG;
1997 	last_state = params.state;
1998 
1999 	io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr = &params;
2000 
2001 	CTL_DEBUG_PRINT(("ctl_ioctl_submit_wait\n"));
2002 
2003 	/* This shouldn't happen */
2004 	if ((retval = ctl_queue(io)) != CTL_RETVAL_COMPLETE)
2005 		return (retval);
2006 
2007 	done = 0;
2008 
2009 	do {
2010 		mtx_lock(&params.ioctl_mtx);
2011 		/*
2012 		 * Check the state here, and don't sleep if the state has
2013 		 * already changed (i.e. wakeup has already occured, but we
2014 		 * weren't waiting yet).
2015 		 */
2016 		if (params.state == last_state) {
2017 			/* XXX KDM cv_wait_sig instead? */
2018 			cv_wait(&params.sem, &params.ioctl_mtx);
2019 		}
2020 		last_state = params.state;
2021 
2022 		switch (params.state) {
2023 		case CTL_IOCTL_INPROG:
2024 			/* Why did we wake up? */
2025 			/* XXX KDM error here? */
2026 			mtx_unlock(&params.ioctl_mtx);
2027 			break;
2028 		case CTL_IOCTL_DATAMOVE:
2029 			CTL_DEBUG_PRINT(("got CTL_IOCTL_DATAMOVE\n"));
2030 
2031 			/*
2032 			 * change last_state back to INPROG to avoid
2033 			 * deadlock on subsequent data moves.
2034 			 */
2035 			params.state = last_state = CTL_IOCTL_INPROG;
2036 
2037 			mtx_unlock(&params.ioctl_mtx);
2038 			ctl_ioctl_do_datamove(&io->scsiio);
2039 			/*
2040 			 * Note that in some cases, most notably writes,
2041 			 * this will queue the I/O and call us back later.
2042 			 * In other cases, generally reads, this routine
2043 			 * will immediately call back and wake us up,
2044 			 * probably using our own context.
2045 			 */
2046 			io->scsiio.be_move_done(io);
2047 			break;
2048 		case CTL_IOCTL_DONE:
2049 			mtx_unlock(&params.ioctl_mtx);
2050 			CTL_DEBUG_PRINT(("got CTL_IOCTL_DONE\n"));
2051 			done = 1;
2052 			break;
2053 		default:
2054 			mtx_unlock(&params.ioctl_mtx);
2055 			/* XXX KDM error here? */
2056 			break;
2057 		}
2058 	} while (done == 0);
2059 
2060 	mtx_destroy(&params.ioctl_mtx);
2061 	cv_destroy(&params.sem);
2062 
2063 	return (CTL_RETVAL_COMPLETE);
2064 }
2065 
2066 static void
2067 ctl_ioctl_datamove(union ctl_io *io)
2068 {
2069 	struct ctl_fe_ioctl_params *params;
2070 
2071 	params = (struct ctl_fe_ioctl_params *)
2072 		io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr;
2073 
2074 	mtx_lock(&params->ioctl_mtx);
2075 	params->state = CTL_IOCTL_DATAMOVE;
2076 	cv_broadcast(&params->sem);
2077 	mtx_unlock(&params->ioctl_mtx);
2078 }
2079 
2080 static void
2081 ctl_ioctl_done(union ctl_io *io)
2082 {
2083 	struct ctl_fe_ioctl_params *params;
2084 
2085 	params = (struct ctl_fe_ioctl_params *)
2086 		io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr;
2087 
2088 	mtx_lock(&params->ioctl_mtx);
2089 	params->state = CTL_IOCTL_DONE;
2090 	cv_broadcast(&params->sem);
2091 	mtx_unlock(&params->ioctl_mtx);
2092 }
2093 
2094 static void
2095 ctl_ioctl_hard_startstop_callback(void *arg, struct cfi_metatask *metatask)
2096 {
2097 	struct ctl_fe_ioctl_startstop_info *sd_info;
2098 
2099 	sd_info = (struct ctl_fe_ioctl_startstop_info *)arg;
2100 
2101 	sd_info->hs_info.status = metatask->status;
2102 	sd_info->hs_info.total_luns = metatask->taskinfo.startstop.total_luns;
2103 	sd_info->hs_info.luns_complete =
2104 		metatask->taskinfo.startstop.luns_complete;
2105 	sd_info->hs_info.luns_failed = metatask->taskinfo.startstop.luns_failed;
2106 
2107 	cv_broadcast(&sd_info->sem);
2108 }
2109 
2110 static void
2111 ctl_ioctl_bbrread_callback(void *arg, struct cfi_metatask *metatask)
2112 {
2113 	struct ctl_fe_ioctl_bbrread_info *fe_bbr_info;
2114 
2115 	fe_bbr_info = (struct ctl_fe_ioctl_bbrread_info *)arg;
2116 
2117 	mtx_lock(fe_bbr_info->lock);
2118 	fe_bbr_info->bbr_info->status = metatask->status;
2119 	fe_bbr_info->bbr_info->bbr_status = metatask->taskinfo.bbrread.status;
2120 	fe_bbr_info->wakeup_done = 1;
2121 	mtx_unlock(fe_bbr_info->lock);
2122 
2123 	cv_broadcast(&fe_bbr_info->sem);
2124 }
2125 
2126 /*
2127  * Returns 0 for success, errno for failure.
2128  */
2129 static int
2130 ctl_ioctl_fill_ooa(struct ctl_lun *lun, uint32_t *cur_fill_num,
2131 		   struct ctl_ooa *ooa_hdr, struct ctl_ooa_entry *kern_entries)
2132 {
2133 	union ctl_io *io;
2134 	int retval;
2135 
2136 	retval = 0;
2137 
2138 	mtx_lock(&lun->lun_lock);
2139 	for (io = (union ctl_io *)TAILQ_FIRST(&lun->ooa_queue); (io != NULL);
2140 	     (*cur_fill_num)++, io = (union ctl_io *)TAILQ_NEXT(&io->io_hdr,
2141 	     ooa_links)) {
2142 		struct ctl_ooa_entry *entry;
2143 
2144 		/*
2145 		 * If we've got more than we can fit, just count the
2146 		 * remaining entries.
2147 		 */
2148 		if (*cur_fill_num >= ooa_hdr->alloc_num)
2149 			continue;
2150 
2151 		entry = &kern_entries[*cur_fill_num];
2152 
2153 		entry->tag_num = io->scsiio.tag_num;
2154 		entry->lun_num = lun->lun;
2155 #ifdef CTL_TIME_IO
2156 		entry->start_bt = io->io_hdr.start_bt;
2157 #endif
2158 		bcopy(io->scsiio.cdb, entry->cdb, io->scsiio.cdb_len);
2159 		entry->cdb_len = io->scsiio.cdb_len;
2160 		if (io->io_hdr.flags & CTL_FLAG_BLOCKED)
2161 			entry->cmd_flags |= CTL_OOACMD_FLAG_BLOCKED;
2162 
2163 		if (io->io_hdr.flags & CTL_FLAG_DMA_INPROG)
2164 			entry->cmd_flags |= CTL_OOACMD_FLAG_DMA;
2165 
2166 		if (io->io_hdr.flags & CTL_FLAG_ABORT)
2167 			entry->cmd_flags |= CTL_OOACMD_FLAG_ABORT;
2168 
2169 		if (io->io_hdr.flags & CTL_FLAG_IS_WAS_ON_RTR)
2170 			entry->cmd_flags |= CTL_OOACMD_FLAG_RTR;
2171 
2172 		if (io->io_hdr.flags & CTL_FLAG_DMA_QUEUED)
2173 			entry->cmd_flags |= CTL_OOACMD_FLAG_DMA_QUEUED;
2174 	}
2175 	mtx_unlock(&lun->lun_lock);
2176 
2177 	return (retval);
2178 }
2179 
2180 static void *
2181 ctl_copyin_alloc(void *user_addr, int len, char *error_str,
2182 		 size_t error_str_len)
2183 {
2184 	void *kptr;
2185 
2186 	kptr = malloc(len, M_CTL, M_WAITOK | M_ZERO);
2187 
2188 	if (copyin(user_addr, kptr, len) != 0) {
2189 		snprintf(error_str, error_str_len, "Error copying %d bytes "
2190 			 "from user address %p to kernel address %p", len,
2191 			 user_addr, kptr);
2192 		free(kptr, M_CTL);
2193 		return (NULL);
2194 	}
2195 
2196 	return (kptr);
2197 }
2198 
2199 static void
2200 ctl_free_args(int num_args, struct ctl_be_arg *args)
2201 {
2202 	int i;
2203 
2204 	if (args == NULL)
2205 		return;
2206 
2207 	for (i = 0; i < num_args; i++) {
2208 		free(args[i].kname, M_CTL);
2209 		free(args[i].kvalue, M_CTL);
2210 	}
2211 
2212 	free(args, M_CTL);
2213 }
2214 
2215 static struct ctl_be_arg *
2216 ctl_copyin_args(int num_args, struct ctl_be_arg *uargs,
2217 		char *error_str, size_t error_str_len)
2218 {
2219 	struct ctl_be_arg *args;
2220 	int i;
2221 
2222 	args = ctl_copyin_alloc(uargs, num_args * sizeof(*args),
2223 				error_str, error_str_len);
2224 
2225 	if (args == NULL)
2226 		goto bailout;
2227 
2228 	for (i = 0; i < num_args; i++) {
2229 		args[i].kname = NULL;
2230 		args[i].kvalue = NULL;
2231 	}
2232 
2233 	for (i = 0; i < num_args; i++) {
2234 		uint8_t *tmpptr;
2235 
2236 		args[i].kname = ctl_copyin_alloc(args[i].name,
2237 			args[i].namelen, error_str, error_str_len);
2238 		if (args[i].kname == NULL)
2239 			goto bailout;
2240 
2241 		if (args[i].kname[args[i].namelen - 1] != '\0') {
2242 			snprintf(error_str, error_str_len, "Argument %d "
2243 				 "name is not NUL-terminated", i);
2244 			goto bailout;
2245 		}
2246 
2247 		if (args[i].flags & CTL_BEARG_RD) {
2248 			tmpptr = ctl_copyin_alloc(args[i].value,
2249 				args[i].vallen, error_str, error_str_len);
2250 			if (tmpptr == NULL)
2251 				goto bailout;
2252 			if ((args[i].flags & CTL_BEARG_ASCII)
2253 			 && (tmpptr[args[i].vallen - 1] != '\0')) {
2254 				snprintf(error_str, error_str_len, "Argument "
2255 				    "%d value is not NUL-terminated", i);
2256 				goto bailout;
2257 			}
2258 			args[i].kvalue = tmpptr;
2259 		} else {
2260 			args[i].kvalue = malloc(args[i].vallen,
2261 			    M_CTL, M_WAITOK | M_ZERO);
2262 		}
2263 	}
2264 
2265 	return (args);
2266 bailout:
2267 
2268 	ctl_free_args(num_args, args);
2269 
2270 	return (NULL);
2271 }
2272 
2273 static void
2274 ctl_copyout_args(int num_args, struct ctl_be_arg *args)
2275 {
2276 	int i;
2277 
2278 	for (i = 0; i < num_args; i++) {
2279 		if (args[i].flags & CTL_BEARG_WR)
2280 			copyout(args[i].kvalue, args[i].value, args[i].vallen);
2281 	}
2282 }
2283 
2284 /*
2285  * Escape characters that are illegal or not recommended in XML.
2286  */
2287 int
2288 ctl_sbuf_printf_esc(struct sbuf *sb, char *str, int size)
2289 {
2290 	char *end = str + size;
2291 	int retval;
2292 
2293 	retval = 0;
2294 
2295 	for (; *str && str < end; str++) {
2296 		switch (*str) {
2297 		case '&':
2298 			retval = sbuf_printf(sb, "&amp;");
2299 			break;
2300 		case '>':
2301 			retval = sbuf_printf(sb, "&gt;");
2302 			break;
2303 		case '<':
2304 			retval = sbuf_printf(sb, "&lt;");
2305 			break;
2306 		default:
2307 			retval = sbuf_putc(sb, *str);
2308 			break;
2309 		}
2310 
2311 		if (retval != 0)
2312 			break;
2313 
2314 	}
2315 
2316 	return (retval);
2317 }
2318 
2319 static void
2320 ctl_id_sbuf(struct ctl_devid *id, struct sbuf *sb)
2321 {
2322 	struct scsi_vpd_id_descriptor *desc;
2323 	int i;
2324 
2325 	if (id == NULL || id->len < 4)
2326 		return;
2327 	desc = (struct scsi_vpd_id_descriptor *)id->data;
2328 	switch (desc->id_type & SVPD_ID_TYPE_MASK) {
2329 	case SVPD_ID_TYPE_T10:
2330 		sbuf_printf(sb, "t10.");
2331 		break;
2332 	case SVPD_ID_TYPE_EUI64:
2333 		sbuf_printf(sb, "eui.");
2334 		break;
2335 	case SVPD_ID_TYPE_NAA:
2336 		sbuf_printf(sb, "naa.");
2337 		break;
2338 	case SVPD_ID_TYPE_SCSI_NAME:
2339 		break;
2340 	}
2341 	switch (desc->proto_codeset & SVPD_ID_CODESET_MASK) {
2342 	case SVPD_ID_CODESET_BINARY:
2343 		for (i = 0; i < desc->length; i++)
2344 			sbuf_printf(sb, "%02x", desc->identifier[i]);
2345 		break;
2346 	case SVPD_ID_CODESET_ASCII:
2347 		sbuf_printf(sb, "%.*s", (int)desc->length,
2348 		    (char *)desc->identifier);
2349 		break;
2350 	case SVPD_ID_CODESET_UTF8:
2351 		sbuf_printf(sb, "%s", (char *)desc->identifier);
2352 		break;
2353 	}
2354 }
2355 
2356 static int
2357 ctl_ioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flag,
2358 	  struct thread *td)
2359 {
2360 	struct ctl_softc *softc;
2361 	int retval;
2362 
2363 	softc = control_softc;
2364 
2365 	retval = 0;
2366 
2367 	switch (cmd) {
2368 	case CTL_IO: {
2369 		union ctl_io *io;
2370 		void *pool_tmp;
2371 
2372 		/*
2373 		 * If we haven't been "enabled", don't allow any SCSI I/O
2374 		 * to this FETD.
2375 		 */
2376 		if ((softc->ioctl_info.flags & CTL_IOCTL_FLAG_ENABLED) == 0) {
2377 			retval = EPERM;
2378 			break;
2379 		}
2380 
2381 		io = ctl_alloc_io(softc->ioctl_info.port.ctl_pool_ref);
2382 
2383 		/*
2384 		 * Need to save the pool reference so it doesn't get
2385 		 * spammed by the user's ctl_io.
2386 		 */
2387 		pool_tmp = io->io_hdr.pool;
2388 		memcpy(io, (void *)addr, sizeof(*io));
2389 		io->io_hdr.pool = pool_tmp;
2390 
2391 		/*
2392 		 * No status yet, so make sure the status is set properly.
2393 		 */
2394 		io->io_hdr.status = CTL_STATUS_NONE;
2395 
2396 		/*
2397 		 * The user sets the initiator ID, target and LUN IDs.
2398 		 */
2399 		io->io_hdr.nexus.targ_port = softc->ioctl_info.port.targ_port;
2400 		io->io_hdr.flags |= CTL_FLAG_USER_REQ;
2401 		if ((io->io_hdr.io_type == CTL_IO_SCSI)
2402 		 && (io->scsiio.tag_type != CTL_TAG_UNTAGGED))
2403 			io->scsiio.tag_num = softc->ioctl_info.cur_tag_num++;
2404 
2405 		retval = ctl_ioctl_submit_wait(io);
2406 
2407 		if (retval != 0) {
2408 			ctl_free_io(io);
2409 			break;
2410 		}
2411 
2412 		memcpy((void *)addr, io, sizeof(*io));
2413 
2414 		/* return this to our pool */
2415 		ctl_free_io(io);
2416 
2417 		break;
2418 	}
2419 	case CTL_ENABLE_PORT:
2420 	case CTL_DISABLE_PORT:
2421 	case CTL_SET_PORT_WWNS: {
2422 		struct ctl_port *port;
2423 		struct ctl_port_entry *entry;
2424 
2425 		entry = (struct ctl_port_entry *)addr;
2426 
2427 		mtx_lock(&softc->ctl_lock);
2428 		STAILQ_FOREACH(port, &softc->port_list, links) {
2429 			int action, done;
2430 
2431 			action = 0;
2432 			done = 0;
2433 
2434 			if ((entry->port_type == CTL_PORT_NONE)
2435 			 && (entry->targ_port == port->targ_port)) {
2436 				/*
2437 				 * If the user only wants to enable or
2438 				 * disable or set WWNs on a specific port,
2439 				 * do the operation and we're done.
2440 				 */
2441 				action = 1;
2442 				done = 1;
2443 			} else if (entry->port_type & port->port_type) {
2444 				/*
2445 				 * Compare the user's type mask with the
2446 				 * particular frontend type to see if we
2447 				 * have a match.
2448 				 */
2449 				action = 1;
2450 				done = 0;
2451 
2452 				/*
2453 				 * Make sure the user isn't trying to set
2454 				 * WWNs on multiple ports at the same time.
2455 				 */
2456 				if (cmd == CTL_SET_PORT_WWNS) {
2457 					printf("%s: Can't set WWNs on "
2458 					       "multiple ports\n", __func__);
2459 					retval = EINVAL;
2460 					break;
2461 				}
2462 			}
2463 			if (action != 0) {
2464 				/*
2465 				 * XXX KDM we have to drop the lock here,
2466 				 * because the online/offline operations
2467 				 * can potentially block.  We need to
2468 				 * reference count the frontends so they
2469 				 * can't go away,
2470 				 */
2471 				mtx_unlock(&softc->ctl_lock);
2472 
2473 				if (cmd == CTL_ENABLE_PORT) {
2474 					ctl_port_online(port);
2475 				} else if (cmd == CTL_DISABLE_PORT) {
2476 					ctl_port_offline(port);
2477 				}
2478 
2479 				mtx_lock(&softc->ctl_lock);
2480 
2481 				if (cmd == CTL_SET_PORT_WWNS)
2482 					ctl_port_set_wwns(port,
2483 					    (entry->flags & CTL_PORT_WWNN_VALID) ?
2484 					    1 : 0, entry->wwnn,
2485 					    (entry->flags & CTL_PORT_WWPN_VALID) ?
2486 					    1 : 0, entry->wwpn);
2487 			}
2488 			if (done != 0)
2489 				break;
2490 		}
2491 		mtx_unlock(&softc->ctl_lock);
2492 		break;
2493 	}
2494 	case CTL_GET_PORT_LIST: {
2495 		struct ctl_port *port;
2496 		struct ctl_port_list *list;
2497 		int i;
2498 
2499 		list = (struct ctl_port_list *)addr;
2500 
2501 		if (list->alloc_len != (list->alloc_num *
2502 		    sizeof(struct ctl_port_entry))) {
2503 			printf("%s: CTL_GET_PORT_LIST: alloc_len %u != "
2504 			       "alloc_num %u * sizeof(struct ctl_port_entry) "
2505 			       "%zu\n", __func__, list->alloc_len,
2506 			       list->alloc_num, sizeof(struct ctl_port_entry));
2507 			retval = EINVAL;
2508 			break;
2509 		}
2510 		list->fill_len = 0;
2511 		list->fill_num = 0;
2512 		list->dropped_num = 0;
2513 		i = 0;
2514 		mtx_lock(&softc->ctl_lock);
2515 		STAILQ_FOREACH(port, &softc->port_list, links) {
2516 			struct ctl_port_entry entry, *list_entry;
2517 
2518 			if (list->fill_num >= list->alloc_num) {
2519 				list->dropped_num++;
2520 				continue;
2521 			}
2522 
2523 			entry.port_type = port->port_type;
2524 			strlcpy(entry.port_name, port->port_name,
2525 				sizeof(entry.port_name));
2526 			entry.targ_port = port->targ_port;
2527 			entry.physical_port = port->physical_port;
2528 			entry.virtual_port = port->virtual_port;
2529 			entry.wwnn = port->wwnn;
2530 			entry.wwpn = port->wwpn;
2531 			if (port->status & CTL_PORT_STATUS_ONLINE)
2532 				entry.online = 1;
2533 			else
2534 				entry.online = 0;
2535 
2536 			list_entry = &list->entries[i];
2537 
2538 			retval = copyout(&entry, list_entry, sizeof(entry));
2539 			if (retval != 0) {
2540 				printf("%s: CTL_GET_PORT_LIST: copyout "
2541 				       "returned %d\n", __func__, retval);
2542 				break;
2543 			}
2544 			i++;
2545 			list->fill_num++;
2546 			list->fill_len += sizeof(entry);
2547 		}
2548 		mtx_unlock(&softc->ctl_lock);
2549 
2550 		/*
2551 		 * If this is non-zero, we had a copyout fault, so there's
2552 		 * probably no point in attempting to set the status inside
2553 		 * the structure.
2554 		 */
2555 		if (retval != 0)
2556 			break;
2557 
2558 		if (list->dropped_num > 0)
2559 			list->status = CTL_PORT_LIST_NEED_MORE_SPACE;
2560 		else
2561 			list->status = CTL_PORT_LIST_OK;
2562 		break;
2563 	}
2564 	case CTL_DUMP_OOA: {
2565 		struct ctl_lun *lun;
2566 		union ctl_io *io;
2567 		char printbuf[128];
2568 		struct sbuf sb;
2569 
2570 		mtx_lock(&softc->ctl_lock);
2571 		printf("Dumping OOA queues:\n");
2572 		STAILQ_FOREACH(lun, &softc->lun_list, links) {
2573 			mtx_lock(&lun->lun_lock);
2574 			for (io = (union ctl_io *)TAILQ_FIRST(
2575 			     &lun->ooa_queue); io != NULL;
2576 			     io = (union ctl_io *)TAILQ_NEXT(&io->io_hdr,
2577 			     ooa_links)) {
2578 				sbuf_new(&sb, printbuf, sizeof(printbuf),
2579 					 SBUF_FIXEDLEN);
2580 				sbuf_printf(&sb, "LUN %jd tag 0x%04x%s%s%s%s: ",
2581 					    (intmax_t)lun->lun,
2582 					    io->scsiio.tag_num,
2583 					    (io->io_hdr.flags &
2584 					    CTL_FLAG_BLOCKED) ? "" : " BLOCKED",
2585 					    (io->io_hdr.flags &
2586 					    CTL_FLAG_DMA_INPROG) ? " DMA" : "",
2587 					    (io->io_hdr.flags &
2588 					    CTL_FLAG_ABORT) ? " ABORT" : "",
2589 			                    (io->io_hdr.flags &
2590 		                        CTL_FLAG_IS_WAS_ON_RTR) ? " RTR" : "");
2591 				ctl_scsi_command_string(&io->scsiio, NULL, &sb);
2592 				sbuf_finish(&sb);
2593 				printf("%s\n", sbuf_data(&sb));
2594 			}
2595 			mtx_unlock(&lun->lun_lock);
2596 		}
2597 		printf("OOA queues dump done\n");
2598 		mtx_unlock(&softc->ctl_lock);
2599 		break;
2600 	}
2601 	case CTL_GET_OOA: {
2602 		struct ctl_lun *lun;
2603 		struct ctl_ooa *ooa_hdr;
2604 		struct ctl_ooa_entry *entries;
2605 		uint32_t cur_fill_num;
2606 
2607 		ooa_hdr = (struct ctl_ooa *)addr;
2608 
2609 		if ((ooa_hdr->alloc_len == 0)
2610 		 || (ooa_hdr->alloc_num == 0)) {
2611 			printf("%s: CTL_GET_OOA: alloc len %u and alloc num %u "
2612 			       "must be non-zero\n", __func__,
2613 			       ooa_hdr->alloc_len, ooa_hdr->alloc_num);
2614 			retval = EINVAL;
2615 			break;
2616 		}
2617 
2618 		if (ooa_hdr->alloc_len != (ooa_hdr->alloc_num *
2619 		    sizeof(struct ctl_ooa_entry))) {
2620 			printf("%s: CTL_GET_OOA: alloc len %u must be alloc "
2621 			       "num %d * sizeof(struct ctl_ooa_entry) %zd\n",
2622 			       __func__, ooa_hdr->alloc_len,
2623 			       ooa_hdr->alloc_num,sizeof(struct ctl_ooa_entry));
2624 			retval = EINVAL;
2625 			break;
2626 		}
2627 
2628 		entries = malloc(ooa_hdr->alloc_len, M_CTL, M_WAITOK | M_ZERO);
2629 		if (entries == NULL) {
2630 			printf("%s: could not allocate %d bytes for OOA "
2631 			       "dump\n", __func__, ooa_hdr->alloc_len);
2632 			retval = ENOMEM;
2633 			break;
2634 		}
2635 
2636 		mtx_lock(&softc->ctl_lock);
2637 		if (((ooa_hdr->flags & CTL_OOA_FLAG_ALL_LUNS) == 0)
2638 		 && ((ooa_hdr->lun_num >= CTL_MAX_LUNS)
2639 		  || (softc->ctl_luns[ooa_hdr->lun_num] == NULL))) {
2640 			mtx_unlock(&softc->ctl_lock);
2641 			free(entries, M_CTL);
2642 			printf("%s: CTL_GET_OOA: invalid LUN %ju\n",
2643 			       __func__, (uintmax_t)ooa_hdr->lun_num);
2644 			retval = EINVAL;
2645 			break;
2646 		}
2647 
2648 		cur_fill_num = 0;
2649 
2650 		if (ooa_hdr->flags & CTL_OOA_FLAG_ALL_LUNS) {
2651 			STAILQ_FOREACH(lun, &softc->lun_list, links) {
2652 				retval = ctl_ioctl_fill_ooa(lun, &cur_fill_num,
2653 					ooa_hdr, entries);
2654 				if (retval != 0)
2655 					break;
2656 			}
2657 			if (retval != 0) {
2658 				mtx_unlock(&softc->ctl_lock);
2659 				free(entries, M_CTL);
2660 				break;
2661 			}
2662 		} else {
2663 			lun = softc->ctl_luns[ooa_hdr->lun_num];
2664 
2665 			retval = ctl_ioctl_fill_ooa(lun, &cur_fill_num,ooa_hdr,
2666 						    entries);
2667 		}
2668 		mtx_unlock(&softc->ctl_lock);
2669 
2670 		ooa_hdr->fill_num = min(cur_fill_num, ooa_hdr->alloc_num);
2671 		ooa_hdr->fill_len = ooa_hdr->fill_num *
2672 			sizeof(struct ctl_ooa_entry);
2673 		retval = copyout(entries, ooa_hdr->entries, ooa_hdr->fill_len);
2674 		if (retval != 0) {
2675 			printf("%s: error copying out %d bytes for OOA dump\n",
2676 			       __func__, ooa_hdr->fill_len);
2677 		}
2678 
2679 		getbintime(&ooa_hdr->cur_bt);
2680 
2681 		if (cur_fill_num > ooa_hdr->alloc_num) {
2682 			ooa_hdr->dropped_num = cur_fill_num -ooa_hdr->alloc_num;
2683 			ooa_hdr->status = CTL_OOA_NEED_MORE_SPACE;
2684 		} else {
2685 			ooa_hdr->dropped_num = 0;
2686 			ooa_hdr->status = CTL_OOA_OK;
2687 		}
2688 
2689 		free(entries, M_CTL);
2690 		break;
2691 	}
2692 	case CTL_CHECK_OOA: {
2693 		union ctl_io *io;
2694 		struct ctl_lun *lun;
2695 		struct ctl_ooa_info *ooa_info;
2696 
2697 
2698 		ooa_info = (struct ctl_ooa_info *)addr;
2699 
2700 		if (ooa_info->lun_id >= CTL_MAX_LUNS) {
2701 			ooa_info->status = CTL_OOA_INVALID_LUN;
2702 			break;
2703 		}
2704 		mtx_lock(&softc->ctl_lock);
2705 		lun = softc->ctl_luns[ooa_info->lun_id];
2706 		if (lun == NULL) {
2707 			mtx_unlock(&softc->ctl_lock);
2708 			ooa_info->status = CTL_OOA_INVALID_LUN;
2709 			break;
2710 		}
2711 		mtx_lock(&lun->lun_lock);
2712 		mtx_unlock(&softc->ctl_lock);
2713 		ooa_info->num_entries = 0;
2714 		for (io = (union ctl_io *)TAILQ_FIRST(&lun->ooa_queue);
2715 		     io != NULL; io = (union ctl_io *)TAILQ_NEXT(
2716 		     &io->io_hdr, ooa_links)) {
2717 			ooa_info->num_entries++;
2718 		}
2719 		mtx_unlock(&lun->lun_lock);
2720 
2721 		ooa_info->status = CTL_OOA_SUCCESS;
2722 
2723 		break;
2724 	}
2725 	case CTL_HARD_START:
2726 	case CTL_HARD_STOP: {
2727 		struct ctl_fe_ioctl_startstop_info ss_info;
2728 		struct cfi_metatask *metatask;
2729 		struct mtx hs_mtx;
2730 
2731 		mtx_init(&hs_mtx, "HS Mutex", NULL, MTX_DEF);
2732 
2733 		cv_init(&ss_info.sem, "hard start/stop cv" );
2734 
2735 		metatask = cfi_alloc_metatask(/*can_wait*/ 1);
2736 		if (metatask == NULL) {
2737 			retval = ENOMEM;
2738 			mtx_destroy(&hs_mtx);
2739 			break;
2740 		}
2741 
2742 		if (cmd == CTL_HARD_START)
2743 			metatask->tasktype = CFI_TASK_STARTUP;
2744 		else
2745 			metatask->tasktype = CFI_TASK_SHUTDOWN;
2746 
2747 		metatask->callback = ctl_ioctl_hard_startstop_callback;
2748 		metatask->callback_arg = &ss_info;
2749 
2750 		cfi_action(metatask);
2751 
2752 		/* Wait for the callback */
2753 		mtx_lock(&hs_mtx);
2754 		cv_wait_sig(&ss_info.sem, &hs_mtx);
2755 		mtx_unlock(&hs_mtx);
2756 
2757 		/*
2758 		 * All information has been copied from the metatask by the
2759 		 * time cv_broadcast() is called, so we free the metatask here.
2760 		 */
2761 		cfi_free_metatask(metatask);
2762 
2763 		memcpy((void *)addr, &ss_info.hs_info, sizeof(ss_info.hs_info));
2764 
2765 		mtx_destroy(&hs_mtx);
2766 		break;
2767 	}
2768 	case CTL_BBRREAD: {
2769 		struct ctl_bbrread_info *bbr_info;
2770 		struct ctl_fe_ioctl_bbrread_info fe_bbr_info;
2771 		struct mtx bbr_mtx;
2772 		struct cfi_metatask *metatask;
2773 
2774 		bbr_info = (struct ctl_bbrread_info *)addr;
2775 
2776 		bzero(&fe_bbr_info, sizeof(fe_bbr_info));
2777 
2778 		bzero(&bbr_mtx, sizeof(bbr_mtx));
2779 		mtx_init(&bbr_mtx, "BBR Mutex", NULL, MTX_DEF);
2780 
2781 		fe_bbr_info.bbr_info = bbr_info;
2782 		fe_bbr_info.lock = &bbr_mtx;
2783 
2784 		cv_init(&fe_bbr_info.sem, "BBR read cv");
2785 		metatask = cfi_alloc_metatask(/*can_wait*/ 1);
2786 
2787 		if (metatask == NULL) {
2788 			mtx_destroy(&bbr_mtx);
2789 			cv_destroy(&fe_bbr_info.sem);
2790 			retval = ENOMEM;
2791 			break;
2792 		}
2793 		metatask->tasktype = CFI_TASK_BBRREAD;
2794 		metatask->callback = ctl_ioctl_bbrread_callback;
2795 		metatask->callback_arg = &fe_bbr_info;
2796 		metatask->taskinfo.bbrread.lun_num = bbr_info->lun_num;
2797 		metatask->taskinfo.bbrread.lba = bbr_info->lba;
2798 		metatask->taskinfo.bbrread.len = bbr_info->len;
2799 
2800 		cfi_action(metatask);
2801 
2802 		mtx_lock(&bbr_mtx);
2803 		while (fe_bbr_info.wakeup_done == 0)
2804 			cv_wait_sig(&fe_bbr_info.sem, &bbr_mtx);
2805 		mtx_unlock(&bbr_mtx);
2806 
2807 		bbr_info->status = metatask->status;
2808 		bbr_info->bbr_status = metatask->taskinfo.bbrread.status;
2809 		bbr_info->scsi_status = metatask->taskinfo.bbrread.scsi_status;
2810 		memcpy(&bbr_info->sense_data,
2811 		       &metatask->taskinfo.bbrread.sense_data,
2812 		       MIN(sizeof(bbr_info->sense_data),
2813 			   sizeof(metatask->taskinfo.bbrread.sense_data)));
2814 
2815 		cfi_free_metatask(metatask);
2816 
2817 		mtx_destroy(&bbr_mtx);
2818 		cv_destroy(&fe_bbr_info.sem);
2819 
2820 		break;
2821 	}
2822 	case CTL_DELAY_IO: {
2823 		struct ctl_io_delay_info *delay_info;
2824 #ifdef CTL_IO_DELAY
2825 		struct ctl_lun *lun;
2826 #endif /* CTL_IO_DELAY */
2827 
2828 		delay_info = (struct ctl_io_delay_info *)addr;
2829 
2830 #ifdef CTL_IO_DELAY
2831 		mtx_lock(&softc->ctl_lock);
2832 
2833 		if ((delay_info->lun_id >= CTL_MAX_LUNS)
2834 		 || (softc->ctl_luns[delay_info->lun_id] == NULL)) {
2835 			delay_info->status = CTL_DELAY_STATUS_INVALID_LUN;
2836 		} else {
2837 			lun = softc->ctl_luns[delay_info->lun_id];
2838 			mtx_lock(&lun->lun_lock);
2839 
2840 			delay_info->status = CTL_DELAY_STATUS_OK;
2841 
2842 			switch (delay_info->delay_type) {
2843 			case CTL_DELAY_TYPE_CONT:
2844 				break;
2845 			case CTL_DELAY_TYPE_ONESHOT:
2846 				break;
2847 			default:
2848 				delay_info->status =
2849 					CTL_DELAY_STATUS_INVALID_TYPE;
2850 				break;
2851 			}
2852 
2853 			switch (delay_info->delay_loc) {
2854 			case CTL_DELAY_LOC_DATAMOVE:
2855 				lun->delay_info.datamove_type =
2856 					delay_info->delay_type;
2857 				lun->delay_info.datamove_delay =
2858 					delay_info->delay_secs;
2859 				break;
2860 			case CTL_DELAY_LOC_DONE:
2861 				lun->delay_info.done_type =
2862 					delay_info->delay_type;
2863 				lun->delay_info.done_delay =
2864 					delay_info->delay_secs;
2865 				break;
2866 			default:
2867 				delay_info->status =
2868 					CTL_DELAY_STATUS_INVALID_LOC;
2869 				break;
2870 			}
2871 			mtx_unlock(&lun->lun_lock);
2872 		}
2873 
2874 		mtx_unlock(&softc->ctl_lock);
2875 #else
2876 		delay_info->status = CTL_DELAY_STATUS_NOT_IMPLEMENTED;
2877 #endif /* CTL_IO_DELAY */
2878 		break;
2879 	}
2880 	case CTL_REALSYNC_SET: {
2881 		int *syncstate;
2882 
2883 		syncstate = (int *)addr;
2884 
2885 		mtx_lock(&softc->ctl_lock);
2886 		switch (*syncstate) {
2887 		case 0:
2888 			softc->flags &= ~CTL_FLAG_REAL_SYNC;
2889 			break;
2890 		case 1:
2891 			softc->flags |= CTL_FLAG_REAL_SYNC;
2892 			break;
2893 		default:
2894 			retval = EINVAL;
2895 			break;
2896 		}
2897 		mtx_unlock(&softc->ctl_lock);
2898 		break;
2899 	}
2900 	case CTL_REALSYNC_GET: {
2901 		int *syncstate;
2902 
2903 		syncstate = (int*)addr;
2904 
2905 		mtx_lock(&softc->ctl_lock);
2906 		if (softc->flags & CTL_FLAG_REAL_SYNC)
2907 			*syncstate = 1;
2908 		else
2909 			*syncstate = 0;
2910 		mtx_unlock(&softc->ctl_lock);
2911 
2912 		break;
2913 	}
2914 	case CTL_SETSYNC:
2915 	case CTL_GETSYNC: {
2916 		struct ctl_sync_info *sync_info;
2917 		struct ctl_lun *lun;
2918 
2919 		sync_info = (struct ctl_sync_info *)addr;
2920 
2921 		mtx_lock(&softc->ctl_lock);
2922 		lun = softc->ctl_luns[sync_info->lun_id];
2923 		if (lun == NULL) {
2924 			mtx_unlock(&softc->ctl_lock);
2925 			sync_info->status = CTL_GS_SYNC_NO_LUN;
2926 		}
2927 		/*
2928 		 * Get or set the sync interval.  We're not bounds checking
2929 		 * in the set case, hopefully the user won't do something
2930 		 * silly.
2931 		 */
2932 		mtx_lock(&lun->lun_lock);
2933 		mtx_unlock(&softc->ctl_lock);
2934 		if (cmd == CTL_GETSYNC)
2935 			sync_info->sync_interval = lun->sync_interval;
2936 		else
2937 			lun->sync_interval = sync_info->sync_interval;
2938 		mtx_unlock(&lun->lun_lock);
2939 
2940 		sync_info->status = CTL_GS_SYNC_OK;
2941 
2942 		break;
2943 	}
2944 	case CTL_GETSTATS: {
2945 		struct ctl_stats *stats;
2946 		struct ctl_lun *lun;
2947 		int i;
2948 
2949 		stats = (struct ctl_stats *)addr;
2950 
2951 		if ((sizeof(struct ctl_lun_io_stats) * softc->num_luns) >
2952 		     stats->alloc_len) {
2953 			stats->status = CTL_SS_NEED_MORE_SPACE;
2954 			stats->num_luns = softc->num_luns;
2955 			break;
2956 		}
2957 		/*
2958 		 * XXX KDM no locking here.  If the LUN list changes,
2959 		 * things can blow up.
2960 		 */
2961 		for (i = 0, lun = STAILQ_FIRST(&softc->lun_list); lun != NULL;
2962 		     i++, lun = STAILQ_NEXT(lun, links)) {
2963 			retval = copyout(&lun->stats, &stats->lun_stats[i],
2964 					 sizeof(lun->stats));
2965 			if (retval != 0)
2966 				break;
2967 		}
2968 		stats->num_luns = softc->num_luns;
2969 		stats->fill_len = sizeof(struct ctl_lun_io_stats) *
2970 				 softc->num_luns;
2971 		stats->status = CTL_SS_OK;
2972 #ifdef CTL_TIME_IO
2973 		stats->flags = CTL_STATS_FLAG_TIME_VALID;
2974 #else
2975 		stats->flags = CTL_STATS_FLAG_NONE;
2976 #endif
2977 		getnanouptime(&stats->timestamp);
2978 		break;
2979 	}
2980 	case CTL_ERROR_INJECT: {
2981 		struct ctl_error_desc *err_desc, *new_err_desc;
2982 		struct ctl_lun *lun;
2983 
2984 		err_desc = (struct ctl_error_desc *)addr;
2985 
2986 		new_err_desc = malloc(sizeof(*new_err_desc), M_CTL,
2987 				      M_WAITOK | M_ZERO);
2988 		bcopy(err_desc, new_err_desc, sizeof(*new_err_desc));
2989 
2990 		mtx_lock(&softc->ctl_lock);
2991 		lun = softc->ctl_luns[err_desc->lun_id];
2992 		if (lun == NULL) {
2993 			mtx_unlock(&softc->ctl_lock);
2994 			free(new_err_desc, M_CTL);
2995 			printf("%s: CTL_ERROR_INJECT: invalid LUN %ju\n",
2996 			       __func__, (uintmax_t)err_desc->lun_id);
2997 			retval = EINVAL;
2998 			break;
2999 		}
3000 		mtx_lock(&lun->lun_lock);
3001 		mtx_unlock(&softc->ctl_lock);
3002 
3003 		/*
3004 		 * We could do some checking here to verify the validity
3005 		 * of the request, but given the complexity of error
3006 		 * injection requests, the checking logic would be fairly
3007 		 * complex.
3008 		 *
3009 		 * For now, if the request is invalid, it just won't get
3010 		 * executed and might get deleted.
3011 		 */
3012 		STAILQ_INSERT_TAIL(&lun->error_list, new_err_desc, links);
3013 
3014 		/*
3015 		 * XXX KDM check to make sure the serial number is unique,
3016 		 * in case we somehow manage to wrap.  That shouldn't
3017 		 * happen for a very long time, but it's the right thing to
3018 		 * do.
3019 		 */
3020 		new_err_desc->serial = lun->error_serial;
3021 		err_desc->serial = lun->error_serial;
3022 		lun->error_serial++;
3023 
3024 		mtx_unlock(&lun->lun_lock);
3025 		break;
3026 	}
3027 	case CTL_ERROR_INJECT_DELETE: {
3028 		struct ctl_error_desc *delete_desc, *desc, *desc2;
3029 		struct ctl_lun *lun;
3030 		int delete_done;
3031 
3032 		delete_desc = (struct ctl_error_desc *)addr;
3033 		delete_done = 0;
3034 
3035 		mtx_lock(&softc->ctl_lock);
3036 		lun = softc->ctl_luns[delete_desc->lun_id];
3037 		if (lun == NULL) {
3038 			mtx_unlock(&softc->ctl_lock);
3039 			printf("%s: CTL_ERROR_INJECT_DELETE: invalid LUN %ju\n",
3040 			       __func__, (uintmax_t)delete_desc->lun_id);
3041 			retval = EINVAL;
3042 			break;
3043 		}
3044 		mtx_lock(&lun->lun_lock);
3045 		mtx_unlock(&softc->ctl_lock);
3046 		STAILQ_FOREACH_SAFE(desc, &lun->error_list, links, desc2) {
3047 			if (desc->serial != delete_desc->serial)
3048 				continue;
3049 
3050 			STAILQ_REMOVE(&lun->error_list, desc, ctl_error_desc,
3051 				      links);
3052 			free(desc, M_CTL);
3053 			delete_done = 1;
3054 		}
3055 		mtx_unlock(&lun->lun_lock);
3056 		if (delete_done == 0) {
3057 			printf("%s: CTL_ERROR_INJECT_DELETE: can't find "
3058 			       "error serial %ju on LUN %u\n", __func__,
3059 			       delete_desc->serial, delete_desc->lun_id);
3060 			retval = EINVAL;
3061 			break;
3062 		}
3063 		break;
3064 	}
3065 	case CTL_DUMP_STRUCTS: {
3066 		int i, j, k;
3067 		struct ctl_port *port;
3068 		struct ctl_frontend *fe;
3069 
3070 		mtx_lock(&softc->ctl_lock);
3071 		printf("CTL Persistent Reservation information start:\n");
3072 		for (i = 0; i < CTL_MAX_LUNS; i++) {
3073 			struct ctl_lun *lun;
3074 
3075 			lun = softc->ctl_luns[i];
3076 
3077 			if ((lun == NULL)
3078 			 || ((lun->flags & CTL_LUN_DISABLED) != 0))
3079 				continue;
3080 
3081 			for (j = 0; j < (CTL_MAX_PORTS * 2); j++) {
3082 				if (lun->pr_keys[j] == NULL)
3083 					continue;
3084 				for (k = 0; k < CTL_MAX_INIT_PER_PORT; k++){
3085 					if (lun->pr_keys[j][k] == 0)
3086 						continue;
3087 					printf("  LUN %d port %d iid %d key "
3088 					       "%#jx\n", i, j, k,
3089 					       (uintmax_t)lun->pr_keys[j][k]);
3090 				}
3091 			}
3092 		}
3093 		printf("CTL Persistent Reservation information end\n");
3094 		printf("CTL Ports:\n");
3095 		STAILQ_FOREACH(port, &softc->port_list, links) {
3096 			printf("  Port %d '%s' Frontend '%s' Type %u pp %d vp %d WWNN "
3097 			       "%#jx WWPN %#jx\n", port->targ_port, port->port_name,
3098 			       port->frontend->name, port->port_type,
3099 			       port->physical_port, port->virtual_port,
3100 			       (uintmax_t)port->wwnn, (uintmax_t)port->wwpn);
3101 			for (j = 0; j < CTL_MAX_INIT_PER_PORT; j++) {
3102 				if (port->wwpn_iid[j].in_use == 0 &&
3103 				    port->wwpn_iid[j].wwpn == 0 &&
3104 				    port->wwpn_iid[j].name == NULL)
3105 					continue;
3106 
3107 				printf("    iid %u use %d WWPN %#jx '%s'\n",
3108 				    j, port->wwpn_iid[j].in_use,
3109 				    (uintmax_t)port->wwpn_iid[j].wwpn,
3110 				    port->wwpn_iid[j].name);
3111 			}
3112 		}
3113 		printf("CTL Port information end\n");
3114 		mtx_unlock(&softc->ctl_lock);
3115 		/*
3116 		 * XXX KDM calling this without a lock.  We'd likely want
3117 		 * to drop the lock before calling the frontend's dump
3118 		 * routine anyway.
3119 		 */
3120 		printf("CTL Frontends:\n");
3121 		STAILQ_FOREACH(fe, &softc->fe_list, links) {
3122 			printf("  Frontend '%s'\n", fe->name);
3123 			if (fe->fe_dump != NULL)
3124 				fe->fe_dump();
3125 		}
3126 		printf("CTL Frontend information end\n");
3127 		break;
3128 	}
3129 	case CTL_LUN_REQ: {
3130 		struct ctl_lun_req *lun_req;
3131 		struct ctl_backend_driver *backend;
3132 
3133 		lun_req = (struct ctl_lun_req *)addr;
3134 
3135 		backend = ctl_backend_find(lun_req->backend);
3136 		if (backend == NULL) {
3137 			lun_req->status = CTL_LUN_ERROR;
3138 			snprintf(lun_req->error_str,
3139 				 sizeof(lun_req->error_str),
3140 				 "Backend \"%s\" not found.",
3141 				 lun_req->backend);
3142 			break;
3143 		}
3144 		if (lun_req->num_be_args > 0) {
3145 			lun_req->kern_be_args = ctl_copyin_args(
3146 				lun_req->num_be_args,
3147 				lun_req->be_args,
3148 				lun_req->error_str,
3149 				sizeof(lun_req->error_str));
3150 			if (lun_req->kern_be_args == NULL) {
3151 				lun_req->status = CTL_LUN_ERROR;
3152 				break;
3153 			}
3154 		}
3155 
3156 		retval = backend->ioctl(dev, cmd, addr, flag, td);
3157 
3158 		if (lun_req->num_be_args > 0) {
3159 			ctl_copyout_args(lun_req->num_be_args,
3160 				      lun_req->kern_be_args);
3161 			ctl_free_args(lun_req->num_be_args,
3162 				      lun_req->kern_be_args);
3163 		}
3164 		break;
3165 	}
3166 	case CTL_LUN_LIST: {
3167 		struct sbuf *sb;
3168 		struct ctl_lun *lun;
3169 		struct ctl_lun_list *list;
3170 		struct ctl_option *opt;
3171 
3172 		list = (struct ctl_lun_list *)addr;
3173 
3174 		/*
3175 		 * Allocate a fixed length sbuf here, based on the length
3176 		 * of the user's buffer.  We could allocate an auto-extending
3177 		 * buffer, and then tell the user how much larger our
3178 		 * amount of data is than his buffer, but that presents
3179 		 * some problems:
3180 		 *
3181 		 * 1.  The sbuf(9) routines use a blocking malloc, and so
3182 		 *     we can't hold a lock while calling them with an
3183 		 *     auto-extending buffer.
3184  		 *
3185 		 * 2.  There is not currently a LUN reference counting
3186 		 *     mechanism, outside of outstanding transactions on
3187 		 *     the LUN's OOA queue.  So a LUN could go away on us
3188 		 *     while we're getting the LUN number, backend-specific
3189 		 *     information, etc.  Thus, given the way things
3190 		 *     currently work, we need to hold the CTL lock while
3191 		 *     grabbing LUN information.
3192 		 *
3193 		 * So, from the user's standpoint, the best thing to do is
3194 		 * allocate what he thinks is a reasonable buffer length,
3195 		 * and then if he gets a CTL_LUN_LIST_NEED_MORE_SPACE error,
3196 		 * double the buffer length and try again.  (And repeat
3197 		 * that until he succeeds.)
3198 		 */
3199 		sb = sbuf_new(NULL, NULL, list->alloc_len, SBUF_FIXEDLEN);
3200 		if (sb == NULL) {
3201 			list->status = CTL_LUN_LIST_ERROR;
3202 			snprintf(list->error_str, sizeof(list->error_str),
3203 				 "Unable to allocate %d bytes for LUN list",
3204 				 list->alloc_len);
3205 			break;
3206 		}
3207 
3208 		sbuf_printf(sb, "<ctllunlist>\n");
3209 
3210 		mtx_lock(&softc->ctl_lock);
3211 		STAILQ_FOREACH(lun, &softc->lun_list, links) {
3212 			mtx_lock(&lun->lun_lock);
3213 			retval = sbuf_printf(sb, "<lun id=\"%ju\">\n",
3214 					     (uintmax_t)lun->lun);
3215 
3216 			/*
3217 			 * Bail out as soon as we see that we've overfilled
3218 			 * the buffer.
3219 			 */
3220 			if (retval != 0)
3221 				break;
3222 
3223 			retval = sbuf_printf(sb, "\t<backend_type>%s"
3224 					     "</backend_type>\n",
3225 					     (lun->backend == NULL) ?  "none" :
3226 					     lun->backend->name);
3227 
3228 			if (retval != 0)
3229 				break;
3230 
3231 			retval = sbuf_printf(sb, "\t<lun_type>%d</lun_type>\n",
3232 					     lun->be_lun->lun_type);
3233 
3234 			if (retval != 0)
3235 				break;
3236 
3237 			if (lun->backend == NULL) {
3238 				retval = sbuf_printf(sb, "</lun>\n");
3239 				if (retval != 0)
3240 					break;
3241 				continue;
3242 			}
3243 
3244 			retval = sbuf_printf(sb, "\t<size>%ju</size>\n",
3245 					     (lun->be_lun->maxlba > 0) ?
3246 					     lun->be_lun->maxlba + 1 : 0);
3247 
3248 			if (retval != 0)
3249 				break;
3250 
3251 			retval = sbuf_printf(sb, "\t<blocksize>%u</blocksize>\n",
3252 					     lun->be_lun->blocksize);
3253 
3254 			if (retval != 0)
3255 				break;
3256 
3257 			retval = sbuf_printf(sb, "\t<serial_number>");
3258 
3259 			if (retval != 0)
3260 				break;
3261 
3262 			retval = ctl_sbuf_printf_esc(sb,
3263 			    lun->be_lun->serial_num,
3264 			    sizeof(lun->be_lun->serial_num));
3265 
3266 			if (retval != 0)
3267 				break;
3268 
3269 			retval = sbuf_printf(sb, "</serial_number>\n");
3270 
3271 			if (retval != 0)
3272 				break;
3273 
3274 			retval = sbuf_printf(sb, "\t<device_id>");
3275 
3276 			if (retval != 0)
3277 				break;
3278 
3279 			retval = ctl_sbuf_printf_esc(sb,
3280 			    lun->be_lun->device_id,
3281 			    sizeof(lun->be_lun->device_id));
3282 
3283 			if (retval != 0)
3284 				break;
3285 
3286 			retval = sbuf_printf(sb, "</device_id>\n");
3287 
3288 			if (retval != 0)
3289 				break;
3290 
3291 			if (lun->backend->lun_info != NULL) {
3292 				retval = lun->backend->lun_info(lun->be_lun->be_lun, sb);
3293 				if (retval != 0)
3294 					break;
3295 			}
3296 			STAILQ_FOREACH(opt, &lun->be_lun->options, links) {
3297 				retval = sbuf_printf(sb, "\t<%s>%s</%s>\n",
3298 				    opt->name, opt->value, opt->name);
3299 				if (retval != 0)
3300 					break;
3301 			}
3302 
3303 			retval = sbuf_printf(sb, "</lun>\n");
3304 
3305 			if (retval != 0)
3306 				break;
3307 			mtx_unlock(&lun->lun_lock);
3308 		}
3309 		if (lun != NULL)
3310 			mtx_unlock(&lun->lun_lock);
3311 		mtx_unlock(&softc->ctl_lock);
3312 
3313 		if ((retval != 0)
3314 		 || ((retval = sbuf_printf(sb, "</ctllunlist>\n")) != 0)) {
3315 			retval = 0;
3316 			sbuf_delete(sb);
3317 			list->status = CTL_LUN_LIST_NEED_MORE_SPACE;
3318 			snprintf(list->error_str, sizeof(list->error_str),
3319 				 "Out of space, %d bytes is too small",
3320 				 list->alloc_len);
3321 			break;
3322 		}
3323 
3324 		sbuf_finish(sb);
3325 
3326 		retval = copyout(sbuf_data(sb), list->lun_xml,
3327 				 sbuf_len(sb) + 1);
3328 
3329 		list->fill_len = sbuf_len(sb) + 1;
3330 		list->status = CTL_LUN_LIST_OK;
3331 		sbuf_delete(sb);
3332 		break;
3333 	}
3334 	case CTL_ISCSI: {
3335 		struct ctl_iscsi *ci;
3336 		struct ctl_frontend *fe;
3337 
3338 		ci = (struct ctl_iscsi *)addr;
3339 
3340 		fe = ctl_frontend_find("iscsi");
3341 		if (fe == NULL) {
3342 			ci->status = CTL_ISCSI_ERROR;
3343 			snprintf(ci->error_str, sizeof(ci->error_str),
3344 			    "Frontend \"iscsi\" not found.");
3345 			break;
3346 		}
3347 
3348 		retval = fe->ioctl(dev, cmd, addr, flag, td);
3349 		break;
3350 	}
3351 	case CTL_PORT_REQ: {
3352 		struct ctl_req *req;
3353 		struct ctl_frontend *fe;
3354 
3355 		req = (struct ctl_req *)addr;
3356 
3357 		fe = ctl_frontend_find(req->driver);
3358 		if (fe == NULL) {
3359 			req->status = CTL_LUN_ERROR;
3360 			snprintf(req->error_str, sizeof(req->error_str),
3361 			    "Frontend \"%s\" not found.", req->driver);
3362 			break;
3363 		}
3364 		if (req->num_args > 0) {
3365 			req->kern_args = ctl_copyin_args(req->num_args,
3366 			    req->args, req->error_str, sizeof(req->error_str));
3367 			if (req->kern_args == NULL) {
3368 				req->status = CTL_LUN_ERROR;
3369 				break;
3370 			}
3371 		}
3372 
3373 		retval = fe->ioctl(dev, cmd, addr, flag, td);
3374 
3375 		if (req->num_args > 0) {
3376 			ctl_copyout_args(req->num_args, req->kern_args);
3377 			ctl_free_args(req->num_args, req->kern_args);
3378 		}
3379 		break;
3380 	}
3381 	case CTL_PORT_LIST: {
3382 		struct sbuf *sb;
3383 		struct ctl_port *port;
3384 		struct ctl_lun_list *list;
3385 		struct ctl_option *opt;
3386 		int j;
3387 		uint32_t plun;
3388 
3389 		list = (struct ctl_lun_list *)addr;
3390 
3391 		sb = sbuf_new(NULL, NULL, list->alloc_len, SBUF_FIXEDLEN);
3392 		if (sb == NULL) {
3393 			list->status = CTL_LUN_LIST_ERROR;
3394 			snprintf(list->error_str, sizeof(list->error_str),
3395 				 "Unable to allocate %d bytes for LUN list",
3396 				 list->alloc_len);
3397 			break;
3398 		}
3399 
3400 		sbuf_printf(sb, "<ctlportlist>\n");
3401 
3402 		mtx_lock(&softc->ctl_lock);
3403 		STAILQ_FOREACH(port, &softc->port_list, links) {
3404 			retval = sbuf_printf(sb, "<targ_port id=\"%ju\">\n",
3405 					     (uintmax_t)port->targ_port);
3406 
3407 			/*
3408 			 * Bail out as soon as we see that we've overfilled
3409 			 * the buffer.
3410 			 */
3411 			if (retval != 0)
3412 				break;
3413 
3414 			retval = sbuf_printf(sb, "\t<frontend_type>%s"
3415 			    "</frontend_type>\n", port->frontend->name);
3416 			if (retval != 0)
3417 				break;
3418 
3419 			retval = sbuf_printf(sb, "\t<port_type>%d</port_type>\n",
3420 					     port->port_type);
3421 			if (retval != 0)
3422 				break;
3423 
3424 			retval = sbuf_printf(sb, "\t<online>%s</online>\n",
3425 			    (port->status & CTL_PORT_STATUS_ONLINE) ? "YES" : "NO");
3426 			if (retval != 0)
3427 				break;
3428 
3429 			retval = sbuf_printf(sb, "\t<port_name>%s</port_name>\n",
3430 			    port->port_name);
3431 			if (retval != 0)
3432 				break;
3433 
3434 			retval = sbuf_printf(sb, "\t<physical_port>%d</physical_port>\n",
3435 			    port->physical_port);
3436 			if (retval != 0)
3437 				break;
3438 
3439 			retval = sbuf_printf(sb, "\t<virtual_port>%d</virtual_port>\n",
3440 			    port->virtual_port);
3441 			if (retval != 0)
3442 				break;
3443 
3444 			if (port->target_devid != NULL) {
3445 				sbuf_printf(sb, "\t<target>");
3446 				ctl_id_sbuf(port->target_devid, sb);
3447 				sbuf_printf(sb, "</target>\n");
3448 			}
3449 
3450 			if (port->port_devid != NULL) {
3451 				sbuf_printf(sb, "\t<port>");
3452 				ctl_id_sbuf(port->port_devid, sb);
3453 				sbuf_printf(sb, "</port>\n");
3454 			}
3455 
3456 			if (port->port_info != NULL) {
3457 				retval = port->port_info(port->onoff_arg, sb);
3458 				if (retval != 0)
3459 					break;
3460 			}
3461 			STAILQ_FOREACH(opt, &port->options, links) {
3462 				retval = sbuf_printf(sb, "\t<%s>%s</%s>\n",
3463 				    opt->name, opt->value, opt->name);
3464 				if (retval != 0)
3465 					break;
3466 			}
3467 
3468 			if (port->lun_map != NULL) {
3469 				sbuf_printf(sb, "\t<lun_map>on</lun_map>\n");
3470 				for (j = 0; j < CTL_MAX_LUNS; j++) {
3471 					plun = ctl_lun_map_from_port(port, j);
3472 					if (plun >= CTL_MAX_LUNS)
3473 						continue;
3474 					sbuf_printf(sb,
3475 					    "\t<lun id=\"%u\">%u</lun>\n",
3476 					    j, plun);
3477 				}
3478 			}
3479 
3480 			for (j = 0; j < CTL_MAX_INIT_PER_PORT; j++) {
3481 				if (port->wwpn_iid[j].in_use == 0 ||
3482 				    (port->wwpn_iid[j].wwpn == 0 &&
3483 				     port->wwpn_iid[j].name == NULL))
3484 					continue;
3485 
3486 				if (port->wwpn_iid[j].name != NULL)
3487 					retval = sbuf_printf(sb,
3488 					    "\t<initiator id=\"%u\">%s</initiator>\n",
3489 					    j, port->wwpn_iid[j].name);
3490 				else
3491 					retval = sbuf_printf(sb,
3492 					    "\t<initiator id=\"%u\">naa.%08jx</initiator>\n",
3493 					    j, port->wwpn_iid[j].wwpn);
3494 				if (retval != 0)
3495 					break;
3496 			}
3497 			if (retval != 0)
3498 				break;
3499 
3500 			retval = sbuf_printf(sb, "</targ_port>\n");
3501 			if (retval != 0)
3502 				break;
3503 		}
3504 		mtx_unlock(&softc->ctl_lock);
3505 
3506 		if ((retval != 0)
3507 		 || ((retval = sbuf_printf(sb, "</ctlportlist>\n")) != 0)) {
3508 			retval = 0;
3509 			sbuf_delete(sb);
3510 			list->status = CTL_LUN_LIST_NEED_MORE_SPACE;
3511 			snprintf(list->error_str, sizeof(list->error_str),
3512 				 "Out of space, %d bytes is too small",
3513 				 list->alloc_len);
3514 			break;
3515 		}
3516 
3517 		sbuf_finish(sb);
3518 
3519 		retval = copyout(sbuf_data(sb), list->lun_xml,
3520 				 sbuf_len(sb) + 1);
3521 
3522 		list->fill_len = sbuf_len(sb) + 1;
3523 		list->status = CTL_LUN_LIST_OK;
3524 		sbuf_delete(sb);
3525 		break;
3526 	}
3527 	case CTL_LUN_MAP: {
3528 		struct ctl_lun_map *lm  = (struct ctl_lun_map *)addr;
3529 		struct ctl_port *port;
3530 
3531 		mtx_lock(&softc->ctl_lock);
3532 		if (lm->port >= CTL_MAX_PORTS ||
3533 		    (port = softc->ctl_ports[lm->port]) == NULL) {
3534 			mtx_unlock(&softc->ctl_lock);
3535 			return (ENXIO);
3536 		}
3537 		mtx_unlock(&softc->ctl_lock); // XXX: port_enable sleeps
3538 		if (lm->plun < CTL_MAX_LUNS) {
3539 			if (lm->lun == UINT32_MAX)
3540 				retval = ctl_lun_map_unset(port, lm->plun);
3541 			else if (lm->lun < CTL_MAX_LUNS &&
3542 			    softc->ctl_luns[lm->lun] != NULL)
3543 				retval = ctl_lun_map_set(port, lm->plun, lm->lun);
3544 			else
3545 				return (ENXIO);
3546 		} else if (lm->plun == UINT32_MAX) {
3547 			if (lm->lun == UINT32_MAX)
3548 				retval = ctl_lun_map_deinit(port);
3549 			else
3550 				retval = ctl_lun_map_init(port);
3551 		} else
3552 			return (ENXIO);
3553 		break;
3554 	}
3555 	default: {
3556 		/* XXX KDM should we fix this? */
3557 #if 0
3558 		struct ctl_backend_driver *backend;
3559 		unsigned int type;
3560 		int found;
3561 
3562 		found = 0;
3563 
3564 		/*
3565 		 * We encode the backend type as the ioctl type for backend
3566 		 * ioctls.  So parse it out here, and then search for a
3567 		 * backend of this type.
3568 		 */
3569 		type = _IOC_TYPE(cmd);
3570 
3571 		STAILQ_FOREACH(backend, &softc->be_list, links) {
3572 			if (backend->type == type) {
3573 				found = 1;
3574 				break;
3575 			}
3576 		}
3577 		if (found == 0) {
3578 			printf("ctl: unknown ioctl command %#lx or backend "
3579 			       "%d\n", cmd, type);
3580 			retval = EINVAL;
3581 			break;
3582 		}
3583 		retval = backend->ioctl(dev, cmd, addr, flag, td);
3584 #endif
3585 		retval = ENOTTY;
3586 		break;
3587 	}
3588 	}
3589 	return (retval);
3590 }
3591 
3592 uint32_t
3593 ctl_get_initindex(struct ctl_nexus *nexus)
3594 {
3595 	if (nexus->targ_port < CTL_MAX_PORTS)
3596 		return (nexus->initid.id +
3597 			(nexus->targ_port * CTL_MAX_INIT_PER_PORT));
3598 	else
3599 		return (nexus->initid.id +
3600 		       ((nexus->targ_port - CTL_MAX_PORTS) *
3601 			CTL_MAX_INIT_PER_PORT));
3602 }
3603 
3604 uint32_t
3605 ctl_get_resindex(struct ctl_nexus *nexus)
3606 {
3607 	return (nexus->initid.id + (nexus->targ_port * CTL_MAX_INIT_PER_PORT));
3608 }
3609 
3610 uint32_t
3611 ctl_port_idx(int port_num)
3612 {
3613 	if (port_num < CTL_MAX_PORTS)
3614 		return(port_num);
3615 	else
3616 		return(port_num - CTL_MAX_PORTS);
3617 }
3618 
3619 int
3620 ctl_lun_map_init(struct ctl_port *port)
3621 {
3622 	struct ctl_softc *softc = control_softc;
3623 	struct ctl_lun *lun;
3624 	uint32_t i;
3625 
3626 	if (port->lun_map == NULL)
3627 		port->lun_map = malloc(sizeof(uint32_t) * CTL_MAX_LUNS,
3628 		    M_CTL, M_NOWAIT);
3629 	if (port->lun_map == NULL)
3630 		return (ENOMEM);
3631 	for (i = 0; i < CTL_MAX_LUNS; i++)
3632 		port->lun_map[i] = UINT32_MAX;
3633 	if (port->status & CTL_PORT_STATUS_ONLINE) {
3634 		STAILQ_FOREACH(lun, &softc->lun_list, links)
3635 			port->lun_disable(port->targ_lun_arg, lun->lun);
3636 	}
3637 	return (0);
3638 }
3639 
3640 int
3641 ctl_lun_map_deinit(struct ctl_port *port)
3642 {
3643 	struct ctl_softc *softc = control_softc;
3644 	struct ctl_lun *lun;
3645 
3646 	if (port->lun_map == NULL)
3647 		return (0);
3648 	free(port->lun_map, M_CTL);
3649 	port->lun_map = NULL;
3650 	if (port->status & CTL_PORT_STATUS_ONLINE) {
3651 		STAILQ_FOREACH(lun, &softc->lun_list, links)
3652 			port->lun_enable(port->targ_lun_arg, lun->lun);
3653 	}
3654 	return (0);
3655 }
3656 
3657 int
3658 ctl_lun_map_set(struct ctl_port *port, uint32_t plun, uint32_t glun)
3659 {
3660 	int status;
3661 	uint32_t old;
3662 
3663 	if (port->lun_map == NULL) {
3664 		status = ctl_lun_map_init(port);
3665 		if (status != 0)
3666 			return (status);
3667 	}
3668 	old = port->lun_map[plun];
3669 	port->lun_map[plun] = glun;
3670 	if ((port->status & CTL_PORT_STATUS_ONLINE) && old >= CTL_MAX_LUNS)
3671 		port->lun_enable(port->targ_lun_arg, plun);
3672 	return (0);
3673 }
3674 
3675 int
3676 ctl_lun_map_unset(struct ctl_port *port, uint32_t plun)
3677 {
3678 	uint32_t old;
3679 
3680 	if (port->lun_map == NULL)
3681 		return (0);
3682 	old = port->lun_map[plun];
3683 	port->lun_map[plun] = UINT32_MAX;
3684 	if ((port->status & CTL_PORT_STATUS_ONLINE) && old < CTL_MAX_LUNS)
3685 		port->lun_disable(port->targ_lun_arg, plun);
3686 	return (0);
3687 }
3688 
3689 uint32_t
3690 ctl_lun_map_from_port(struct ctl_port *port, uint32_t lun_id)
3691 {
3692 
3693 	if (port == NULL)
3694 		return (UINT32_MAX);
3695 	if (port->lun_map == NULL || lun_id >= CTL_MAX_LUNS)
3696 		return (lun_id);
3697 	return (port->lun_map[lun_id]);
3698 }
3699 
3700 uint32_t
3701 ctl_lun_map_to_port(struct ctl_port *port, uint32_t lun_id)
3702 {
3703 	uint32_t i;
3704 
3705 	if (port == NULL)
3706 		return (UINT32_MAX);
3707 	if (port->lun_map == NULL)
3708 		return (lun_id);
3709 	for (i = 0; i < CTL_MAX_LUNS; i++) {
3710 		if (port->lun_map[i] == lun_id)
3711 			return (i);
3712 	}
3713 	return (UINT32_MAX);
3714 }
3715 
3716 static struct ctl_port *
3717 ctl_io_port(struct ctl_io_hdr *io_hdr)
3718 {
3719 	int port_num;
3720 
3721 	port_num = io_hdr->nexus.targ_port;
3722 	return (control_softc->ctl_ports[ctl_port_idx(port_num)]);
3723 }
3724 
3725 /*
3726  * Note:  This only works for bitmask sizes that are at least 32 bits, and
3727  * that are a power of 2.
3728  */
3729 int
3730 ctl_ffz(uint32_t *mask, uint32_t size)
3731 {
3732 	uint32_t num_chunks, num_pieces;
3733 	int i, j;
3734 
3735 	num_chunks = (size >> 5);
3736 	if (num_chunks == 0)
3737 		num_chunks++;
3738 	num_pieces = MIN((sizeof(uint32_t) * 8), size);
3739 
3740 	for (i = 0; i < num_chunks; i++) {
3741 		for (j = 0; j < num_pieces; j++) {
3742 			if ((mask[i] & (1 << j)) == 0)
3743 				return ((i << 5) + j);
3744 		}
3745 	}
3746 
3747 	return (-1);
3748 }
3749 
3750 int
3751 ctl_set_mask(uint32_t *mask, uint32_t bit)
3752 {
3753 	uint32_t chunk, piece;
3754 
3755 	chunk = bit >> 5;
3756 	piece = bit % (sizeof(uint32_t) * 8);
3757 
3758 	if ((mask[chunk] & (1 << piece)) != 0)
3759 		return (-1);
3760 	else
3761 		mask[chunk] |= (1 << piece);
3762 
3763 	return (0);
3764 }
3765 
3766 int
3767 ctl_clear_mask(uint32_t *mask, uint32_t bit)
3768 {
3769 	uint32_t chunk, piece;
3770 
3771 	chunk = bit >> 5;
3772 	piece = bit % (sizeof(uint32_t) * 8);
3773 
3774 	if ((mask[chunk] & (1 << piece)) == 0)
3775 		return (-1);
3776 	else
3777 		mask[chunk] &= ~(1 << piece);
3778 
3779 	return (0);
3780 }
3781 
3782 int
3783 ctl_is_set(uint32_t *mask, uint32_t bit)
3784 {
3785 	uint32_t chunk, piece;
3786 
3787 	chunk = bit >> 5;
3788 	piece = bit % (sizeof(uint32_t) * 8);
3789 
3790 	if ((mask[chunk] & (1 << piece)) == 0)
3791 		return (0);
3792 	else
3793 		return (1);
3794 }
3795 
3796 static uint64_t
3797 ctl_get_prkey(struct ctl_lun *lun, uint32_t residx)
3798 {
3799 	uint64_t *t;
3800 
3801 	t = lun->pr_keys[residx/CTL_MAX_INIT_PER_PORT];
3802 	if (t == NULL)
3803 		return (0);
3804 	return (t[residx % CTL_MAX_INIT_PER_PORT]);
3805 }
3806 
3807 static void
3808 ctl_clr_prkey(struct ctl_lun *lun, uint32_t residx)
3809 {
3810 	uint64_t *t;
3811 
3812 	t = lun->pr_keys[residx/CTL_MAX_INIT_PER_PORT];
3813 	if (t == NULL)
3814 		return;
3815 	t[residx % CTL_MAX_INIT_PER_PORT] = 0;
3816 }
3817 
3818 static void
3819 ctl_alloc_prkey(struct ctl_lun *lun, uint32_t residx)
3820 {
3821 	uint64_t *p;
3822 	u_int i;
3823 
3824 	i = residx/CTL_MAX_INIT_PER_PORT;
3825 	if (lun->pr_keys[i] != NULL)
3826 		return;
3827 	mtx_unlock(&lun->lun_lock);
3828 	p = malloc(sizeof(uint64_t) * CTL_MAX_INIT_PER_PORT, M_CTL,
3829 	    M_WAITOK | M_ZERO);
3830 	mtx_lock(&lun->lun_lock);
3831 	if (lun->pr_keys[i] == NULL)
3832 		lun->pr_keys[i] = p;
3833 	else
3834 		free(p, M_CTL);
3835 }
3836 
3837 static void
3838 ctl_set_prkey(struct ctl_lun *lun, uint32_t residx, uint64_t key)
3839 {
3840 	uint64_t *t;
3841 
3842 	t = lun->pr_keys[residx/CTL_MAX_INIT_PER_PORT];
3843 	KASSERT(t != NULL, ("prkey %d is not allocated", residx));
3844 	t[residx % CTL_MAX_INIT_PER_PORT] = key;
3845 }
3846 
3847 /*
3848  * ctl_softc, pool_name, total_ctl_io are passed in.
3849  * npool is passed out.
3850  */
3851 int
3852 ctl_pool_create(struct ctl_softc *ctl_softc, const char *pool_name,
3853 		uint32_t total_ctl_io, void **npool)
3854 {
3855 #ifdef IO_POOLS
3856 	struct ctl_io_pool *pool;
3857 
3858 	pool = (struct ctl_io_pool *)malloc(sizeof(*pool), M_CTL,
3859 					    M_NOWAIT | M_ZERO);
3860 	if (pool == NULL)
3861 		return (ENOMEM);
3862 
3863 	snprintf(pool->name, sizeof(pool->name), "CTL IO %s", pool_name);
3864 	pool->ctl_softc = ctl_softc;
3865 	pool->zone = uma_zsecond_create(pool->name, NULL,
3866 	    NULL, NULL, NULL, ctl_softc->io_zone);
3867 	/* uma_prealloc(pool->zone, total_ctl_io); */
3868 
3869 	*npool = pool;
3870 #else
3871 	*npool = ctl_softc->io_zone;
3872 #endif
3873 	return (0);
3874 }
3875 
3876 void
3877 ctl_pool_free(struct ctl_io_pool *pool)
3878 {
3879 
3880 	if (pool == NULL)
3881 		return;
3882 
3883 #ifdef IO_POOLS
3884 	uma_zdestroy(pool->zone);
3885 	free(pool, M_CTL);
3886 #endif
3887 }
3888 
3889 union ctl_io *
3890 ctl_alloc_io(void *pool_ref)
3891 {
3892 	union ctl_io *io;
3893 #ifdef IO_POOLS
3894 	struct ctl_io_pool *pool = (struct ctl_io_pool *)pool_ref;
3895 
3896 	io = uma_zalloc(pool->zone, M_WAITOK);
3897 #else
3898 	io = uma_zalloc((uma_zone_t)pool_ref, M_WAITOK);
3899 #endif
3900 	if (io != NULL)
3901 		io->io_hdr.pool = pool_ref;
3902 	return (io);
3903 }
3904 
3905 union ctl_io *
3906 ctl_alloc_io_nowait(void *pool_ref)
3907 {
3908 	union ctl_io *io;
3909 #ifdef IO_POOLS
3910 	struct ctl_io_pool *pool = (struct ctl_io_pool *)pool_ref;
3911 
3912 	io = uma_zalloc(pool->zone, M_NOWAIT);
3913 #else
3914 	io = uma_zalloc((uma_zone_t)pool_ref, M_NOWAIT);
3915 #endif
3916 	if (io != NULL)
3917 		io->io_hdr.pool = pool_ref;
3918 	return (io);
3919 }
3920 
3921 void
3922 ctl_free_io(union ctl_io *io)
3923 {
3924 #ifdef IO_POOLS
3925 	struct ctl_io_pool *pool;
3926 #endif
3927 
3928 	if (io == NULL)
3929 		return;
3930 
3931 #ifdef IO_POOLS
3932 	pool = (struct ctl_io_pool *)io->io_hdr.pool;
3933 	uma_zfree(pool->zone, io);
3934 #else
3935 	uma_zfree((uma_zone_t)io->io_hdr.pool, io);
3936 #endif
3937 }
3938 
3939 void
3940 ctl_zero_io(union ctl_io *io)
3941 {
3942 	void *pool_ref;
3943 
3944 	if (io == NULL)
3945 		return;
3946 
3947 	/*
3948 	 * May need to preserve linked list pointers at some point too.
3949 	 */
3950 	pool_ref = io->io_hdr.pool;
3951 	memset(io, 0, sizeof(*io));
3952 	io->io_hdr.pool = pool_ref;
3953 }
3954 
3955 /*
3956  * This routine is currently used for internal copies of ctl_ios that need
3957  * to persist for some reason after we've already returned status to the
3958  * FETD.  (Thus the flag set.)
3959  *
3960  * XXX XXX
3961  * Note that this makes a blind copy of all fields in the ctl_io, except
3962  * for the pool reference.  This includes any memory that has been
3963  * allocated!  That memory will no longer be valid after done has been
3964  * called, so this would be VERY DANGEROUS for command that actually does
3965  * any reads or writes.  Right now (11/7/2005), this is only used for immediate
3966  * start and stop commands, which don't transfer any data, so this is not a
3967  * problem.  If it is used for anything else, the caller would also need to
3968  * allocate data buffer space and this routine would need to be modified to
3969  * copy the data buffer(s) as well.
3970  */
3971 void
3972 ctl_copy_io(union ctl_io *src, union ctl_io *dest)
3973 {
3974 	void *pool_ref;
3975 
3976 	if ((src == NULL)
3977 	 || (dest == NULL))
3978 		return;
3979 
3980 	/*
3981 	 * May need to preserve linked list pointers at some point too.
3982 	 */
3983 	pool_ref = dest->io_hdr.pool;
3984 
3985 	memcpy(dest, src, MIN(sizeof(*src), sizeof(*dest)));
3986 
3987 	dest->io_hdr.pool = pool_ref;
3988 	/*
3989 	 * We need to know that this is an internal copy, and doesn't need
3990 	 * to get passed back to the FETD that allocated it.
3991 	 */
3992 	dest->io_hdr.flags |= CTL_FLAG_INT_COPY;
3993 }
3994 
3995 int
3996 ctl_expand_number(const char *buf, uint64_t *num)
3997 {
3998 	char *endptr;
3999 	uint64_t number;
4000 	unsigned shift;
4001 
4002 	number = strtoq(buf, &endptr, 0);
4003 
4004 	switch (tolower((unsigned char)*endptr)) {
4005 	case 'e':
4006 		shift = 60;
4007 		break;
4008 	case 'p':
4009 		shift = 50;
4010 		break;
4011 	case 't':
4012 		shift = 40;
4013 		break;
4014 	case 'g':
4015 		shift = 30;
4016 		break;
4017 	case 'm':
4018 		shift = 20;
4019 		break;
4020 	case 'k':
4021 		shift = 10;
4022 		break;
4023 	case 'b':
4024 	case '\0': /* No unit. */
4025 		*num = number;
4026 		return (0);
4027 	default:
4028 		/* Unrecognized unit. */
4029 		return (-1);
4030 	}
4031 
4032 	if ((number << shift) >> shift != number) {
4033 		/* Overflow */
4034 		return (-1);
4035 	}
4036 	*num = number << shift;
4037 	return (0);
4038 }
4039 
4040 
4041 /*
4042  * This routine could be used in the future to load default and/or saved
4043  * mode page parameters for a particuar lun.
4044  */
4045 static int
4046 ctl_init_page_index(struct ctl_lun *lun)
4047 {
4048 	int i;
4049 	struct ctl_page_index *page_index;
4050 	const char *value;
4051 	uint64_t ival;
4052 
4053 	memcpy(&lun->mode_pages.index, page_index_template,
4054 	       sizeof(page_index_template));
4055 
4056 	for (i = 0; i < CTL_NUM_MODE_PAGES; i++) {
4057 
4058 		page_index = &lun->mode_pages.index[i];
4059 		/*
4060 		 * If this is a disk-only mode page, there's no point in
4061 		 * setting it up.  For some pages, we have to have some
4062 		 * basic information about the disk in order to calculate the
4063 		 * mode page data.
4064 		 */
4065 		if ((lun->be_lun->lun_type != T_DIRECT)
4066 		 && (page_index->page_flags & CTL_PAGE_FLAG_DISK_ONLY))
4067 			continue;
4068 
4069 		switch (page_index->page_code & SMPH_PC_MASK) {
4070 		case SMS_RW_ERROR_RECOVERY_PAGE: {
4071 			if (page_index->subpage != SMS_SUBPAGE_PAGE_0)
4072 				panic("subpage is incorrect!");
4073 			memcpy(&lun->mode_pages.rw_er_page[CTL_PAGE_CURRENT],
4074 			       &rw_er_page_default,
4075 			       sizeof(rw_er_page_default));
4076 			memcpy(&lun->mode_pages.rw_er_page[CTL_PAGE_CHANGEABLE],
4077 			       &rw_er_page_changeable,
4078 			       sizeof(rw_er_page_changeable));
4079 			memcpy(&lun->mode_pages.rw_er_page[CTL_PAGE_DEFAULT],
4080 			       &rw_er_page_default,
4081 			       sizeof(rw_er_page_default));
4082 			memcpy(&lun->mode_pages.rw_er_page[CTL_PAGE_SAVED],
4083 			       &rw_er_page_default,
4084 			       sizeof(rw_er_page_default));
4085 			page_index->page_data =
4086 				(uint8_t *)lun->mode_pages.rw_er_page;
4087 			break;
4088 		}
4089 		case SMS_FORMAT_DEVICE_PAGE: {
4090 			struct scsi_format_page *format_page;
4091 
4092 			if (page_index->subpage != SMS_SUBPAGE_PAGE_0)
4093 				panic("subpage is incorrect!");
4094 
4095 			/*
4096 			 * Sectors per track are set above.  Bytes per
4097 			 * sector need to be set here on a per-LUN basis.
4098 			 */
4099 			memcpy(&lun->mode_pages.format_page[CTL_PAGE_CURRENT],
4100 			       &format_page_default,
4101 			       sizeof(format_page_default));
4102 			memcpy(&lun->mode_pages.format_page[
4103 			       CTL_PAGE_CHANGEABLE], &format_page_changeable,
4104 			       sizeof(format_page_changeable));
4105 			memcpy(&lun->mode_pages.format_page[CTL_PAGE_DEFAULT],
4106 			       &format_page_default,
4107 			       sizeof(format_page_default));
4108 			memcpy(&lun->mode_pages.format_page[CTL_PAGE_SAVED],
4109 			       &format_page_default,
4110 			       sizeof(format_page_default));
4111 
4112 			format_page = &lun->mode_pages.format_page[
4113 				CTL_PAGE_CURRENT];
4114 			scsi_ulto2b(lun->be_lun->blocksize,
4115 				    format_page->bytes_per_sector);
4116 
4117 			format_page = &lun->mode_pages.format_page[
4118 				CTL_PAGE_DEFAULT];
4119 			scsi_ulto2b(lun->be_lun->blocksize,
4120 				    format_page->bytes_per_sector);
4121 
4122 			format_page = &lun->mode_pages.format_page[
4123 				CTL_PAGE_SAVED];
4124 			scsi_ulto2b(lun->be_lun->blocksize,
4125 				    format_page->bytes_per_sector);
4126 
4127 			page_index->page_data =
4128 				(uint8_t *)lun->mode_pages.format_page;
4129 			break;
4130 		}
4131 		case SMS_RIGID_DISK_PAGE: {
4132 			struct scsi_rigid_disk_page *rigid_disk_page;
4133 			uint32_t sectors_per_cylinder;
4134 			uint64_t cylinders;
4135 #ifndef	__XSCALE__
4136 			int shift;
4137 #endif /* !__XSCALE__ */
4138 
4139 			if (page_index->subpage != SMS_SUBPAGE_PAGE_0)
4140 				panic("invalid subpage value %d",
4141 				      page_index->subpage);
4142 
4143 			/*
4144 			 * Rotation rate and sectors per track are set
4145 			 * above.  We calculate the cylinders here based on
4146 			 * capacity.  Due to the number of heads and
4147 			 * sectors per track we're using, smaller arrays
4148 			 * may turn out to have 0 cylinders.  Linux and
4149 			 * FreeBSD don't pay attention to these mode pages
4150 			 * to figure out capacity, but Solaris does.  It
4151 			 * seems to deal with 0 cylinders just fine, and
4152 			 * works out a fake geometry based on the capacity.
4153 			 */
4154 			memcpy(&lun->mode_pages.rigid_disk_page[
4155 			       CTL_PAGE_DEFAULT], &rigid_disk_page_default,
4156 			       sizeof(rigid_disk_page_default));
4157 			memcpy(&lun->mode_pages.rigid_disk_page[
4158 			       CTL_PAGE_CHANGEABLE],&rigid_disk_page_changeable,
4159 			       sizeof(rigid_disk_page_changeable));
4160 
4161 			sectors_per_cylinder = CTL_DEFAULT_SECTORS_PER_TRACK *
4162 				CTL_DEFAULT_HEADS;
4163 
4164 			/*
4165 			 * The divide method here will be more accurate,
4166 			 * probably, but results in floating point being
4167 			 * used in the kernel on i386 (__udivdi3()).  On the
4168 			 * XScale, though, __udivdi3() is implemented in
4169 			 * software.
4170 			 *
4171 			 * The shift method for cylinder calculation is
4172 			 * accurate if sectors_per_cylinder is a power of
4173 			 * 2.  Otherwise it might be slightly off -- you
4174 			 * might have a bit of a truncation problem.
4175 			 */
4176 #ifdef	__XSCALE__
4177 			cylinders = (lun->be_lun->maxlba + 1) /
4178 				sectors_per_cylinder;
4179 #else
4180 			for (shift = 31; shift > 0; shift--) {
4181 				if (sectors_per_cylinder & (1 << shift))
4182 					break;
4183 			}
4184 			cylinders = (lun->be_lun->maxlba + 1) >> shift;
4185 #endif
4186 
4187 			/*
4188 			 * We've basically got 3 bytes, or 24 bits for the
4189 			 * cylinder size in the mode page.  If we're over,
4190 			 * just round down to 2^24.
4191 			 */
4192 			if (cylinders > 0xffffff)
4193 				cylinders = 0xffffff;
4194 
4195 			rigid_disk_page = &lun->mode_pages.rigid_disk_page[
4196 				CTL_PAGE_DEFAULT];
4197 			scsi_ulto3b(cylinders, rigid_disk_page->cylinders);
4198 
4199 			if ((value = ctl_get_opt(&lun->be_lun->options,
4200 			    "rpm")) != NULL) {
4201 				scsi_ulto2b(strtol(value, NULL, 0),
4202 				     rigid_disk_page->rotation_rate);
4203 			}
4204 
4205 			memcpy(&lun->mode_pages.rigid_disk_page[CTL_PAGE_CURRENT],
4206 			       &lun->mode_pages.rigid_disk_page[CTL_PAGE_DEFAULT],
4207 			       sizeof(rigid_disk_page_default));
4208 			memcpy(&lun->mode_pages.rigid_disk_page[CTL_PAGE_SAVED],
4209 			       &lun->mode_pages.rigid_disk_page[CTL_PAGE_DEFAULT],
4210 			       sizeof(rigid_disk_page_default));
4211 
4212 			page_index->page_data =
4213 				(uint8_t *)lun->mode_pages.rigid_disk_page;
4214 			break;
4215 		}
4216 		case SMS_CACHING_PAGE: {
4217 			struct scsi_caching_page *caching_page;
4218 
4219 			if (page_index->subpage != SMS_SUBPAGE_PAGE_0)
4220 				panic("invalid subpage value %d",
4221 				      page_index->subpage);
4222 			memcpy(&lun->mode_pages.caching_page[CTL_PAGE_DEFAULT],
4223 			       &caching_page_default,
4224 			       sizeof(caching_page_default));
4225 			memcpy(&lun->mode_pages.caching_page[
4226 			       CTL_PAGE_CHANGEABLE], &caching_page_changeable,
4227 			       sizeof(caching_page_changeable));
4228 			memcpy(&lun->mode_pages.caching_page[CTL_PAGE_SAVED],
4229 			       &caching_page_default,
4230 			       sizeof(caching_page_default));
4231 			caching_page = &lun->mode_pages.caching_page[
4232 			    CTL_PAGE_SAVED];
4233 			value = ctl_get_opt(&lun->be_lun->options, "writecache");
4234 			if (value != NULL && strcmp(value, "off") == 0)
4235 				caching_page->flags1 &= ~SCP_WCE;
4236 			value = ctl_get_opt(&lun->be_lun->options, "readcache");
4237 			if (value != NULL && strcmp(value, "off") == 0)
4238 				caching_page->flags1 |= SCP_RCD;
4239 			memcpy(&lun->mode_pages.caching_page[CTL_PAGE_CURRENT],
4240 			       &lun->mode_pages.caching_page[CTL_PAGE_SAVED],
4241 			       sizeof(caching_page_default));
4242 			page_index->page_data =
4243 				(uint8_t *)lun->mode_pages.caching_page;
4244 			break;
4245 		}
4246 		case SMS_CONTROL_MODE_PAGE: {
4247 			struct scsi_control_page *control_page;
4248 
4249 			if (page_index->subpage != SMS_SUBPAGE_PAGE_0)
4250 				panic("invalid subpage value %d",
4251 				      page_index->subpage);
4252 
4253 			memcpy(&lun->mode_pages.control_page[CTL_PAGE_DEFAULT],
4254 			       &control_page_default,
4255 			       sizeof(control_page_default));
4256 			memcpy(&lun->mode_pages.control_page[
4257 			       CTL_PAGE_CHANGEABLE], &control_page_changeable,
4258 			       sizeof(control_page_changeable));
4259 			memcpy(&lun->mode_pages.control_page[CTL_PAGE_SAVED],
4260 			       &control_page_default,
4261 			       sizeof(control_page_default));
4262 			control_page = &lun->mode_pages.control_page[
4263 			    CTL_PAGE_SAVED];
4264 			value = ctl_get_opt(&lun->be_lun->options, "reordering");
4265 			if (value != NULL && strcmp(value, "unrestricted") == 0) {
4266 				control_page->queue_flags &= ~SCP_QUEUE_ALG_MASK;
4267 				control_page->queue_flags |= SCP_QUEUE_ALG_UNRESTRICTED;
4268 			}
4269 			memcpy(&lun->mode_pages.control_page[CTL_PAGE_CURRENT],
4270 			       &lun->mode_pages.control_page[CTL_PAGE_SAVED],
4271 			       sizeof(control_page_default));
4272 			page_index->page_data =
4273 				(uint8_t *)lun->mode_pages.control_page;
4274 			break;
4275 
4276 		}
4277 		case SMS_INFO_EXCEPTIONS_PAGE: {
4278 			switch (page_index->subpage) {
4279 			case SMS_SUBPAGE_PAGE_0:
4280 				memcpy(&lun->mode_pages.ie_page[CTL_PAGE_CURRENT],
4281 				       &ie_page_default,
4282 				       sizeof(ie_page_default));
4283 				memcpy(&lun->mode_pages.ie_page[
4284 				       CTL_PAGE_CHANGEABLE], &ie_page_changeable,
4285 				       sizeof(ie_page_changeable));
4286 				memcpy(&lun->mode_pages.ie_page[CTL_PAGE_DEFAULT],
4287 				       &ie_page_default,
4288 				       sizeof(ie_page_default));
4289 				memcpy(&lun->mode_pages.ie_page[CTL_PAGE_SAVED],
4290 				       &ie_page_default,
4291 				       sizeof(ie_page_default));
4292 				page_index->page_data =
4293 					(uint8_t *)lun->mode_pages.ie_page;
4294 				break;
4295 			case 0x02: {
4296 				struct ctl_logical_block_provisioning_page *page;
4297 
4298 				memcpy(&lun->mode_pages.lbp_page[CTL_PAGE_DEFAULT],
4299 				       &lbp_page_default,
4300 				       sizeof(lbp_page_default));
4301 				memcpy(&lun->mode_pages.lbp_page[
4302 				       CTL_PAGE_CHANGEABLE], &lbp_page_changeable,
4303 				       sizeof(lbp_page_changeable));
4304 				memcpy(&lun->mode_pages.lbp_page[CTL_PAGE_SAVED],
4305 				       &lbp_page_default,
4306 				       sizeof(lbp_page_default));
4307 				page = &lun->mode_pages.lbp_page[CTL_PAGE_SAVED];
4308 				value = ctl_get_opt(&lun->be_lun->options,
4309 				    "avail-threshold");
4310 				if (value != NULL &&
4311 				    ctl_expand_number(value, &ival) == 0) {
4312 					page->descr[0].flags |= SLBPPD_ENABLED |
4313 					    SLBPPD_ARMING_DEC;
4314 					if (lun->be_lun->blocksize)
4315 						ival /= lun->be_lun->blocksize;
4316 					else
4317 						ival /= 512;
4318 					scsi_ulto4b(ival >> CTL_LBP_EXPONENT,
4319 					    page->descr[0].count);
4320 				}
4321 				value = ctl_get_opt(&lun->be_lun->options,
4322 				    "used-threshold");
4323 				if (value != NULL &&
4324 				    ctl_expand_number(value, &ival) == 0) {
4325 					page->descr[1].flags |= SLBPPD_ENABLED |
4326 					    SLBPPD_ARMING_INC;
4327 					if (lun->be_lun->blocksize)
4328 						ival /= lun->be_lun->blocksize;
4329 					else
4330 						ival /= 512;
4331 					scsi_ulto4b(ival >> CTL_LBP_EXPONENT,
4332 					    page->descr[1].count);
4333 				}
4334 				value = ctl_get_opt(&lun->be_lun->options,
4335 				    "pool-avail-threshold");
4336 				if (value != NULL &&
4337 				    ctl_expand_number(value, &ival) == 0) {
4338 					page->descr[2].flags |= SLBPPD_ENABLED |
4339 					    SLBPPD_ARMING_DEC;
4340 					if (lun->be_lun->blocksize)
4341 						ival /= lun->be_lun->blocksize;
4342 					else
4343 						ival /= 512;
4344 					scsi_ulto4b(ival >> CTL_LBP_EXPONENT,
4345 					    page->descr[2].count);
4346 				}
4347 				value = ctl_get_opt(&lun->be_lun->options,
4348 				    "pool-used-threshold");
4349 				if (value != NULL &&
4350 				    ctl_expand_number(value, &ival) == 0) {
4351 					page->descr[3].flags |= SLBPPD_ENABLED |
4352 					    SLBPPD_ARMING_INC;
4353 					if (lun->be_lun->blocksize)
4354 						ival /= lun->be_lun->blocksize;
4355 					else
4356 						ival /= 512;
4357 					scsi_ulto4b(ival >> CTL_LBP_EXPONENT,
4358 					    page->descr[3].count);
4359 				}
4360 				memcpy(&lun->mode_pages.lbp_page[CTL_PAGE_CURRENT],
4361 				       &lun->mode_pages.lbp_page[CTL_PAGE_SAVED],
4362 				       sizeof(lbp_page_default));
4363 				page_index->page_data =
4364 					(uint8_t *)lun->mode_pages.lbp_page;
4365 			}}
4366 			break;
4367 		}
4368 		case SMS_VENDOR_SPECIFIC_PAGE:{
4369 			switch (page_index->subpage) {
4370 			case DBGCNF_SUBPAGE_CODE: {
4371 				struct copan_debugconf_subpage *current_page,
4372 							       *saved_page;
4373 
4374 				memcpy(&lun->mode_pages.debugconf_subpage[
4375 				       CTL_PAGE_CURRENT],
4376 				       &debugconf_page_default,
4377 				       sizeof(debugconf_page_default));
4378 				memcpy(&lun->mode_pages.debugconf_subpage[
4379 				       CTL_PAGE_CHANGEABLE],
4380 				       &debugconf_page_changeable,
4381 				       sizeof(debugconf_page_changeable));
4382 				memcpy(&lun->mode_pages.debugconf_subpage[
4383 				       CTL_PAGE_DEFAULT],
4384 				       &debugconf_page_default,
4385 				       sizeof(debugconf_page_default));
4386 				memcpy(&lun->mode_pages.debugconf_subpage[
4387 				       CTL_PAGE_SAVED],
4388 				       &debugconf_page_default,
4389 				       sizeof(debugconf_page_default));
4390 				page_index->page_data =
4391 					(uint8_t *)lun->mode_pages.debugconf_subpage;
4392 
4393 				current_page = (struct copan_debugconf_subpage *)
4394 					(page_index->page_data +
4395 					 (page_index->page_len *
4396 					  CTL_PAGE_CURRENT));
4397 				saved_page = (struct copan_debugconf_subpage *)
4398 					(page_index->page_data +
4399 					 (page_index->page_len *
4400 					  CTL_PAGE_SAVED));
4401 				break;
4402 			}
4403 			default:
4404 				panic("invalid subpage value %d",
4405 				      page_index->subpage);
4406 				break;
4407 			}
4408    			break;
4409 		}
4410 		default:
4411 			panic("invalid page value %d",
4412 			      page_index->page_code & SMPH_PC_MASK);
4413 			break;
4414     	}
4415 	}
4416 
4417 	return (CTL_RETVAL_COMPLETE);
4418 }
4419 
4420 static int
4421 ctl_init_log_page_index(struct ctl_lun *lun)
4422 {
4423 	struct ctl_page_index *page_index;
4424 	int i, j, k, prev;
4425 
4426 	memcpy(&lun->log_pages.index, log_page_index_template,
4427 	       sizeof(log_page_index_template));
4428 
4429 	prev = -1;
4430 	for (i = 0, j = 0, k = 0; i < CTL_NUM_LOG_PAGES; i++) {
4431 
4432 		page_index = &lun->log_pages.index[i];
4433 		/*
4434 		 * If this is a disk-only mode page, there's no point in
4435 		 * setting it up.  For some pages, we have to have some
4436 		 * basic information about the disk in order to calculate the
4437 		 * mode page data.
4438 		 */
4439 		if ((lun->be_lun->lun_type != T_DIRECT)
4440 		 && (page_index->page_flags & CTL_PAGE_FLAG_DISK_ONLY))
4441 			continue;
4442 
4443 		if (page_index->page_code == SLS_LOGICAL_BLOCK_PROVISIONING &&
4444 		     lun->backend->lun_attr == NULL)
4445 			continue;
4446 
4447 		if (page_index->page_code != prev) {
4448 			lun->log_pages.pages_page[j] = page_index->page_code;
4449 			prev = page_index->page_code;
4450 			j++;
4451 		}
4452 		lun->log_pages.subpages_page[k*2] = page_index->page_code;
4453 		lun->log_pages.subpages_page[k*2+1] = page_index->subpage;
4454 		k++;
4455 	}
4456 	lun->log_pages.index[0].page_data = &lun->log_pages.pages_page[0];
4457 	lun->log_pages.index[0].page_len = j;
4458 	lun->log_pages.index[1].page_data = &lun->log_pages.subpages_page[0];
4459 	lun->log_pages.index[1].page_len = k * 2;
4460 	lun->log_pages.index[2].page_data = &lun->log_pages.lbp_page[0];
4461 	lun->log_pages.index[2].page_len = 12*CTL_NUM_LBP_PARAMS;
4462 	lun->log_pages.index[3].page_data = (uint8_t *)&lun->log_pages.stat_page;
4463 	lun->log_pages.index[3].page_len = sizeof(lun->log_pages.stat_page);
4464 
4465 	return (CTL_RETVAL_COMPLETE);
4466 }
4467 
4468 static int
4469 hex2bin(const char *str, uint8_t *buf, int buf_size)
4470 {
4471 	int i;
4472 	u_char c;
4473 
4474 	memset(buf, 0, buf_size);
4475 	while (isspace(str[0]))
4476 		str++;
4477 	if (str[0] == '0' && (str[1] == 'x' || str[1] == 'X'))
4478 		str += 2;
4479 	buf_size *= 2;
4480 	for (i = 0; str[i] != 0 && i < buf_size; i++) {
4481 		c = str[i];
4482 		if (isdigit(c))
4483 			c -= '0';
4484 		else if (isalpha(c))
4485 			c -= isupper(c) ? 'A' - 10 : 'a' - 10;
4486 		else
4487 			break;
4488 		if (c >= 16)
4489 			break;
4490 		if ((i & 1) == 0)
4491 			buf[i / 2] |= (c << 4);
4492 		else
4493 			buf[i / 2] |= c;
4494 	}
4495 	return ((i + 1) / 2);
4496 }
4497 
4498 /*
4499  * LUN allocation.
4500  *
4501  * Requirements:
4502  * - caller allocates and zeros LUN storage, or passes in a NULL LUN if he
4503  *   wants us to allocate the LUN and he can block.
4504  * - ctl_softc is always set
4505  * - be_lun is set if the LUN has a backend (needed for disk LUNs)
4506  *
4507  * Returns 0 for success, non-zero (errno) for failure.
4508  */
4509 static int
4510 ctl_alloc_lun(struct ctl_softc *ctl_softc, struct ctl_lun *ctl_lun,
4511 	      struct ctl_be_lun *const be_lun)
4512 {
4513 	struct ctl_lun *nlun, *lun;
4514 	struct scsi_vpd_id_descriptor *desc;
4515 	struct scsi_vpd_id_t10 *t10id;
4516 	const char *eui, *naa, *scsiname, *vendor, *value;
4517 	int lun_number, i, lun_malloced;
4518 	int devidlen, idlen1, idlen2 = 0, len;
4519 
4520 	if (be_lun == NULL)
4521 		return (EINVAL);
4522 
4523 	/*
4524 	 * We currently only support Direct Access or Processor LUN types.
4525 	 */
4526 	switch (be_lun->lun_type) {
4527 	case T_DIRECT:
4528 		break;
4529 	case T_PROCESSOR:
4530 		break;
4531 	case T_SEQUENTIAL:
4532 	case T_CHANGER:
4533 	default:
4534 		be_lun->lun_config_status(be_lun->be_lun,
4535 					  CTL_LUN_CONFIG_FAILURE);
4536 		break;
4537 	}
4538 	if (ctl_lun == NULL) {
4539 		lun = malloc(sizeof(*lun), M_CTL, M_WAITOK);
4540 		lun_malloced = 1;
4541 	} else {
4542 		lun_malloced = 0;
4543 		lun = ctl_lun;
4544 	}
4545 
4546 	memset(lun, 0, sizeof(*lun));
4547 	if (lun_malloced)
4548 		lun->flags = CTL_LUN_MALLOCED;
4549 
4550 	/* Generate LUN ID. */
4551 	devidlen = max(CTL_DEVID_MIN_LEN,
4552 	    strnlen(be_lun->device_id, CTL_DEVID_LEN));
4553 	idlen1 = sizeof(*t10id) + devidlen;
4554 	len = sizeof(struct scsi_vpd_id_descriptor) + idlen1;
4555 	scsiname = ctl_get_opt(&be_lun->options, "scsiname");
4556 	if (scsiname != NULL) {
4557 		idlen2 = roundup2(strlen(scsiname) + 1, 4);
4558 		len += sizeof(struct scsi_vpd_id_descriptor) + idlen2;
4559 	}
4560 	eui = ctl_get_opt(&be_lun->options, "eui");
4561 	if (eui != NULL) {
4562 		len += sizeof(struct scsi_vpd_id_descriptor) + 16;
4563 	}
4564 	naa = ctl_get_opt(&be_lun->options, "naa");
4565 	if (naa != NULL) {
4566 		len += sizeof(struct scsi_vpd_id_descriptor) + 16;
4567 	}
4568 	lun->lun_devid = malloc(sizeof(struct ctl_devid) + len,
4569 	    M_CTL, M_WAITOK | M_ZERO);
4570 	desc = (struct scsi_vpd_id_descriptor *)lun->lun_devid->data;
4571 	desc->proto_codeset = SVPD_ID_CODESET_ASCII;
4572 	desc->id_type = SVPD_ID_PIV | SVPD_ID_ASSOC_LUN | SVPD_ID_TYPE_T10;
4573 	desc->length = idlen1;
4574 	t10id = (struct scsi_vpd_id_t10 *)&desc->identifier[0];
4575 	memset(t10id->vendor, ' ', sizeof(t10id->vendor));
4576 	if ((vendor = ctl_get_opt(&be_lun->options, "vendor")) == NULL) {
4577 		strncpy((char *)t10id->vendor, CTL_VENDOR, sizeof(t10id->vendor));
4578 	} else {
4579 		strncpy(t10id->vendor, vendor,
4580 		    min(sizeof(t10id->vendor), strlen(vendor)));
4581 	}
4582 	strncpy((char *)t10id->vendor_spec_id,
4583 	    (char *)be_lun->device_id, devidlen);
4584 	if (scsiname != NULL) {
4585 		desc = (struct scsi_vpd_id_descriptor *)(&desc->identifier[0] +
4586 		    desc->length);
4587 		desc->proto_codeset = SVPD_ID_CODESET_UTF8;
4588 		desc->id_type = SVPD_ID_PIV | SVPD_ID_ASSOC_LUN |
4589 		    SVPD_ID_TYPE_SCSI_NAME;
4590 		desc->length = idlen2;
4591 		strlcpy(desc->identifier, scsiname, idlen2);
4592 	}
4593 	if (eui != NULL) {
4594 		desc = (struct scsi_vpd_id_descriptor *)(&desc->identifier[0] +
4595 		    desc->length);
4596 		desc->proto_codeset = SVPD_ID_CODESET_BINARY;
4597 		desc->id_type = SVPD_ID_PIV | SVPD_ID_ASSOC_LUN |
4598 		    SVPD_ID_TYPE_EUI64;
4599 		desc->length = hex2bin(eui, desc->identifier, 16);
4600 		desc->length = desc->length > 12 ? 16 :
4601 		    (desc->length > 8 ? 12 : 8);
4602 		len -= 16 - desc->length;
4603 	}
4604 	if (naa != NULL) {
4605 		desc = (struct scsi_vpd_id_descriptor *)(&desc->identifier[0] +
4606 		    desc->length);
4607 		desc->proto_codeset = SVPD_ID_CODESET_BINARY;
4608 		desc->id_type = SVPD_ID_PIV | SVPD_ID_ASSOC_LUN |
4609 		    SVPD_ID_TYPE_NAA;
4610 		desc->length = hex2bin(naa, desc->identifier, 16);
4611 		desc->length = desc->length > 8 ? 16 : 8;
4612 		len -= 16 - desc->length;
4613 	}
4614 	lun->lun_devid->len = len;
4615 
4616 	mtx_lock(&ctl_softc->ctl_lock);
4617 	/*
4618 	 * See if the caller requested a particular LUN number.  If so, see
4619 	 * if it is available.  Otherwise, allocate the first available LUN.
4620 	 */
4621 	if (be_lun->flags & CTL_LUN_FLAG_ID_REQ) {
4622 		if ((be_lun->req_lun_id > (CTL_MAX_LUNS - 1))
4623 		 || (ctl_is_set(ctl_softc->ctl_lun_mask, be_lun->req_lun_id))) {
4624 			mtx_unlock(&ctl_softc->ctl_lock);
4625 			if (be_lun->req_lun_id > (CTL_MAX_LUNS - 1)) {
4626 				printf("ctl: requested LUN ID %d is higher "
4627 				       "than CTL_MAX_LUNS - 1 (%d)\n",
4628 				       be_lun->req_lun_id, CTL_MAX_LUNS - 1);
4629 			} else {
4630 				/*
4631 				 * XXX KDM return an error, or just assign
4632 				 * another LUN ID in this case??
4633 				 */
4634 				printf("ctl: requested LUN ID %d is already "
4635 				       "in use\n", be_lun->req_lun_id);
4636 			}
4637 			if (lun->flags & CTL_LUN_MALLOCED)
4638 				free(lun, M_CTL);
4639 			be_lun->lun_config_status(be_lun->be_lun,
4640 						  CTL_LUN_CONFIG_FAILURE);
4641 			return (ENOSPC);
4642 		}
4643 		lun_number = be_lun->req_lun_id;
4644 	} else {
4645 		lun_number = ctl_ffz(ctl_softc->ctl_lun_mask, CTL_MAX_LUNS);
4646 		if (lun_number == -1) {
4647 			mtx_unlock(&ctl_softc->ctl_lock);
4648 			printf("ctl: can't allocate LUN, out of LUNs\n");
4649 			if (lun->flags & CTL_LUN_MALLOCED)
4650 				free(lun, M_CTL);
4651 			be_lun->lun_config_status(be_lun->be_lun,
4652 						  CTL_LUN_CONFIG_FAILURE);
4653 			return (ENOSPC);
4654 		}
4655 	}
4656 	ctl_set_mask(ctl_softc->ctl_lun_mask, lun_number);
4657 
4658 	mtx_init(&lun->lun_lock, "CTL LUN", NULL, MTX_DEF);
4659 	lun->lun = lun_number;
4660 	lun->be_lun = be_lun;
4661 	/*
4662 	 * The processor LUN is always enabled.  Disk LUNs come on line
4663 	 * disabled, and must be enabled by the backend.
4664 	 */
4665 	lun->flags |= CTL_LUN_DISABLED;
4666 	lun->backend = be_lun->be;
4667 	be_lun->ctl_lun = lun;
4668 	be_lun->lun_id = lun_number;
4669 	atomic_add_int(&be_lun->be->num_luns, 1);
4670 	if (be_lun->flags & CTL_LUN_FLAG_OFFLINE)
4671 		lun->flags |= CTL_LUN_OFFLINE;
4672 
4673 	if (be_lun->flags & CTL_LUN_FLAG_POWERED_OFF)
4674 		lun->flags |= CTL_LUN_STOPPED;
4675 
4676 	if (be_lun->flags & CTL_LUN_FLAG_INOPERABLE)
4677 		lun->flags |= CTL_LUN_INOPERABLE;
4678 
4679 	if (be_lun->flags & CTL_LUN_FLAG_PRIMARY)
4680 		lun->flags |= CTL_LUN_PRIMARY_SC;
4681 
4682 	value = ctl_get_opt(&be_lun->options, "readonly");
4683 	if (value != NULL && strcmp(value, "on") == 0)
4684 		lun->flags |= CTL_LUN_READONLY;
4685 
4686 	lun->serseq = CTL_LUN_SERSEQ_OFF;
4687 	if (be_lun->flags & CTL_LUN_FLAG_SERSEQ_READ)
4688 		lun->serseq = CTL_LUN_SERSEQ_READ;
4689 	value = ctl_get_opt(&be_lun->options, "serseq");
4690 	if (value != NULL && strcmp(value, "on") == 0)
4691 		lun->serseq = CTL_LUN_SERSEQ_ON;
4692 	else if (value != NULL && strcmp(value, "read") == 0)
4693 		lun->serseq = CTL_LUN_SERSEQ_READ;
4694 	else if (value != NULL && strcmp(value, "off") == 0)
4695 		lun->serseq = CTL_LUN_SERSEQ_OFF;
4696 
4697 	lun->ctl_softc = ctl_softc;
4698 #ifdef CTL_TIME_IO
4699 	lun->last_busy = getsbinuptime();
4700 #endif
4701 	TAILQ_INIT(&lun->ooa_queue);
4702 	TAILQ_INIT(&lun->blocked_queue);
4703 	STAILQ_INIT(&lun->error_list);
4704 	ctl_tpc_lun_init(lun);
4705 
4706 	/*
4707 	 * Initialize the mode and log page index.
4708 	 */
4709 	ctl_init_page_index(lun);
4710 	ctl_init_log_page_index(lun);
4711 
4712 	/*
4713 	 * Now, before we insert this lun on the lun list, set the lun
4714 	 * inventory changed UA for all other luns.
4715 	 */
4716 	STAILQ_FOREACH(nlun, &ctl_softc->lun_list, links) {
4717 		mtx_lock(&nlun->lun_lock);
4718 		ctl_est_ua_all(nlun, -1, CTL_UA_LUN_CHANGE);
4719 		mtx_unlock(&nlun->lun_lock);
4720 	}
4721 
4722 	STAILQ_INSERT_TAIL(&ctl_softc->lun_list, lun, links);
4723 
4724 	ctl_softc->ctl_luns[lun_number] = lun;
4725 
4726 	ctl_softc->num_luns++;
4727 
4728 	/* Setup statistics gathering */
4729 	lun->stats.device_type = be_lun->lun_type;
4730 	lun->stats.lun_number = lun_number;
4731 	if (lun->stats.device_type == T_DIRECT)
4732 		lun->stats.blocksize = be_lun->blocksize;
4733 	else
4734 		lun->stats.flags = CTL_LUN_STATS_NO_BLOCKSIZE;
4735 	for (i = 0;i < CTL_MAX_PORTS;i++)
4736 		lun->stats.ports[i].targ_port = i;
4737 
4738 	mtx_unlock(&ctl_softc->ctl_lock);
4739 
4740 	lun->be_lun->lun_config_status(lun->be_lun->be_lun, CTL_LUN_CONFIG_OK);
4741 	return (0);
4742 }
4743 
4744 /*
4745  * Delete a LUN.
4746  * Assumptions:
4747  * - LUN has already been marked invalid and any pending I/O has been taken
4748  *   care of.
4749  */
4750 static int
4751 ctl_free_lun(struct ctl_lun *lun)
4752 {
4753 	struct ctl_softc *softc;
4754 	struct ctl_lun *nlun;
4755 	int i;
4756 
4757 	softc = lun->ctl_softc;
4758 
4759 	mtx_assert(&softc->ctl_lock, MA_OWNED);
4760 
4761 	STAILQ_REMOVE(&softc->lun_list, lun, ctl_lun, links);
4762 
4763 	ctl_clear_mask(softc->ctl_lun_mask, lun->lun);
4764 
4765 	softc->ctl_luns[lun->lun] = NULL;
4766 
4767 	if (!TAILQ_EMPTY(&lun->ooa_queue))
4768 		panic("Freeing a LUN %p with outstanding I/O!!\n", lun);
4769 
4770 	softc->num_luns--;
4771 
4772 	/*
4773 	 * Tell the backend to free resources, if this LUN has a backend.
4774 	 */
4775 	atomic_subtract_int(&lun->be_lun->be->num_luns, 1);
4776 	lun->be_lun->lun_shutdown(lun->be_lun->be_lun);
4777 
4778 	ctl_tpc_lun_shutdown(lun);
4779 	mtx_destroy(&lun->lun_lock);
4780 	free(lun->lun_devid, M_CTL);
4781 	for (i = 0; i < CTL_MAX_PORTS; i++)
4782 		free(lun->pending_ua[i], M_CTL);
4783 	for (i = 0; i < 2 * CTL_MAX_PORTS; i++)
4784 		free(lun->pr_keys[i], M_CTL);
4785 	free(lun->write_buffer, M_CTL);
4786 	if (lun->flags & CTL_LUN_MALLOCED)
4787 		free(lun, M_CTL);
4788 
4789 	STAILQ_FOREACH(nlun, &softc->lun_list, links) {
4790 		mtx_lock(&nlun->lun_lock);
4791 		ctl_est_ua_all(nlun, -1, CTL_UA_LUN_CHANGE);
4792 		mtx_unlock(&nlun->lun_lock);
4793 	}
4794 
4795 	return (0);
4796 }
4797 
4798 static void
4799 ctl_create_lun(struct ctl_be_lun *be_lun)
4800 {
4801 	struct ctl_softc *softc;
4802 
4803 	softc = control_softc;
4804 
4805 	/*
4806 	 * ctl_alloc_lun() should handle all potential failure cases.
4807 	 */
4808 	ctl_alloc_lun(softc, NULL, be_lun);
4809 }
4810 
4811 int
4812 ctl_add_lun(struct ctl_be_lun *be_lun)
4813 {
4814 	struct ctl_softc *softc = control_softc;
4815 
4816 	mtx_lock(&softc->ctl_lock);
4817 	STAILQ_INSERT_TAIL(&softc->pending_lun_queue, be_lun, links);
4818 	mtx_unlock(&softc->ctl_lock);
4819 	wakeup(&softc->pending_lun_queue);
4820 
4821 	return (0);
4822 }
4823 
4824 int
4825 ctl_enable_lun(struct ctl_be_lun *be_lun)
4826 {
4827 	struct ctl_softc *softc;
4828 	struct ctl_port *port, *nport;
4829 	struct ctl_lun *lun;
4830 	int retval;
4831 
4832 	lun = (struct ctl_lun *)be_lun->ctl_lun;
4833 	softc = lun->ctl_softc;
4834 
4835 	mtx_lock(&softc->ctl_lock);
4836 	mtx_lock(&lun->lun_lock);
4837 	if ((lun->flags & CTL_LUN_DISABLED) == 0) {
4838 		/*
4839 		 * eh?  Why did we get called if the LUN is already
4840 		 * enabled?
4841 		 */
4842 		mtx_unlock(&lun->lun_lock);
4843 		mtx_unlock(&softc->ctl_lock);
4844 		return (0);
4845 	}
4846 	lun->flags &= ~CTL_LUN_DISABLED;
4847 	mtx_unlock(&lun->lun_lock);
4848 
4849 	for (port = STAILQ_FIRST(&softc->port_list); port != NULL; port = nport) {
4850 		nport = STAILQ_NEXT(port, links);
4851 		if ((port->status & CTL_PORT_STATUS_ONLINE) == 0 ||
4852 		    port->lun_map != NULL)
4853 			continue;
4854 
4855 		/*
4856 		 * Drop the lock while we call the FETD's enable routine.
4857 		 * This can lead to a callback into CTL (at least in the
4858 		 * case of the internal initiator frontend.
4859 		 */
4860 		mtx_unlock(&softc->ctl_lock);
4861 		retval = port->lun_enable(port->targ_lun_arg, lun->lun);
4862 		mtx_lock(&softc->ctl_lock);
4863 		if (retval != 0) {
4864 			printf("%s: FETD %s port %d returned error "
4865 			       "%d for lun_enable on lun %jd\n",
4866 			       __func__, port->port_name, port->targ_port,
4867 			       retval, (intmax_t)lun->lun);
4868 		}
4869 	}
4870 
4871 	mtx_unlock(&softc->ctl_lock);
4872 
4873 	return (0);
4874 }
4875 
4876 int
4877 ctl_disable_lun(struct ctl_be_lun *be_lun)
4878 {
4879 	struct ctl_softc *softc;
4880 	struct ctl_port *port;
4881 	struct ctl_lun *lun;
4882 	int retval;
4883 
4884 	lun = (struct ctl_lun *)be_lun->ctl_lun;
4885 	softc = lun->ctl_softc;
4886 
4887 	mtx_lock(&softc->ctl_lock);
4888 	mtx_lock(&lun->lun_lock);
4889 	if (lun->flags & CTL_LUN_DISABLED) {
4890 		mtx_unlock(&lun->lun_lock);
4891 		mtx_unlock(&softc->ctl_lock);
4892 		return (0);
4893 	}
4894 	lun->flags |= CTL_LUN_DISABLED;
4895 	mtx_unlock(&lun->lun_lock);
4896 
4897 	STAILQ_FOREACH(port, &softc->port_list, links) {
4898 		if ((port->status & CTL_PORT_STATUS_ONLINE) == 0 ||
4899 		    port->lun_map != NULL)
4900 			continue;
4901 		mtx_unlock(&softc->ctl_lock);
4902 		/*
4903 		 * Drop the lock before we call the frontend's disable
4904 		 * routine, to avoid lock order reversals.
4905 		 *
4906 		 * XXX KDM what happens if the frontend list changes while
4907 		 * we're traversing it?  It's unlikely, but should be handled.
4908 		 */
4909 		retval = port->lun_disable(port->targ_lun_arg, lun->lun);
4910 		mtx_lock(&softc->ctl_lock);
4911 		if (retval != 0) {
4912 			printf("%s: FETD %s port %d returned error "
4913 			       "%d for lun_disable on lun %jd\n",
4914 			       __func__, port->port_name, port->targ_port,
4915 			       retval, (intmax_t)lun->lun);
4916 		}
4917 	}
4918 
4919 	mtx_unlock(&softc->ctl_lock);
4920 
4921 	return (0);
4922 }
4923 
4924 int
4925 ctl_start_lun(struct ctl_be_lun *be_lun)
4926 {
4927 	struct ctl_lun *lun = (struct ctl_lun *)be_lun->ctl_lun;
4928 
4929 	mtx_lock(&lun->lun_lock);
4930 	lun->flags &= ~CTL_LUN_STOPPED;
4931 	mtx_unlock(&lun->lun_lock);
4932 	return (0);
4933 }
4934 
4935 int
4936 ctl_stop_lun(struct ctl_be_lun *be_lun)
4937 {
4938 	struct ctl_lun *lun = (struct ctl_lun *)be_lun->ctl_lun;
4939 
4940 	mtx_lock(&lun->lun_lock);
4941 	lun->flags |= CTL_LUN_STOPPED;
4942 	mtx_unlock(&lun->lun_lock);
4943 	return (0);
4944 }
4945 
4946 int
4947 ctl_lun_offline(struct ctl_be_lun *be_lun)
4948 {
4949 	struct ctl_lun *lun = (struct ctl_lun *)be_lun->ctl_lun;
4950 
4951 	mtx_lock(&lun->lun_lock);
4952 	lun->flags |= CTL_LUN_OFFLINE;
4953 	mtx_unlock(&lun->lun_lock);
4954 	return (0);
4955 }
4956 
4957 int
4958 ctl_lun_online(struct ctl_be_lun *be_lun)
4959 {
4960 	struct ctl_lun *lun = (struct ctl_lun *)be_lun->ctl_lun;
4961 
4962 	mtx_lock(&lun->lun_lock);
4963 	lun->flags &= ~CTL_LUN_OFFLINE;
4964 	mtx_unlock(&lun->lun_lock);
4965 	return (0);
4966 }
4967 
4968 int
4969 ctl_invalidate_lun(struct ctl_be_lun *be_lun)
4970 {
4971 	struct ctl_softc *softc;
4972 	struct ctl_lun *lun;
4973 
4974 	lun = (struct ctl_lun *)be_lun->ctl_lun;
4975 	softc = lun->ctl_softc;
4976 
4977 	mtx_lock(&lun->lun_lock);
4978 
4979 	/*
4980 	 * The LUN needs to be disabled before it can be marked invalid.
4981 	 */
4982 	if ((lun->flags & CTL_LUN_DISABLED) == 0) {
4983 		mtx_unlock(&lun->lun_lock);
4984 		return (-1);
4985 	}
4986 	/*
4987 	 * Mark the LUN invalid.
4988 	 */
4989 	lun->flags |= CTL_LUN_INVALID;
4990 
4991 	/*
4992 	 * If there is nothing in the OOA queue, go ahead and free the LUN.
4993 	 * If we have something in the OOA queue, we'll free it when the
4994 	 * last I/O completes.
4995 	 */
4996 	if (TAILQ_EMPTY(&lun->ooa_queue)) {
4997 		mtx_unlock(&lun->lun_lock);
4998 		mtx_lock(&softc->ctl_lock);
4999 		ctl_free_lun(lun);
5000 		mtx_unlock(&softc->ctl_lock);
5001 	} else
5002 		mtx_unlock(&lun->lun_lock);
5003 
5004 	return (0);
5005 }
5006 
5007 int
5008 ctl_lun_inoperable(struct ctl_be_lun *be_lun)
5009 {
5010 	struct ctl_lun *lun = (struct ctl_lun *)be_lun->ctl_lun;
5011 
5012 	mtx_lock(&lun->lun_lock);
5013 	lun->flags |= CTL_LUN_INOPERABLE;
5014 	mtx_unlock(&lun->lun_lock);
5015 	return (0);
5016 }
5017 
5018 int
5019 ctl_lun_operable(struct ctl_be_lun *be_lun)
5020 {
5021 	struct ctl_lun *lun = (struct ctl_lun *)be_lun->ctl_lun;
5022 
5023 	mtx_lock(&lun->lun_lock);
5024 	lun->flags &= ~CTL_LUN_INOPERABLE;
5025 	mtx_unlock(&lun->lun_lock);
5026 	return (0);
5027 }
5028 
5029 void
5030 ctl_lun_capacity_changed(struct ctl_be_lun *be_lun)
5031 {
5032 	struct ctl_lun *lun = (struct ctl_lun *)be_lun->ctl_lun;
5033 
5034 	mtx_lock(&lun->lun_lock);
5035 	ctl_est_ua_all(lun, -1, CTL_UA_CAPACITY_CHANGED);
5036 	mtx_unlock(&lun->lun_lock);
5037 }
5038 
5039 /*
5040  * Backend "memory move is complete" callback for requests that never
5041  * make it down to say RAIDCore's configuration code.
5042  */
5043 int
5044 ctl_config_move_done(union ctl_io *io)
5045 {
5046 	int retval;
5047 
5048 	CTL_DEBUG_PRINT(("ctl_config_move_done\n"));
5049 	KASSERT(io->io_hdr.io_type == CTL_IO_SCSI,
5050 	    ("Config I/O type isn't CTL_IO_SCSI (%d)!", io->io_hdr.io_type));
5051 
5052 	if ((io->io_hdr.port_status != 0) &&
5053 	    ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_STATUS_NONE ||
5054 	     (io->io_hdr.status & CTL_STATUS_MASK) == CTL_SUCCESS)) {
5055 		/*
5056 		 * For hardware error sense keys, the sense key
5057 		 * specific value is defined to be a retry count,
5058 		 * but we use it to pass back an internal FETD
5059 		 * error code.  XXX KDM  Hopefully the FETD is only
5060 		 * using 16 bits for an error code, since that's
5061 		 * all the space we have in the sks field.
5062 		 */
5063 		ctl_set_internal_failure(&io->scsiio,
5064 					 /*sks_valid*/ 1,
5065 					 /*retry_count*/
5066 					 io->io_hdr.port_status);
5067 	}
5068 
5069 	if (((io->io_hdr.flags & CTL_FLAG_DATA_MASK) == CTL_FLAG_DATA_IN) ||
5070 	    ((io->io_hdr.status & CTL_STATUS_MASK) != CTL_STATUS_NONE &&
5071 	     (io->io_hdr.status & CTL_STATUS_MASK) != CTL_SUCCESS) ||
5072 	    ((io->io_hdr.flags & CTL_FLAG_ABORT) != 0)) {
5073 		/*
5074 		 * XXX KDM just assuming a single pointer here, and not a
5075 		 * S/G list.  If we start using S/G lists for config data,
5076 		 * we'll need to know how to clean them up here as well.
5077 		 */
5078 		if (io->io_hdr.flags & CTL_FLAG_ALLOCATED)
5079 			free(io->scsiio.kern_data_ptr, M_CTL);
5080 		ctl_done(io);
5081 		retval = CTL_RETVAL_COMPLETE;
5082 	} else {
5083 		/*
5084 		 * XXX KDM now we need to continue data movement.  Some
5085 		 * options:
5086 		 * - call ctl_scsiio() again?  We don't do this for data
5087 		 *   writes, because for those at least we know ahead of
5088 		 *   time where the write will go and how long it is.  For
5089 		 *   config writes, though, that information is largely
5090 		 *   contained within the write itself, thus we need to
5091 		 *   parse out the data again.
5092 		 *
5093 		 * - Call some other function once the data is in?
5094 		 */
5095 		if (ctl_debug & CTL_DEBUG_CDB_DATA)
5096 			ctl_data_print(io);
5097 
5098 		/*
5099 		 * XXX KDM call ctl_scsiio() again for now, and check flag
5100 		 * bits to see whether we're allocated or not.
5101 		 */
5102 		retval = ctl_scsiio(&io->scsiio);
5103 	}
5104 	return (retval);
5105 }
5106 
5107 /*
5108  * This gets called by a backend driver when it is done with a
5109  * data_submit method.
5110  */
5111 void
5112 ctl_data_submit_done(union ctl_io *io)
5113 {
5114 	/*
5115 	 * If the IO_CONT flag is set, we need to call the supplied
5116 	 * function to continue processing the I/O, instead of completing
5117 	 * the I/O just yet.
5118 	 *
5119 	 * If there is an error, though, we don't want to keep processing.
5120 	 * Instead, just send status back to the initiator.
5121 	 */
5122 	if ((io->io_hdr.flags & CTL_FLAG_IO_CONT) &&
5123 	    (io->io_hdr.flags & CTL_FLAG_ABORT) == 0 &&
5124 	    ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_STATUS_NONE ||
5125 	     (io->io_hdr.status & CTL_STATUS_MASK) == CTL_SUCCESS)) {
5126 		io->scsiio.io_cont(io);
5127 		return;
5128 	}
5129 	ctl_done(io);
5130 }
5131 
5132 /*
5133  * This gets called by a backend driver when it is done with a
5134  * configuration write.
5135  */
5136 void
5137 ctl_config_write_done(union ctl_io *io)
5138 {
5139 	uint8_t *buf;
5140 
5141 	/*
5142 	 * If the IO_CONT flag is set, we need to call the supplied
5143 	 * function to continue processing the I/O, instead of completing
5144 	 * the I/O just yet.
5145 	 *
5146 	 * If there is an error, though, we don't want to keep processing.
5147 	 * Instead, just send status back to the initiator.
5148 	 */
5149 	if ((io->io_hdr.flags & CTL_FLAG_IO_CONT) &&
5150 	    (io->io_hdr.flags & CTL_FLAG_ABORT) == 0 &&
5151 	    ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_STATUS_NONE ||
5152 	     (io->io_hdr.status & CTL_STATUS_MASK) == CTL_SUCCESS)) {
5153 		io->scsiio.io_cont(io);
5154 		return;
5155 	}
5156 	/*
5157 	 * Since a configuration write can be done for commands that actually
5158 	 * have data allocated, like write buffer, and commands that have
5159 	 * no data, like start/stop unit, we need to check here.
5160 	 */
5161 	if (io->io_hdr.flags & CTL_FLAG_ALLOCATED)
5162 		buf = io->scsiio.kern_data_ptr;
5163 	else
5164 		buf = NULL;
5165 	ctl_done(io);
5166 	if (buf)
5167 		free(buf, M_CTL);
5168 }
5169 
5170 void
5171 ctl_config_read_done(union ctl_io *io)
5172 {
5173 	uint8_t *buf;
5174 
5175 	/*
5176 	 * If there is some error -- we are done, skip data transfer.
5177 	 */
5178 	if ((io->io_hdr.flags & CTL_FLAG_ABORT) != 0 ||
5179 	    ((io->io_hdr.status & CTL_STATUS_MASK) != CTL_STATUS_NONE &&
5180 	     (io->io_hdr.status & CTL_STATUS_MASK) != CTL_SUCCESS)) {
5181 		if (io->io_hdr.flags & CTL_FLAG_ALLOCATED)
5182 			buf = io->scsiio.kern_data_ptr;
5183 		else
5184 			buf = NULL;
5185 		ctl_done(io);
5186 		if (buf)
5187 			free(buf, M_CTL);
5188 		return;
5189 	}
5190 
5191 	/*
5192 	 * If the IO_CONT flag is set, we need to call the supplied
5193 	 * function to continue processing the I/O, instead of completing
5194 	 * the I/O just yet.
5195 	 */
5196 	if (io->io_hdr.flags & CTL_FLAG_IO_CONT) {
5197 		io->scsiio.io_cont(io);
5198 		return;
5199 	}
5200 
5201 	ctl_datamove(io);
5202 }
5203 
5204 /*
5205  * SCSI release command.
5206  */
5207 int
5208 ctl_scsi_release(struct ctl_scsiio *ctsio)
5209 {
5210 	int length, longid, thirdparty_id, resv_id;
5211 	struct ctl_lun *lun;
5212 	uint32_t residx;
5213 
5214 	length = 0;
5215 	resv_id = 0;
5216 
5217 	CTL_DEBUG_PRINT(("ctl_scsi_release\n"));
5218 
5219 	residx = ctl_get_resindex(&ctsio->io_hdr.nexus);
5220 	lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
5221 
5222 	switch (ctsio->cdb[0]) {
5223 	case RELEASE_10: {
5224 		struct scsi_release_10 *cdb;
5225 
5226 		cdb = (struct scsi_release_10 *)ctsio->cdb;
5227 
5228 		if (cdb->byte2 & SR10_LONGID)
5229 			longid = 1;
5230 		else
5231 			thirdparty_id = cdb->thirdparty_id;
5232 
5233 		resv_id = cdb->resv_id;
5234 		length = scsi_2btoul(cdb->length);
5235 		break;
5236 	}
5237 	}
5238 
5239 
5240 	/*
5241 	 * XXX KDM right now, we only support LUN reservation.  We don't
5242 	 * support 3rd party reservations, or extent reservations, which
5243 	 * might actually need the parameter list.  If we've gotten this
5244 	 * far, we've got a LUN reservation.  Anything else got kicked out
5245 	 * above.  So, according to SPC, ignore the length.
5246 	 */
5247 	length = 0;
5248 
5249 	if (((ctsio->io_hdr.flags & CTL_FLAG_ALLOCATED) == 0)
5250 	 && (length > 0)) {
5251 		ctsio->kern_data_ptr = malloc(length, M_CTL, M_WAITOK);
5252 		ctsio->kern_data_len = length;
5253 		ctsio->kern_total_len = length;
5254 		ctsio->kern_data_resid = 0;
5255 		ctsio->kern_rel_offset = 0;
5256 		ctsio->kern_sg_entries = 0;
5257 		ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
5258 		ctsio->be_move_done = ctl_config_move_done;
5259 		ctl_datamove((union ctl_io *)ctsio);
5260 
5261 		return (CTL_RETVAL_COMPLETE);
5262 	}
5263 
5264 	if (length > 0)
5265 		thirdparty_id = scsi_8btou64(ctsio->kern_data_ptr);
5266 
5267 	mtx_lock(&lun->lun_lock);
5268 
5269 	/*
5270 	 * According to SPC, it is not an error for an intiator to attempt
5271 	 * to release a reservation on a LUN that isn't reserved, or that
5272 	 * is reserved by another initiator.  The reservation can only be
5273 	 * released, though, by the initiator who made it or by one of
5274 	 * several reset type events.
5275 	 */
5276 	if ((lun->flags & CTL_LUN_RESERVED) && (lun->res_idx == residx))
5277 			lun->flags &= ~CTL_LUN_RESERVED;
5278 
5279 	mtx_unlock(&lun->lun_lock);
5280 
5281 	if (ctsio->io_hdr.flags & CTL_FLAG_ALLOCATED) {
5282 		free(ctsio->kern_data_ptr, M_CTL);
5283 		ctsio->io_hdr.flags &= ~CTL_FLAG_ALLOCATED;
5284 	}
5285 
5286 	ctl_set_success(ctsio);
5287 	ctl_done((union ctl_io *)ctsio);
5288 	return (CTL_RETVAL_COMPLETE);
5289 }
5290 
5291 int
5292 ctl_scsi_reserve(struct ctl_scsiio *ctsio)
5293 {
5294 	int extent, thirdparty, longid;
5295 	int resv_id, length;
5296 	uint64_t thirdparty_id;
5297 	struct ctl_lun *lun;
5298 	uint32_t residx;
5299 
5300 	extent = 0;
5301 	thirdparty = 0;
5302 	longid = 0;
5303 	resv_id = 0;
5304 	length = 0;
5305 	thirdparty_id = 0;
5306 
5307 	CTL_DEBUG_PRINT(("ctl_reserve\n"));
5308 
5309 	residx = ctl_get_resindex(&ctsio->io_hdr.nexus);
5310 	lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
5311 
5312 	switch (ctsio->cdb[0]) {
5313 	case RESERVE_10: {
5314 		struct scsi_reserve_10 *cdb;
5315 
5316 		cdb = (struct scsi_reserve_10 *)ctsio->cdb;
5317 
5318 		if (cdb->byte2 & SR10_LONGID)
5319 			longid = 1;
5320 		else
5321 			thirdparty_id = cdb->thirdparty_id;
5322 
5323 		resv_id = cdb->resv_id;
5324 		length = scsi_2btoul(cdb->length);
5325 		break;
5326 	}
5327 	}
5328 
5329 	/*
5330 	 * XXX KDM right now, we only support LUN reservation.  We don't
5331 	 * support 3rd party reservations, or extent reservations, which
5332 	 * might actually need the parameter list.  If we've gotten this
5333 	 * far, we've got a LUN reservation.  Anything else got kicked out
5334 	 * above.  So, according to SPC, ignore the length.
5335 	 */
5336 	length = 0;
5337 
5338 	if (((ctsio->io_hdr.flags & CTL_FLAG_ALLOCATED) == 0)
5339 	 && (length > 0)) {
5340 		ctsio->kern_data_ptr = malloc(length, M_CTL, M_WAITOK);
5341 		ctsio->kern_data_len = length;
5342 		ctsio->kern_total_len = length;
5343 		ctsio->kern_data_resid = 0;
5344 		ctsio->kern_rel_offset = 0;
5345 		ctsio->kern_sg_entries = 0;
5346 		ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
5347 		ctsio->be_move_done = ctl_config_move_done;
5348 		ctl_datamove((union ctl_io *)ctsio);
5349 
5350 		return (CTL_RETVAL_COMPLETE);
5351 	}
5352 
5353 	if (length > 0)
5354 		thirdparty_id = scsi_8btou64(ctsio->kern_data_ptr);
5355 
5356 	mtx_lock(&lun->lun_lock);
5357 	if ((lun->flags & CTL_LUN_RESERVED) && (lun->res_idx != residx)) {
5358 		ctl_set_reservation_conflict(ctsio);
5359 		goto bailout;
5360 	}
5361 
5362 	lun->flags |= CTL_LUN_RESERVED;
5363 	lun->res_idx = residx;
5364 
5365 	ctl_set_success(ctsio);
5366 
5367 bailout:
5368 	mtx_unlock(&lun->lun_lock);
5369 
5370 	if (ctsio->io_hdr.flags & CTL_FLAG_ALLOCATED) {
5371 		free(ctsio->kern_data_ptr, M_CTL);
5372 		ctsio->io_hdr.flags &= ~CTL_FLAG_ALLOCATED;
5373 	}
5374 
5375 	ctl_done((union ctl_io *)ctsio);
5376 	return (CTL_RETVAL_COMPLETE);
5377 }
5378 
5379 int
5380 ctl_start_stop(struct ctl_scsiio *ctsio)
5381 {
5382 	struct scsi_start_stop_unit *cdb;
5383 	struct ctl_lun *lun;
5384 	int retval;
5385 
5386 	CTL_DEBUG_PRINT(("ctl_start_stop\n"));
5387 
5388 	lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
5389 	retval = 0;
5390 
5391 	cdb = (struct scsi_start_stop_unit *)ctsio->cdb;
5392 
5393 	/*
5394 	 * XXX KDM
5395 	 * We don't support the immediate bit on a stop unit.  In order to
5396 	 * do that, we would need to code up a way to know that a stop is
5397 	 * pending, and hold off any new commands until it completes, one
5398 	 * way or another.  Then we could accept or reject those commands
5399 	 * depending on its status.  We would almost need to do the reverse
5400 	 * of what we do below for an immediate start -- return the copy of
5401 	 * the ctl_io to the FETD with status to send to the host (and to
5402 	 * free the copy!) and then free the original I/O once the stop
5403 	 * actually completes.  That way, the OOA queue mechanism can work
5404 	 * to block commands that shouldn't proceed.  Another alternative
5405 	 * would be to put the copy in the queue in place of the original,
5406 	 * and return the original back to the caller.  That could be
5407 	 * slightly safer..
5408 	 */
5409 	if ((cdb->byte2 & SSS_IMMED)
5410 	 && ((cdb->how & SSS_START) == 0)) {
5411 		ctl_set_invalid_field(ctsio,
5412 				      /*sks_valid*/ 1,
5413 				      /*command*/ 1,
5414 				      /*field*/ 1,
5415 				      /*bit_valid*/ 1,
5416 				      /*bit*/ 0);
5417 		ctl_done((union ctl_io *)ctsio);
5418 		return (CTL_RETVAL_COMPLETE);
5419 	}
5420 
5421 	if ((lun->flags & CTL_LUN_PR_RESERVED)
5422 	 && ((cdb->how & SSS_START)==0)) {
5423 		uint32_t residx;
5424 
5425 		residx = ctl_get_resindex(&ctsio->io_hdr.nexus);
5426 		if (ctl_get_prkey(lun, residx) == 0
5427 		 || (lun->pr_res_idx!=residx && lun->res_type < 4)) {
5428 
5429 			ctl_set_reservation_conflict(ctsio);
5430 			ctl_done((union ctl_io *)ctsio);
5431 			return (CTL_RETVAL_COMPLETE);
5432 		}
5433 	}
5434 
5435 	/*
5436 	 * If there is no backend on this device, we can't start or stop
5437 	 * it.  In theory we shouldn't get any start/stop commands in the
5438 	 * first place at this level if the LUN doesn't have a backend.
5439 	 * That should get stopped by the command decode code.
5440 	 */
5441 	if (lun->backend == NULL) {
5442 		ctl_set_invalid_opcode(ctsio);
5443 		ctl_done((union ctl_io *)ctsio);
5444 		return (CTL_RETVAL_COMPLETE);
5445 	}
5446 
5447 	/*
5448 	 * XXX KDM Copan-specific offline behavior.
5449 	 * Figure out a reasonable way to port this?
5450 	 */
5451 #ifdef NEEDTOPORT
5452 	mtx_lock(&lun->lun_lock);
5453 
5454 	if (((cdb->byte2 & SSS_ONOFFLINE) == 0)
5455 	 && (lun->flags & CTL_LUN_OFFLINE)) {
5456 		/*
5457 		 * If the LUN is offline, and the on/offline bit isn't set,
5458 		 * reject the start or stop.  Otherwise, let it through.
5459 		 */
5460 		mtx_unlock(&lun->lun_lock);
5461 		ctl_set_lun_not_ready(ctsio);
5462 		ctl_done((union ctl_io *)ctsio);
5463 	} else {
5464 		mtx_unlock(&lun->lun_lock);
5465 #endif /* NEEDTOPORT */
5466 		/*
5467 		 * This could be a start or a stop when we're online,
5468 		 * or a stop/offline or start/online.  A start or stop when
5469 		 * we're offline is covered in the case above.
5470 		 */
5471 		/*
5472 		 * In the non-immediate case, we send the request to
5473 		 * the backend and return status to the user when
5474 		 * it is done.
5475 		 *
5476 		 * In the immediate case, we allocate a new ctl_io
5477 		 * to hold a copy of the request, and send that to
5478 		 * the backend.  We then set good status on the
5479 		 * user's request and return it immediately.
5480 		 */
5481 		if (cdb->byte2 & SSS_IMMED) {
5482 			union ctl_io *new_io;
5483 
5484 			new_io = ctl_alloc_io(ctsio->io_hdr.pool);
5485 			ctl_copy_io((union ctl_io *)ctsio, new_io);
5486 			retval = lun->backend->config_write(new_io);
5487 			ctl_set_success(ctsio);
5488 			ctl_done((union ctl_io *)ctsio);
5489 		} else {
5490 			retval = lun->backend->config_write(
5491 				(union ctl_io *)ctsio);
5492 		}
5493 #ifdef NEEDTOPORT
5494 	}
5495 #endif
5496 	return (retval);
5497 }
5498 
5499 /*
5500  * We support the SYNCHRONIZE CACHE command (10 and 16 byte versions), but
5501  * we don't really do anything with the LBA and length fields if the user
5502  * passes them in.  Instead we'll just flush out the cache for the entire
5503  * LUN.
5504  */
5505 int
5506 ctl_sync_cache(struct ctl_scsiio *ctsio)
5507 {
5508 	struct ctl_lun *lun;
5509 	struct ctl_softc *softc;
5510 	uint64_t starting_lba;
5511 	uint32_t block_count;
5512 	int retval;
5513 
5514 	CTL_DEBUG_PRINT(("ctl_sync_cache\n"));
5515 
5516 	lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
5517 	softc = lun->ctl_softc;
5518 	retval = 0;
5519 
5520 	switch (ctsio->cdb[0]) {
5521 	case SYNCHRONIZE_CACHE: {
5522 		struct scsi_sync_cache *cdb;
5523 		cdb = (struct scsi_sync_cache *)ctsio->cdb;
5524 
5525 		starting_lba = scsi_4btoul(cdb->begin_lba);
5526 		block_count = scsi_2btoul(cdb->lb_count);
5527 		break;
5528 	}
5529 	case SYNCHRONIZE_CACHE_16: {
5530 		struct scsi_sync_cache_16 *cdb;
5531 		cdb = (struct scsi_sync_cache_16 *)ctsio->cdb;
5532 
5533 		starting_lba = scsi_8btou64(cdb->begin_lba);
5534 		block_count = scsi_4btoul(cdb->lb_count);
5535 		break;
5536 	}
5537 	default:
5538 		ctl_set_invalid_opcode(ctsio);
5539 		ctl_done((union ctl_io *)ctsio);
5540 		goto bailout;
5541 		break; /* NOTREACHED */
5542 	}
5543 
5544 	/*
5545 	 * We check the LBA and length, but don't do anything with them.
5546 	 * A SYNCHRONIZE CACHE will cause the entire cache for this lun to
5547 	 * get flushed.  This check will just help satisfy anyone who wants
5548 	 * to see an error for an out of range LBA.
5549 	 */
5550 	if ((starting_lba + block_count) > (lun->be_lun->maxlba + 1)) {
5551 		ctl_set_lba_out_of_range(ctsio);
5552 		ctl_done((union ctl_io *)ctsio);
5553 		goto bailout;
5554 	}
5555 
5556 	/*
5557 	 * If this LUN has no backend, we can't flush the cache anyway.
5558 	 */
5559 	if (lun->backend == NULL) {
5560 		ctl_set_invalid_opcode(ctsio);
5561 		ctl_done((union ctl_io *)ctsio);
5562 		goto bailout;
5563 	}
5564 
5565 	/*
5566 	 * Check to see whether we're configured to send the SYNCHRONIZE
5567 	 * CACHE command directly to the back end.
5568 	 */
5569 	mtx_lock(&lun->lun_lock);
5570 	if ((softc->flags & CTL_FLAG_REAL_SYNC)
5571 	 && (++(lun->sync_count) >= lun->sync_interval)) {
5572 		lun->sync_count = 0;
5573 		mtx_unlock(&lun->lun_lock);
5574 		retval = lun->backend->config_write((union ctl_io *)ctsio);
5575 	} else {
5576 		mtx_unlock(&lun->lun_lock);
5577 		ctl_set_success(ctsio);
5578 		ctl_done((union ctl_io *)ctsio);
5579 	}
5580 
5581 bailout:
5582 
5583 	return (retval);
5584 }
5585 
5586 int
5587 ctl_format(struct ctl_scsiio *ctsio)
5588 {
5589 	struct scsi_format *cdb;
5590 	struct ctl_lun *lun;
5591 	int length, defect_list_len;
5592 
5593 	CTL_DEBUG_PRINT(("ctl_format\n"));
5594 
5595 	lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
5596 
5597 	cdb = (struct scsi_format *)ctsio->cdb;
5598 
5599 	length = 0;
5600 	if (cdb->byte2 & SF_FMTDATA) {
5601 		if (cdb->byte2 & SF_LONGLIST)
5602 			length = sizeof(struct scsi_format_header_long);
5603 		else
5604 			length = sizeof(struct scsi_format_header_short);
5605 	}
5606 
5607 	if (((ctsio->io_hdr.flags & CTL_FLAG_ALLOCATED) == 0)
5608 	 && (length > 0)) {
5609 		ctsio->kern_data_ptr = malloc(length, M_CTL, M_WAITOK);
5610 		ctsio->kern_data_len = length;
5611 		ctsio->kern_total_len = length;
5612 		ctsio->kern_data_resid = 0;
5613 		ctsio->kern_rel_offset = 0;
5614 		ctsio->kern_sg_entries = 0;
5615 		ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
5616 		ctsio->be_move_done = ctl_config_move_done;
5617 		ctl_datamove((union ctl_io *)ctsio);
5618 
5619 		return (CTL_RETVAL_COMPLETE);
5620 	}
5621 
5622 	defect_list_len = 0;
5623 
5624 	if (cdb->byte2 & SF_FMTDATA) {
5625 		if (cdb->byte2 & SF_LONGLIST) {
5626 			struct scsi_format_header_long *header;
5627 
5628 			header = (struct scsi_format_header_long *)
5629 				ctsio->kern_data_ptr;
5630 
5631 			defect_list_len = scsi_4btoul(header->defect_list_len);
5632 			if (defect_list_len != 0) {
5633 				ctl_set_invalid_field(ctsio,
5634 						      /*sks_valid*/ 1,
5635 						      /*command*/ 0,
5636 						      /*field*/ 2,
5637 						      /*bit_valid*/ 0,
5638 						      /*bit*/ 0);
5639 				goto bailout;
5640 			}
5641 		} else {
5642 			struct scsi_format_header_short *header;
5643 
5644 			header = (struct scsi_format_header_short *)
5645 				ctsio->kern_data_ptr;
5646 
5647 			defect_list_len = scsi_2btoul(header->defect_list_len);
5648 			if (defect_list_len != 0) {
5649 				ctl_set_invalid_field(ctsio,
5650 						      /*sks_valid*/ 1,
5651 						      /*command*/ 0,
5652 						      /*field*/ 2,
5653 						      /*bit_valid*/ 0,
5654 						      /*bit*/ 0);
5655 				goto bailout;
5656 			}
5657 		}
5658 	}
5659 
5660 	/*
5661 	 * The format command will clear out the "Medium format corrupted"
5662 	 * status if set by the configuration code.  That status is really
5663 	 * just a way to notify the host that we have lost the media, and
5664 	 * get them to issue a command that will basically make them think
5665 	 * they're blowing away the media.
5666 	 */
5667 	mtx_lock(&lun->lun_lock);
5668 	lun->flags &= ~CTL_LUN_INOPERABLE;
5669 	mtx_unlock(&lun->lun_lock);
5670 
5671 	ctl_set_success(ctsio);
5672 bailout:
5673 
5674 	if (ctsio->io_hdr.flags & CTL_FLAG_ALLOCATED) {
5675 		free(ctsio->kern_data_ptr, M_CTL);
5676 		ctsio->io_hdr.flags &= ~CTL_FLAG_ALLOCATED;
5677 	}
5678 
5679 	ctl_done((union ctl_io *)ctsio);
5680 	return (CTL_RETVAL_COMPLETE);
5681 }
5682 
5683 int
5684 ctl_read_buffer(struct ctl_scsiio *ctsio)
5685 {
5686 	struct scsi_read_buffer *cdb;
5687 	struct ctl_lun *lun;
5688 	int buffer_offset, len;
5689 	static uint8_t descr[4];
5690 	static uint8_t echo_descr[4] = { 0 };
5691 
5692 	CTL_DEBUG_PRINT(("ctl_read_buffer\n"));
5693 
5694 	lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
5695 	cdb = (struct scsi_read_buffer *)ctsio->cdb;
5696 
5697 	if ((cdb->byte2 & RWB_MODE) != RWB_MODE_DATA &&
5698 	    (cdb->byte2 & RWB_MODE) != RWB_MODE_ECHO_DESCR &&
5699 	    (cdb->byte2 & RWB_MODE) != RWB_MODE_DESCR) {
5700 		ctl_set_invalid_field(ctsio,
5701 				      /*sks_valid*/ 1,
5702 				      /*command*/ 1,
5703 				      /*field*/ 1,
5704 				      /*bit_valid*/ 1,
5705 				      /*bit*/ 4);
5706 		ctl_done((union ctl_io *)ctsio);
5707 		return (CTL_RETVAL_COMPLETE);
5708 	}
5709 
5710 	len = scsi_3btoul(cdb->length);
5711 	buffer_offset = scsi_3btoul(cdb->offset);
5712 
5713 	if (buffer_offset + len > CTL_WRITE_BUFFER_SIZE) {
5714 		ctl_set_invalid_field(ctsio,
5715 				      /*sks_valid*/ 1,
5716 				      /*command*/ 1,
5717 				      /*field*/ 6,
5718 				      /*bit_valid*/ 0,
5719 				      /*bit*/ 0);
5720 		ctl_done((union ctl_io *)ctsio);
5721 		return (CTL_RETVAL_COMPLETE);
5722 	}
5723 
5724 	if ((cdb->byte2 & RWB_MODE) == RWB_MODE_DESCR) {
5725 		descr[0] = 0;
5726 		scsi_ulto3b(CTL_WRITE_BUFFER_SIZE, &descr[1]);
5727 		ctsio->kern_data_ptr = descr;
5728 		len = min(len, sizeof(descr));
5729 	} else if ((cdb->byte2 & RWB_MODE) == RWB_MODE_ECHO_DESCR) {
5730 		ctsio->kern_data_ptr = echo_descr;
5731 		len = min(len, sizeof(echo_descr));
5732 	} else {
5733 		if (lun->write_buffer == NULL) {
5734 			lun->write_buffer = malloc(CTL_WRITE_BUFFER_SIZE,
5735 			    M_CTL, M_WAITOK);
5736 		}
5737 		ctsio->kern_data_ptr = lun->write_buffer + buffer_offset;
5738 	}
5739 	ctsio->kern_data_len = len;
5740 	ctsio->kern_total_len = len;
5741 	ctsio->kern_data_resid = 0;
5742 	ctsio->kern_rel_offset = 0;
5743 	ctsio->kern_sg_entries = 0;
5744 	ctl_set_success(ctsio);
5745 	ctsio->be_move_done = ctl_config_move_done;
5746 	ctl_datamove((union ctl_io *)ctsio);
5747 	return (CTL_RETVAL_COMPLETE);
5748 }
5749 
5750 int
5751 ctl_write_buffer(struct ctl_scsiio *ctsio)
5752 {
5753 	struct scsi_write_buffer *cdb;
5754 	struct ctl_lun *lun;
5755 	int buffer_offset, len;
5756 
5757 	CTL_DEBUG_PRINT(("ctl_write_buffer\n"));
5758 
5759 	lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
5760 	cdb = (struct scsi_write_buffer *)ctsio->cdb;
5761 
5762 	if ((cdb->byte2 & RWB_MODE) != RWB_MODE_DATA) {
5763 		ctl_set_invalid_field(ctsio,
5764 				      /*sks_valid*/ 1,
5765 				      /*command*/ 1,
5766 				      /*field*/ 1,
5767 				      /*bit_valid*/ 1,
5768 				      /*bit*/ 4);
5769 		ctl_done((union ctl_io *)ctsio);
5770 		return (CTL_RETVAL_COMPLETE);
5771 	}
5772 
5773 	len = scsi_3btoul(cdb->length);
5774 	buffer_offset = scsi_3btoul(cdb->offset);
5775 
5776 	if (buffer_offset + len > CTL_WRITE_BUFFER_SIZE) {
5777 		ctl_set_invalid_field(ctsio,
5778 				      /*sks_valid*/ 1,
5779 				      /*command*/ 1,
5780 				      /*field*/ 6,
5781 				      /*bit_valid*/ 0,
5782 				      /*bit*/ 0);
5783 		ctl_done((union ctl_io *)ctsio);
5784 		return (CTL_RETVAL_COMPLETE);
5785 	}
5786 
5787 	/*
5788 	 * If we've got a kernel request that hasn't been malloced yet,
5789 	 * malloc it and tell the caller the data buffer is here.
5790 	 */
5791 	if ((ctsio->io_hdr.flags & CTL_FLAG_ALLOCATED) == 0) {
5792 		if (lun->write_buffer == NULL) {
5793 			lun->write_buffer = malloc(CTL_WRITE_BUFFER_SIZE,
5794 			    M_CTL, M_WAITOK);
5795 		}
5796 		ctsio->kern_data_ptr = lun->write_buffer + buffer_offset;
5797 		ctsio->kern_data_len = len;
5798 		ctsio->kern_total_len = len;
5799 		ctsio->kern_data_resid = 0;
5800 		ctsio->kern_rel_offset = 0;
5801 		ctsio->kern_sg_entries = 0;
5802 		ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
5803 		ctsio->be_move_done = ctl_config_move_done;
5804 		ctl_datamove((union ctl_io *)ctsio);
5805 
5806 		return (CTL_RETVAL_COMPLETE);
5807 	}
5808 
5809 	ctl_set_success(ctsio);
5810 	ctl_done((union ctl_io *)ctsio);
5811 	return (CTL_RETVAL_COMPLETE);
5812 }
5813 
5814 int
5815 ctl_write_same(struct ctl_scsiio *ctsio)
5816 {
5817 	struct ctl_lun *lun;
5818 	struct ctl_lba_len_flags *lbalen;
5819 	uint64_t lba;
5820 	uint32_t num_blocks;
5821 	int len, retval;
5822 	uint8_t byte2;
5823 
5824 	retval = CTL_RETVAL_COMPLETE;
5825 
5826 	CTL_DEBUG_PRINT(("ctl_write_same\n"));
5827 
5828 	lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
5829 
5830 	switch (ctsio->cdb[0]) {
5831 	case WRITE_SAME_10: {
5832 		struct scsi_write_same_10 *cdb;
5833 
5834 		cdb = (struct scsi_write_same_10 *)ctsio->cdb;
5835 
5836 		lba = scsi_4btoul(cdb->addr);
5837 		num_blocks = scsi_2btoul(cdb->length);
5838 		byte2 = cdb->byte2;
5839 		break;
5840 	}
5841 	case WRITE_SAME_16: {
5842 		struct scsi_write_same_16 *cdb;
5843 
5844 		cdb = (struct scsi_write_same_16 *)ctsio->cdb;
5845 
5846 		lba = scsi_8btou64(cdb->addr);
5847 		num_blocks = scsi_4btoul(cdb->length);
5848 		byte2 = cdb->byte2;
5849 		break;
5850 	}
5851 	default:
5852 		/*
5853 		 * We got a command we don't support.  This shouldn't
5854 		 * happen, commands should be filtered out above us.
5855 		 */
5856 		ctl_set_invalid_opcode(ctsio);
5857 		ctl_done((union ctl_io *)ctsio);
5858 
5859 		return (CTL_RETVAL_COMPLETE);
5860 		break; /* NOTREACHED */
5861 	}
5862 
5863 	/* NDOB and ANCHOR flags can be used only together with UNMAP */
5864 	if ((byte2 & SWS_UNMAP) == 0 &&
5865 	    (byte2 & (SWS_NDOB | SWS_ANCHOR)) != 0) {
5866 		ctl_set_invalid_field(ctsio, /*sks_valid*/ 1,
5867 		    /*command*/ 1, /*field*/ 1, /*bit_valid*/ 1, /*bit*/ 0);
5868 		ctl_done((union ctl_io *)ctsio);
5869 		return (CTL_RETVAL_COMPLETE);
5870 	}
5871 
5872 	/*
5873 	 * The first check is to make sure we're in bounds, the second
5874 	 * check is to catch wrap-around problems.  If the lba + num blocks
5875 	 * is less than the lba, then we've wrapped around and the block
5876 	 * range is invalid anyway.
5877 	 */
5878 	if (((lba + num_blocks) > (lun->be_lun->maxlba + 1))
5879 	 || ((lba + num_blocks) < lba)) {
5880 		ctl_set_lba_out_of_range(ctsio);
5881 		ctl_done((union ctl_io *)ctsio);
5882 		return (CTL_RETVAL_COMPLETE);
5883 	}
5884 
5885 	/* Zero number of blocks means "to the last logical block" */
5886 	if (num_blocks == 0) {
5887 		if ((lun->be_lun->maxlba + 1) - lba > UINT32_MAX) {
5888 			ctl_set_invalid_field(ctsio,
5889 					      /*sks_valid*/ 0,
5890 					      /*command*/ 1,
5891 					      /*field*/ 0,
5892 					      /*bit_valid*/ 0,
5893 					      /*bit*/ 0);
5894 			ctl_done((union ctl_io *)ctsio);
5895 			return (CTL_RETVAL_COMPLETE);
5896 		}
5897 		num_blocks = (lun->be_lun->maxlba + 1) - lba;
5898 	}
5899 
5900 	len = lun->be_lun->blocksize;
5901 
5902 	/*
5903 	 * If we've got a kernel request that hasn't been malloced yet,
5904 	 * malloc it and tell the caller the data buffer is here.
5905 	 */
5906 	if ((byte2 & SWS_NDOB) == 0 &&
5907 	    (ctsio->io_hdr.flags & CTL_FLAG_ALLOCATED) == 0) {
5908 		ctsio->kern_data_ptr = malloc(len, M_CTL, M_WAITOK);;
5909 		ctsio->kern_data_len = len;
5910 		ctsio->kern_total_len = len;
5911 		ctsio->kern_data_resid = 0;
5912 		ctsio->kern_rel_offset = 0;
5913 		ctsio->kern_sg_entries = 0;
5914 		ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
5915 		ctsio->be_move_done = ctl_config_move_done;
5916 		ctl_datamove((union ctl_io *)ctsio);
5917 
5918 		return (CTL_RETVAL_COMPLETE);
5919 	}
5920 
5921 	lbalen = (struct ctl_lba_len_flags *)&ctsio->io_hdr.ctl_private[CTL_PRIV_LBA_LEN];
5922 	lbalen->lba = lba;
5923 	lbalen->len = num_blocks;
5924 	lbalen->flags = byte2;
5925 	retval = lun->backend->config_write((union ctl_io *)ctsio);
5926 
5927 	return (retval);
5928 }
5929 
5930 int
5931 ctl_unmap(struct ctl_scsiio *ctsio)
5932 {
5933 	struct ctl_lun *lun;
5934 	struct scsi_unmap *cdb;
5935 	struct ctl_ptr_len_flags *ptrlen;
5936 	struct scsi_unmap_header *hdr;
5937 	struct scsi_unmap_desc *buf, *end, *endnz, *range;
5938 	uint64_t lba;
5939 	uint32_t num_blocks;
5940 	int len, retval;
5941 	uint8_t byte2;
5942 
5943 	retval = CTL_RETVAL_COMPLETE;
5944 
5945 	CTL_DEBUG_PRINT(("ctl_unmap\n"));
5946 
5947 	lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
5948 	cdb = (struct scsi_unmap *)ctsio->cdb;
5949 
5950 	len = scsi_2btoul(cdb->length);
5951 	byte2 = cdb->byte2;
5952 
5953 	/*
5954 	 * If we've got a kernel request that hasn't been malloced yet,
5955 	 * malloc it and tell the caller the data buffer is here.
5956 	 */
5957 	if ((ctsio->io_hdr.flags & CTL_FLAG_ALLOCATED) == 0) {
5958 		ctsio->kern_data_ptr = malloc(len, M_CTL, M_WAITOK);;
5959 		ctsio->kern_data_len = len;
5960 		ctsio->kern_total_len = len;
5961 		ctsio->kern_data_resid = 0;
5962 		ctsio->kern_rel_offset = 0;
5963 		ctsio->kern_sg_entries = 0;
5964 		ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
5965 		ctsio->be_move_done = ctl_config_move_done;
5966 		ctl_datamove((union ctl_io *)ctsio);
5967 
5968 		return (CTL_RETVAL_COMPLETE);
5969 	}
5970 
5971 	len = ctsio->kern_total_len - ctsio->kern_data_resid;
5972 	hdr = (struct scsi_unmap_header *)ctsio->kern_data_ptr;
5973 	if (len < sizeof (*hdr) ||
5974 	    len < (scsi_2btoul(hdr->length) + sizeof(hdr->length)) ||
5975 	    len < (scsi_2btoul(hdr->desc_length) + sizeof (*hdr)) ||
5976 	    scsi_2btoul(hdr->desc_length) % sizeof(*buf) != 0) {
5977 		ctl_set_invalid_field(ctsio,
5978 				      /*sks_valid*/ 0,
5979 				      /*command*/ 0,
5980 				      /*field*/ 0,
5981 				      /*bit_valid*/ 0,
5982 				      /*bit*/ 0);
5983 		goto done;
5984 	}
5985 	len = scsi_2btoul(hdr->desc_length);
5986 	buf = (struct scsi_unmap_desc *)(hdr + 1);
5987 	end = buf + len / sizeof(*buf);
5988 
5989 	endnz = buf;
5990 	for (range = buf; range < end; range++) {
5991 		lba = scsi_8btou64(range->lba);
5992 		num_blocks = scsi_4btoul(range->length);
5993 		if (((lba + num_blocks) > (lun->be_lun->maxlba + 1))
5994 		 || ((lba + num_blocks) < lba)) {
5995 			ctl_set_lba_out_of_range(ctsio);
5996 			ctl_done((union ctl_io *)ctsio);
5997 			return (CTL_RETVAL_COMPLETE);
5998 		}
5999 		if (num_blocks != 0)
6000 			endnz = range + 1;
6001 	}
6002 
6003 	/*
6004 	 * Block backend can not handle zero last range.
6005 	 * Filter it out and return if there is nothing left.
6006 	 */
6007 	len = (uint8_t *)endnz - (uint8_t *)buf;
6008 	if (len == 0) {
6009 		ctl_set_success(ctsio);
6010 		goto done;
6011 	}
6012 
6013 	mtx_lock(&lun->lun_lock);
6014 	ptrlen = (struct ctl_ptr_len_flags *)
6015 	    &ctsio->io_hdr.ctl_private[CTL_PRIV_LBA_LEN];
6016 	ptrlen->ptr = (void *)buf;
6017 	ptrlen->len = len;
6018 	ptrlen->flags = byte2;
6019 	ctl_check_blocked(lun);
6020 	mtx_unlock(&lun->lun_lock);
6021 
6022 	retval = lun->backend->config_write((union ctl_io *)ctsio);
6023 	return (retval);
6024 
6025 done:
6026 	if (ctsio->io_hdr.flags & CTL_FLAG_ALLOCATED) {
6027 		free(ctsio->kern_data_ptr, M_CTL);
6028 		ctsio->io_hdr.flags &= ~CTL_FLAG_ALLOCATED;
6029 	}
6030 	ctl_done((union ctl_io *)ctsio);
6031 	return (CTL_RETVAL_COMPLETE);
6032 }
6033 
6034 /*
6035  * Note that this function currently doesn't actually do anything inside
6036  * CTL to enforce things if the DQue bit is turned on.
6037  *
6038  * Also note that this function can't be used in the default case, because
6039  * the DQue bit isn't set in the changeable mask for the control mode page
6040  * anyway.  This is just here as an example for how to implement a page
6041  * handler, and a placeholder in case we want to allow the user to turn
6042  * tagged queueing on and off.
6043  *
6044  * The D_SENSE bit handling is functional, however, and will turn
6045  * descriptor sense on and off for a given LUN.
6046  */
6047 int
6048 ctl_control_page_handler(struct ctl_scsiio *ctsio,
6049 			 struct ctl_page_index *page_index, uint8_t *page_ptr)
6050 {
6051 	struct scsi_control_page *current_cp, *saved_cp, *user_cp;
6052 	struct ctl_lun *lun;
6053 	int set_ua;
6054 	uint32_t initidx;
6055 
6056 	lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
6057 	initidx = ctl_get_initindex(&ctsio->io_hdr.nexus);
6058 	set_ua = 0;
6059 
6060 	user_cp = (struct scsi_control_page *)page_ptr;
6061 	current_cp = (struct scsi_control_page *)
6062 		(page_index->page_data + (page_index->page_len *
6063 		CTL_PAGE_CURRENT));
6064 	saved_cp = (struct scsi_control_page *)
6065 		(page_index->page_data + (page_index->page_len *
6066 		CTL_PAGE_SAVED));
6067 
6068 	mtx_lock(&lun->lun_lock);
6069 	if (((current_cp->rlec & SCP_DSENSE) == 0)
6070 	 && ((user_cp->rlec & SCP_DSENSE) != 0)) {
6071 		/*
6072 		 * Descriptor sense is currently turned off and the user
6073 		 * wants to turn it on.
6074 		 */
6075 		current_cp->rlec |= SCP_DSENSE;
6076 		saved_cp->rlec |= SCP_DSENSE;
6077 		lun->flags |= CTL_LUN_SENSE_DESC;
6078 		set_ua = 1;
6079 	} else if (((current_cp->rlec & SCP_DSENSE) != 0)
6080 		&& ((user_cp->rlec & SCP_DSENSE) == 0)) {
6081 		/*
6082 		 * Descriptor sense is currently turned on, and the user
6083 		 * wants to turn it off.
6084 		 */
6085 		current_cp->rlec &= ~SCP_DSENSE;
6086 		saved_cp->rlec &= ~SCP_DSENSE;
6087 		lun->flags &= ~CTL_LUN_SENSE_DESC;
6088 		set_ua = 1;
6089 	}
6090 	if ((current_cp->queue_flags & SCP_QUEUE_ALG_MASK) !=
6091 	    (user_cp->queue_flags & SCP_QUEUE_ALG_MASK)) {
6092 		current_cp->queue_flags &= ~SCP_QUEUE_ALG_MASK;
6093 		current_cp->queue_flags |= user_cp->queue_flags & SCP_QUEUE_ALG_MASK;
6094 		saved_cp->queue_flags &= ~SCP_QUEUE_ALG_MASK;
6095 		saved_cp->queue_flags |= user_cp->queue_flags & SCP_QUEUE_ALG_MASK;
6096 		set_ua = 1;
6097 	}
6098 	if ((current_cp->eca_and_aen & SCP_SWP) !=
6099 	    (user_cp->eca_and_aen & SCP_SWP)) {
6100 		current_cp->eca_and_aen &= ~SCP_SWP;
6101 		current_cp->eca_and_aen |= user_cp->eca_and_aen & SCP_SWP;
6102 		saved_cp->eca_and_aen &= ~SCP_SWP;
6103 		saved_cp->eca_and_aen |= user_cp->eca_and_aen & SCP_SWP;
6104 		set_ua = 1;
6105 	}
6106 	if (set_ua != 0)
6107 		ctl_est_ua_all(lun, initidx, CTL_UA_MODE_CHANGE);
6108 	mtx_unlock(&lun->lun_lock);
6109 
6110 	return (0);
6111 }
6112 
6113 int
6114 ctl_caching_sp_handler(struct ctl_scsiio *ctsio,
6115 		     struct ctl_page_index *page_index, uint8_t *page_ptr)
6116 {
6117 	struct scsi_caching_page *current_cp, *saved_cp, *user_cp;
6118 	struct ctl_lun *lun;
6119 	int set_ua;
6120 	uint32_t initidx;
6121 
6122 	lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
6123 	initidx = ctl_get_initindex(&ctsio->io_hdr.nexus);
6124 	set_ua = 0;
6125 
6126 	user_cp = (struct scsi_caching_page *)page_ptr;
6127 	current_cp = (struct scsi_caching_page *)
6128 		(page_index->page_data + (page_index->page_len *
6129 		CTL_PAGE_CURRENT));
6130 	saved_cp = (struct scsi_caching_page *)
6131 		(page_index->page_data + (page_index->page_len *
6132 		CTL_PAGE_SAVED));
6133 
6134 	mtx_lock(&lun->lun_lock);
6135 	if ((current_cp->flags1 & (SCP_WCE | SCP_RCD)) !=
6136 	    (user_cp->flags1 & (SCP_WCE | SCP_RCD))) {
6137 		current_cp->flags1 &= ~(SCP_WCE | SCP_RCD);
6138 		current_cp->flags1 |= user_cp->flags1 & (SCP_WCE | SCP_RCD);
6139 		saved_cp->flags1 &= ~(SCP_WCE | SCP_RCD);
6140 		saved_cp->flags1 |= user_cp->flags1 & (SCP_WCE | SCP_RCD);
6141 		set_ua = 1;
6142 	}
6143 	if (set_ua != 0)
6144 		ctl_est_ua_all(lun, initidx, CTL_UA_MODE_CHANGE);
6145 	mtx_unlock(&lun->lun_lock);
6146 
6147 	return (0);
6148 }
6149 
6150 int
6151 ctl_debugconf_sp_select_handler(struct ctl_scsiio *ctsio,
6152 				struct ctl_page_index *page_index,
6153 				uint8_t *page_ptr)
6154 {
6155 	uint8_t *c;
6156 	int i;
6157 
6158 	c = ((struct copan_debugconf_subpage *)page_ptr)->ctl_time_io_secs;
6159 	ctl_time_io_secs =
6160 		(c[0] << 8) |
6161 		(c[1] << 0) |
6162 		0;
6163 	CTL_DEBUG_PRINT(("set ctl_time_io_secs to %d\n", ctl_time_io_secs));
6164 	printf("set ctl_time_io_secs to %d\n", ctl_time_io_secs);
6165 	printf("page data:");
6166 	for (i=0; i<8; i++)
6167 		printf(" %.2x",page_ptr[i]);
6168 	printf("\n");
6169 	return (0);
6170 }
6171 
6172 int
6173 ctl_debugconf_sp_sense_handler(struct ctl_scsiio *ctsio,
6174 			       struct ctl_page_index *page_index,
6175 			       int pc)
6176 {
6177 	struct copan_debugconf_subpage *page;
6178 
6179 	page = (struct copan_debugconf_subpage *)page_index->page_data +
6180 		(page_index->page_len * pc);
6181 
6182 	switch (pc) {
6183 	case SMS_PAGE_CTRL_CHANGEABLE >> 6:
6184 	case SMS_PAGE_CTRL_DEFAULT >> 6:
6185 	case SMS_PAGE_CTRL_SAVED >> 6:
6186 		/*
6187 		 * We don't update the changable or default bits for this page.
6188 		 */
6189 		break;
6190 	case SMS_PAGE_CTRL_CURRENT >> 6:
6191 		page->ctl_time_io_secs[0] = ctl_time_io_secs >> 8;
6192 		page->ctl_time_io_secs[1] = ctl_time_io_secs >> 0;
6193 		break;
6194 	default:
6195 #ifdef NEEDTOPORT
6196 		EPRINT(0, "Invalid PC %d!!", pc);
6197 #endif /* NEEDTOPORT */
6198 		break;
6199 	}
6200 	return (0);
6201 }
6202 
6203 
6204 static int
6205 ctl_do_mode_select(union ctl_io *io)
6206 {
6207 	struct scsi_mode_page_header *page_header;
6208 	struct ctl_page_index *page_index;
6209 	struct ctl_scsiio *ctsio;
6210 	int control_dev, page_len;
6211 	int page_len_offset, page_len_size;
6212 	union ctl_modepage_info *modepage_info;
6213 	struct ctl_lun *lun;
6214 	int *len_left, *len_used;
6215 	int retval, i;
6216 
6217 	ctsio = &io->scsiio;
6218 	page_index = NULL;
6219 	page_len = 0;
6220 	retval = CTL_RETVAL_COMPLETE;
6221 
6222 	lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
6223 
6224 	if (lun->be_lun->lun_type != T_DIRECT)
6225 		control_dev = 1;
6226 	else
6227 		control_dev = 0;
6228 
6229 	modepage_info = (union ctl_modepage_info *)
6230 		ctsio->io_hdr.ctl_private[CTL_PRIV_MODEPAGE].bytes;
6231 	len_left = &modepage_info->header.len_left;
6232 	len_used = &modepage_info->header.len_used;
6233 
6234 do_next_page:
6235 
6236 	page_header = (struct scsi_mode_page_header *)
6237 		(ctsio->kern_data_ptr + *len_used);
6238 
6239 	if (*len_left == 0) {
6240 		free(ctsio->kern_data_ptr, M_CTL);
6241 		ctl_set_success(ctsio);
6242 		ctl_done((union ctl_io *)ctsio);
6243 		return (CTL_RETVAL_COMPLETE);
6244 	} else if (*len_left < sizeof(struct scsi_mode_page_header)) {
6245 
6246 		free(ctsio->kern_data_ptr, M_CTL);
6247 		ctl_set_param_len_error(ctsio);
6248 		ctl_done((union ctl_io *)ctsio);
6249 		return (CTL_RETVAL_COMPLETE);
6250 
6251 	} else if ((page_header->page_code & SMPH_SPF)
6252 		&& (*len_left < sizeof(struct scsi_mode_page_header_sp))) {
6253 
6254 		free(ctsio->kern_data_ptr, M_CTL);
6255 		ctl_set_param_len_error(ctsio);
6256 		ctl_done((union ctl_io *)ctsio);
6257 		return (CTL_RETVAL_COMPLETE);
6258 	}
6259 
6260 
6261 	/*
6262 	 * XXX KDM should we do something with the block descriptor?
6263 	 */
6264 	for (i = 0; i < CTL_NUM_MODE_PAGES; i++) {
6265 
6266 		if ((control_dev != 0)
6267 		 && (lun->mode_pages.index[i].page_flags &
6268 		     CTL_PAGE_FLAG_DISK_ONLY))
6269 			continue;
6270 
6271 		if ((lun->mode_pages.index[i].page_code & SMPH_PC_MASK) !=
6272 		    (page_header->page_code & SMPH_PC_MASK))
6273 			continue;
6274 
6275 		/*
6276 		 * If neither page has a subpage code, then we've got a
6277 		 * match.
6278 		 */
6279 		if (((lun->mode_pages.index[i].page_code & SMPH_SPF) == 0)
6280 		 && ((page_header->page_code & SMPH_SPF) == 0)) {
6281 			page_index = &lun->mode_pages.index[i];
6282 			page_len = page_header->page_length;
6283 			break;
6284 		}
6285 
6286 		/*
6287 		 * If both pages have subpages, then the subpage numbers
6288 		 * have to match.
6289 		 */
6290 		if ((lun->mode_pages.index[i].page_code & SMPH_SPF)
6291 		  && (page_header->page_code & SMPH_SPF)) {
6292 			struct scsi_mode_page_header_sp *sph;
6293 
6294 			sph = (struct scsi_mode_page_header_sp *)page_header;
6295 
6296 			if (lun->mode_pages.index[i].subpage ==
6297 			    sph->subpage) {
6298 				page_index = &lun->mode_pages.index[i];
6299 				page_len = scsi_2btoul(sph->page_length);
6300 				break;
6301 			}
6302 		}
6303 	}
6304 
6305 	/*
6306 	 * If we couldn't find the page, or if we don't have a mode select
6307 	 * handler for it, send back an error to the user.
6308 	 */
6309 	if ((page_index == NULL)
6310 	 || (page_index->select_handler == NULL)) {
6311 		ctl_set_invalid_field(ctsio,
6312 				      /*sks_valid*/ 1,
6313 				      /*command*/ 0,
6314 				      /*field*/ *len_used,
6315 				      /*bit_valid*/ 0,
6316 				      /*bit*/ 0);
6317 		free(ctsio->kern_data_ptr, M_CTL);
6318 		ctl_done((union ctl_io *)ctsio);
6319 		return (CTL_RETVAL_COMPLETE);
6320 	}
6321 
6322 	if (page_index->page_code & SMPH_SPF) {
6323 		page_len_offset = 2;
6324 		page_len_size = 2;
6325 	} else {
6326 		page_len_size = 1;
6327 		page_len_offset = 1;
6328 	}
6329 
6330 	/*
6331 	 * If the length the initiator gives us isn't the one we specify in
6332 	 * the mode page header, or if they didn't specify enough data in
6333 	 * the CDB to avoid truncating this page, kick out the request.
6334 	 */
6335 	if ((page_len != (page_index->page_len - page_len_offset -
6336 			  page_len_size))
6337 	 || (*len_left < page_index->page_len)) {
6338 
6339 
6340 		ctl_set_invalid_field(ctsio,
6341 				      /*sks_valid*/ 1,
6342 				      /*command*/ 0,
6343 				      /*field*/ *len_used + page_len_offset,
6344 				      /*bit_valid*/ 0,
6345 				      /*bit*/ 0);
6346 		free(ctsio->kern_data_ptr, M_CTL);
6347 		ctl_done((union ctl_io *)ctsio);
6348 		return (CTL_RETVAL_COMPLETE);
6349 	}
6350 
6351 	/*
6352 	 * Run through the mode page, checking to make sure that the bits
6353 	 * the user changed are actually legal for him to change.
6354 	 */
6355 	for (i = 0; i < page_index->page_len; i++) {
6356 		uint8_t *user_byte, *change_mask, *current_byte;
6357 		int bad_bit;
6358 		int j;
6359 
6360 		user_byte = (uint8_t *)page_header + i;
6361 		change_mask = page_index->page_data +
6362 			      (page_index->page_len * CTL_PAGE_CHANGEABLE) + i;
6363 		current_byte = page_index->page_data +
6364 			       (page_index->page_len * CTL_PAGE_CURRENT) + i;
6365 
6366 		/*
6367 		 * Check to see whether the user set any bits in this byte
6368 		 * that he is not allowed to set.
6369 		 */
6370 		if ((*user_byte & ~(*change_mask)) ==
6371 		    (*current_byte & ~(*change_mask)))
6372 			continue;
6373 
6374 		/*
6375 		 * Go through bit by bit to determine which one is illegal.
6376 		 */
6377 		bad_bit = 0;
6378 		for (j = 7; j >= 0; j--) {
6379 			if ((((1 << i) & ~(*change_mask)) & *user_byte) !=
6380 			    (((1 << i) & ~(*change_mask)) & *current_byte)) {
6381 				bad_bit = i;
6382 				break;
6383 			}
6384 		}
6385 		ctl_set_invalid_field(ctsio,
6386 				      /*sks_valid*/ 1,
6387 				      /*command*/ 0,
6388 				      /*field*/ *len_used + i,
6389 				      /*bit_valid*/ 1,
6390 				      /*bit*/ bad_bit);
6391 		free(ctsio->kern_data_ptr, M_CTL);
6392 		ctl_done((union ctl_io *)ctsio);
6393 		return (CTL_RETVAL_COMPLETE);
6394 	}
6395 
6396 	/*
6397 	 * Decrement these before we call the page handler, since we may
6398 	 * end up getting called back one way or another before the handler
6399 	 * returns to this context.
6400 	 */
6401 	*len_left -= page_index->page_len;
6402 	*len_used += page_index->page_len;
6403 
6404 	retval = page_index->select_handler(ctsio, page_index,
6405 					    (uint8_t *)page_header);
6406 
6407 	/*
6408 	 * If the page handler returns CTL_RETVAL_QUEUED, then we need to
6409 	 * wait until this queued command completes to finish processing
6410 	 * the mode page.  If it returns anything other than
6411 	 * CTL_RETVAL_COMPLETE (e.g. CTL_RETVAL_ERROR), then it should have
6412 	 * already set the sense information, freed the data pointer, and
6413 	 * completed the io for us.
6414 	 */
6415 	if (retval != CTL_RETVAL_COMPLETE)
6416 		goto bailout_no_done;
6417 
6418 	/*
6419 	 * If the initiator sent us more than one page, parse the next one.
6420 	 */
6421 	if (*len_left > 0)
6422 		goto do_next_page;
6423 
6424 	ctl_set_success(ctsio);
6425 	free(ctsio->kern_data_ptr, M_CTL);
6426 	ctl_done((union ctl_io *)ctsio);
6427 
6428 bailout_no_done:
6429 
6430 	return (CTL_RETVAL_COMPLETE);
6431 
6432 }
6433 
6434 int
6435 ctl_mode_select(struct ctl_scsiio *ctsio)
6436 {
6437 	int param_len, pf, sp;
6438 	int header_size, bd_len;
6439 	int len_left, len_used;
6440 	struct ctl_page_index *page_index;
6441 	struct ctl_lun *lun;
6442 	int control_dev, page_len;
6443 	union ctl_modepage_info *modepage_info;
6444 	int retval;
6445 
6446 	pf = 0;
6447 	sp = 0;
6448 	page_len = 0;
6449 	len_used = 0;
6450 	len_left = 0;
6451 	retval = 0;
6452 	bd_len = 0;
6453 	page_index = NULL;
6454 
6455 	lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
6456 
6457 	if (lun->be_lun->lun_type != T_DIRECT)
6458 		control_dev = 1;
6459 	else
6460 		control_dev = 0;
6461 
6462 	switch (ctsio->cdb[0]) {
6463 	case MODE_SELECT_6: {
6464 		struct scsi_mode_select_6 *cdb;
6465 
6466 		cdb = (struct scsi_mode_select_6 *)ctsio->cdb;
6467 
6468 		pf = (cdb->byte2 & SMS_PF) ? 1 : 0;
6469 		sp = (cdb->byte2 & SMS_SP) ? 1 : 0;
6470 
6471 		param_len = cdb->length;
6472 		header_size = sizeof(struct scsi_mode_header_6);
6473 		break;
6474 	}
6475 	case MODE_SELECT_10: {
6476 		struct scsi_mode_select_10 *cdb;
6477 
6478 		cdb = (struct scsi_mode_select_10 *)ctsio->cdb;
6479 
6480 		pf = (cdb->byte2 & SMS_PF) ? 1 : 0;
6481 		sp = (cdb->byte2 & SMS_SP) ? 1 : 0;
6482 
6483 		param_len = scsi_2btoul(cdb->length);
6484 		header_size = sizeof(struct scsi_mode_header_10);
6485 		break;
6486 	}
6487 	default:
6488 		ctl_set_invalid_opcode(ctsio);
6489 		ctl_done((union ctl_io *)ctsio);
6490 		return (CTL_RETVAL_COMPLETE);
6491 		break; /* NOTREACHED */
6492 	}
6493 
6494 	/*
6495 	 * From SPC-3:
6496 	 * "A parameter list length of zero indicates that the Data-Out Buffer
6497 	 * shall be empty. This condition shall not be considered as an error."
6498 	 */
6499 	if (param_len == 0) {
6500 		ctl_set_success(ctsio);
6501 		ctl_done((union ctl_io *)ctsio);
6502 		return (CTL_RETVAL_COMPLETE);
6503 	}
6504 
6505 	/*
6506 	 * Since we'll hit this the first time through, prior to
6507 	 * allocation, we don't need to free a data buffer here.
6508 	 */
6509 	if (param_len < header_size) {
6510 		ctl_set_param_len_error(ctsio);
6511 		ctl_done((union ctl_io *)ctsio);
6512 		return (CTL_RETVAL_COMPLETE);
6513 	}
6514 
6515 	/*
6516 	 * Allocate the data buffer and grab the user's data.  In theory,
6517 	 * we shouldn't have to sanity check the parameter list length here
6518 	 * because the maximum size is 64K.  We should be able to malloc
6519 	 * that much without too many problems.
6520 	 */
6521 	if ((ctsio->io_hdr.flags & CTL_FLAG_ALLOCATED) == 0) {
6522 		ctsio->kern_data_ptr = malloc(param_len, M_CTL, M_WAITOK);
6523 		ctsio->kern_data_len = param_len;
6524 		ctsio->kern_total_len = param_len;
6525 		ctsio->kern_data_resid = 0;
6526 		ctsio->kern_rel_offset = 0;
6527 		ctsio->kern_sg_entries = 0;
6528 		ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
6529 		ctsio->be_move_done = ctl_config_move_done;
6530 		ctl_datamove((union ctl_io *)ctsio);
6531 
6532 		return (CTL_RETVAL_COMPLETE);
6533 	}
6534 
6535 	switch (ctsio->cdb[0]) {
6536 	case MODE_SELECT_6: {
6537 		struct scsi_mode_header_6 *mh6;
6538 
6539 		mh6 = (struct scsi_mode_header_6 *)ctsio->kern_data_ptr;
6540 		bd_len = mh6->blk_desc_len;
6541 		break;
6542 	}
6543 	case MODE_SELECT_10: {
6544 		struct scsi_mode_header_10 *mh10;
6545 
6546 		mh10 = (struct scsi_mode_header_10 *)ctsio->kern_data_ptr;
6547 		bd_len = scsi_2btoul(mh10->blk_desc_len);
6548 		break;
6549 	}
6550 	default:
6551 		panic("Invalid CDB type %#x", ctsio->cdb[0]);
6552 		break;
6553 	}
6554 
6555 	if (param_len < (header_size + bd_len)) {
6556 		free(ctsio->kern_data_ptr, M_CTL);
6557 		ctl_set_param_len_error(ctsio);
6558 		ctl_done((union ctl_io *)ctsio);
6559 		return (CTL_RETVAL_COMPLETE);
6560 	}
6561 
6562 	/*
6563 	 * Set the IO_CONT flag, so that if this I/O gets passed to
6564 	 * ctl_config_write_done(), it'll get passed back to
6565 	 * ctl_do_mode_select() for further processing, or completion if
6566 	 * we're all done.
6567 	 */
6568 	ctsio->io_hdr.flags |= CTL_FLAG_IO_CONT;
6569 	ctsio->io_cont = ctl_do_mode_select;
6570 
6571 	modepage_info = (union ctl_modepage_info *)
6572 		ctsio->io_hdr.ctl_private[CTL_PRIV_MODEPAGE].bytes;
6573 
6574 	memset(modepage_info, 0, sizeof(*modepage_info));
6575 
6576 	len_left = param_len - header_size - bd_len;
6577 	len_used = header_size + bd_len;
6578 
6579 	modepage_info->header.len_left = len_left;
6580 	modepage_info->header.len_used = len_used;
6581 
6582 	return (ctl_do_mode_select((union ctl_io *)ctsio));
6583 }
6584 
6585 int
6586 ctl_mode_sense(struct ctl_scsiio *ctsio)
6587 {
6588 	struct ctl_lun *lun;
6589 	int pc, page_code, dbd, llba, subpage;
6590 	int alloc_len, page_len, header_len, total_len;
6591 	struct scsi_mode_block_descr *block_desc;
6592 	struct ctl_page_index *page_index;
6593 	int control_dev;
6594 
6595 	dbd = 0;
6596 	llba = 0;
6597 	block_desc = NULL;
6598 	page_index = NULL;
6599 
6600 	CTL_DEBUG_PRINT(("ctl_mode_sense\n"));
6601 
6602 	lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
6603 
6604 	if (lun->be_lun->lun_type != T_DIRECT)
6605 		control_dev = 1;
6606 	else
6607 		control_dev = 0;
6608 
6609 	switch (ctsio->cdb[0]) {
6610 	case MODE_SENSE_6: {
6611 		struct scsi_mode_sense_6 *cdb;
6612 
6613 		cdb = (struct scsi_mode_sense_6 *)ctsio->cdb;
6614 
6615 		header_len = sizeof(struct scsi_mode_hdr_6);
6616 		if (cdb->byte2 & SMS_DBD)
6617 			dbd = 1;
6618 		else
6619 			header_len += sizeof(struct scsi_mode_block_descr);
6620 
6621 		pc = (cdb->page & SMS_PAGE_CTRL_MASK) >> 6;
6622 		page_code = cdb->page & SMS_PAGE_CODE;
6623 		subpage = cdb->subpage;
6624 		alloc_len = cdb->length;
6625 		break;
6626 	}
6627 	case MODE_SENSE_10: {
6628 		struct scsi_mode_sense_10 *cdb;
6629 
6630 		cdb = (struct scsi_mode_sense_10 *)ctsio->cdb;
6631 
6632 		header_len = sizeof(struct scsi_mode_hdr_10);
6633 
6634 		if (cdb->byte2 & SMS_DBD)
6635 			dbd = 1;
6636 		else
6637 			header_len += sizeof(struct scsi_mode_block_descr);
6638 		if (cdb->byte2 & SMS10_LLBAA)
6639 			llba = 1;
6640 		pc = (cdb->page & SMS_PAGE_CTRL_MASK) >> 6;
6641 		page_code = cdb->page & SMS_PAGE_CODE;
6642 		subpage = cdb->subpage;
6643 		alloc_len = scsi_2btoul(cdb->length);
6644 		break;
6645 	}
6646 	default:
6647 		ctl_set_invalid_opcode(ctsio);
6648 		ctl_done((union ctl_io *)ctsio);
6649 		return (CTL_RETVAL_COMPLETE);
6650 		break; /* NOTREACHED */
6651 	}
6652 
6653 	/*
6654 	 * We have to make a first pass through to calculate the size of
6655 	 * the pages that match the user's query.  Then we allocate enough
6656 	 * memory to hold it, and actually copy the data into the buffer.
6657 	 */
6658 	switch (page_code) {
6659 	case SMS_ALL_PAGES_PAGE: {
6660 		int i;
6661 
6662 		page_len = 0;
6663 
6664 		/*
6665 		 * At the moment, values other than 0 and 0xff here are
6666 		 * reserved according to SPC-3.
6667 		 */
6668 		if ((subpage != SMS_SUBPAGE_PAGE_0)
6669 		 && (subpage != SMS_SUBPAGE_ALL)) {
6670 			ctl_set_invalid_field(ctsio,
6671 					      /*sks_valid*/ 1,
6672 					      /*command*/ 1,
6673 					      /*field*/ 3,
6674 					      /*bit_valid*/ 0,
6675 					      /*bit*/ 0);
6676 			ctl_done((union ctl_io *)ctsio);
6677 			return (CTL_RETVAL_COMPLETE);
6678 		}
6679 
6680 		for (i = 0; i < CTL_NUM_MODE_PAGES; i++) {
6681 			if ((control_dev != 0)
6682 			 && (lun->mode_pages.index[i].page_flags &
6683 			     CTL_PAGE_FLAG_DISK_ONLY))
6684 				continue;
6685 
6686 			/*
6687 			 * We don't use this subpage if the user didn't
6688 			 * request all subpages.
6689 			 */
6690 			if ((lun->mode_pages.index[i].subpage != 0)
6691 			 && (subpage == SMS_SUBPAGE_PAGE_0))
6692 				continue;
6693 
6694 #if 0
6695 			printf("found page %#x len %d\n",
6696 			       lun->mode_pages.index[i].page_code &
6697 			       SMPH_PC_MASK,
6698 			       lun->mode_pages.index[i].page_len);
6699 #endif
6700 			page_len += lun->mode_pages.index[i].page_len;
6701 		}
6702 		break;
6703 	}
6704 	default: {
6705 		int i;
6706 
6707 		page_len = 0;
6708 
6709 		for (i = 0; i < CTL_NUM_MODE_PAGES; i++) {
6710 			/* Look for the right page code */
6711 			if ((lun->mode_pages.index[i].page_code &
6712 			     SMPH_PC_MASK) != page_code)
6713 				continue;
6714 
6715 			/* Look for the right subpage or the subpage wildcard*/
6716 			if ((lun->mode_pages.index[i].subpage != subpage)
6717 			 && (subpage != SMS_SUBPAGE_ALL))
6718 				continue;
6719 
6720 			/* Make sure the page is supported for this dev type */
6721 			if ((control_dev != 0)
6722 			 && (lun->mode_pages.index[i].page_flags &
6723 			     CTL_PAGE_FLAG_DISK_ONLY))
6724 				continue;
6725 
6726 #if 0
6727 			printf("found page %#x len %d\n",
6728 			       lun->mode_pages.index[i].page_code &
6729 			       SMPH_PC_MASK,
6730 			       lun->mode_pages.index[i].page_len);
6731 #endif
6732 
6733 			page_len += lun->mode_pages.index[i].page_len;
6734 		}
6735 
6736 		if (page_len == 0) {
6737 			ctl_set_invalid_field(ctsio,
6738 					      /*sks_valid*/ 1,
6739 					      /*command*/ 1,
6740 					      /*field*/ 2,
6741 					      /*bit_valid*/ 1,
6742 					      /*bit*/ 5);
6743 			ctl_done((union ctl_io *)ctsio);
6744 			return (CTL_RETVAL_COMPLETE);
6745 		}
6746 		break;
6747 	}
6748 	}
6749 
6750 	total_len = header_len + page_len;
6751 #if 0
6752 	printf("header_len = %d, page_len = %d, total_len = %d\n",
6753 	       header_len, page_len, total_len);
6754 #endif
6755 
6756 	ctsio->kern_data_ptr = malloc(total_len, M_CTL, M_WAITOK | M_ZERO);
6757 	ctsio->kern_sg_entries = 0;
6758 	ctsio->kern_data_resid = 0;
6759 	ctsio->kern_rel_offset = 0;
6760 	if (total_len < alloc_len) {
6761 		ctsio->residual = alloc_len - total_len;
6762 		ctsio->kern_data_len = total_len;
6763 		ctsio->kern_total_len = total_len;
6764 	} else {
6765 		ctsio->residual = 0;
6766 		ctsio->kern_data_len = alloc_len;
6767 		ctsio->kern_total_len = alloc_len;
6768 	}
6769 
6770 	switch (ctsio->cdb[0]) {
6771 	case MODE_SENSE_6: {
6772 		struct scsi_mode_hdr_6 *header;
6773 
6774 		header = (struct scsi_mode_hdr_6 *)ctsio->kern_data_ptr;
6775 
6776 		header->datalen = MIN(total_len - 1, 254);
6777 		if (control_dev == 0) {
6778 			header->dev_specific = 0x10; /* DPOFUA */
6779 			if ((lun->flags & CTL_LUN_READONLY) ||
6780 			    (lun->mode_pages.control_page[CTL_PAGE_CURRENT]
6781 			    .eca_and_aen & SCP_SWP) != 0)
6782 				    header->dev_specific |= 0x80; /* WP */
6783 		}
6784 		if (dbd)
6785 			header->block_descr_len = 0;
6786 		else
6787 			header->block_descr_len =
6788 				sizeof(struct scsi_mode_block_descr);
6789 		block_desc = (struct scsi_mode_block_descr *)&header[1];
6790 		break;
6791 	}
6792 	case MODE_SENSE_10: {
6793 		struct scsi_mode_hdr_10 *header;
6794 		int datalen;
6795 
6796 		header = (struct scsi_mode_hdr_10 *)ctsio->kern_data_ptr;
6797 
6798 		datalen = MIN(total_len - 2, 65533);
6799 		scsi_ulto2b(datalen, header->datalen);
6800 		if (control_dev == 0) {
6801 			header->dev_specific = 0x10; /* DPOFUA */
6802 			if ((lun->flags & CTL_LUN_READONLY) ||
6803 			    (lun->mode_pages.control_page[CTL_PAGE_CURRENT]
6804 			    .eca_and_aen & SCP_SWP) != 0)
6805 				    header->dev_specific |= 0x80; /* WP */
6806 		}
6807 		if (dbd)
6808 			scsi_ulto2b(0, header->block_descr_len);
6809 		else
6810 			scsi_ulto2b(sizeof(struct scsi_mode_block_descr),
6811 				    header->block_descr_len);
6812 		block_desc = (struct scsi_mode_block_descr *)&header[1];
6813 		break;
6814 	}
6815 	default:
6816 		panic("invalid CDB type %#x", ctsio->cdb[0]);
6817 		break; /* NOTREACHED */
6818 	}
6819 
6820 	/*
6821 	 * If we've got a disk, use its blocksize in the block
6822 	 * descriptor.  Otherwise, just set it to 0.
6823 	 */
6824 	if (dbd == 0) {
6825 		if (control_dev == 0)
6826 			scsi_ulto3b(lun->be_lun->blocksize,
6827 				    block_desc->block_len);
6828 		else
6829 			scsi_ulto3b(0, block_desc->block_len);
6830 	}
6831 
6832 	switch (page_code) {
6833 	case SMS_ALL_PAGES_PAGE: {
6834 		int i, data_used;
6835 
6836 		data_used = header_len;
6837 		for (i = 0; i < CTL_NUM_MODE_PAGES; i++) {
6838 			struct ctl_page_index *page_index;
6839 
6840 			page_index = &lun->mode_pages.index[i];
6841 
6842 			if ((control_dev != 0)
6843 			 && (page_index->page_flags &
6844 			    CTL_PAGE_FLAG_DISK_ONLY))
6845 				continue;
6846 
6847 			/*
6848 			 * We don't use this subpage if the user didn't
6849 			 * request all subpages.  We already checked (above)
6850 			 * to make sure the user only specified a subpage
6851 			 * of 0 or 0xff in the SMS_ALL_PAGES_PAGE case.
6852 			 */
6853 			if ((page_index->subpage != 0)
6854 			 && (subpage == SMS_SUBPAGE_PAGE_0))
6855 				continue;
6856 
6857 			/*
6858 			 * Call the handler, if it exists, to update the
6859 			 * page to the latest values.
6860 			 */
6861 			if (page_index->sense_handler != NULL)
6862 				page_index->sense_handler(ctsio, page_index,pc);
6863 
6864 			memcpy(ctsio->kern_data_ptr + data_used,
6865 			       page_index->page_data +
6866 			       (page_index->page_len * pc),
6867 			       page_index->page_len);
6868 			data_used += page_index->page_len;
6869 		}
6870 		break;
6871 	}
6872 	default: {
6873 		int i, data_used;
6874 
6875 		data_used = header_len;
6876 
6877 		for (i = 0; i < CTL_NUM_MODE_PAGES; i++) {
6878 			struct ctl_page_index *page_index;
6879 
6880 			page_index = &lun->mode_pages.index[i];
6881 
6882 			/* Look for the right page code */
6883 			if ((page_index->page_code & SMPH_PC_MASK) != page_code)
6884 				continue;
6885 
6886 			/* Look for the right subpage or the subpage wildcard*/
6887 			if ((page_index->subpage != subpage)
6888 			 && (subpage != SMS_SUBPAGE_ALL))
6889 				continue;
6890 
6891 			/* Make sure the page is supported for this dev type */
6892 			if ((control_dev != 0)
6893 			 && (page_index->page_flags &
6894 			     CTL_PAGE_FLAG_DISK_ONLY))
6895 				continue;
6896 
6897 			/*
6898 			 * Call the handler, if it exists, to update the
6899 			 * page to the latest values.
6900 			 */
6901 			if (page_index->sense_handler != NULL)
6902 				page_index->sense_handler(ctsio, page_index,pc);
6903 
6904 			memcpy(ctsio->kern_data_ptr + data_used,
6905 			       page_index->page_data +
6906 			       (page_index->page_len * pc),
6907 			       page_index->page_len);
6908 			data_used += page_index->page_len;
6909 		}
6910 		break;
6911 	}
6912 	}
6913 
6914 	ctl_set_success(ctsio);
6915 	ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
6916 	ctsio->be_move_done = ctl_config_move_done;
6917 	ctl_datamove((union ctl_io *)ctsio);
6918 	return (CTL_RETVAL_COMPLETE);
6919 }
6920 
6921 int
6922 ctl_lbp_log_sense_handler(struct ctl_scsiio *ctsio,
6923 			       struct ctl_page_index *page_index,
6924 			       int pc)
6925 {
6926 	struct ctl_lun *lun;
6927 	struct scsi_log_param_header *phdr;
6928 	uint8_t *data;
6929 	uint64_t val;
6930 
6931 	lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
6932 	data = page_index->page_data;
6933 
6934 	if (lun->backend->lun_attr != NULL &&
6935 	    (val = lun->backend->lun_attr(lun->be_lun->be_lun, "blocksavail"))
6936 	     != UINT64_MAX) {
6937 		phdr = (struct scsi_log_param_header *)data;
6938 		scsi_ulto2b(0x0001, phdr->param_code);
6939 		phdr->param_control = SLP_LBIN | SLP_LP;
6940 		phdr->param_len = 8;
6941 		data = (uint8_t *)(phdr + 1);
6942 		scsi_ulto4b(val >> CTL_LBP_EXPONENT, data);
6943 		data[4] = 0x02; /* per-pool */
6944 		data += phdr->param_len;
6945 	}
6946 
6947 	if (lun->backend->lun_attr != NULL &&
6948 	    (val = lun->backend->lun_attr(lun->be_lun->be_lun, "blocksused"))
6949 	     != UINT64_MAX) {
6950 		phdr = (struct scsi_log_param_header *)data;
6951 		scsi_ulto2b(0x0002, phdr->param_code);
6952 		phdr->param_control = SLP_LBIN | SLP_LP;
6953 		phdr->param_len = 8;
6954 		data = (uint8_t *)(phdr + 1);
6955 		scsi_ulto4b(val >> CTL_LBP_EXPONENT, data);
6956 		data[4] = 0x01; /* per-LUN */
6957 		data += phdr->param_len;
6958 	}
6959 
6960 	if (lun->backend->lun_attr != NULL &&
6961 	    (val = lun->backend->lun_attr(lun->be_lun->be_lun, "poolblocksavail"))
6962 	     != UINT64_MAX) {
6963 		phdr = (struct scsi_log_param_header *)data;
6964 		scsi_ulto2b(0x00f1, phdr->param_code);
6965 		phdr->param_control = SLP_LBIN | SLP_LP;
6966 		phdr->param_len = 8;
6967 		data = (uint8_t *)(phdr + 1);
6968 		scsi_ulto4b(val >> CTL_LBP_EXPONENT, data);
6969 		data[4] = 0x02; /* per-pool */
6970 		data += phdr->param_len;
6971 	}
6972 
6973 	if (lun->backend->lun_attr != NULL &&
6974 	    (val = lun->backend->lun_attr(lun->be_lun->be_lun, "poolblocksused"))
6975 	     != UINT64_MAX) {
6976 		phdr = (struct scsi_log_param_header *)data;
6977 		scsi_ulto2b(0x00f2, phdr->param_code);
6978 		phdr->param_control = SLP_LBIN | SLP_LP;
6979 		phdr->param_len = 8;
6980 		data = (uint8_t *)(phdr + 1);
6981 		scsi_ulto4b(val >> CTL_LBP_EXPONENT, data);
6982 		data[4] = 0x02; /* per-pool */
6983 		data += phdr->param_len;
6984 	}
6985 
6986 	page_index->page_len = data - page_index->page_data;
6987 	return (0);
6988 }
6989 
6990 int
6991 ctl_sap_log_sense_handler(struct ctl_scsiio *ctsio,
6992 			       struct ctl_page_index *page_index,
6993 			       int pc)
6994 {
6995 	struct ctl_lun *lun;
6996 	struct stat_page *data;
6997 	uint64_t rn, wn, rb, wb;
6998 	struct bintime rt, wt;
6999 	int i;
7000 
7001 	lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
7002 	data = (struct stat_page *)page_index->page_data;
7003 
7004 	scsi_ulto2b(SLP_SAP, data->sap.hdr.param_code);
7005 	data->sap.hdr.param_control = SLP_LBIN;
7006 	data->sap.hdr.param_len = sizeof(struct scsi_log_stat_and_perf) -
7007 	    sizeof(struct scsi_log_param_header);
7008 	rn = wn = rb = wb = 0;
7009 	bintime_clear(&rt);
7010 	bintime_clear(&wt);
7011 	for (i = 0; i < CTL_MAX_PORTS; i++) {
7012 		rn += lun->stats.ports[i].operations[CTL_STATS_READ];
7013 		wn += lun->stats.ports[i].operations[CTL_STATS_WRITE];
7014 		rb += lun->stats.ports[i].bytes[CTL_STATS_READ];
7015 		wb += lun->stats.ports[i].bytes[CTL_STATS_WRITE];
7016 		bintime_add(&rt, &lun->stats.ports[i].time[CTL_STATS_READ]);
7017 		bintime_add(&wt, &lun->stats.ports[i].time[CTL_STATS_WRITE]);
7018 	}
7019 	scsi_u64to8b(rn, data->sap.read_num);
7020 	scsi_u64to8b(wn, data->sap.write_num);
7021 	if (lun->stats.blocksize > 0) {
7022 		scsi_u64to8b(wb / lun->stats.blocksize,
7023 		    data->sap.recvieved_lba);
7024 		scsi_u64to8b(rb / lun->stats.blocksize,
7025 		    data->sap.transmitted_lba);
7026 	}
7027 	scsi_u64to8b((uint64_t)rt.sec * 1000 + rt.frac / (UINT64_MAX / 1000),
7028 	    data->sap.read_int);
7029 	scsi_u64to8b((uint64_t)wt.sec * 1000 + wt.frac / (UINT64_MAX / 1000),
7030 	    data->sap.write_int);
7031 	scsi_u64to8b(0, data->sap.weighted_num);
7032 	scsi_u64to8b(0, data->sap.weighted_int);
7033 	scsi_ulto2b(SLP_IT, data->it.hdr.param_code);
7034 	data->it.hdr.param_control = SLP_LBIN;
7035 	data->it.hdr.param_len = sizeof(struct scsi_log_idle_time) -
7036 	    sizeof(struct scsi_log_param_header);
7037 #ifdef CTL_TIME_IO
7038 	scsi_u64to8b(lun->idle_time / SBT_1MS, data->it.idle_int);
7039 #endif
7040 	scsi_ulto2b(SLP_TI, data->ti.hdr.param_code);
7041 	data->it.hdr.param_control = SLP_LBIN;
7042 	data->ti.hdr.param_len = sizeof(struct scsi_log_time_interval) -
7043 	    sizeof(struct scsi_log_param_header);
7044 	scsi_ulto4b(3, data->ti.exponent);
7045 	scsi_ulto4b(1, data->ti.integer);
7046 
7047 	page_index->page_len = sizeof(*data);
7048 	return (0);
7049 }
7050 
7051 int
7052 ctl_log_sense(struct ctl_scsiio *ctsio)
7053 {
7054 	struct ctl_lun *lun;
7055 	int i, pc, page_code, subpage;
7056 	int alloc_len, total_len;
7057 	struct ctl_page_index *page_index;
7058 	struct scsi_log_sense *cdb;
7059 	struct scsi_log_header *header;
7060 
7061 	CTL_DEBUG_PRINT(("ctl_log_sense\n"));
7062 
7063 	lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
7064 	cdb = (struct scsi_log_sense *)ctsio->cdb;
7065 	pc = (cdb->page & SLS_PAGE_CTRL_MASK) >> 6;
7066 	page_code = cdb->page & SLS_PAGE_CODE;
7067 	subpage = cdb->subpage;
7068 	alloc_len = scsi_2btoul(cdb->length);
7069 
7070 	page_index = NULL;
7071 	for (i = 0; i < CTL_NUM_LOG_PAGES; i++) {
7072 		page_index = &lun->log_pages.index[i];
7073 
7074 		/* Look for the right page code */
7075 		if ((page_index->page_code & SL_PAGE_CODE) != page_code)
7076 			continue;
7077 
7078 		/* Look for the right subpage or the subpage wildcard*/
7079 		if (page_index->subpage != subpage)
7080 			continue;
7081 
7082 		break;
7083 	}
7084 	if (i >= CTL_NUM_LOG_PAGES) {
7085 		ctl_set_invalid_field(ctsio,
7086 				      /*sks_valid*/ 1,
7087 				      /*command*/ 1,
7088 				      /*field*/ 2,
7089 				      /*bit_valid*/ 0,
7090 				      /*bit*/ 0);
7091 		ctl_done((union ctl_io *)ctsio);
7092 		return (CTL_RETVAL_COMPLETE);
7093 	}
7094 
7095 	total_len = sizeof(struct scsi_log_header) + page_index->page_len;
7096 
7097 	ctsio->kern_data_ptr = malloc(total_len, M_CTL, M_WAITOK | M_ZERO);
7098 	ctsio->kern_sg_entries = 0;
7099 	ctsio->kern_data_resid = 0;
7100 	ctsio->kern_rel_offset = 0;
7101 	if (total_len < alloc_len) {
7102 		ctsio->residual = alloc_len - total_len;
7103 		ctsio->kern_data_len = total_len;
7104 		ctsio->kern_total_len = total_len;
7105 	} else {
7106 		ctsio->residual = 0;
7107 		ctsio->kern_data_len = alloc_len;
7108 		ctsio->kern_total_len = alloc_len;
7109 	}
7110 
7111 	header = (struct scsi_log_header *)ctsio->kern_data_ptr;
7112 	header->page = page_index->page_code;
7113 	if (page_index->subpage) {
7114 		header->page |= SL_SPF;
7115 		header->subpage = page_index->subpage;
7116 	}
7117 	scsi_ulto2b(page_index->page_len, header->datalen);
7118 
7119 	/*
7120 	 * Call the handler, if it exists, to update the
7121 	 * page to the latest values.
7122 	 */
7123 	if (page_index->sense_handler != NULL)
7124 		page_index->sense_handler(ctsio, page_index, pc);
7125 
7126 	memcpy(header + 1, page_index->page_data, page_index->page_len);
7127 
7128 	ctl_set_success(ctsio);
7129 	ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
7130 	ctsio->be_move_done = ctl_config_move_done;
7131 	ctl_datamove((union ctl_io *)ctsio);
7132 	return (CTL_RETVAL_COMPLETE);
7133 }
7134 
7135 int
7136 ctl_read_capacity(struct ctl_scsiio *ctsio)
7137 {
7138 	struct scsi_read_capacity *cdb;
7139 	struct scsi_read_capacity_data *data;
7140 	struct ctl_lun *lun;
7141 	uint32_t lba;
7142 
7143 	CTL_DEBUG_PRINT(("ctl_read_capacity\n"));
7144 
7145 	cdb = (struct scsi_read_capacity *)ctsio->cdb;
7146 
7147 	lba = scsi_4btoul(cdb->addr);
7148 	if (((cdb->pmi & SRC_PMI) == 0)
7149 	 && (lba != 0)) {
7150 		ctl_set_invalid_field(/*ctsio*/ ctsio,
7151 				      /*sks_valid*/ 1,
7152 				      /*command*/ 1,
7153 				      /*field*/ 2,
7154 				      /*bit_valid*/ 0,
7155 				      /*bit*/ 0);
7156 		ctl_done((union ctl_io *)ctsio);
7157 		return (CTL_RETVAL_COMPLETE);
7158 	}
7159 
7160 	lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
7161 
7162 	ctsio->kern_data_ptr = malloc(sizeof(*data), M_CTL, M_WAITOK | M_ZERO);
7163 	data = (struct scsi_read_capacity_data *)ctsio->kern_data_ptr;
7164 	ctsio->residual = 0;
7165 	ctsio->kern_data_len = sizeof(*data);
7166 	ctsio->kern_total_len = sizeof(*data);
7167 	ctsio->kern_data_resid = 0;
7168 	ctsio->kern_rel_offset = 0;
7169 	ctsio->kern_sg_entries = 0;
7170 
7171 	/*
7172 	 * If the maximum LBA is greater than 0xfffffffe, the user must
7173 	 * issue a SERVICE ACTION IN (16) command, with the read capacity
7174 	 * serivce action set.
7175 	 */
7176 	if (lun->be_lun->maxlba > 0xfffffffe)
7177 		scsi_ulto4b(0xffffffff, data->addr);
7178 	else
7179 		scsi_ulto4b(lun->be_lun->maxlba, data->addr);
7180 
7181 	/*
7182 	 * XXX KDM this may not be 512 bytes...
7183 	 */
7184 	scsi_ulto4b(lun->be_lun->blocksize, data->length);
7185 
7186 	ctl_set_success(ctsio);
7187 	ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
7188 	ctsio->be_move_done = ctl_config_move_done;
7189 	ctl_datamove((union ctl_io *)ctsio);
7190 	return (CTL_RETVAL_COMPLETE);
7191 }
7192 
7193 int
7194 ctl_read_capacity_16(struct ctl_scsiio *ctsio)
7195 {
7196 	struct scsi_read_capacity_16 *cdb;
7197 	struct scsi_read_capacity_data_long *data;
7198 	struct ctl_lun *lun;
7199 	uint64_t lba;
7200 	uint32_t alloc_len;
7201 
7202 	CTL_DEBUG_PRINT(("ctl_read_capacity_16\n"));
7203 
7204 	cdb = (struct scsi_read_capacity_16 *)ctsio->cdb;
7205 
7206 	alloc_len = scsi_4btoul(cdb->alloc_len);
7207 	lba = scsi_8btou64(cdb->addr);
7208 
7209 	if ((cdb->reladr & SRC16_PMI)
7210 	 && (lba != 0)) {
7211 		ctl_set_invalid_field(/*ctsio*/ ctsio,
7212 				      /*sks_valid*/ 1,
7213 				      /*command*/ 1,
7214 				      /*field*/ 2,
7215 				      /*bit_valid*/ 0,
7216 				      /*bit*/ 0);
7217 		ctl_done((union ctl_io *)ctsio);
7218 		return (CTL_RETVAL_COMPLETE);
7219 	}
7220 
7221 	lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
7222 
7223 	ctsio->kern_data_ptr = malloc(sizeof(*data), M_CTL, M_WAITOK | M_ZERO);
7224 	data = (struct scsi_read_capacity_data_long *)ctsio->kern_data_ptr;
7225 
7226 	if (sizeof(*data) < alloc_len) {
7227 		ctsio->residual = alloc_len - sizeof(*data);
7228 		ctsio->kern_data_len = sizeof(*data);
7229 		ctsio->kern_total_len = sizeof(*data);
7230 	} else {
7231 		ctsio->residual = 0;
7232 		ctsio->kern_data_len = alloc_len;
7233 		ctsio->kern_total_len = alloc_len;
7234 	}
7235 	ctsio->kern_data_resid = 0;
7236 	ctsio->kern_rel_offset = 0;
7237 	ctsio->kern_sg_entries = 0;
7238 
7239 	scsi_u64to8b(lun->be_lun->maxlba, data->addr);
7240 	/* XXX KDM this may not be 512 bytes... */
7241 	scsi_ulto4b(lun->be_lun->blocksize, data->length);
7242 	data->prot_lbppbe = lun->be_lun->pblockexp & SRC16_LBPPBE;
7243 	scsi_ulto2b(lun->be_lun->pblockoff & SRC16_LALBA_A, data->lalba_lbp);
7244 	if (lun->be_lun->flags & CTL_LUN_FLAG_UNMAP)
7245 		data->lalba_lbp[0] |= SRC16_LBPME | SRC16_LBPRZ;
7246 
7247 	ctl_set_success(ctsio);
7248 	ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
7249 	ctsio->be_move_done = ctl_config_move_done;
7250 	ctl_datamove((union ctl_io *)ctsio);
7251 	return (CTL_RETVAL_COMPLETE);
7252 }
7253 
7254 int
7255 ctl_get_lba_status(struct ctl_scsiio *ctsio)
7256 {
7257 	struct scsi_get_lba_status *cdb;
7258 	struct scsi_get_lba_status_data *data;
7259 	struct ctl_lun *lun;
7260 	struct ctl_lba_len_flags *lbalen;
7261 	uint64_t lba;
7262 	uint32_t alloc_len, total_len;
7263 	int retval;
7264 
7265 	CTL_DEBUG_PRINT(("ctl_get_lba_status\n"));
7266 
7267 	lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
7268 	cdb = (struct scsi_get_lba_status *)ctsio->cdb;
7269 	lba = scsi_8btou64(cdb->addr);
7270 	alloc_len = scsi_4btoul(cdb->alloc_len);
7271 
7272 	if (lba > lun->be_lun->maxlba) {
7273 		ctl_set_lba_out_of_range(ctsio);
7274 		ctl_done((union ctl_io *)ctsio);
7275 		return (CTL_RETVAL_COMPLETE);
7276 	}
7277 
7278 	total_len = sizeof(*data) + sizeof(data->descr[0]);
7279 	ctsio->kern_data_ptr = malloc(total_len, M_CTL, M_WAITOK | M_ZERO);
7280 	data = (struct scsi_get_lba_status_data *)ctsio->kern_data_ptr;
7281 
7282 	if (total_len < alloc_len) {
7283 		ctsio->residual = alloc_len - total_len;
7284 		ctsio->kern_data_len = total_len;
7285 		ctsio->kern_total_len = total_len;
7286 	} else {
7287 		ctsio->residual = 0;
7288 		ctsio->kern_data_len = alloc_len;
7289 		ctsio->kern_total_len = alloc_len;
7290 	}
7291 	ctsio->kern_data_resid = 0;
7292 	ctsio->kern_rel_offset = 0;
7293 	ctsio->kern_sg_entries = 0;
7294 
7295 	/* Fill dummy data in case backend can't tell anything. */
7296 	scsi_ulto4b(4 + sizeof(data->descr[0]), data->length);
7297 	scsi_u64to8b(lba, data->descr[0].addr);
7298 	scsi_ulto4b(MIN(UINT32_MAX, lun->be_lun->maxlba + 1 - lba),
7299 	    data->descr[0].length);
7300 	data->descr[0].status = 0; /* Mapped or unknown. */
7301 
7302 	ctl_set_success(ctsio);
7303 	ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
7304 	ctsio->be_move_done = ctl_config_move_done;
7305 
7306 	lbalen = (struct ctl_lba_len_flags *)&ctsio->io_hdr.ctl_private[CTL_PRIV_LBA_LEN];
7307 	lbalen->lba = lba;
7308 	lbalen->len = total_len;
7309 	lbalen->flags = 0;
7310 	retval = lun->backend->config_read((union ctl_io *)ctsio);
7311 	return (CTL_RETVAL_COMPLETE);
7312 }
7313 
7314 int
7315 ctl_read_defect(struct ctl_scsiio *ctsio)
7316 {
7317 	struct scsi_read_defect_data_10 *ccb10;
7318 	struct scsi_read_defect_data_12 *ccb12;
7319 	struct scsi_read_defect_data_hdr_10 *data10;
7320 	struct scsi_read_defect_data_hdr_12 *data12;
7321 	uint32_t alloc_len, data_len;
7322 	uint8_t format;
7323 
7324 	CTL_DEBUG_PRINT(("ctl_read_defect\n"));
7325 
7326 	if (ctsio->cdb[0] == READ_DEFECT_DATA_10) {
7327 		ccb10 = (struct scsi_read_defect_data_10 *)&ctsio->cdb;
7328 		format = ccb10->format;
7329 		alloc_len = scsi_2btoul(ccb10->alloc_length);
7330 		data_len = sizeof(*data10);
7331 	} else {
7332 		ccb12 = (struct scsi_read_defect_data_12 *)&ctsio->cdb;
7333 		format = ccb12->format;
7334 		alloc_len = scsi_4btoul(ccb12->alloc_length);
7335 		data_len = sizeof(*data12);
7336 	}
7337 	if (alloc_len == 0) {
7338 		ctl_set_success(ctsio);
7339 		ctl_done((union ctl_io *)ctsio);
7340 		return (CTL_RETVAL_COMPLETE);
7341 	}
7342 
7343 	ctsio->kern_data_ptr = malloc(data_len, M_CTL, M_WAITOK | M_ZERO);
7344 	if (data_len < alloc_len) {
7345 		ctsio->residual = alloc_len - data_len;
7346 		ctsio->kern_data_len = data_len;
7347 		ctsio->kern_total_len = data_len;
7348 	} else {
7349 		ctsio->residual = 0;
7350 		ctsio->kern_data_len = alloc_len;
7351 		ctsio->kern_total_len = alloc_len;
7352 	}
7353 	ctsio->kern_data_resid = 0;
7354 	ctsio->kern_rel_offset = 0;
7355 	ctsio->kern_sg_entries = 0;
7356 
7357 	if (ctsio->cdb[0] == READ_DEFECT_DATA_10) {
7358 		data10 = (struct scsi_read_defect_data_hdr_10 *)
7359 		    ctsio->kern_data_ptr;
7360 		data10->format = format;
7361 		scsi_ulto2b(0, data10->length);
7362 	} else {
7363 		data12 = (struct scsi_read_defect_data_hdr_12 *)
7364 		    ctsio->kern_data_ptr;
7365 		data12->format = format;
7366 		scsi_ulto2b(0, data12->generation);
7367 		scsi_ulto4b(0, data12->length);
7368 	}
7369 
7370 	ctl_set_success(ctsio);
7371 	ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
7372 	ctsio->be_move_done = ctl_config_move_done;
7373 	ctl_datamove((union ctl_io *)ctsio);
7374 	return (CTL_RETVAL_COMPLETE);
7375 }
7376 
7377 int
7378 ctl_report_tagret_port_groups(struct ctl_scsiio *ctsio)
7379 {
7380 	struct scsi_maintenance_in *cdb;
7381 	int retval;
7382 	int alloc_len, ext, total_len = 0, g, p, pc, pg, gs, os;
7383 	int num_target_port_groups, num_target_ports;
7384 	struct ctl_lun *lun;
7385 	struct ctl_softc *softc;
7386 	struct ctl_port *port;
7387 	struct scsi_target_group_data *rtg_ptr;
7388 	struct scsi_target_group_data_extended *rtg_ext_ptr;
7389 	struct scsi_target_port_group_descriptor *tpg_desc;
7390 
7391 	CTL_DEBUG_PRINT(("ctl_report_tagret_port_groups\n"));
7392 
7393 	cdb = (struct scsi_maintenance_in *)ctsio->cdb;
7394 	lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
7395 	softc = lun->ctl_softc;
7396 
7397 	retval = CTL_RETVAL_COMPLETE;
7398 
7399 	switch (cdb->byte2 & STG_PDF_MASK) {
7400 	case STG_PDF_LENGTH:
7401 		ext = 0;
7402 		break;
7403 	case STG_PDF_EXTENDED:
7404 		ext = 1;
7405 		break;
7406 	default:
7407 		ctl_set_invalid_field(/*ctsio*/ ctsio,
7408 				      /*sks_valid*/ 1,
7409 				      /*command*/ 1,
7410 				      /*field*/ 2,
7411 				      /*bit_valid*/ 1,
7412 				      /*bit*/ 5);
7413 		ctl_done((union ctl_io *)ctsio);
7414 		return(retval);
7415 	}
7416 
7417 	if (softc->is_single)
7418 		num_target_port_groups = 1;
7419 	else
7420 		num_target_port_groups = NUM_TARGET_PORT_GROUPS;
7421 	num_target_ports = 0;
7422 	mtx_lock(&softc->ctl_lock);
7423 	STAILQ_FOREACH(port, &softc->port_list, links) {
7424 		if ((port->status & CTL_PORT_STATUS_ONLINE) == 0)
7425 			continue;
7426 		if (ctl_lun_map_to_port(port, lun->lun) >= CTL_MAX_LUNS)
7427 			continue;
7428 		num_target_ports++;
7429 	}
7430 	mtx_unlock(&softc->ctl_lock);
7431 
7432 	if (ext)
7433 		total_len = sizeof(struct scsi_target_group_data_extended);
7434 	else
7435 		total_len = sizeof(struct scsi_target_group_data);
7436 	total_len += sizeof(struct scsi_target_port_group_descriptor) *
7437 		num_target_port_groups +
7438 	    sizeof(struct scsi_target_port_descriptor) *
7439 		num_target_ports * num_target_port_groups;
7440 
7441 	alloc_len = scsi_4btoul(cdb->length);
7442 
7443 	ctsio->kern_data_ptr = malloc(total_len, M_CTL, M_WAITOK | M_ZERO);
7444 
7445 	ctsio->kern_sg_entries = 0;
7446 
7447 	if (total_len < alloc_len) {
7448 		ctsio->residual = alloc_len - total_len;
7449 		ctsio->kern_data_len = total_len;
7450 		ctsio->kern_total_len = total_len;
7451 	} else {
7452 		ctsio->residual = 0;
7453 		ctsio->kern_data_len = alloc_len;
7454 		ctsio->kern_total_len = alloc_len;
7455 	}
7456 	ctsio->kern_data_resid = 0;
7457 	ctsio->kern_rel_offset = 0;
7458 
7459 	if (ext) {
7460 		rtg_ext_ptr = (struct scsi_target_group_data_extended *)
7461 		    ctsio->kern_data_ptr;
7462 		scsi_ulto4b(total_len - 4, rtg_ext_ptr->length);
7463 		rtg_ext_ptr->format_type = 0x10;
7464 		rtg_ext_ptr->implicit_transition_time = 0;
7465 		tpg_desc = &rtg_ext_ptr->groups[0];
7466 	} else {
7467 		rtg_ptr = (struct scsi_target_group_data *)
7468 		    ctsio->kern_data_ptr;
7469 		scsi_ulto4b(total_len - 4, rtg_ptr->length);
7470 		tpg_desc = &rtg_ptr->groups[0];
7471 	}
7472 
7473 	mtx_lock(&softc->ctl_lock);
7474 	pg = softc->port_offset / CTL_MAX_PORTS;
7475 	if (softc->flags & CTL_FLAG_ACTIVE_SHELF) {
7476 		if (softc->ha_mode == CTL_HA_MODE_ACT_STBY) {
7477 			gs = TPG_ASYMMETRIC_ACCESS_OPTIMIZED;
7478 			os = TPG_ASYMMETRIC_ACCESS_STANDBY;
7479 		} else if (lun->flags & CTL_LUN_PRIMARY_SC) {
7480 			gs = TPG_ASYMMETRIC_ACCESS_OPTIMIZED;
7481 			os = TPG_ASYMMETRIC_ACCESS_NONOPTIMIZED;
7482 		} else {
7483 			gs = TPG_ASYMMETRIC_ACCESS_NONOPTIMIZED;
7484 			os = TPG_ASYMMETRIC_ACCESS_OPTIMIZED;
7485 		}
7486 	} else {
7487 		gs = TPG_ASYMMETRIC_ACCESS_STANDBY;
7488 		os = TPG_ASYMMETRIC_ACCESS_OPTIMIZED;
7489 	}
7490 	for (g = 0; g < num_target_port_groups; g++) {
7491 		tpg_desc->pref_state = (g == pg) ? gs : os;
7492 		tpg_desc->support = TPG_AO_SUP | TPG_AN_SUP | TPG_S_SUP;
7493 		scsi_ulto2b(g + 1, tpg_desc->target_port_group);
7494 		tpg_desc->status = TPG_IMPLICIT;
7495 		pc = 0;
7496 		STAILQ_FOREACH(port, &softc->port_list, links) {
7497 			if ((port->status & CTL_PORT_STATUS_ONLINE) == 0)
7498 				continue;
7499 			if (ctl_lun_map_to_port(port, lun->lun) >= CTL_MAX_LUNS)
7500 				continue;
7501 			p = port->targ_port % CTL_MAX_PORTS + g * CTL_MAX_PORTS;
7502 			scsi_ulto2b(p, tpg_desc->descriptors[pc].
7503 			    relative_target_port_identifier);
7504 			pc++;
7505 		}
7506 		tpg_desc->target_port_count = pc;
7507 		tpg_desc = (struct scsi_target_port_group_descriptor *)
7508 		    &tpg_desc->descriptors[pc];
7509 	}
7510 	mtx_unlock(&softc->ctl_lock);
7511 
7512 	ctl_set_success(ctsio);
7513 	ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
7514 	ctsio->be_move_done = ctl_config_move_done;
7515 	ctl_datamove((union ctl_io *)ctsio);
7516 	return(retval);
7517 }
7518 
7519 int
7520 ctl_report_supported_opcodes(struct ctl_scsiio *ctsio)
7521 {
7522 	struct ctl_lun *lun;
7523 	struct scsi_report_supported_opcodes *cdb;
7524 	const struct ctl_cmd_entry *entry, *sentry;
7525 	struct scsi_report_supported_opcodes_all *all;
7526 	struct scsi_report_supported_opcodes_descr *descr;
7527 	struct scsi_report_supported_opcodes_one *one;
7528 	int retval;
7529 	int alloc_len, total_len;
7530 	int opcode, service_action, i, j, num;
7531 
7532 	CTL_DEBUG_PRINT(("ctl_report_supported_opcodes\n"));
7533 
7534 	cdb = (struct scsi_report_supported_opcodes *)ctsio->cdb;
7535 	lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
7536 
7537 	retval = CTL_RETVAL_COMPLETE;
7538 
7539 	opcode = cdb->requested_opcode;
7540 	service_action = scsi_2btoul(cdb->requested_service_action);
7541 	switch (cdb->options & RSO_OPTIONS_MASK) {
7542 	case RSO_OPTIONS_ALL:
7543 		num = 0;
7544 		for (i = 0; i < 256; i++) {
7545 			entry = &ctl_cmd_table[i];
7546 			if (entry->flags & CTL_CMD_FLAG_SA5) {
7547 				for (j = 0; j < 32; j++) {
7548 					sentry = &((const struct ctl_cmd_entry *)
7549 					    entry->execute)[j];
7550 					if (ctl_cmd_applicable(
7551 					    lun->be_lun->lun_type, sentry))
7552 						num++;
7553 				}
7554 			} else {
7555 				if (ctl_cmd_applicable(lun->be_lun->lun_type,
7556 				    entry))
7557 					num++;
7558 			}
7559 		}
7560 		total_len = sizeof(struct scsi_report_supported_opcodes_all) +
7561 		    num * sizeof(struct scsi_report_supported_opcodes_descr);
7562 		break;
7563 	case RSO_OPTIONS_OC:
7564 		if (ctl_cmd_table[opcode].flags & CTL_CMD_FLAG_SA5) {
7565 			ctl_set_invalid_field(/*ctsio*/ ctsio,
7566 					      /*sks_valid*/ 1,
7567 					      /*command*/ 1,
7568 					      /*field*/ 2,
7569 					      /*bit_valid*/ 1,
7570 					      /*bit*/ 2);
7571 			ctl_done((union ctl_io *)ctsio);
7572 			return (CTL_RETVAL_COMPLETE);
7573 		}
7574 		total_len = sizeof(struct scsi_report_supported_opcodes_one) + 32;
7575 		break;
7576 	case RSO_OPTIONS_OC_SA:
7577 		if ((ctl_cmd_table[opcode].flags & CTL_CMD_FLAG_SA5) == 0 ||
7578 		    service_action >= 32) {
7579 			ctl_set_invalid_field(/*ctsio*/ ctsio,
7580 					      /*sks_valid*/ 1,
7581 					      /*command*/ 1,
7582 					      /*field*/ 2,
7583 					      /*bit_valid*/ 1,
7584 					      /*bit*/ 2);
7585 			ctl_done((union ctl_io *)ctsio);
7586 			return (CTL_RETVAL_COMPLETE);
7587 		}
7588 		total_len = sizeof(struct scsi_report_supported_opcodes_one) + 32;
7589 		break;
7590 	default:
7591 		ctl_set_invalid_field(/*ctsio*/ ctsio,
7592 				      /*sks_valid*/ 1,
7593 				      /*command*/ 1,
7594 				      /*field*/ 2,
7595 				      /*bit_valid*/ 1,
7596 				      /*bit*/ 2);
7597 		ctl_done((union ctl_io *)ctsio);
7598 		return (CTL_RETVAL_COMPLETE);
7599 	}
7600 
7601 	alloc_len = scsi_4btoul(cdb->length);
7602 
7603 	ctsio->kern_data_ptr = malloc(total_len, M_CTL, M_WAITOK | M_ZERO);
7604 
7605 	ctsio->kern_sg_entries = 0;
7606 
7607 	if (total_len < alloc_len) {
7608 		ctsio->residual = alloc_len - total_len;
7609 		ctsio->kern_data_len = total_len;
7610 		ctsio->kern_total_len = total_len;
7611 	} else {
7612 		ctsio->residual = 0;
7613 		ctsio->kern_data_len = alloc_len;
7614 		ctsio->kern_total_len = alloc_len;
7615 	}
7616 	ctsio->kern_data_resid = 0;
7617 	ctsio->kern_rel_offset = 0;
7618 
7619 	switch (cdb->options & RSO_OPTIONS_MASK) {
7620 	case RSO_OPTIONS_ALL:
7621 		all = (struct scsi_report_supported_opcodes_all *)
7622 		    ctsio->kern_data_ptr;
7623 		num = 0;
7624 		for (i = 0; i < 256; i++) {
7625 			entry = &ctl_cmd_table[i];
7626 			if (entry->flags & CTL_CMD_FLAG_SA5) {
7627 				for (j = 0; j < 32; j++) {
7628 					sentry = &((const struct ctl_cmd_entry *)
7629 					    entry->execute)[j];
7630 					if (!ctl_cmd_applicable(
7631 					    lun->be_lun->lun_type, sentry))
7632 						continue;
7633 					descr = &all->descr[num++];
7634 					descr->opcode = i;
7635 					scsi_ulto2b(j, descr->service_action);
7636 					descr->flags = RSO_SERVACTV;
7637 					scsi_ulto2b(sentry->length,
7638 					    descr->cdb_length);
7639 				}
7640 			} else {
7641 				if (!ctl_cmd_applicable(lun->be_lun->lun_type,
7642 				    entry))
7643 					continue;
7644 				descr = &all->descr[num++];
7645 				descr->opcode = i;
7646 				scsi_ulto2b(0, descr->service_action);
7647 				descr->flags = 0;
7648 				scsi_ulto2b(entry->length, descr->cdb_length);
7649 			}
7650 		}
7651 		scsi_ulto4b(
7652 		    num * sizeof(struct scsi_report_supported_opcodes_descr),
7653 		    all->length);
7654 		break;
7655 	case RSO_OPTIONS_OC:
7656 		one = (struct scsi_report_supported_opcodes_one *)
7657 		    ctsio->kern_data_ptr;
7658 		entry = &ctl_cmd_table[opcode];
7659 		goto fill_one;
7660 	case RSO_OPTIONS_OC_SA:
7661 		one = (struct scsi_report_supported_opcodes_one *)
7662 		    ctsio->kern_data_ptr;
7663 		entry = &ctl_cmd_table[opcode];
7664 		entry = &((const struct ctl_cmd_entry *)
7665 		    entry->execute)[service_action];
7666 fill_one:
7667 		if (ctl_cmd_applicable(lun->be_lun->lun_type, entry)) {
7668 			one->support = 3;
7669 			scsi_ulto2b(entry->length, one->cdb_length);
7670 			one->cdb_usage[0] = opcode;
7671 			memcpy(&one->cdb_usage[1], entry->usage,
7672 			    entry->length - 1);
7673 		} else
7674 			one->support = 1;
7675 		break;
7676 	}
7677 
7678 	ctl_set_success(ctsio);
7679 	ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
7680 	ctsio->be_move_done = ctl_config_move_done;
7681 	ctl_datamove((union ctl_io *)ctsio);
7682 	return(retval);
7683 }
7684 
7685 int
7686 ctl_report_supported_tmf(struct ctl_scsiio *ctsio)
7687 {
7688 	struct scsi_report_supported_tmf *cdb;
7689 	struct scsi_report_supported_tmf_data *data;
7690 	int retval;
7691 	int alloc_len, total_len;
7692 
7693 	CTL_DEBUG_PRINT(("ctl_report_supported_tmf\n"));
7694 
7695 	cdb = (struct scsi_report_supported_tmf *)ctsio->cdb;
7696 
7697 	retval = CTL_RETVAL_COMPLETE;
7698 
7699 	total_len = sizeof(struct scsi_report_supported_tmf_data);
7700 	alloc_len = scsi_4btoul(cdb->length);
7701 
7702 	ctsio->kern_data_ptr = malloc(total_len, M_CTL, M_WAITOK | M_ZERO);
7703 
7704 	ctsio->kern_sg_entries = 0;
7705 
7706 	if (total_len < alloc_len) {
7707 		ctsio->residual = alloc_len - total_len;
7708 		ctsio->kern_data_len = total_len;
7709 		ctsio->kern_total_len = total_len;
7710 	} else {
7711 		ctsio->residual = 0;
7712 		ctsio->kern_data_len = alloc_len;
7713 		ctsio->kern_total_len = alloc_len;
7714 	}
7715 	ctsio->kern_data_resid = 0;
7716 	ctsio->kern_rel_offset = 0;
7717 
7718 	data = (struct scsi_report_supported_tmf_data *)ctsio->kern_data_ptr;
7719 	data->byte1 |= RST_ATS | RST_ATSS | RST_CTSS | RST_LURS | RST_TRS;
7720 	data->byte2 |= RST_ITNRS;
7721 
7722 	ctl_set_success(ctsio);
7723 	ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
7724 	ctsio->be_move_done = ctl_config_move_done;
7725 	ctl_datamove((union ctl_io *)ctsio);
7726 	return (retval);
7727 }
7728 
7729 int
7730 ctl_report_timestamp(struct ctl_scsiio *ctsio)
7731 {
7732 	struct scsi_report_timestamp *cdb;
7733 	struct scsi_report_timestamp_data *data;
7734 	struct timeval tv;
7735 	int64_t timestamp;
7736 	int retval;
7737 	int alloc_len, total_len;
7738 
7739 	CTL_DEBUG_PRINT(("ctl_report_timestamp\n"));
7740 
7741 	cdb = (struct scsi_report_timestamp *)ctsio->cdb;
7742 
7743 	retval = CTL_RETVAL_COMPLETE;
7744 
7745 	total_len = sizeof(struct scsi_report_timestamp_data);
7746 	alloc_len = scsi_4btoul(cdb->length);
7747 
7748 	ctsio->kern_data_ptr = malloc(total_len, M_CTL, M_WAITOK | M_ZERO);
7749 
7750 	ctsio->kern_sg_entries = 0;
7751 
7752 	if (total_len < alloc_len) {
7753 		ctsio->residual = alloc_len - total_len;
7754 		ctsio->kern_data_len = total_len;
7755 		ctsio->kern_total_len = total_len;
7756 	} else {
7757 		ctsio->residual = 0;
7758 		ctsio->kern_data_len = alloc_len;
7759 		ctsio->kern_total_len = alloc_len;
7760 	}
7761 	ctsio->kern_data_resid = 0;
7762 	ctsio->kern_rel_offset = 0;
7763 
7764 	data = (struct scsi_report_timestamp_data *)ctsio->kern_data_ptr;
7765 	scsi_ulto2b(sizeof(*data) - 2, data->length);
7766 	data->origin = RTS_ORIG_OUTSIDE;
7767 	getmicrotime(&tv);
7768 	timestamp = (int64_t)tv.tv_sec * 1000 + tv.tv_usec / 1000;
7769 	scsi_ulto4b(timestamp >> 16, data->timestamp);
7770 	scsi_ulto2b(timestamp & 0xffff, &data->timestamp[4]);
7771 
7772 	ctl_set_success(ctsio);
7773 	ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
7774 	ctsio->be_move_done = ctl_config_move_done;
7775 	ctl_datamove((union ctl_io *)ctsio);
7776 	return (retval);
7777 }
7778 
7779 int
7780 ctl_persistent_reserve_in(struct ctl_scsiio *ctsio)
7781 {
7782 	struct scsi_per_res_in *cdb;
7783 	int alloc_len, total_len = 0;
7784 	/* struct scsi_per_res_in_rsrv in_data; */
7785 	struct ctl_lun *lun;
7786 	struct ctl_softc *softc;
7787 	uint64_t key;
7788 
7789 	CTL_DEBUG_PRINT(("ctl_persistent_reserve_in\n"));
7790 
7791 	cdb = (struct scsi_per_res_in *)ctsio->cdb;
7792 
7793 	alloc_len = scsi_2btoul(cdb->length);
7794 
7795 	lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
7796 	softc = lun->ctl_softc;
7797 
7798 retry:
7799 	mtx_lock(&lun->lun_lock);
7800 	switch (cdb->action) {
7801 	case SPRI_RK: /* read keys */
7802 		total_len = sizeof(struct scsi_per_res_in_keys) +
7803 			lun->pr_key_count *
7804 			sizeof(struct scsi_per_res_key);
7805 		break;
7806 	case SPRI_RR: /* read reservation */
7807 		if (lun->flags & CTL_LUN_PR_RESERVED)
7808 			total_len = sizeof(struct scsi_per_res_in_rsrv);
7809 		else
7810 			total_len = sizeof(struct scsi_per_res_in_header);
7811 		break;
7812 	case SPRI_RC: /* report capabilities */
7813 		total_len = sizeof(struct scsi_per_res_cap);
7814 		break;
7815 	case SPRI_RS: /* read full status */
7816 		total_len = sizeof(struct scsi_per_res_in_header) +
7817 		    (sizeof(struct scsi_per_res_in_full_desc) + 256) *
7818 		    lun->pr_key_count;
7819 		break;
7820 	default:
7821 		panic("Invalid PR type %x", cdb->action);
7822 	}
7823 	mtx_unlock(&lun->lun_lock);
7824 
7825 	ctsio->kern_data_ptr = malloc(total_len, M_CTL, M_WAITOK | M_ZERO);
7826 
7827 	if (total_len < alloc_len) {
7828 		ctsio->residual = alloc_len - total_len;
7829 		ctsio->kern_data_len = total_len;
7830 		ctsio->kern_total_len = total_len;
7831 	} else {
7832 		ctsio->residual = 0;
7833 		ctsio->kern_data_len = alloc_len;
7834 		ctsio->kern_total_len = alloc_len;
7835 	}
7836 
7837 	ctsio->kern_data_resid = 0;
7838 	ctsio->kern_rel_offset = 0;
7839 	ctsio->kern_sg_entries = 0;
7840 
7841 	mtx_lock(&lun->lun_lock);
7842 	switch (cdb->action) {
7843 	case SPRI_RK: { // read keys
7844         struct scsi_per_res_in_keys *res_keys;
7845 		int i, key_count;
7846 
7847 		res_keys = (struct scsi_per_res_in_keys*)ctsio->kern_data_ptr;
7848 
7849 		/*
7850 		 * We had to drop the lock to allocate our buffer, which
7851 		 * leaves time for someone to come in with another
7852 		 * persistent reservation.  (That is unlikely, though,
7853 		 * since this should be the only persistent reservation
7854 		 * command active right now.)
7855 		 */
7856 		if (total_len != (sizeof(struct scsi_per_res_in_keys) +
7857 		    (lun->pr_key_count *
7858 		     sizeof(struct scsi_per_res_key)))){
7859 			mtx_unlock(&lun->lun_lock);
7860 			free(ctsio->kern_data_ptr, M_CTL);
7861 			printf("%s: reservation length changed, retrying\n",
7862 			       __func__);
7863 			goto retry;
7864 		}
7865 
7866 		scsi_ulto4b(lun->PRGeneration, res_keys->header.generation);
7867 
7868 		scsi_ulto4b(sizeof(struct scsi_per_res_key) *
7869 			     lun->pr_key_count, res_keys->header.length);
7870 
7871 		for (i = 0, key_count = 0; i < 2*CTL_MAX_INITIATORS; i++) {
7872 			if ((key = ctl_get_prkey(lun, i)) == 0)
7873 				continue;
7874 
7875 			/*
7876 			 * We used lun->pr_key_count to calculate the
7877 			 * size to allocate.  If it turns out the number of
7878 			 * initiators with the registered flag set is
7879 			 * larger than that (i.e. they haven't been kept in
7880 			 * sync), we've got a problem.
7881 			 */
7882 			if (key_count >= lun->pr_key_count) {
7883 #ifdef NEEDTOPORT
7884 				csevent_log(CSC_CTL | CSC_SHELF_SW |
7885 					    CTL_PR_ERROR,
7886 					    csevent_LogType_Fault,
7887 					    csevent_AlertLevel_Yellow,
7888 					    csevent_FRU_ShelfController,
7889 					    csevent_FRU_Firmware,
7890 				        csevent_FRU_Unknown,
7891 					    "registered keys %d >= key "
7892 					    "count %d", key_count,
7893 					    lun->pr_key_count);
7894 #endif
7895 				key_count++;
7896 				continue;
7897 			}
7898 			scsi_u64to8b(key, res_keys->keys[key_count].key);
7899 			key_count++;
7900 		}
7901 		break;
7902 	}
7903 	case SPRI_RR: { // read reservation
7904 		struct scsi_per_res_in_rsrv *res;
7905 		int tmp_len, header_only;
7906 
7907 		res = (struct scsi_per_res_in_rsrv *)ctsio->kern_data_ptr;
7908 
7909 		scsi_ulto4b(lun->PRGeneration, res->header.generation);
7910 
7911 		if (lun->flags & CTL_LUN_PR_RESERVED)
7912 		{
7913 			tmp_len = sizeof(struct scsi_per_res_in_rsrv);
7914 			scsi_ulto4b(sizeof(struct scsi_per_res_in_rsrv_data),
7915 				    res->header.length);
7916 			header_only = 0;
7917 		} else {
7918 			tmp_len = sizeof(struct scsi_per_res_in_header);
7919 			scsi_ulto4b(0, res->header.length);
7920 			header_only = 1;
7921 		}
7922 
7923 		/*
7924 		 * We had to drop the lock to allocate our buffer, which
7925 		 * leaves time for someone to come in with another
7926 		 * persistent reservation.  (That is unlikely, though,
7927 		 * since this should be the only persistent reservation
7928 		 * command active right now.)
7929 		 */
7930 		if (tmp_len != total_len) {
7931 			mtx_unlock(&lun->lun_lock);
7932 			free(ctsio->kern_data_ptr, M_CTL);
7933 			printf("%s: reservation status changed, retrying\n",
7934 			       __func__);
7935 			goto retry;
7936 		}
7937 
7938 		/*
7939 		 * No reservation held, so we're done.
7940 		 */
7941 		if (header_only != 0)
7942 			break;
7943 
7944 		/*
7945 		 * If the registration is an All Registrants type, the key
7946 		 * is 0, since it doesn't really matter.
7947 		 */
7948 		if (lun->pr_res_idx != CTL_PR_ALL_REGISTRANTS) {
7949 			scsi_u64to8b(ctl_get_prkey(lun, lun->pr_res_idx),
7950 			    res->data.reservation);
7951 		}
7952 		res->data.scopetype = lun->res_type;
7953 		break;
7954 	}
7955 	case SPRI_RC:     //report capabilities
7956 	{
7957 		struct scsi_per_res_cap *res_cap;
7958 		uint16_t type_mask;
7959 
7960 		res_cap = (struct scsi_per_res_cap *)ctsio->kern_data_ptr;
7961 		scsi_ulto2b(sizeof(*res_cap), res_cap->length);
7962 		res_cap->flags2 |= SPRI_TMV | SPRI_ALLOW_5;
7963 		type_mask = SPRI_TM_WR_EX_AR |
7964 			    SPRI_TM_EX_AC_RO |
7965 			    SPRI_TM_WR_EX_RO |
7966 			    SPRI_TM_EX_AC |
7967 			    SPRI_TM_WR_EX |
7968 			    SPRI_TM_EX_AC_AR;
7969 		scsi_ulto2b(type_mask, res_cap->type_mask);
7970 		break;
7971 	}
7972 	case SPRI_RS: { // read full status
7973 		struct scsi_per_res_in_full *res_status;
7974 		struct scsi_per_res_in_full_desc *res_desc;
7975 		struct ctl_port *port;
7976 		int i, len;
7977 
7978 		res_status = (struct scsi_per_res_in_full*)ctsio->kern_data_ptr;
7979 
7980 		/*
7981 		 * We had to drop the lock to allocate our buffer, which
7982 		 * leaves time for someone to come in with another
7983 		 * persistent reservation.  (That is unlikely, though,
7984 		 * since this should be the only persistent reservation
7985 		 * command active right now.)
7986 		 */
7987 		if (total_len < (sizeof(struct scsi_per_res_in_header) +
7988 		    (sizeof(struct scsi_per_res_in_full_desc) + 256) *
7989 		     lun->pr_key_count)){
7990 			mtx_unlock(&lun->lun_lock);
7991 			free(ctsio->kern_data_ptr, M_CTL);
7992 			printf("%s: reservation length changed, retrying\n",
7993 			       __func__);
7994 			goto retry;
7995 		}
7996 
7997 		scsi_ulto4b(lun->PRGeneration, res_status->header.generation);
7998 
7999 		res_desc = &res_status->desc[0];
8000 		for (i = 0; i < 2*CTL_MAX_INITIATORS; i++) {
8001 			if ((key = ctl_get_prkey(lun, i)) == 0)
8002 				continue;
8003 
8004 			scsi_u64to8b(key, res_desc->res_key.key);
8005 			if ((lun->flags & CTL_LUN_PR_RESERVED) &&
8006 			    (lun->pr_res_idx == i ||
8007 			     lun->pr_res_idx == CTL_PR_ALL_REGISTRANTS)) {
8008 				res_desc->flags = SPRI_FULL_R_HOLDER;
8009 				res_desc->scopetype = lun->res_type;
8010 			}
8011 			scsi_ulto2b(i / CTL_MAX_INIT_PER_PORT,
8012 			    res_desc->rel_trgt_port_id);
8013 			len = 0;
8014 			port = softc->ctl_ports[
8015 			    ctl_port_idx(i / CTL_MAX_INIT_PER_PORT)];
8016 			if (port != NULL)
8017 				len = ctl_create_iid(port,
8018 				    i % CTL_MAX_INIT_PER_PORT,
8019 				    res_desc->transport_id);
8020 			scsi_ulto4b(len, res_desc->additional_length);
8021 			res_desc = (struct scsi_per_res_in_full_desc *)
8022 			    &res_desc->transport_id[len];
8023 		}
8024 		scsi_ulto4b((uint8_t *)res_desc - (uint8_t *)&res_status->desc[0],
8025 		    res_status->header.length);
8026 		break;
8027 	}
8028 	default:
8029 		/*
8030 		 * This is a bug, because we just checked for this above,
8031 		 * and should have returned an error.
8032 		 */
8033 		panic("Invalid PR type %x", cdb->action);
8034 		break; /* NOTREACHED */
8035 	}
8036 	mtx_unlock(&lun->lun_lock);
8037 
8038 	ctl_set_success(ctsio);
8039 	ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
8040 	ctsio->be_move_done = ctl_config_move_done;
8041 	ctl_datamove((union ctl_io *)ctsio);
8042 	return (CTL_RETVAL_COMPLETE);
8043 }
8044 
8045 static void
8046 ctl_est_res_ua(struct ctl_lun *lun, uint32_t residx, ctl_ua_type ua)
8047 {
8048 	int off = lun->ctl_softc->persis_offset;
8049 
8050 	if (residx >= off && residx < off + CTL_MAX_INITIATORS)
8051 		ctl_est_ua(lun, residx - off, ua);
8052 }
8053 
8054 /*
8055  * Returns 0 if ctl_persistent_reserve_out() should continue, non-zero if
8056  * it should return.
8057  */
8058 static int
8059 ctl_pro_preempt(struct ctl_softc *softc, struct ctl_lun *lun, uint64_t res_key,
8060 		uint64_t sa_res_key, uint8_t type, uint32_t residx,
8061 		struct ctl_scsiio *ctsio, struct scsi_per_res_out *cdb,
8062 		struct scsi_per_res_out_parms* param)
8063 {
8064 	union ctl_ha_msg persis_io;
8065 	int retval, i;
8066 	int isc_retval;
8067 
8068 	retval = 0;
8069 
8070 	mtx_lock(&lun->lun_lock);
8071 	if (sa_res_key == 0) {
8072 		if (lun->pr_res_idx == CTL_PR_ALL_REGISTRANTS) {
8073 			/* validate scope and type */
8074 			if ((cdb->scope_type & SPR_SCOPE_MASK) !=
8075 			     SPR_LU_SCOPE) {
8076 				mtx_unlock(&lun->lun_lock);
8077 				ctl_set_invalid_field(/*ctsio*/ ctsio,
8078 						      /*sks_valid*/ 1,
8079 						      /*command*/ 1,
8080 						      /*field*/ 2,
8081 						      /*bit_valid*/ 1,
8082 						      /*bit*/ 4);
8083 				ctl_done((union ctl_io *)ctsio);
8084 				return (1);
8085 			}
8086 
8087 		        if (type>8 || type==2 || type==4 || type==0) {
8088 				mtx_unlock(&lun->lun_lock);
8089 				ctl_set_invalid_field(/*ctsio*/ ctsio,
8090        	           				      /*sks_valid*/ 1,
8091 						      /*command*/ 1,
8092 						      /*field*/ 2,
8093 						      /*bit_valid*/ 1,
8094 						      /*bit*/ 0);
8095 				ctl_done((union ctl_io *)ctsio);
8096 				return (1);
8097 		        }
8098 
8099 			/*
8100 			 * Unregister everybody else and build UA for
8101 			 * them
8102 			 */
8103 			for(i=0; i < 2*CTL_MAX_INITIATORS; i++) {
8104 				if (i == residx || ctl_get_prkey(lun, i) == 0)
8105 					continue;
8106 
8107 				ctl_clr_prkey(lun, i);
8108 				ctl_est_res_ua(lun, i, CTL_UA_REG_PREEMPT);
8109 			}
8110 			lun->pr_key_count = 1;
8111 			lun->res_type = type;
8112 			if (lun->res_type != SPR_TYPE_WR_EX_AR
8113 			 && lun->res_type != SPR_TYPE_EX_AC_AR)
8114 				lun->pr_res_idx = residx;
8115 
8116 			/* send msg to other side */
8117 			persis_io.hdr.nexus = ctsio->io_hdr.nexus;
8118 			persis_io.hdr.msg_type = CTL_MSG_PERS_ACTION;
8119 			persis_io.pr.pr_info.action = CTL_PR_PREEMPT;
8120 			persis_io.pr.pr_info.residx = lun->pr_res_idx;
8121 			persis_io.pr.pr_info.res_type = type;
8122 			memcpy(persis_io.pr.pr_info.sa_res_key,
8123 			       param->serv_act_res_key,
8124 			       sizeof(param->serv_act_res_key));
8125 			if ((isc_retval=ctl_ha_msg_send(CTL_HA_CHAN_CTL,
8126 			     &persis_io, sizeof(persis_io), 0)) >
8127 			     CTL_HA_STATUS_SUCCESS) {
8128 				printf("CTL:Persis Out error returned "
8129 				       "from ctl_ha_msg_send %d\n",
8130 				       isc_retval);
8131 			}
8132 		} else {
8133 			/* not all registrants */
8134 			mtx_unlock(&lun->lun_lock);
8135 			free(ctsio->kern_data_ptr, M_CTL);
8136 			ctl_set_invalid_field(ctsio,
8137 					      /*sks_valid*/ 1,
8138 					      /*command*/ 0,
8139 					      /*field*/ 8,
8140 					      /*bit_valid*/ 0,
8141 					      /*bit*/ 0);
8142 			ctl_done((union ctl_io *)ctsio);
8143 			return (1);
8144 		}
8145 	} else if (lun->pr_res_idx == CTL_PR_ALL_REGISTRANTS
8146 		|| !(lun->flags & CTL_LUN_PR_RESERVED)) {
8147 		int found = 0;
8148 
8149 		if (res_key == sa_res_key) {
8150 			/* special case */
8151 			/*
8152 			 * The spec implies this is not good but doesn't
8153 			 * say what to do. There are two choices either
8154 			 * generate a res conflict or check condition
8155 			 * with illegal field in parameter data. Since
8156 			 * that is what is done when the sa_res_key is
8157 			 * zero I'll take that approach since this has
8158 			 * to do with the sa_res_key.
8159 			 */
8160 			mtx_unlock(&lun->lun_lock);
8161 			free(ctsio->kern_data_ptr, M_CTL);
8162 			ctl_set_invalid_field(ctsio,
8163 					      /*sks_valid*/ 1,
8164 					      /*command*/ 0,
8165 					      /*field*/ 8,
8166 					      /*bit_valid*/ 0,
8167 					      /*bit*/ 0);
8168 			ctl_done((union ctl_io *)ctsio);
8169 			return (1);
8170 		}
8171 
8172 		for (i=0; i < 2*CTL_MAX_INITIATORS; i++) {
8173 			if (ctl_get_prkey(lun, i) != sa_res_key)
8174 				continue;
8175 
8176 			found = 1;
8177 			ctl_clr_prkey(lun, i);
8178 			lun->pr_key_count--;
8179 			ctl_est_res_ua(lun, i, CTL_UA_REG_PREEMPT);
8180 		}
8181 		if (!found) {
8182 			mtx_unlock(&lun->lun_lock);
8183 			free(ctsio->kern_data_ptr, M_CTL);
8184 			ctl_set_reservation_conflict(ctsio);
8185 			ctl_done((union ctl_io *)ctsio);
8186 			return (CTL_RETVAL_COMPLETE);
8187 		}
8188 		/* send msg to other side */
8189 		persis_io.hdr.nexus = ctsio->io_hdr.nexus;
8190 		persis_io.hdr.msg_type = CTL_MSG_PERS_ACTION;
8191 		persis_io.pr.pr_info.action = CTL_PR_PREEMPT;
8192 		persis_io.pr.pr_info.residx = lun->pr_res_idx;
8193 		persis_io.pr.pr_info.res_type = type;
8194 		memcpy(persis_io.pr.pr_info.sa_res_key,
8195 		       param->serv_act_res_key,
8196 		       sizeof(param->serv_act_res_key));
8197 		if ((isc_retval=ctl_ha_msg_send(CTL_HA_CHAN_CTL,
8198 		     &persis_io, sizeof(persis_io), 0)) >
8199 		     CTL_HA_STATUS_SUCCESS) {
8200 			printf("CTL:Persis Out error returned from "
8201 			       "ctl_ha_msg_send %d\n", isc_retval);
8202 		}
8203 	} else {
8204 		/* Reserved but not all registrants */
8205 		/* sa_res_key is res holder */
8206 		if (sa_res_key == ctl_get_prkey(lun, lun->pr_res_idx)) {
8207 			/* validate scope and type */
8208 			if ((cdb->scope_type & SPR_SCOPE_MASK) !=
8209 			     SPR_LU_SCOPE) {
8210 				mtx_unlock(&lun->lun_lock);
8211 				ctl_set_invalid_field(/*ctsio*/ ctsio,
8212 						      /*sks_valid*/ 1,
8213 						      /*command*/ 1,
8214 						      /*field*/ 2,
8215 						      /*bit_valid*/ 1,
8216 						      /*bit*/ 4);
8217 				ctl_done((union ctl_io *)ctsio);
8218 				return (1);
8219 			}
8220 
8221 			if (type>8 || type==2 || type==4 || type==0) {
8222 				mtx_unlock(&lun->lun_lock);
8223 				ctl_set_invalid_field(/*ctsio*/ ctsio,
8224 						      /*sks_valid*/ 1,
8225 						      /*command*/ 1,
8226 						      /*field*/ 2,
8227 						      /*bit_valid*/ 1,
8228 						      /*bit*/ 0);
8229 				ctl_done((union ctl_io *)ctsio);
8230 				return (1);
8231 			}
8232 
8233 			/*
8234 			 * Do the following:
8235 			 * if sa_res_key != res_key remove all
8236 			 * registrants w/sa_res_key and generate UA
8237 			 * for these registrants(Registrations
8238 			 * Preempted) if it wasn't an exclusive
8239 			 * reservation generate UA(Reservations
8240 			 * Preempted) for all other registered nexuses
8241 			 * if the type has changed. Establish the new
8242 			 * reservation and holder. If res_key and
8243 			 * sa_res_key are the same do the above
8244 			 * except don't unregister the res holder.
8245 			 */
8246 
8247 			for(i=0; i < 2*CTL_MAX_INITIATORS; i++) {
8248 				if (i == residx || ctl_get_prkey(lun, i) == 0)
8249 					continue;
8250 
8251 				if (sa_res_key == ctl_get_prkey(lun, i)) {
8252 					ctl_clr_prkey(lun, i);
8253 					lun->pr_key_count--;
8254 					ctl_est_res_ua(lun, i, CTL_UA_REG_PREEMPT);
8255 				} else if (type != lun->res_type
8256 					&& (lun->res_type == SPR_TYPE_WR_EX_RO
8257 					 || lun->res_type ==SPR_TYPE_EX_AC_RO)){
8258 					ctl_est_res_ua(lun, i, CTL_UA_RES_RELEASE);
8259 				}
8260 			}
8261 			lun->res_type = type;
8262 			if (lun->res_type != SPR_TYPE_WR_EX_AR
8263 			 && lun->res_type != SPR_TYPE_EX_AC_AR)
8264 				lun->pr_res_idx = residx;
8265 			else
8266 				lun->pr_res_idx = CTL_PR_ALL_REGISTRANTS;
8267 
8268 			persis_io.hdr.nexus = ctsio->io_hdr.nexus;
8269 			persis_io.hdr.msg_type = CTL_MSG_PERS_ACTION;
8270 			persis_io.pr.pr_info.action = CTL_PR_PREEMPT;
8271 			persis_io.pr.pr_info.residx = lun->pr_res_idx;
8272 			persis_io.pr.pr_info.res_type = type;
8273 			memcpy(persis_io.pr.pr_info.sa_res_key,
8274 			       param->serv_act_res_key,
8275 			       sizeof(param->serv_act_res_key));
8276 			if ((isc_retval=ctl_ha_msg_send(CTL_HA_CHAN_CTL,
8277 			     &persis_io, sizeof(persis_io), 0)) >
8278 			     CTL_HA_STATUS_SUCCESS) {
8279 				printf("CTL:Persis Out error returned "
8280 				       "from ctl_ha_msg_send %d\n",
8281 				       isc_retval);
8282 			}
8283 		} else {
8284 			/*
8285 			 * sa_res_key is not the res holder just
8286 			 * remove registrants
8287 			 */
8288 			int found=0;
8289 
8290 			for (i=0; i < 2*CTL_MAX_INITIATORS; i++) {
8291 				if (sa_res_key != ctl_get_prkey(lun, i))
8292 					continue;
8293 
8294 				found = 1;
8295 				ctl_clr_prkey(lun, i);
8296 				lun->pr_key_count--;
8297 				ctl_est_res_ua(lun, i, CTL_UA_REG_PREEMPT);
8298 			}
8299 
8300 			if (!found) {
8301 				mtx_unlock(&lun->lun_lock);
8302 				free(ctsio->kern_data_ptr, M_CTL);
8303 				ctl_set_reservation_conflict(ctsio);
8304 				ctl_done((union ctl_io *)ctsio);
8305 		        	return (1);
8306 			}
8307 			persis_io.hdr.nexus = ctsio->io_hdr.nexus;
8308 			persis_io.hdr.msg_type = CTL_MSG_PERS_ACTION;
8309 			persis_io.pr.pr_info.action = CTL_PR_PREEMPT;
8310 			persis_io.pr.pr_info.residx = lun->pr_res_idx;
8311 			persis_io.pr.pr_info.res_type = type;
8312 			memcpy(persis_io.pr.pr_info.sa_res_key,
8313 			       param->serv_act_res_key,
8314 			       sizeof(param->serv_act_res_key));
8315 			if ((isc_retval=ctl_ha_msg_send(CTL_HA_CHAN_CTL,
8316 			     &persis_io, sizeof(persis_io), 0)) >
8317 			     CTL_HA_STATUS_SUCCESS) {
8318 				printf("CTL:Persis Out error returned "
8319 				       "from ctl_ha_msg_send %d\n",
8320 				isc_retval);
8321 			}
8322 		}
8323 	}
8324 
8325 	lun->PRGeneration++;
8326 	mtx_unlock(&lun->lun_lock);
8327 
8328 	return (retval);
8329 }
8330 
8331 static void
8332 ctl_pro_preempt_other(struct ctl_lun *lun, union ctl_ha_msg *msg)
8333 {
8334 	uint64_t sa_res_key;
8335 	int i;
8336 
8337 	sa_res_key = scsi_8btou64(msg->pr.pr_info.sa_res_key);
8338 
8339 	if (lun->pr_res_idx == CTL_PR_ALL_REGISTRANTS
8340 	 || lun->pr_res_idx == CTL_PR_NO_RESERVATION
8341 	 || sa_res_key != ctl_get_prkey(lun, lun->pr_res_idx)) {
8342 		if (sa_res_key == 0) {
8343 			/*
8344 			 * Unregister everybody else and build UA for
8345 			 * them
8346 			 */
8347 			for(i=0; i < 2*CTL_MAX_INITIATORS; i++) {
8348 				if (i == msg->pr.pr_info.residx ||
8349 				    ctl_get_prkey(lun, i) == 0)
8350 					continue;
8351 
8352 				ctl_clr_prkey(lun, i);
8353 				ctl_est_res_ua(lun, i, CTL_UA_REG_PREEMPT);
8354 			}
8355 
8356 			lun->pr_key_count = 1;
8357 			lun->res_type = msg->pr.pr_info.res_type;
8358 			if (lun->res_type != SPR_TYPE_WR_EX_AR
8359 			 && lun->res_type != SPR_TYPE_EX_AC_AR)
8360 				lun->pr_res_idx = msg->pr.pr_info.residx;
8361 		} else {
8362 		        for (i=0; i < 2*CTL_MAX_INITIATORS; i++) {
8363 				if (sa_res_key == ctl_get_prkey(lun, i))
8364 					continue;
8365 
8366 				ctl_clr_prkey(lun, i);
8367 				lun->pr_key_count--;
8368 				ctl_est_res_ua(lun, i, CTL_UA_REG_PREEMPT);
8369 			}
8370 		}
8371 	} else {
8372 		for (i=0; i < 2*CTL_MAX_INITIATORS; i++) {
8373 			if (i == msg->pr.pr_info.residx ||
8374 			    ctl_get_prkey(lun, i) == 0)
8375 				continue;
8376 
8377 			if (sa_res_key == ctl_get_prkey(lun, i)) {
8378 				ctl_clr_prkey(lun, i);
8379 				lun->pr_key_count--;
8380 				ctl_est_res_ua(lun, i, CTL_UA_REG_PREEMPT);
8381 			} else if (msg->pr.pr_info.res_type != lun->res_type
8382 				&& (lun->res_type == SPR_TYPE_WR_EX_RO
8383 				 || lun->res_type == SPR_TYPE_EX_AC_RO)) {
8384 				ctl_est_res_ua(lun, i, CTL_UA_RES_RELEASE);
8385 			}
8386 		}
8387 		lun->res_type = msg->pr.pr_info.res_type;
8388 		if (lun->res_type != SPR_TYPE_WR_EX_AR
8389 		 && lun->res_type != SPR_TYPE_EX_AC_AR)
8390 			lun->pr_res_idx = msg->pr.pr_info.residx;
8391 		else
8392 			lun->pr_res_idx = CTL_PR_ALL_REGISTRANTS;
8393 	}
8394 	lun->PRGeneration++;
8395 
8396 }
8397 
8398 
8399 int
8400 ctl_persistent_reserve_out(struct ctl_scsiio *ctsio)
8401 {
8402 	int retval;
8403 	int isc_retval;
8404 	u_int32_t param_len;
8405 	struct scsi_per_res_out *cdb;
8406 	struct ctl_lun *lun;
8407 	struct scsi_per_res_out_parms* param;
8408 	struct ctl_softc *softc;
8409 	uint32_t residx;
8410 	uint64_t res_key, sa_res_key, key;
8411 	uint8_t type;
8412 	union ctl_ha_msg persis_io;
8413 	int    i;
8414 
8415 	CTL_DEBUG_PRINT(("ctl_persistent_reserve_out\n"));
8416 
8417 	retval = CTL_RETVAL_COMPLETE;
8418 
8419 	cdb = (struct scsi_per_res_out *)ctsio->cdb;
8420 	lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
8421 	softc = lun->ctl_softc;
8422 
8423 	/*
8424 	 * We only support whole-LUN scope.  The scope & type are ignored for
8425 	 * register, register and ignore existing key and clear.
8426 	 * We sometimes ignore scope and type on preempts too!!
8427 	 * Verify reservation type here as well.
8428 	 */
8429 	type = cdb->scope_type & SPR_TYPE_MASK;
8430 	if ((cdb->action == SPRO_RESERVE)
8431 	 || (cdb->action == SPRO_RELEASE)) {
8432 		if ((cdb->scope_type & SPR_SCOPE_MASK) != SPR_LU_SCOPE) {
8433 			ctl_set_invalid_field(/*ctsio*/ ctsio,
8434 					      /*sks_valid*/ 1,
8435 					      /*command*/ 1,
8436 					      /*field*/ 2,
8437 					      /*bit_valid*/ 1,
8438 					      /*bit*/ 4);
8439 			ctl_done((union ctl_io *)ctsio);
8440 			return (CTL_RETVAL_COMPLETE);
8441 		}
8442 
8443 		if (type>8 || type==2 || type==4 || type==0) {
8444 			ctl_set_invalid_field(/*ctsio*/ ctsio,
8445 					      /*sks_valid*/ 1,
8446 					      /*command*/ 1,
8447 					      /*field*/ 2,
8448 					      /*bit_valid*/ 1,
8449 					      /*bit*/ 0);
8450 			ctl_done((union ctl_io *)ctsio);
8451 			return (CTL_RETVAL_COMPLETE);
8452 		}
8453 	}
8454 
8455 	param_len = scsi_4btoul(cdb->length);
8456 
8457 	if ((ctsio->io_hdr.flags & CTL_FLAG_ALLOCATED) == 0) {
8458 		ctsio->kern_data_ptr = malloc(param_len, M_CTL, M_WAITOK);
8459 		ctsio->kern_data_len = param_len;
8460 		ctsio->kern_total_len = param_len;
8461 		ctsio->kern_data_resid = 0;
8462 		ctsio->kern_rel_offset = 0;
8463 		ctsio->kern_sg_entries = 0;
8464 		ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
8465 		ctsio->be_move_done = ctl_config_move_done;
8466 		ctl_datamove((union ctl_io *)ctsio);
8467 
8468 		return (CTL_RETVAL_COMPLETE);
8469 	}
8470 
8471 	param = (struct scsi_per_res_out_parms *)ctsio->kern_data_ptr;
8472 
8473 	residx = ctl_get_resindex(&ctsio->io_hdr.nexus);
8474 	res_key = scsi_8btou64(param->res_key.key);
8475 	sa_res_key = scsi_8btou64(param->serv_act_res_key);
8476 
8477 	/*
8478 	 * Validate the reservation key here except for SPRO_REG_IGNO
8479 	 * This must be done for all other service actions
8480 	 */
8481 	if ((cdb->action & SPRO_ACTION_MASK) != SPRO_REG_IGNO) {
8482 		mtx_lock(&lun->lun_lock);
8483 		if ((key = ctl_get_prkey(lun, residx)) != 0) {
8484 			if (res_key != key) {
8485 				/*
8486 				 * The current key passed in doesn't match
8487 				 * the one the initiator previously
8488 				 * registered.
8489 				 */
8490 				mtx_unlock(&lun->lun_lock);
8491 				free(ctsio->kern_data_ptr, M_CTL);
8492 				ctl_set_reservation_conflict(ctsio);
8493 				ctl_done((union ctl_io *)ctsio);
8494 				return (CTL_RETVAL_COMPLETE);
8495 			}
8496 		} else if ((cdb->action & SPRO_ACTION_MASK) != SPRO_REGISTER) {
8497 			/*
8498 			 * We are not registered
8499 			 */
8500 			mtx_unlock(&lun->lun_lock);
8501 			free(ctsio->kern_data_ptr, M_CTL);
8502 			ctl_set_reservation_conflict(ctsio);
8503 			ctl_done((union ctl_io *)ctsio);
8504 			return (CTL_RETVAL_COMPLETE);
8505 		} else if (res_key != 0) {
8506 			/*
8507 			 * We are not registered and trying to register but
8508 			 * the register key isn't zero.
8509 			 */
8510 			mtx_unlock(&lun->lun_lock);
8511 			free(ctsio->kern_data_ptr, M_CTL);
8512 			ctl_set_reservation_conflict(ctsio);
8513 			ctl_done((union ctl_io *)ctsio);
8514 			return (CTL_RETVAL_COMPLETE);
8515 		}
8516 		mtx_unlock(&lun->lun_lock);
8517 	}
8518 
8519 	switch (cdb->action & SPRO_ACTION_MASK) {
8520 	case SPRO_REGISTER:
8521 	case SPRO_REG_IGNO: {
8522 
8523 #if 0
8524 		printf("Registration received\n");
8525 #endif
8526 
8527 		/*
8528 		 * We don't support any of these options, as we report in
8529 		 * the read capabilities request (see
8530 		 * ctl_persistent_reserve_in(), above).
8531 		 */
8532 		if ((param->flags & SPR_SPEC_I_PT)
8533 		 || (param->flags & SPR_ALL_TG_PT)
8534 		 || (param->flags & SPR_APTPL)) {
8535 			int bit_ptr;
8536 
8537 			if (param->flags & SPR_APTPL)
8538 				bit_ptr = 0;
8539 			else if (param->flags & SPR_ALL_TG_PT)
8540 				bit_ptr = 2;
8541 			else /* SPR_SPEC_I_PT */
8542 				bit_ptr = 3;
8543 
8544 			free(ctsio->kern_data_ptr, M_CTL);
8545 			ctl_set_invalid_field(ctsio,
8546 					      /*sks_valid*/ 1,
8547 					      /*command*/ 0,
8548 					      /*field*/ 20,
8549 					      /*bit_valid*/ 1,
8550 					      /*bit*/ bit_ptr);
8551 			ctl_done((union ctl_io *)ctsio);
8552 			return (CTL_RETVAL_COMPLETE);
8553 		}
8554 
8555 		mtx_lock(&lun->lun_lock);
8556 
8557 		/*
8558 		 * The initiator wants to clear the
8559 		 * key/unregister.
8560 		 */
8561 		if (sa_res_key == 0) {
8562 			if ((res_key == 0
8563 			  && (cdb->action & SPRO_ACTION_MASK) == SPRO_REGISTER)
8564 			 || ((cdb->action & SPRO_ACTION_MASK) == SPRO_REG_IGNO
8565 			  && ctl_get_prkey(lun, residx) == 0)) {
8566 				mtx_unlock(&lun->lun_lock);
8567 				goto done;
8568 			}
8569 
8570 			ctl_clr_prkey(lun, residx);
8571 			lun->pr_key_count--;
8572 
8573 			if (residx == lun->pr_res_idx) {
8574 				lun->flags &= ~CTL_LUN_PR_RESERVED;
8575 				lun->pr_res_idx = CTL_PR_NO_RESERVATION;
8576 
8577 				if ((lun->res_type == SPR_TYPE_WR_EX_RO
8578 				  || lun->res_type == SPR_TYPE_EX_AC_RO)
8579 				 && lun->pr_key_count) {
8580 					/*
8581 					 * If the reservation is a registrants
8582 					 * only type we need to generate a UA
8583 					 * for other registered inits.  The
8584 					 * sense code should be RESERVATIONS
8585 					 * RELEASED
8586 					 */
8587 
8588 					for (i = 0; i < CTL_MAX_INITIATORS;i++){
8589 						if (ctl_get_prkey(lun, i +
8590 						    softc->persis_offset) == 0)
8591 							continue;
8592 						ctl_est_ua(lun, i,
8593 						    CTL_UA_RES_RELEASE);
8594 					}
8595 				}
8596 				lun->res_type = 0;
8597 			} else if (lun->pr_res_idx == CTL_PR_ALL_REGISTRANTS) {
8598 				if (lun->pr_key_count==0) {
8599 					lun->flags &= ~CTL_LUN_PR_RESERVED;
8600 					lun->res_type = 0;
8601 					lun->pr_res_idx = CTL_PR_NO_RESERVATION;
8602 				}
8603 			}
8604 			persis_io.hdr.nexus = ctsio->io_hdr.nexus;
8605 			persis_io.hdr.msg_type = CTL_MSG_PERS_ACTION;
8606 			persis_io.pr.pr_info.action = CTL_PR_UNREG_KEY;
8607 			persis_io.pr.pr_info.residx = residx;
8608 			if ((isc_retval = ctl_ha_msg_send(CTL_HA_CHAN_CTL,
8609 			     &persis_io, sizeof(persis_io), 0 )) >
8610 			     CTL_HA_STATUS_SUCCESS) {
8611 				printf("CTL:Persis Out error returned from "
8612 				       "ctl_ha_msg_send %d\n", isc_retval);
8613 			}
8614 		} else /* sa_res_key != 0 */ {
8615 
8616 			/*
8617 			 * If we aren't registered currently then increment
8618 			 * the key count and set the registered flag.
8619 			 */
8620 			ctl_alloc_prkey(lun, residx);
8621 			if (ctl_get_prkey(lun, residx) == 0)
8622 				lun->pr_key_count++;
8623 			ctl_set_prkey(lun, residx, sa_res_key);
8624 
8625 			persis_io.hdr.nexus = ctsio->io_hdr.nexus;
8626 			persis_io.hdr.msg_type = CTL_MSG_PERS_ACTION;
8627 			persis_io.pr.pr_info.action = CTL_PR_REG_KEY;
8628 			persis_io.pr.pr_info.residx = residx;
8629 			memcpy(persis_io.pr.pr_info.sa_res_key,
8630 			       param->serv_act_res_key,
8631 			       sizeof(param->serv_act_res_key));
8632 			if ((isc_retval=ctl_ha_msg_send(CTL_HA_CHAN_CTL,
8633 			     &persis_io, sizeof(persis_io), 0)) >
8634 			     CTL_HA_STATUS_SUCCESS) {
8635 				printf("CTL:Persis Out error returned from "
8636 				       "ctl_ha_msg_send %d\n", isc_retval);
8637 			}
8638 		}
8639 		lun->PRGeneration++;
8640 		mtx_unlock(&lun->lun_lock);
8641 
8642 		break;
8643 	}
8644 	case SPRO_RESERVE:
8645 #if 0
8646                 printf("Reserve executed type %d\n", type);
8647 #endif
8648 		mtx_lock(&lun->lun_lock);
8649 		if (lun->flags & CTL_LUN_PR_RESERVED) {
8650 			/*
8651 			 * if this isn't the reservation holder and it's
8652 			 * not a "all registrants" type or if the type is
8653 			 * different then we have a conflict
8654 			 */
8655 			if ((lun->pr_res_idx != residx
8656 			  && lun->pr_res_idx != CTL_PR_ALL_REGISTRANTS)
8657 			 || lun->res_type != type) {
8658 				mtx_unlock(&lun->lun_lock);
8659 				free(ctsio->kern_data_ptr, M_CTL);
8660 				ctl_set_reservation_conflict(ctsio);
8661 				ctl_done((union ctl_io *)ctsio);
8662 				return (CTL_RETVAL_COMPLETE);
8663 			}
8664 			mtx_unlock(&lun->lun_lock);
8665 		} else /* create a reservation */ {
8666 			/*
8667 			 * If it's not an "all registrants" type record
8668 			 * reservation holder
8669 			 */
8670 			if (type != SPR_TYPE_WR_EX_AR
8671 			 && type != SPR_TYPE_EX_AC_AR)
8672 				lun->pr_res_idx = residx; /* Res holder */
8673 			else
8674 				lun->pr_res_idx = CTL_PR_ALL_REGISTRANTS;
8675 
8676 			lun->flags |= CTL_LUN_PR_RESERVED;
8677 			lun->res_type = type;
8678 
8679 			mtx_unlock(&lun->lun_lock);
8680 
8681 			/* send msg to other side */
8682 			persis_io.hdr.nexus = ctsio->io_hdr.nexus;
8683 			persis_io.hdr.msg_type = CTL_MSG_PERS_ACTION;
8684 			persis_io.pr.pr_info.action = CTL_PR_RESERVE;
8685 			persis_io.pr.pr_info.residx = lun->pr_res_idx;
8686 			persis_io.pr.pr_info.res_type = type;
8687 			if ((isc_retval=ctl_ha_msg_send(CTL_HA_CHAN_CTL,
8688 			     &persis_io, sizeof(persis_io), 0)) >
8689 			     CTL_HA_STATUS_SUCCESS) {
8690 				printf("CTL:Persis Out error returned from "
8691 				       "ctl_ha_msg_send %d\n", isc_retval);
8692 			}
8693 		}
8694 		break;
8695 
8696 	case SPRO_RELEASE:
8697 		mtx_lock(&lun->lun_lock);
8698 		if ((lun->flags & CTL_LUN_PR_RESERVED) == 0) {
8699 			/* No reservation exists return good status */
8700 			mtx_unlock(&lun->lun_lock);
8701 			goto done;
8702 		}
8703 		/*
8704 		 * Is this nexus a reservation holder?
8705 		 */
8706 		if (lun->pr_res_idx != residx
8707 		 && lun->pr_res_idx != CTL_PR_ALL_REGISTRANTS) {
8708 			/*
8709 			 * not a res holder return good status but
8710 			 * do nothing
8711 			 */
8712 			mtx_unlock(&lun->lun_lock);
8713 			goto done;
8714 		}
8715 
8716 		if (lun->res_type != type) {
8717 			mtx_unlock(&lun->lun_lock);
8718 			free(ctsio->kern_data_ptr, M_CTL);
8719 			ctl_set_illegal_pr_release(ctsio);
8720 			ctl_done((union ctl_io *)ctsio);
8721 			return (CTL_RETVAL_COMPLETE);
8722 		}
8723 
8724 		/* okay to release */
8725 		lun->flags &= ~CTL_LUN_PR_RESERVED;
8726 		lun->pr_res_idx = CTL_PR_NO_RESERVATION;
8727 		lun->res_type = 0;
8728 
8729 		/*
8730 		 * if this isn't an exclusive access
8731 		 * res generate UA for all other
8732 		 * registrants.
8733 		 */
8734 		if (type != SPR_TYPE_EX_AC
8735 		 && type != SPR_TYPE_WR_EX) {
8736 			for (i = 0; i < CTL_MAX_INITIATORS; i++) {
8737 				if (i == residx ||
8738 				    ctl_get_prkey(lun,
8739 				     i + softc->persis_offset) == 0)
8740 					continue;
8741 				ctl_est_ua(lun, i, CTL_UA_RES_RELEASE);
8742 			}
8743 		}
8744 		mtx_unlock(&lun->lun_lock);
8745 		/* Send msg to other side */
8746 		persis_io.hdr.nexus = ctsio->io_hdr.nexus;
8747 		persis_io.hdr.msg_type = CTL_MSG_PERS_ACTION;
8748 		persis_io.pr.pr_info.action = CTL_PR_RELEASE;
8749 		if ((isc_retval=ctl_ha_msg_send( CTL_HA_CHAN_CTL, &persis_io,
8750 		     sizeof(persis_io), 0)) > CTL_HA_STATUS_SUCCESS) {
8751 			printf("CTL:Persis Out error returned from "
8752 			       "ctl_ha_msg_send %d\n", isc_retval);
8753 		}
8754 		break;
8755 
8756 	case SPRO_CLEAR:
8757 		/* send msg to other side */
8758 
8759 		mtx_lock(&lun->lun_lock);
8760 		lun->flags &= ~CTL_LUN_PR_RESERVED;
8761 		lun->res_type = 0;
8762 		lun->pr_key_count = 0;
8763 		lun->pr_res_idx = CTL_PR_NO_RESERVATION;
8764 
8765 		ctl_clr_prkey(lun, residx);
8766 		for (i=0; i < 2*CTL_MAX_INITIATORS; i++)
8767 			if (ctl_get_prkey(lun, i) != 0) {
8768 				ctl_clr_prkey(lun, i);
8769 				ctl_est_res_ua(lun, i, CTL_UA_REG_PREEMPT);
8770 			}
8771 		lun->PRGeneration++;
8772 		mtx_unlock(&lun->lun_lock);
8773 		persis_io.hdr.nexus = ctsio->io_hdr.nexus;
8774 		persis_io.hdr.msg_type = CTL_MSG_PERS_ACTION;
8775 		persis_io.pr.pr_info.action = CTL_PR_CLEAR;
8776 		if ((isc_retval=ctl_ha_msg_send(CTL_HA_CHAN_CTL, &persis_io,
8777 		     sizeof(persis_io), 0)) > CTL_HA_STATUS_SUCCESS) {
8778 			printf("CTL:Persis Out error returned from "
8779 			       "ctl_ha_msg_send %d\n", isc_retval);
8780 		}
8781 		break;
8782 
8783 	case SPRO_PREEMPT:
8784 	case SPRO_PRE_ABO: {
8785 		int nretval;
8786 
8787 		nretval = ctl_pro_preempt(softc, lun, res_key, sa_res_key, type,
8788 					  residx, ctsio, cdb, param);
8789 		if (nretval != 0)
8790 			return (CTL_RETVAL_COMPLETE);
8791 		break;
8792 	}
8793 	default:
8794 		panic("Invalid PR type %x", cdb->action);
8795 	}
8796 
8797 done:
8798 	free(ctsio->kern_data_ptr, M_CTL);
8799 	ctl_set_success(ctsio);
8800 	ctl_done((union ctl_io *)ctsio);
8801 
8802 	return (retval);
8803 }
8804 
8805 /*
8806  * This routine is for handling a message from the other SC pertaining to
8807  * persistent reserve out. All the error checking will have been done
8808  * so only perorming the action need be done here to keep the two
8809  * in sync.
8810  */
8811 static void
8812 ctl_hndl_per_res_out_on_other_sc(union ctl_ha_msg *msg)
8813 {
8814 	struct ctl_lun *lun;
8815 	struct ctl_softc *softc;
8816 	int i;
8817 	uint32_t targ_lun;
8818 
8819 	softc = control_softc;
8820 
8821 	targ_lun = msg->hdr.nexus.targ_mapped_lun;
8822 	lun = softc->ctl_luns[targ_lun];
8823 	mtx_lock(&lun->lun_lock);
8824 	switch(msg->pr.pr_info.action) {
8825 	case CTL_PR_REG_KEY:
8826 		ctl_alloc_prkey(lun, msg->pr.pr_info.residx);
8827 		if (ctl_get_prkey(lun, msg->pr.pr_info.residx) == 0)
8828 			lun->pr_key_count++;
8829 		ctl_set_prkey(lun, msg->pr.pr_info.residx,
8830 		    scsi_8btou64(msg->pr.pr_info.sa_res_key));
8831 		lun->PRGeneration++;
8832 		break;
8833 
8834 	case CTL_PR_UNREG_KEY:
8835 		ctl_clr_prkey(lun, msg->pr.pr_info.residx);
8836 		lun->pr_key_count--;
8837 
8838 		/* XXX Need to see if the reservation has been released */
8839 		/* if so do we need to generate UA? */
8840 		if (msg->pr.pr_info.residx == lun->pr_res_idx) {
8841 			lun->flags &= ~CTL_LUN_PR_RESERVED;
8842 			lun->pr_res_idx = CTL_PR_NO_RESERVATION;
8843 
8844 			if ((lun->res_type == SPR_TYPE_WR_EX_RO
8845 			  || lun->res_type == SPR_TYPE_EX_AC_RO)
8846 			 && lun->pr_key_count) {
8847 				/*
8848 				 * If the reservation is a registrants
8849 				 * only type we need to generate a UA
8850 				 * for other registered inits.  The
8851 				 * sense code should be RESERVATIONS
8852 				 * RELEASED
8853 				 */
8854 
8855 				for (i = 0; i < CTL_MAX_INITIATORS; i++) {
8856 					if (ctl_get_prkey(lun, i +
8857 					    softc->persis_offset) == 0)
8858 						continue;
8859 
8860 					ctl_est_ua(lun, i, CTL_UA_RES_RELEASE);
8861 				}
8862 			}
8863 			lun->res_type = 0;
8864 		} else if (lun->pr_res_idx == CTL_PR_ALL_REGISTRANTS) {
8865 			if (lun->pr_key_count==0) {
8866 				lun->flags &= ~CTL_LUN_PR_RESERVED;
8867 				lun->res_type = 0;
8868 				lun->pr_res_idx = CTL_PR_NO_RESERVATION;
8869 			}
8870 		}
8871 		lun->PRGeneration++;
8872 		break;
8873 
8874 	case CTL_PR_RESERVE:
8875 		lun->flags |= CTL_LUN_PR_RESERVED;
8876 		lun->res_type = msg->pr.pr_info.res_type;
8877 		lun->pr_res_idx = msg->pr.pr_info.residx;
8878 
8879 		break;
8880 
8881 	case CTL_PR_RELEASE:
8882 		/*
8883 		 * if this isn't an exclusive access res generate UA for all
8884 		 * other registrants.
8885 		 */
8886 		if (lun->res_type != SPR_TYPE_EX_AC
8887 		 && lun->res_type != SPR_TYPE_WR_EX) {
8888 			for (i = 0; i < CTL_MAX_INITIATORS; i++)
8889 				if (ctl_get_prkey(lun, i + softc->persis_offset) != 0)
8890 					ctl_est_ua(lun, i, CTL_UA_RES_RELEASE);
8891 		}
8892 
8893 		lun->flags &= ~CTL_LUN_PR_RESERVED;
8894 		lun->pr_res_idx = CTL_PR_NO_RESERVATION;
8895 		lun->res_type = 0;
8896 		break;
8897 
8898 	case CTL_PR_PREEMPT:
8899 		ctl_pro_preempt_other(lun, msg);
8900 		break;
8901 	case CTL_PR_CLEAR:
8902 		lun->flags &= ~CTL_LUN_PR_RESERVED;
8903 		lun->res_type = 0;
8904 		lun->pr_key_count = 0;
8905 		lun->pr_res_idx = CTL_PR_NO_RESERVATION;
8906 
8907 		for (i=0; i < 2*CTL_MAX_INITIATORS; i++) {
8908 			if (ctl_get_prkey(lun, i) == 0)
8909 				continue;
8910 			ctl_clr_prkey(lun, i);
8911 			ctl_est_res_ua(lun, i, CTL_UA_REG_PREEMPT);
8912 		}
8913 		lun->PRGeneration++;
8914 		break;
8915 	}
8916 
8917 	mtx_unlock(&lun->lun_lock);
8918 }
8919 
8920 int
8921 ctl_read_write(struct ctl_scsiio *ctsio)
8922 {
8923 	struct ctl_lun *lun;
8924 	struct ctl_lba_len_flags *lbalen;
8925 	uint64_t lba;
8926 	uint32_t num_blocks;
8927 	int flags, retval;
8928 	int isread;
8929 
8930 	lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
8931 
8932 	CTL_DEBUG_PRINT(("ctl_read_write: command: %#x\n", ctsio->cdb[0]));
8933 
8934 	flags = 0;
8935 	retval = CTL_RETVAL_COMPLETE;
8936 
8937 	isread = ctsio->cdb[0] == READ_6  || ctsio->cdb[0] == READ_10
8938 	      || ctsio->cdb[0] == READ_12 || ctsio->cdb[0] == READ_16;
8939 	switch (ctsio->cdb[0]) {
8940 	case READ_6:
8941 	case WRITE_6: {
8942 		struct scsi_rw_6 *cdb;
8943 
8944 		cdb = (struct scsi_rw_6 *)ctsio->cdb;
8945 
8946 		lba = scsi_3btoul(cdb->addr);
8947 		/* only 5 bits are valid in the most significant address byte */
8948 		lba &= 0x1fffff;
8949 		num_blocks = cdb->length;
8950 		/*
8951 		 * This is correct according to SBC-2.
8952 		 */
8953 		if (num_blocks == 0)
8954 			num_blocks = 256;
8955 		break;
8956 	}
8957 	case READ_10:
8958 	case WRITE_10: {
8959 		struct scsi_rw_10 *cdb;
8960 
8961 		cdb = (struct scsi_rw_10 *)ctsio->cdb;
8962 		if (cdb->byte2 & SRW10_FUA)
8963 			flags |= CTL_LLF_FUA;
8964 		if (cdb->byte2 & SRW10_DPO)
8965 			flags |= CTL_LLF_DPO;
8966 		lba = scsi_4btoul(cdb->addr);
8967 		num_blocks = scsi_2btoul(cdb->length);
8968 		break;
8969 	}
8970 	case WRITE_VERIFY_10: {
8971 		struct scsi_write_verify_10 *cdb;
8972 
8973 		cdb = (struct scsi_write_verify_10 *)ctsio->cdb;
8974 		flags |= CTL_LLF_FUA;
8975 		if (cdb->byte2 & SWV_DPO)
8976 			flags |= CTL_LLF_DPO;
8977 		lba = scsi_4btoul(cdb->addr);
8978 		num_blocks = scsi_2btoul(cdb->length);
8979 		break;
8980 	}
8981 	case READ_12:
8982 	case WRITE_12: {
8983 		struct scsi_rw_12 *cdb;
8984 
8985 		cdb = (struct scsi_rw_12 *)ctsio->cdb;
8986 		if (cdb->byte2 & SRW12_FUA)
8987 			flags |= CTL_LLF_FUA;
8988 		if (cdb->byte2 & SRW12_DPO)
8989 			flags |= CTL_LLF_DPO;
8990 		lba = scsi_4btoul(cdb->addr);
8991 		num_blocks = scsi_4btoul(cdb->length);
8992 		break;
8993 	}
8994 	case WRITE_VERIFY_12: {
8995 		struct scsi_write_verify_12 *cdb;
8996 
8997 		cdb = (struct scsi_write_verify_12 *)ctsio->cdb;
8998 		flags |= CTL_LLF_FUA;
8999 		if (cdb->byte2 & SWV_DPO)
9000 			flags |= CTL_LLF_DPO;
9001 		lba = scsi_4btoul(cdb->addr);
9002 		num_blocks = scsi_4btoul(cdb->length);
9003 		break;
9004 	}
9005 	case READ_16:
9006 	case WRITE_16: {
9007 		struct scsi_rw_16 *cdb;
9008 
9009 		cdb = (struct scsi_rw_16 *)ctsio->cdb;
9010 		if (cdb->byte2 & SRW12_FUA)
9011 			flags |= CTL_LLF_FUA;
9012 		if (cdb->byte2 & SRW12_DPO)
9013 			flags |= CTL_LLF_DPO;
9014 		lba = scsi_8btou64(cdb->addr);
9015 		num_blocks = scsi_4btoul(cdb->length);
9016 		break;
9017 	}
9018 	case WRITE_ATOMIC_16: {
9019 		struct scsi_rw_16 *cdb;
9020 
9021 		if (lun->be_lun->atomicblock == 0) {
9022 			ctl_set_invalid_opcode(ctsio);
9023 			ctl_done((union ctl_io *)ctsio);
9024 			return (CTL_RETVAL_COMPLETE);
9025 		}
9026 
9027 		cdb = (struct scsi_rw_16 *)ctsio->cdb;
9028 		if (cdb->byte2 & SRW12_FUA)
9029 			flags |= CTL_LLF_FUA;
9030 		if (cdb->byte2 & SRW12_DPO)
9031 			flags |= CTL_LLF_DPO;
9032 		lba = scsi_8btou64(cdb->addr);
9033 		num_blocks = scsi_4btoul(cdb->length);
9034 		if (num_blocks > lun->be_lun->atomicblock) {
9035 			ctl_set_invalid_field(ctsio, /*sks_valid*/ 1,
9036 			    /*command*/ 1, /*field*/ 12, /*bit_valid*/ 0,
9037 			    /*bit*/ 0);
9038 			ctl_done((union ctl_io *)ctsio);
9039 			return (CTL_RETVAL_COMPLETE);
9040 		}
9041 		break;
9042 	}
9043 	case WRITE_VERIFY_16: {
9044 		struct scsi_write_verify_16 *cdb;
9045 
9046 		cdb = (struct scsi_write_verify_16 *)ctsio->cdb;
9047 		flags |= CTL_LLF_FUA;
9048 		if (cdb->byte2 & SWV_DPO)
9049 			flags |= CTL_LLF_DPO;
9050 		lba = scsi_8btou64(cdb->addr);
9051 		num_blocks = scsi_4btoul(cdb->length);
9052 		break;
9053 	}
9054 	default:
9055 		/*
9056 		 * We got a command we don't support.  This shouldn't
9057 		 * happen, commands should be filtered out above us.
9058 		 */
9059 		ctl_set_invalid_opcode(ctsio);
9060 		ctl_done((union ctl_io *)ctsio);
9061 
9062 		return (CTL_RETVAL_COMPLETE);
9063 		break; /* NOTREACHED */
9064 	}
9065 
9066 	/*
9067 	 * The first check is to make sure we're in bounds, the second
9068 	 * check is to catch wrap-around problems.  If the lba + num blocks
9069 	 * is less than the lba, then we've wrapped around and the block
9070 	 * range is invalid anyway.
9071 	 */
9072 	if (((lba + num_blocks) > (lun->be_lun->maxlba + 1))
9073 	 || ((lba + num_blocks) < lba)) {
9074 		ctl_set_lba_out_of_range(ctsio);
9075 		ctl_done((union ctl_io *)ctsio);
9076 		return (CTL_RETVAL_COMPLETE);
9077 	}
9078 
9079 	/*
9080 	 * According to SBC-3, a transfer length of 0 is not an error.
9081 	 * Note that this cannot happen with WRITE(6) or READ(6), since 0
9082 	 * translates to 256 blocks for those commands.
9083 	 */
9084 	if (num_blocks == 0) {
9085 		ctl_set_success(ctsio);
9086 		ctl_done((union ctl_io *)ctsio);
9087 		return (CTL_RETVAL_COMPLETE);
9088 	}
9089 
9090 	/* Set FUA and/or DPO if caches are disabled. */
9091 	if (isread) {
9092 		if ((lun->mode_pages.caching_page[CTL_PAGE_CURRENT].flags1 &
9093 		    SCP_RCD) != 0)
9094 			flags |= CTL_LLF_FUA | CTL_LLF_DPO;
9095 	} else {
9096 		if ((lun->mode_pages.caching_page[CTL_PAGE_CURRENT].flags1 &
9097 		    SCP_WCE) == 0)
9098 			flags |= CTL_LLF_FUA;
9099 	}
9100 
9101 	lbalen = (struct ctl_lba_len_flags *)
9102 	    &ctsio->io_hdr.ctl_private[CTL_PRIV_LBA_LEN];
9103 	lbalen->lba = lba;
9104 	lbalen->len = num_blocks;
9105 	lbalen->flags = (isread ? CTL_LLF_READ : CTL_LLF_WRITE) | flags;
9106 
9107 	ctsio->kern_total_len = num_blocks * lun->be_lun->blocksize;
9108 	ctsio->kern_rel_offset = 0;
9109 
9110 	CTL_DEBUG_PRINT(("ctl_read_write: calling data_submit()\n"));
9111 
9112 	retval = lun->backend->data_submit((union ctl_io *)ctsio);
9113 
9114 	return (retval);
9115 }
9116 
9117 static int
9118 ctl_cnw_cont(union ctl_io *io)
9119 {
9120 	struct ctl_scsiio *ctsio;
9121 	struct ctl_lun *lun;
9122 	struct ctl_lba_len_flags *lbalen;
9123 	int retval;
9124 
9125 	ctsio = &io->scsiio;
9126 	ctsio->io_hdr.status = CTL_STATUS_NONE;
9127 	ctsio->io_hdr.flags &= ~CTL_FLAG_IO_CONT;
9128 	lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
9129 	lbalen = (struct ctl_lba_len_flags *)
9130 	    &ctsio->io_hdr.ctl_private[CTL_PRIV_LBA_LEN];
9131 	lbalen->flags &= ~CTL_LLF_COMPARE;
9132 	lbalen->flags |= CTL_LLF_WRITE;
9133 
9134 	CTL_DEBUG_PRINT(("ctl_cnw_cont: calling data_submit()\n"));
9135 	retval = lun->backend->data_submit((union ctl_io *)ctsio);
9136 	return (retval);
9137 }
9138 
9139 int
9140 ctl_cnw(struct ctl_scsiio *ctsio)
9141 {
9142 	struct ctl_lun *lun;
9143 	struct ctl_lba_len_flags *lbalen;
9144 	uint64_t lba;
9145 	uint32_t num_blocks;
9146 	int flags, retval;
9147 
9148 	lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
9149 
9150 	CTL_DEBUG_PRINT(("ctl_cnw: command: %#x\n", ctsio->cdb[0]));
9151 
9152 	flags = 0;
9153 	retval = CTL_RETVAL_COMPLETE;
9154 
9155 	switch (ctsio->cdb[0]) {
9156 	case COMPARE_AND_WRITE: {
9157 		struct scsi_compare_and_write *cdb;
9158 
9159 		cdb = (struct scsi_compare_and_write *)ctsio->cdb;
9160 		if (cdb->byte2 & SRW10_FUA)
9161 			flags |= CTL_LLF_FUA;
9162 		if (cdb->byte2 & SRW10_DPO)
9163 			flags |= CTL_LLF_DPO;
9164 		lba = scsi_8btou64(cdb->addr);
9165 		num_blocks = cdb->length;
9166 		break;
9167 	}
9168 	default:
9169 		/*
9170 		 * We got a command we don't support.  This shouldn't
9171 		 * happen, commands should be filtered out above us.
9172 		 */
9173 		ctl_set_invalid_opcode(ctsio);
9174 		ctl_done((union ctl_io *)ctsio);
9175 
9176 		return (CTL_RETVAL_COMPLETE);
9177 		break; /* NOTREACHED */
9178 	}
9179 
9180 	/*
9181 	 * The first check is to make sure we're in bounds, the second
9182 	 * check is to catch wrap-around problems.  If the lba + num blocks
9183 	 * is less than the lba, then we've wrapped around and the block
9184 	 * range is invalid anyway.
9185 	 */
9186 	if (((lba + num_blocks) > (lun->be_lun->maxlba + 1))
9187 	 || ((lba + num_blocks) < lba)) {
9188 		ctl_set_lba_out_of_range(ctsio);
9189 		ctl_done((union ctl_io *)ctsio);
9190 		return (CTL_RETVAL_COMPLETE);
9191 	}
9192 
9193 	/*
9194 	 * According to SBC-3, a transfer length of 0 is not an error.
9195 	 */
9196 	if (num_blocks == 0) {
9197 		ctl_set_success(ctsio);
9198 		ctl_done((union ctl_io *)ctsio);
9199 		return (CTL_RETVAL_COMPLETE);
9200 	}
9201 
9202 	/* Set FUA if write cache is disabled. */
9203 	if ((lun->mode_pages.caching_page[CTL_PAGE_CURRENT].flags1 &
9204 	    SCP_WCE) == 0)
9205 		flags |= CTL_LLF_FUA;
9206 
9207 	ctsio->kern_total_len = 2 * num_blocks * lun->be_lun->blocksize;
9208 	ctsio->kern_rel_offset = 0;
9209 
9210 	/*
9211 	 * Set the IO_CONT flag, so that if this I/O gets passed to
9212 	 * ctl_data_submit_done(), it'll get passed back to
9213 	 * ctl_ctl_cnw_cont() for further processing.
9214 	 */
9215 	ctsio->io_hdr.flags |= CTL_FLAG_IO_CONT;
9216 	ctsio->io_cont = ctl_cnw_cont;
9217 
9218 	lbalen = (struct ctl_lba_len_flags *)
9219 	    &ctsio->io_hdr.ctl_private[CTL_PRIV_LBA_LEN];
9220 	lbalen->lba = lba;
9221 	lbalen->len = num_blocks;
9222 	lbalen->flags = CTL_LLF_COMPARE | flags;
9223 
9224 	CTL_DEBUG_PRINT(("ctl_cnw: calling data_submit()\n"));
9225 	retval = lun->backend->data_submit((union ctl_io *)ctsio);
9226 	return (retval);
9227 }
9228 
9229 int
9230 ctl_verify(struct ctl_scsiio *ctsio)
9231 {
9232 	struct ctl_lun *lun;
9233 	struct ctl_lba_len_flags *lbalen;
9234 	uint64_t lba;
9235 	uint32_t num_blocks;
9236 	int bytchk, flags;
9237 	int retval;
9238 
9239 	lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
9240 
9241 	CTL_DEBUG_PRINT(("ctl_verify: command: %#x\n", ctsio->cdb[0]));
9242 
9243 	bytchk = 0;
9244 	flags = CTL_LLF_FUA;
9245 	retval = CTL_RETVAL_COMPLETE;
9246 
9247 	switch (ctsio->cdb[0]) {
9248 	case VERIFY_10: {
9249 		struct scsi_verify_10 *cdb;
9250 
9251 		cdb = (struct scsi_verify_10 *)ctsio->cdb;
9252 		if (cdb->byte2 & SVFY_BYTCHK)
9253 			bytchk = 1;
9254 		if (cdb->byte2 & SVFY_DPO)
9255 			flags |= CTL_LLF_DPO;
9256 		lba = scsi_4btoul(cdb->addr);
9257 		num_blocks = scsi_2btoul(cdb->length);
9258 		break;
9259 	}
9260 	case VERIFY_12: {
9261 		struct scsi_verify_12 *cdb;
9262 
9263 		cdb = (struct scsi_verify_12 *)ctsio->cdb;
9264 		if (cdb->byte2 & SVFY_BYTCHK)
9265 			bytchk = 1;
9266 		if (cdb->byte2 & SVFY_DPO)
9267 			flags |= CTL_LLF_DPO;
9268 		lba = scsi_4btoul(cdb->addr);
9269 		num_blocks = scsi_4btoul(cdb->length);
9270 		break;
9271 	}
9272 	case VERIFY_16: {
9273 		struct scsi_rw_16 *cdb;
9274 
9275 		cdb = (struct scsi_rw_16 *)ctsio->cdb;
9276 		if (cdb->byte2 & SVFY_BYTCHK)
9277 			bytchk = 1;
9278 		if (cdb->byte2 & SVFY_DPO)
9279 			flags |= CTL_LLF_DPO;
9280 		lba = scsi_8btou64(cdb->addr);
9281 		num_blocks = scsi_4btoul(cdb->length);
9282 		break;
9283 	}
9284 	default:
9285 		/*
9286 		 * We got a command we don't support.  This shouldn't
9287 		 * happen, commands should be filtered out above us.
9288 		 */
9289 		ctl_set_invalid_opcode(ctsio);
9290 		ctl_done((union ctl_io *)ctsio);
9291 		return (CTL_RETVAL_COMPLETE);
9292 	}
9293 
9294 	/*
9295 	 * The first check is to make sure we're in bounds, the second
9296 	 * check is to catch wrap-around problems.  If the lba + num blocks
9297 	 * is less than the lba, then we've wrapped around and the block
9298 	 * range is invalid anyway.
9299 	 */
9300 	if (((lba + num_blocks) > (lun->be_lun->maxlba + 1))
9301 	 || ((lba + num_blocks) < lba)) {
9302 		ctl_set_lba_out_of_range(ctsio);
9303 		ctl_done((union ctl_io *)ctsio);
9304 		return (CTL_RETVAL_COMPLETE);
9305 	}
9306 
9307 	/*
9308 	 * According to SBC-3, a transfer length of 0 is not an error.
9309 	 */
9310 	if (num_blocks == 0) {
9311 		ctl_set_success(ctsio);
9312 		ctl_done((union ctl_io *)ctsio);
9313 		return (CTL_RETVAL_COMPLETE);
9314 	}
9315 
9316 	lbalen = (struct ctl_lba_len_flags *)
9317 	    &ctsio->io_hdr.ctl_private[CTL_PRIV_LBA_LEN];
9318 	lbalen->lba = lba;
9319 	lbalen->len = num_blocks;
9320 	if (bytchk) {
9321 		lbalen->flags = CTL_LLF_COMPARE | flags;
9322 		ctsio->kern_total_len = num_blocks * lun->be_lun->blocksize;
9323 	} else {
9324 		lbalen->flags = CTL_LLF_VERIFY | flags;
9325 		ctsio->kern_total_len = 0;
9326 	}
9327 	ctsio->kern_rel_offset = 0;
9328 
9329 	CTL_DEBUG_PRINT(("ctl_verify: calling data_submit()\n"));
9330 	retval = lun->backend->data_submit((union ctl_io *)ctsio);
9331 	return (retval);
9332 }
9333 
9334 int
9335 ctl_report_luns(struct ctl_scsiio *ctsio)
9336 {
9337 	struct ctl_softc *softc = control_softc;
9338 	struct scsi_report_luns *cdb;
9339 	struct scsi_report_luns_data *lun_data;
9340 	struct ctl_lun *lun, *request_lun;
9341 	struct ctl_port *port;
9342 	int num_luns, retval;
9343 	uint32_t alloc_len, lun_datalen;
9344 	int num_filled, well_known;
9345 	uint32_t initidx, targ_lun_id, lun_id;
9346 
9347 	retval = CTL_RETVAL_COMPLETE;
9348 	well_known = 0;
9349 
9350 	cdb = (struct scsi_report_luns *)ctsio->cdb;
9351 	port = ctl_io_port(&ctsio->io_hdr);
9352 
9353 	CTL_DEBUG_PRINT(("ctl_report_luns\n"));
9354 
9355 	mtx_lock(&softc->ctl_lock);
9356 	num_luns = 0;
9357 	for (targ_lun_id = 0; targ_lun_id < CTL_MAX_LUNS; targ_lun_id++) {
9358 		if (ctl_lun_map_from_port(port, targ_lun_id) < CTL_MAX_LUNS)
9359 			num_luns++;
9360 	}
9361 	mtx_unlock(&softc->ctl_lock);
9362 
9363 	switch (cdb->select_report) {
9364 	case RPL_REPORT_DEFAULT:
9365 	case RPL_REPORT_ALL:
9366 		break;
9367 	case RPL_REPORT_WELLKNOWN:
9368 		well_known = 1;
9369 		num_luns = 0;
9370 		break;
9371 	default:
9372 		ctl_set_invalid_field(ctsio,
9373 				      /*sks_valid*/ 1,
9374 				      /*command*/ 1,
9375 				      /*field*/ 2,
9376 				      /*bit_valid*/ 0,
9377 				      /*bit*/ 0);
9378 		ctl_done((union ctl_io *)ctsio);
9379 		return (retval);
9380 		break; /* NOTREACHED */
9381 	}
9382 
9383 	alloc_len = scsi_4btoul(cdb->length);
9384 	/*
9385 	 * The initiator has to allocate at least 16 bytes for this request,
9386 	 * so he can at least get the header and the first LUN.  Otherwise
9387 	 * we reject the request (per SPC-3 rev 14, section 6.21).
9388 	 */
9389 	if (alloc_len < (sizeof(struct scsi_report_luns_data) +
9390 	    sizeof(struct scsi_report_luns_lundata))) {
9391 		ctl_set_invalid_field(ctsio,
9392 				      /*sks_valid*/ 1,
9393 				      /*command*/ 1,
9394 				      /*field*/ 6,
9395 				      /*bit_valid*/ 0,
9396 				      /*bit*/ 0);
9397 		ctl_done((union ctl_io *)ctsio);
9398 		return (retval);
9399 	}
9400 
9401 	request_lun = (struct ctl_lun *)
9402 		ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
9403 
9404 	lun_datalen = sizeof(*lun_data) +
9405 		(num_luns * sizeof(struct scsi_report_luns_lundata));
9406 
9407 	ctsio->kern_data_ptr = malloc(lun_datalen, M_CTL, M_WAITOK | M_ZERO);
9408 	lun_data = (struct scsi_report_luns_data *)ctsio->kern_data_ptr;
9409 	ctsio->kern_sg_entries = 0;
9410 
9411 	initidx = ctl_get_initindex(&ctsio->io_hdr.nexus);
9412 
9413 	mtx_lock(&softc->ctl_lock);
9414 	for (targ_lun_id = 0, num_filled = 0; targ_lun_id < CTL_MAX_LUNS && num_filled < num_luns; targ_lun_id++) {
9415 		lun_id = ctl_lun_map_from_port(port, targ_lun_id);
9416 		if (lun_id >= CTL_MAX_LUNS)
9417 			continue;
9418 		lun = softc->ctl_luns[lun_id];
9419 		if (lun == NULL)
9420 			continue;
9421 
9422 		if (targ_lun_id <= 0xff) {
9423 			/*
9424 			 * Peripheral addressing method, bus number 0.
9425 			 */
9426 			lun_data->luns[num_filled].lundata[0] =
9427 				RPL_LUNDATA_ATYP_PERIPH;
9428 			lun_data->luns[num_filled].lundata[1] = targ_lun_id;
9429 			num_filled++;
9430 		} else if (targ_lun_id <= 0x3fff) {
9431 			/*
9432 			 * Flat addressing method.
9433 			 */
9434 			lun_data->luns[num_filled].lundata[0] =
9435 				RPL_LUNDATA_ATYP_FLAT | (targ_lun_id >> 8);
9436 			lun_data->luns[num_filled].lundata[1] =
9437 				(targ_lun_id & 0xff);
9438 			num_filled++;
9439 		} else if (targ_lun_id <= 0xffffff) {
9440 			/*
9441 			 * Extended flat addressing method.
9442 			 */
9443 			lun_data->luns[num_filled].lundata[0] =
9444 			    RPL_LUNDATA_ATYP_EXTLUN | 0x12;
9445 			scsi_ulto3b(targ_lun_id,
9446 			    &lun_data->luns[num_filled].lundata[1]);
9447 			num_filled++;
9448 		} else {
9449 			printf("ctl_report_luns: bogus LUN number %jd, "
9450 			       "skipping\n", (intmax_t)targ_lun_id);
9451 		}
9452 		/*
9453 		 * According to SPC-3, rev 14 section 6.21:
9454 		 *
9455 		 * "The execution of a REPORT LUNS command to any valid and
9456 		 * installed logical unit shall clear the REPORTED LUNS DATA
9457 		 * HAS CHANGED unit attention condition for all logical
9458 		 * units of that target with respect to the requesting
9459 		 * initiator. A valid and installed logical unit is one
9460 		 * having a PERIPHERAL QUALIFIER of 000b in the standard
9461 		 * INQUIRY data (see 6.4.2)."
9462 		 *
9463 		 * If request_lun is NULL, the LUN this report luns command
9464 		 * was issued to is either disabled or doesn't exist. In that
9465 		 * case, we shouldn't clear any pending lun change unit
9466 		 * attention.
9467 		 */
9468 		if (request_lun != NULL) {
9469 			mtx_lock(&lun->lun_lock);
9470 			ctl_clr_ua(lun, initidx, CTL_UA_RES_RELEASE);
9471 			mtx_unlock(&lun->lun_lock);
9472 		}
9473 	}
9474 	mtx_unlock(&softc->ctl_lock);
9475 
9476 	/*
9477 	 * It's quite possible that we've returned fewer LUNs than we allocated
9478 	 * space for.  Trim it.
9479 	 */
9480 	lun_datalen = sizeof(*lun_data) +
9481 		(num_filled * sizeof(struct scsi_report_luns_lundata));
9482 
9483 	if (lun_datalen < alloc_len) {
9484 		ctsio->residual = alloc_len - lun_datalen;
9485 		ctsio->kern_data_len = lun_datalen;
9486 		ctsio->kern_total_len = lun_datalen;
9487 	} else {
9488 		ctsio->residual = 0;
9489 		ctsio->kern_data_len = alloc_len;
9490 		ctsio->kern_total_len = alloc_len;
9491 	}
9492 	ctsio->kern_data_resid = 0;
9493 	ctsio->kern_rel_offset = 0;
9494 	ctsio->kern_sg_entries = 0;
9495 
9496 	/*
9497 	 * We set this to the actual data length, regardless of how much
9498 	 * space we actually have to return results.  If the user looks at
9499 	 * this value, he'll know whether or not he allocated enough space
9500 	 * and reissue the command if necessary.  We don't support well
9501 	 * known logical units, so if the user asks for that, return none.
9502 	 */
9503 	scsi_ulto4b(lun_datalen - 8, lun_data->length);
9504 
9505 	/*
9506 	 * We can only return SCSI_STATUS_CHECK_COND when we can't satisfy
9507 	 * this request.
9508 	 */
9509 	ctl_set_success(ctsio);
9510 	ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
9511 	ctsio->be_move_done = ctl_config_move_done;
9512 	ctl_datamove((union ctl_io *)ctsio);
9513 	return (retval);
9514 }
9515 
9516 int
9517 ctl_request_sense(struct ctl_scsiio *ctsio)
9518 {
9519 	struct scsi_request_sense *cdb;
9520 	struct scsi_sense_data *sense_ptr;
9521 	struct ctl_softc *ctl_softc;
9522 	struct ctl_lun *lun;
9523 	uint32_t initidx;
9524 	int have_error;
9525 	scsi_sense_data_type sense_format;
9526 	ctl_ua_type ua_type;
9527 
9528 	cdb = (struct scsi_request_sense *)ctsio->cdb;
9529 
9530 	ctl_softc = control_softc;
9531 	lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
9532 
9533 	CTL_DEBUG_PRINT(("ctl_request_sense\n"));
9534 
9535 	/*
9536 	 * Determine which sense format the user wants.
9537 	 */
9538 	if (cdb->byte2 & SRS_DESC)
9539 		sense_format = SSD_TYPE_DESC;
9540 	else
9541 		sense_format = SSD_TYPE_FIXED;
9542 
9543 	ctsio->kern_data_ptr = malloc(sizeof(*sense_ptr), M_CTL, M_WAITOK);
9544 	sense_ptr = (struct scsi_sense_data *)ctsio->kern_data_ptr;
9545 	ctsio->kern_sg_entries = 0;
9546 
9547 	/*
9548 	 * struct scsi_sense_data, which is currently set to 256 bytes, is
9549 	 * larger than the largest allowed value for the length field in the
9550 	 * REQUEST SENSE CDB, which is 252 bytes as of SPC-4.
9551 	 */
9552 	ctsio->residual = 0;
9553 	ctsio->kern_data_len = cdb->length;
9554 	ctsio->kern_total_len = cdb->length;
9555 
9556 	ctsio->kern_data_resid = 0;
9557 	ctsio->kern_rel_offset = 0;
9558 	ctsio->kern_sg_entries = 0;
9559 
9560 	/*
9561 	 * If we don't have a LUN, we don't have any pending sense.
9562 	 */
9563 	if (lun == NULL)
9564 		goto no_sense;
9565 
9566 	have_error = 0;
9567 	initidx = ctl_get_initindex(&ctsio->io_hdr.nexus);
9568 	/*
9569 	 * Check for pending sense, and then for pending unit attentions.
9570 	 * Pending sense gets returned first, then pending unit attentions.
9571 	 */
9572 	mtx_lock(&lun->lun_lock);
9573 #ifdef CTL_WITH_CA
9574 	if (ctl_is_set(lun->have_ca, initidx)) {
9575 		scsi_sense_data_type stored_format;
9576 
9577 		/*
9578 		 * Check to see which sense format was used for the stored
9579 		 * sense data.
9580 		 */
9581 		stored_format = scsi_sense_type(&lun->pending_sense[initidx]);
9582 
9583 		/*
9584 		 * If the user requested a different sense format than the
9585 		 * one we stored, then we need to convert it to the other
9586 		 * format.  If we're going from descriptor to fixed format
9587 		 * sense data, we may lose things in translation, depending
9588 		 * on what options were used.
9589 		 *
9590 		 * If the stored format is SSD_TYPE_NONE (i.e. invalid),
9591 		 * for some reason we'll just copy it out as-is.
9592 		 */
9593 		if ((stored_format == SSD_TYPE_FIXED)
9594 		 && (sense_format == SSD_TYPE_DESC))
9595 			ctl_sense_to_desc((struct scsi_sense_data_fixed *)
9596 			    &lun->pending_sense[initidx],
9597 			    (struct scsi_sense_data_desc *)sense_ptr);
9598 		else if ((stored_format == SSD_TYPE_DESC)
9599 		      && (sense_format == SSD_TYPE_FIXED))
9600 			ctl_sense_to_fixed((struct scsi_sense_data_desc *)
9601 			    &lun->pending_sense[initidx],
9602 			    (struct scsi_sense_data_fixed *)sense_ptr);
9603 		else
9604 			memcpy(sense_ptr, &lun->pending_sense[initidx],
9605 			       MIN(sizeof(*sense_ptr),
9606 			       sizeof(lun->pending_sense[initidx])));
9607 
9608 		ctl_clear_mask(lun->have_ca, initidx);
9609 		have_error = 1;
9610 	} else
9611 #endif
9612 	{
9613 		ua_type = ctl_build_ua(lun, initidx, sense_ptr, sense_format);
9614 		if (ua_type != CTL_UA_NONE)
9615 			have_error = 1;
9616 		if (ua_type == CTL_UA_LUN_CHANGE) {
9617 			mtx_unlock(&lun->lun_lock);
9618 			mtx_lock(&ctl_softc->ctl_lock);
9619 			ctl_clear_ua(ctl_softc, initidx, ua_type);
9620 			mtx_unlock(&ctl_softc->ctl_lock);
9621 			mtx_lock(&lun->lun_lock);
9622 		}
9623 
9624 	}
9625 	mtx_unlock(&lun->lun_lock);
9626 
9627 	/*
9628 	 * We already have a pending error, return it.
9629 	 */
9630 	if (have_error != 0) {
9631 		/*
9632 		 * We report the SCSI status as OK, since the status of the
9633 		 * request sense command itself is OK.
9634 		 * We report 0 for the sense length, because we aren't doing
9635 		 * autosense in this case.  We're reporting sense as
9636 		 * parameter data.
9637 		 */
9638 		ctl_set_success(ctsio);
9639 		ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
9640 		ctsio->be_move_done = ctl_config_move_done;
9641 		ctl_datamove((union ctl_io *)ctsio);
9642 		return (CTL_RETVAL_COMPLETE);
9643 	}
9644 
9645 no_sense:
9646 
9647 	/*
9648 	 * No sense information to report, so we report that everything is
9649 	 * okay.
9650 	 */
9651 	ctl_set_sense_data(sense_ptr,
9652 			   lun,
9653 			   sense_format,
9654 			   /*current_error*/ 1,
9655 			   /*sense_key*/ SSD_KEY_NO_SENSE,
9656 			   /*asc*/ 0x00,
9657 			   /*ascq*/ 0x00,
9658 			   SSD_ELEM_NONE);
9659 
9660 	/*
9661 	 * We report 0 for the sense length, because we aren't doing
9662 	 * autosense in this case.  We're reporting sense as parameter data.
9663 	 */
9664 	ctl_set_success(ctsio);
9665 	ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
9666 	ctsio->be_move_done = ctl_config_move_done;
9667 	ctl_datamove((union ctl_io *)ctsio);
9668 	return (CTL_RETVAL_COMPLETE);
9669 }
9670 
9671 int
9672 ctl_tur(struct ctl_scsiio *ctsio)
9673 {
9674 
9675 	CTL_DEBUG_PRINT(("ctl_tur\n"));
9676 
9677 	ctl_set_success(ctsio);
9678 	ctl_done((union ctl_io *)ctsio);
9679 
9680 	return (CTL_RETVAL_COMPLETE);
9681 }
9682 
9683 #ifdef notyet
9684 static int
9685 ctl_cmddt_inquiry(struct ctl_scsiio *ctsio)
9686 {
9687 
9688 }
9689 #endif
9690 
9691 /*
9692  * SCSI VPD page 0x00, the Supported VPD Pages page.
9693  */
9694 static int
9695 ctl_inquiry_evpd_supported(struct ctl_scsiio *ctsio, int alloc_len)
9696 {
9697 	struct scsi_vpd_supported_pages *pages;
9698 	int sup_page_size;
9699 	struct ctl_lun *lun;
9700 	int p;
9701 
9702 	lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
9703 
9704 	sup_page_size = sizeof(struct scsi_vpd_supported_pages) *
9705 	    SCSI_EVPD_NUM_SUPPORTED_PAGES;
9706 	ctsio->kern_data_ptr = malloc(sup_page_size, M_CTL, M_WAITOK | M_ZERO);
9707 	pages = (struct scsi_vpd_supported_pages *)ctsio->kern_data_ptr;
9708 	ctsio->kern_sg_entries = 0;
9709 
9710 	if (sup_page_size < alloc_len) {
9711 		ctsio->residual = alloc_len - sup_page_size;
9712 		ctsio->kern_data_len = sup_page_size;
9713 		ctsio->kern_total_len = sup_page_size;
9714 	} else {
9715 		ctsio->residual = 0;
9716 		ctsio->kern_data_len = alloc_len;
9717 		ctsio->kern_total_len = alloc_len;
9718 	}
9719 	ctsio->kern_data_resid = 0;
9720 	ctsio->kern_rel_offset = 0;
9721 	ctsio->kern_sg_entries = 0;
9722 
9723 	/*
9724 	 * The control device is always connected.  The disk device, on the
9725 	 * other hand, may not be online all the time.  Need to change this
9726 	 * to figure out whether the disk device is actually online or not.
9727 	 */
9728 	if (lun != NULL)
9729 		pages->device = (SID_QUAL_LU_CONNECTED << 5) |
9730 				lun->be_lun->lun_type;
9731 	else
9732 		pages->device = (SID_QUAL_LU_OFFLINE << 5) | T_DIRECT;
9733 
9734 	p = 0;
9735 	/* Supported VPD pages */
9736 	pages->page_list[p++] = SVPD_SUPPORTED_PAGES;
9737 	/* Serial Number */
9738 	pages->page_list[p++] = SVPD_UNIT_SERIAL_NUMBER;
9739 	/* Device Identification */
9740 	pages->page_list[p++] = SVPD_DEVICE_ID;
9741 	/* Extended INQUIRY Data */
9742 	pages->page_list[p++] = SVPD_EXTENDED_INQUIRY_DATA;
9743 	/* Mode Page Policy */
9744 	pages->page_list[p++] = SVPD_MODE_PAGE_POLICY;
9745 	/* SCSI Ports */
9746 	pages->page_list[p++] = SVPD_SCSI_PORTS;
9747 	/* Third-party Copy */
9748 	pages->page_list[p++] = SVPD_SCSI_TPC;
9749 	if (lun != NULL && lun->be_lun->lun_type == T_DIRECT) {
9750 		/* Block limits */
9751 		pages->page_list[p++] = SVPD_BLOCK_LIMITS;
9752 		/* Block Device Characteristics */
9753 		pages->page_list[p++] = SVPD_BDC;
9754 		/* Logical Block Provisioning */
9755 		pages->page_list[p++] = SVPD_LBP;
9756 	}
9757 	pages->length = p;
9758 
9759 	ctl_set_success(ctsio);
9760 	ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
9761 	ctsio->be_move_done = ctl_config_move_done;
9762 	ctl_datamove((union ctl_io *)ctsio);
9763 	return (CTL_RETVAL_COMPLETE);
9764 }
9765 
9766 /*
9767  * SCSI VPD page 0x80, the Unit Serial Number page.
9768  */
9769 static int
9770 ctl_inquiry_evpd_serial(struct ctl_scsiio *ctsio, int alloc_len)
9771 {
9772 	struct scsi_vpd_unit_serial_number *sn_ptr;
9773 	struct ctl_lun *lun;
9774 	int data_len;
9775 
9776 	lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
9777 
9778 	data_len = 4 + CTL_SN_LEN;
9779 	ctsio->kern_data_ptr = malloc(data_len, M_CTL, M_WAITOK | M_ZERO);
9780 	sn_ptr = (struct scsi_vpd_unit_serial_number *)ctsio->kern_data_ptr;
9781 	if (data_len < alloc_len) {
9782 		ctsio->residual = alloc_len - data_len;
9783 		ctsio->kern_data_len = data_len;
9784 		ctsio->kern_total_len = data_len;
9785 	} else {
9786 		ctsio->residual = 0;
9787 		ctsio->kern_data_len = alloc_len;
9788 		ctsio->kern_total_len = alloc_len;
9789 	}
9790 	ctsio->kern_data_resid = 0;
9791 	ctsio->kern_rel_offset = 0;
9792 	ctsio->kern_sg_entries = 0;
9793 
9794 	/*
9795 	 * The control device is always connected.  The disk device, on the
9796 	 * other hand, may not be online all the time.  Need to change this
9797 	 * to figure out whether the disk device is actually online or not.
9798 	 */
9799 	if (lun != NULL)
9800 		sn_ptr->device = (SID_QUAL_LU_CONNECTED << 5) |
9801 				  lun->be_lun->lun_type;
9802 	else
9803 		sn_ptr->device = (SID_QUAL_LU_OFFLINE << 5) | T_DIRECT;
9804 
9805 	sn_ptr->page_code = SVPD_UNIT_SERIAL_NUMBER;
9806 	sn_ptr->length = CTL_SN_LEN;
9807 	/*
9808 	 * If we don't have a LUN, we just leave the serial number as
9809 	 * all spaces.
9810 	 */
9811 	if (lun != NULL) {
9812 		strncpy((char *)sn_ptr->serial_num,
9813 			(char *)lun->be_lun->serial_num, CTL_SN_LEN);
9814 	} else
9815 		memset(sn_ptr->serial_num, 0x20, CTL_SN_LEN);
9816 
9817 	ctl_set_success(ctsio);
9818 	ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
9819 	ctsio->be_move_done = ctl_config_move_done;
9820 	ctl_datamove((union ctl_io *)ctsio);
9821 	return (CTL_RETVAL_COMPLETE);
9822 }
9823 
9824 
9825 /*
9826  * SCSI VPD page 0x86, the Extended INQUIRY Data page.
9827  */
9828 static int
9829 ctl_inquiry_evpd_eid(struct ctl_scsiio *ctsio, int alloc_len)
9830 {
9831 	struct scsi_vpd_extended_inquiry_data *eid_ptr;
9832 	struct ctl_lun *lun;
9833 	int data_len;
9834 
9835 	lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
9836 
9837 	data_len = sizeof(struct scsi_vpd_extended_inquiry_data);
9838 	ctsio->kern_data_ptr = malloc(data_len, M_CTL, M_WAITOK | M_ZERO);
9839 	eid_ptr = (struct scsi_vpd_extended_inquiry_data *)ctsio->kern_data_ptr;
9840 	ctsio->kern_sg_entries = 0;
9841 
9842 	if (data_len < alloc_len) {
9843 		ctsio->residual = alloc_len - data_len;
9844 		ctsio->kern_data_len = data_len;
9845 		ctsio->kern_total_len = data_len;
9846 	} else {
9847 		ctsio->residual = 0;
9848 		ctsio->kern_data_len = alloc_len;
9849 		ctsio->kern_total_len = alloc_len;
9850 	}
9851 	ctsio->kern_data_resid = 0;
9852 	ctsio->kern_rel_offset = 0;
9853 	ctsio->kern_sg_entries = 0;
9854 
9855 	/*
9856 	 * The control device is always connected.  The disk device, on the
9857 	 * other hand, may not be online all the time.
9858 	 */
9859 	if (lun != NULL)
9860 		eid_ptr->device = (SID_QUAL_LU_CONNECTED << 5) |
9861 				     lun->be_lun->lun_type;
9862 	else
9863 		eid_ptr->device = (SID_QUAL_LU_OFFLINE << 5) | T_DIRECT;
9864 	eid_ptr->page_code = SVPD_EXTENDED_INQUIRY_DATA;
9865 	scsi_ulto2b(data_len - 4, eid_ptr->page_length);
9866 	/*
9867 	 * We support head of queue, ordered and simple tags.
9868 	 */
9869 	eid_ptr->flags2 = SVPD_EID_HEADSUP | SVPD_EID_ORDSUP | SVPD_EID_SIMPSUP;
9870 	/*
9871 	 * Volatile cache supported.
9872 	 */
9873 	eid_ptr->flags3 = SVPD_EID_V_SUP;
9874 
9875 	/*
9876 	 * This means that we clear the REPORTED LUNS DATA HAS CHANGED unit
9877 	 * attention for a particular IT nexus on all LUNs once we report
9878 	 * it to that nexus once.  This bit is required as of SPC-4.
9879 	 */
9880 	eid_ptr->flags4 = SVPD_EID_LUICLT;
9881 
9882 	/*
9883 	 * XXX KDM in order to correctly answer this, we would need
9884 	 * information from the SIM to determine how much sense data it
9885 	 * can send.  So this would really be a path inquiry field, most
9886 	 * likely.  This can be set to a maximum of 252 according to SPC-4,
9887 	 * but the hardware may or may not be able to support that much.
9888 	 * 0 just means that the maximum sense data length is not reported.
9889 	 */
9890 	eid_ptr->max_sense_length = 0;
9891 
9892 	ctl_set_success(ctsio);
9893 	ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
9894 	ctsio->be_move_done = ctl_config_move_done;
9895 	ctl_datamove((union ctl_io *)ctsio);
9896 	return (CTL_RETVAL_COMPLETE);
9897 }
9898 
9899 static int
9900 ctl_inquiry_evpd_mpp(struct ctl_scsiio *ctsio, int alloc_len)
9901 {
9902 	struct scsi_vpd_mode_page_policy *mpp_ptr;
9903 	struct ctl_lun *lun;
9904 	int data_len;
9905 
9906 	lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
9907 
9908 	data_len = sizeof(struct scsi_vpd_mode_page_policy) +
9909 	    sizeof(struct scsi_vpd_mode_page_policy_descr);
9910 
9911 	ctsio->kern_data_ptr = malloc(data_len, M_CTL, M_WAITOK | M_ZERO);
9912 	mpp_ptr = (struct scsi_vpd_mode_page_policy *)ctsio->kern_data_ptr;
9913 	ctsio->kern_sg_entries = 0;
9914 
9915 	if (data_len < alloc_len) {
9916 		ctsio->residual = alloc_len - data_len;
9917 		ctsio->kern_data_len = data_len;
9918 		ctsio->kern_total_len = data_len;
9919 	} else {
9920 		ctsio->residual = 0;
9921 		ctsio->kern_data_len = alloc_len;
9922 		ctsio->kern_total_len = alloc_len;
9923 	}
9924 	ctsio->kern_data_resid = 0;
9925 	ctsio->kern_rel_offset = 0;
9926 	ctsio->kern_sg_entries = 0;
9927 
9928 	/*
9929 	 * The control device is always connected.  The disk device, on the
9930 	 * other hand, may not be online all the time.
9931 	 */
9932 	if (lun != NULL)
9933 		mpp_ptr->device = (SID_QUAL_LU_CONNECTED << 5) |
9934 				     lun->be_lun->lun_type;
9935 	else
9936 		mpp_ptr->device = (SID_QUAL_LU_OFFLINE << 5) | T_DIRECT;
9937 	mpp_ptr->page_code = SVPD_MODE_PAGE_POLICY;
9938 	scsi_ulto2b(data_len - 4, mpp_ptr->page_length);
9939 	mpp_ptr->descr[0].page_code = 0x3f;
9940 	mpp_ptr->descr[0].subpage_code = 0xff;
9941 	mpp_ptr->descr[0].policy = SVPD_MPP_SHARED;
9942 
9943 	ctl_set_success(ctsio);
9944 	ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
9945 	ctsio->be_move_done = ctl_config_move_done;
9946 	ctl_datamove((union ctl_io *)ctsio);
9947 	return (CTL_RETVAL_COMPLETE);
9948 }
9949 
9950 /*
9951  * SCSI VPD page 0x83, the Device Identification page.
9952  */
9953 static int
9954 ctl_inquiry_evpd_devid(struct ctl_scsiio *ctsio, int alloc_len)
9955 {
9956 	struct scsi_vpd_device_id *devid_ptr;
9957 	struct scsi_vpd_id_descriptor *desc;
9958 	struct ctl_softc *softc;
9959 	struct ctl_lun *lun;
9960 	struct ctl_port *port;
9961 	int data_len;
9962 	uint8_t proto;
9963 
9964 	softc = control_softc;
9965 
9966 	port = softc->ctl_ports[ctl_port_idx(ctsio->io_hdr.nexus.targ_port)];
9967 	lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
9968 
9969 	data_len = sizeof(struct scsi_vpd_device_id) +
9970 	    sizeof(struct scsi_vpd_id_descriptor) +
9971 		sizeof(struct scsi_vpd_id_rel_trgt_port_id) +
9972 	    sizeof(struct scsi_vpd_id_descriptor) +
9973 		sizeof(struct scsi_vpd_id_trgt_port_grp_id);
9974 	if (lun && lun->lun_devid)
9975 		data_len += lun->lun_devid->len;
9976 	if (port->port_devid)
9977 		data_len += port->port_devid->len;
9978 	if (port->target_devid)
9979 		data_len += port->target_devid->len;
9980 
9981 	ctsio->kern_data_ptr = malloc(data_len, M_CTL, M_WAITOK | M_ZERO);
9982 	devid_ptr = (struct scsi_vpd_device_id *)ctsio->kern_data_ptr;
9983 	ctsio->kern_sg_entries = 0;
9984 
9985 	if (data_len < alloc_len) {
9986 		ctsio->residual = alloc_len - data_len;
9987 		ctsio->kern_data_len = data_len;
9988 		ctsio->kern_total_len = data_len;
9989 	} else {
9990 		ctsio->residual = 0;
9991 		ctsio->kern_data_len = alloc_len;
9992 		ctsio->kern_total_len = alloc_len;
9993 	}
9994 	ctsio->kern_data_resid = 0;
9995 	ctsio->kern_rel_offset = 0;
9996 	ctsio->kern_sg_entries = 0;
9997 
9998 	/*
9999 	 * The control device is always connected.  The disk device, on the
10000 	 * other hand, may not be online all the time.
10001 	 */
10002 	if (lun != NULL)
10003 		devid_ptr->device = (SID_QUAL_LU_CONNECTED << 5) |
10004 				     lun->be_lun->lun_type;
10005 	else
10006 		devid_ptr->device = (SID_QUAL_LU_OFFLINE << 5) | T_DIRECT;
10007 	devid_ptr->page_code = SVPD_DEVICE_ID;
10008 	scsi_ulto2b(data_len - 4, devid_ptr->length);
10009 
10010 	if (port->port_type == CTL_PORT_FC)
10011 		proto = SCSI_PROTO_FC << 4;
10012 	else if (port->port_type == CTL_PORT_ISCSI)
10013 		proto = SCSI_PROTO_ISCSI << 4;
10014 	else
10015 		proto = SCSI_PROTO_SPI << 4;
10016 	desc = (struct scsi_vpd_id_descriptor *)devid_ptr->desc_list;
10017 
10018 	/*
10019 	 * We're using a LUN association here.  i.e., this device ID is a
10020 	 * per-LUN identifier.
10021 	 */
10022 	if (lun && lun->lun_devid) {
10023 		memcpy(desc, lun->lun_devid->data, lun->lun_devid->len);
10024 		desc = (struct scsi_vpd_id_descriptor *)((uint8_t *)desc +
10025 		    lun->lun_devid->len);
10026 	}
10027 
10028 	/*
10029 	 * This is for the WWPN which is a port association.
10030 	 */
10031 	if (port->port_devid) {
10032 		memcpy(desc, port->port_devid->data, port->port_devid->len);
10033 		desc = (struct scsi_vpd_id_descriptor *)((uint8_t *)desc +
10034 		    port->port_devid->len);
10035 	}
10036 
10037 	/*
10038 	 * This is for the Relative Target Port(type 4h) identifier
10039 	 */
10040 	desc->proto_codeset = proto | SVPD_ID_CODESET_BINARY;
10041 	desc->id_type = SVPD_ID_PIV | SVPD_ID_ASSOC_PORT |
10042 	    SVPD_ID_TYPE_RELTARG;
10043 	desc->length = 4;
10044 	scsi_ulto2b(ctsio->io_hdr.nexus.targ_port, &desc->identifier[2]);
10045 	desc = (struct scsi_vpd_id_descriptor *)(&desc->identifier[0] +
10046 	    sizeof(struct scsi_vpd_id_rel_trgt_port_id));
10047 
10048 	/*
10049 	 * This is for the Target Port Group(type 5h) identifier
10050 	 */
10051 	desc->proto_codeset = proto | SVPD_ID_CODESET_BINARY;
10052 	desc->id_type = SVPD_ID_PIV | SVPD_ID_ASSOC_PORT |
10053 	    SVPD_ID_TYPE_TPORTGRP;
10054 	desc->length = 4;
10055 	scsi_ulto2b(ctsio->io_hdr.nexus.targ_port / CTL_MAX_PORTS + 1,
10056 	    &desc->identifier[2]);
10057 	desc = (struct scsi_vpd_id_descriptor *)(&desc->identifier[0] +
10058 	    sizeof(struct scsi_vpd_id_trgt_port_grp_id));
10059 
10060 	/*
10061 	 * This is for the Target identifier
10062 	 */
10063 	if (port->target_devid) {
10064 		memcpy(desc, port->target_devid->data, port->target_devid->len);
10065 	}
10066 
10067 	ctl_set_success(ctsio);
10068 	ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
10069 	ctsio->be_move_done = ctl_config_move_done;
10070 	ctl_datamove((union ctl_io *)ctsio);
10071 	return (CTL_RETVAL_COMPLETE);
10072 }
10073 
10074 static int
10075 ctl_inquiry_evpd_scsi_ports(struct ctl_scsiio *ctsio, int alloc_len)
10076 {
10077 	struct ctl_softc *softc = control_softc;
10078 	struct scsi_vpd_scsi_ports *sp;
10079 	struct scsi_vpd_port_designation *pd;
10080 	struct scsi_vpd_port_designation_cont *pdc;
10081 	struct ctl_lun *lun;
10082 	struct ctl_port *port;
10083 	int data_len, num_target_ports, iid_len, id_len, g, pg, p;
10084 	int num_target_port_groups;
10085 
10086 	lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
10087 
10088 	if (softc->is_single)
10089 		num_target_port_groups = 1;
10090 	else
10091 		num_target_port_groups = NUM_TARGET_PORT_GROUPS;
10092 	num_target_ports = 0;
10093 	iid_len = 0;
10094 	id_len = 0;
10095 	mtx_lock(&softc->ctl_lock);
10096 	STAILQ_FOREACH(port, &softc->port_list, links) {
10097 		if ((port->status & CTL_PORT_STATUS_ONLINE) == 0)
10098 			continue;
10099 		if (lun != NULL &&
10100 		    ctl_lun_map_to_port(port, lun->lun) >= CTL_MAX_LUNS)
10101 			continue;
10102 		num_target_ports++;
10103 		if (port->init_devid)
10104 			iid_len += port->init_devid->len;
10105 		if (port->port_devid)
10106 			id_len += port->port_devid->len;
10107 	}
10108 	mtx_unlock(&softc->ctl_lock);
10109 
10110 	data_len = sizeof(struct scsi_vpd_scsi_ports) + num_target_port_groups *
10111 	    num_target_ports * (sizeof(struct scsi_vpd_port_designation) +
10112 	     sizeof(struct scsi_vpd_port_designation_cont)) + iid_len + id_len;
10113 	ctsio->kern_data_ptr = malloc(data_len, M_CTL, M_WAITOK | M_ZERO);
10114 	sp = (struct scsi_vpd_scsi_ports *)ctsio->kern_data_ptr;
10115 	ctsio->kern_sg_entries = 0;
10116 
10117 	if (data_len < alloc_len) {
10118 		ctsio->residual = alloc_len - data_len;
10119 		ctsio->kern_data_len = data_len;
10120 		ctsio->kern_total_len = data_len;
10121 	} else {
10122 		ctsio->residual = 0;
10123 		ctsio->kern_data_len = alloc_len;
10124 		ctsio->kern_total_len = alloc_len;
10125 	}
10126 	ctsio->kern_data_resid = 0;
10127 	ctsio->kern_rel_offset = 0;
10128 	ctsio->kern_sg_entries = 0;
10129 
10130 	/*
10131 	 * The control device is always connected.  The disk device, on the
10132 	 * other hand, may not be online all the time.  Need to change this
10133 	 * to figure out whether the disk device is actually online or not.
10134 	 */
10135 	if (lun != NULL)
10136 		sp->device = (SID_QUAL_LU_CONNECTED << 5) |
10137 				  lun->be_lun->lun_type;
10138 	else
10139 		sp->device = (SID_QUAL_LU_OFFLINE << 5) | T_DIRECT;
10140 
10141 	sp->page_code = SVPD_SCSI_PORTS;
10142 	scsi_ulto2b(data_len - sizeof(struct scsi_vpd_scsi_ports),
10143 	    sp->page_length);
10144 	pd = &sp->design[0];
10145 
10146 	mtx_lock(&softc->ctl_lock);
10147 	pg = softc->port_offset / CTL_MAX_PORTS;
10148 	for (g = 0; g < num_target_port_groups; g++) {
10149 		STAILQ_FOREACH(port, &softc->port_list, links) {
10150 			if ((port->status & CTL_PORT_STATUS_ONLINE) == 0)
10151 				continue;
10152 			if (lun != NULL &&
10153 			    ctl_lun_map_to_port(port, lun->lun) >= CTL_MAX_LUNS)
10154 				continue;
10155 			p = port->targ_port % CTL_MAX_PORTS + g * CTL_MAX_PORTS;
10156 			scsi_ulto2b(p, pd->relative_port_id);
10157 			if (port->init_devid && g == pg) {
10158 				iid_len = port->init_devid->len;
10159 				memcpy(pd->initiator_transportid,
10160 				    port->init_devid->data, port->init_devid->len);
10161 			} else
10162 				iid_len = 0;
10163 			scsi_ulto2b(iid_len, pd->initiator_transportid_length);
10164 			pdc = (struct scsi_vpd_port_designation_cont *)
10165 			    (&pd->initiator_transportid[iid_len]);
10166 			if (port->port_devid && g == pg) {
10167 				id_len = port->port_devid->len;
10168 				memcpy(pdc->target_port_descriptors,
10169 				    port->port_devid->data, port->port_devid->len);
10170 			} else
10171 				id_len = 0;
10172 			scsi_ulto2b(id_len, pdc->target_port_descriptors_length);
10173 			pd = (struct scsi_vpd_port_designation *)
10174 			    ((uint8_t *)pdc->target_port_descriptors + id_len);
10175 		}
10176 	}
10177 	mtx_unlock(&softc->ctl_lock);
10178 
10179 	ctl_set_success(ctsio);
10180 	ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
10181 	ctsio->be_move_done = ctl_config_move_done;
10182 	ctl_datamove((union ctl_io *)ctsio);
10183 	return (CTL_RETVAL_COMPLETE);
10184 }
10185 
10186 static int
10187 ctl_inquiry_evpd_block_limits(struct ctl_scsiio *ctsio, int alloc_len)
10188 {
10189 	struct scsi_vpd_block_limits *bl_ptr;
10190 	struct ctl_lun *lun;
10191 	int bs;
10192 
10193 	lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
10194 
10195 	ctsio->kern_data_ptr = malloc(sizeof(*bl_ptr), M_CTL, M_WAITOK | M_ZERO);
10196 	bl_ptr = (struct scsi_vpd_block_limits *)ctsio->kern_data_ptr;
10197 	ctsio->kern_sg_entries = 0;
10198 
10199 	if (sizeof(*bl_ptr) < alloc_len) {
10200 		ctsio->residual = alloc_len - sizeof(*bl_ptr);
10201 		ctsio->kern_data_len = sizeof(*bl_ptr);
10202 		ctsio->kern_total_len = sizeof(*bl_ptr);
10203 	} else {
10204 		ctsio->residual = 0;
10205 		ctsio->kern_data_len = alloc_len;
10206 		ctsio->kern_total_len = alloc_len;
10207 	}
10208 	ctsio->kern_data_resid = 0;
10209 	ctsio->kern_rel_offset = 0;
10210 	ctsio->kern_sg_entries = 0;
10211 
10212 	/*
10213 	 * The control device is always connected.  The disk device, on the
10214 	 * other hand, may not be online all the time.  Need to change this
10215 	 * to figure out whether the disk device is actually online or not.
10216 	 */
10217 	if (lun != NULL)
10218 		bl_ptr->device = (SID_QUAL_LU_CONNECTED << 5) |
10219 				  lun->be_lun->lun_type;
10220 	else
10221 		bl_ptr->device = (SID_QUAL_LU_OFFLINE << 5) | T_DIRECT;
10222 
10223 	bl_ptr->page_code = SVPD_BLOCK_LIMITS;
10224 	scsi_ulto2b(sizeof(*bl_ptr) - 4, bl_ptr->page_length);
10225 	bl_ptr->max_cmp_write_len = 0xff;
10226 	scsi_ulto4b(0xffffffff, bl_ptr->max_txfer_len);
10227 	if (lun != NULL) {
10228 		bs = lun->be_lun->blocksize;
10229 		scsi_ulto4b(lun->be_lun->opttxferlen, bl_ptr->opt_txfer_len);
10230 		if (lun->be_lun->flags & CTL_LUN_FLAG_UNMAP) {
10231 			scsi_ulto4b(0xffffffff, bl_ptr->max_unmap_lba_cnt);
10232 			scsi_ulto4b(0xffffffff, bl_ptr->max_unmap_blk_cnt);
10233 			if (lun->be_lun->ublockexp != 0) {
10234 				scsi_ulto4b((1 << lun->be_lun->ublockexp),
10235 				    bl_ptr->opt_unmap_grain);
10236 				scsi_ulto4b(0x80000000 | lun->be_lun->ublockoff,
10237 				    bl_ptr->unmap_grain_align);
10238 			}
10239 		}
10240 		scsi_ulto4b(lun->be_lun->atomicblock,
10241 		    bl_ptr->max_atomic_transfer_length);
10242 		scsi_ulto4b(0, bl_ptr->atomic_alignment);
10243 		scsi_ulto4b(0, bl_ptr->atomic_transfer_length_granularity);
10244 	}
10245 	scsi_u64to8b(UINT64_MAX, bl_ptr->max_write_same_length);
10246 
10247 	ctl_set_success(ctsio);
10248 	ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
10249 	ctsio->be_move_done = ctl_config_move_done;
10250 	ctl_datamove((union ctl_io *)ctsio);
10251 	return (CTL_RETVAL_COMPLETE);
10252 }
10253 
10254 static int
10255 ctl_inquiry_evpd_bdc(struct ctl_scsiio *ctsio, int alloc_len)
10256 {
10257 	struct scsi_vpd_block_device_characteristics *bdc_ptr;
10258 	struct ctl_lun *lun;
10259 	const char *value;
10260 	u_int i;
10261 
10262 	lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
10263 
10264 	ctsio->kern_data_ptr = malloc(sizeof(*bdc_ptr), M_CTL, M_WAITOK | M_ZERO);
10265 	bdc_ptr = (struct scsi_vpd_block_device_characteristics *)ctsio->kern_data_ptr;
10266 	ctsio->kern_sg_entries = 0;
10267 
10268 	if (sizeof(*bdc_ptr) < alloc_len) {
10269 		ctsio->residual = alloc_len - sizeof(*bdc_ptr);
10270 		ctsio->kern_data_len = sizeof(*bdc_ptr);
10271 		ctsio->kern_total_len = sizeof(*bdc_ptr);
10272 	} else {
10273 		ctsio->residual = 0;
10274 		ctsio->kern_data_len = alloc_len;
10275 		ctsio->kern_total_len = alloc_len;
10276 	}
10277 	ctsio->kern_data_resid = 0;
10278 	ctsio->kern_rel_offset = 0;
10279 	ctsio->kern_sg_entries = 0;
10280 
10281 	/*
10282 	 * The control device is always connected.  The disk device, on the
10283 	 * other hand, may not be online all the time.  Need to change this
10284 	 * to figure out whether the disk device is actually online or not.
10285 	 */
10286 	if (lun != NULL)
10287 		bdc_ptr->device = (SID_QUAL_LU_CONNECTED << 5) |
10288 				  lun->be_lun->lun_type;
10289 	else
10290 		bdc_ptr->device = (SID_QUAL_LU_OFFLINE << 5) | T_DIRECT;
10291 	bdc_ptr->page_code = SVPD_BDC;
10292 	scsi_ulto2b(sizeof(*bdc_ptr) - 4, bdc_ptr->page_length);
10293 	if (lun != NULL &&
10294 	    (value = ctl_get_opt(&lun->be_lun->options, "rpm")) != NULL)
10295 		i = strtol(value, NULL, 0);
10296 	else
10297 		i = CTL_DEFAULT_ROTATION_RATE;
10298 	scsi_ulto2b(i, bdc_ptr->medium_rotation_rate);
10299 	if (lun != NULL &&
10300 	    (value = ctl_get_opt(&lun->be_lun->options, "formfactor")) != NULL)
10301 		i = strtol(value, NULL, 0);
10302 	else
10303 		i = 0;
10304 	bdc_ptr->wab_wac_ff = (i & 0x0f);
10305 	bdc_ptr->flags = SVPD_FUAB | SVPD_VBULS;
10306 
10307 	ctl_set_success(ctsio);
10308 	ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
10309 	ctsio->be_move_done = ctl_config_move_done;
10310 	ctl_datamove((union ctl_io *)ctsio);
10311 	return (CTL_RETVAL_COMPLETE);
10312 }
10313 
10314 static int
10315 ctl_inquiry_evpd_lbp(struct ctl_scsiio *ctsio, int alloc_len)
10316 {
10317 	struct scsi_vpd_logical_block_prov *lbp_ptr;
10318 	struct ctl_lun *lun;
10319 
10320 	lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
10321 
10322 	ctsio->kern_data_ptr = malloc(sizeof(*lbp_ptr), M_CTL, M_WAITOK | M_ZERO);
10323 	lbp_ptr = (struct scsi_vpd_logical_block_prov *)ctsio->kern_data_ptr;
10324 	ctsio->kern_sg_entries = 0;
10325 
10326 	if (sizeof(*lbp_ptr) < alloc_len) {
10327 		ctsio->residual = alloc_len - sizeof(*lbp_ptr);
10328 		ctsio->kern_data_len = sizeof(*lbp_ptr);
10329 		ctsio->kern_total_len = sizeof(*lbp_ptr);
10330 	} else {
10331 		ctsio->residual = 0;
10332 		ctsio->kern_data_len = alloc_len;
10333 		ctsio->kern_total_len = alloc_len;
10334 	}
10335 	ctsio->kern_data_resid = 0;
10336 	ctsio->kern_rel_offset = 0;
10337 	ctsio->kern_sg_entries = 0;
10338 
10339 	/*
10340 	 * The control device is always connected.  The disk device, on the
10341 	 * other hand, may not be online all the time.  Need to change this
10342 	 * to figure out whether the disk device is actually online or not.
10343 	 */
10344 	if (lun != NULL)
10345 		lbp_ptr->device = (SID_QUAL_LU_CONNECTED << 5) |
10346 				  lun->be_lun->lun_type;
10347 	else
10348 		lbp_ptr->device = (SID_QUAL_LU_OFFLINE << 5) | T_DIRECT;
10349 
10350 	lbp_ptr->page_code = SVPD_LBP;
10351 	scsi_ulto2b(sizeof(*lbp_ptr) - 4, lbp_ptr->page_length);
10352 	lbp_ptr->threshold_exponent = CTL_LBP_EXPONENT;
10353 	if (lun != NULL && lun->be_lun->flags & CTL_LUN_FLAG_UNMAP) {
10354 		lbp_ptr->flags = SVPD_LBP_UNMAP | SVPD_LBP_WS16 |
10355 		    SVPD_LBP_WS10 | SVPD_LBP_RZ | SVPD_LBP_ANC_SUP;
10356 		lbp_ptr->prov_type = SVPD_LBP_THIN;
10357 	}
10358 
10359 	ctl_set_success(ctsio);
10360 	ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
10361 	ctsio->be_move_done = ctl_config_move_done;
10362 	ctl_datamove((union ctl_io *)ctsio);
10363 	return (CTL_RETVAL_COMPLETE);
10364 }
10365 
10366 /*
10367  * INQUIRY with the EVPD bit set.
10368  */
10369 static int
10370 ctl_inquiry_evpd(struct ctl_scsiio *ctsio)
10371 {
10372 	struct ctl_lun *lun;
10373 	struct scsi_inquiry *cdb;
10374 	int alloc_len, retval;
10375 
10376 	lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
10377 	cdb = (struct scsi_inquiry *)ctsio->cdb;
10378 	alloc_len = scsi_2btoul(cdb->length);
10379 
10380 	switch (cdb->page_code) {
10381 	case SVPD_SUPPORTED_PAGES:
10382 		retval = ctl_inquiry_evpd_supported(ctsio, alloc_len);
10383 		break;
10384 	case SVPD_UNIT_SERIAL_NUMBER:
10385 		retval = ctl_inquiry_evpd_serial(ctsio, alloc_len);
10386 		break;
10387 	case SVPD_DEVICE_ID:
10388 		retval = ctl_inquiry_evpd_devid(ctsio, alloc_len);
10389 		break;
10390 	case SVPD_EXTENDED_INQUIRY_DATA:
10391 		retval = ctl_inquiry_evpd_eid(ctsio, alloc_len);
10392 		break;
10393 	case SVPD_MODE_PAGE_POLICY:
10394 		retval = ctl_inquiry_evpd_mpp(ctsio, alloc_len);
10395 		break;
10396 	case SVPD_SCSI_PORTS:
10397 		retval = ctl_inquiry_evpd_scsi_ports(ctsio, alloc_len);
10398 		break;
10399 	case SVPD_SCSI_TPC:
10400 		retval = ctl_inquiry_evpd_tpc(ctsio, alloc_len);
10401 		break;
10402 	case SVPD_BLOCK_LIMITS:
10403 		if (lun == NULL || lun->be_lun->lun_type != T_DIRECT)
10404 			goto err;
10405 		retval = ctl_inquiry_evpd_block_limits(ctsio, alloc_len);
10406 		break;
10407 	case SVPD_BDC:
10408 		if (lun == NULL || lun->be_lun->lun_type != T_DIRECT)
10409 			goto err;
10410 		retval = ctl_inquiry_evpd_bdc(ctsio, alloc_len);
10411 		break;
10412 	case SVPD_LBP:
10413 		if (lun == NULL || lun->be_lun->lun_type != T_DIRECT)
10414 			goto err;
10415 		retval = ctl_inquiry_evpd_lbp(ctsio, alloc_len);
10416 		break;
10417 	default:
10418 err:
10419 		ctl_set_invalid_field(ctsio,
10420 				      /*sks_valid*/ 1,
10421 				      /*command*/ 1,
10422 				      /*field*/ 2,
10423 				      /*bit_valid*/ 0,
10424 				      /*bit*/ 0);
10425 		ctl_done((union ctl_io *)ctsio);
10426 		retval = CTL_RETVAL_COMPLETE;
10427 		break;
10428 	}
10429 
10430 	return (retval);
10431 }
10432 
10433 /*
10434  * Standard INQUIRY data.
10435  */
10436 static int
10437 ctl_inquiry_std(struct ctl_scsiio *ctsio)
10438 {
10439 	struct scsi_inquiry_data *inq_ptr;
10440 	struct scsi_inquiry *cdb;
10441 	struct ctl_softc *softc;
10442 	struct ctl_lun *lun;
10443 	char *val;
10444 	uint32_t alloc_len, data_len;
10445 	ctl_port_type port_type;
10446 
10447 	softc = control_softc;
10448 
10449 	/*
10450 	 * Figure out whether we're talking to a Fibre Channel port or not.
10451 	 * We treat the ioctl front end, and any SCSI adapters, as packetized
10452 	 * SCSI front ends.
10453 	 */
10454 	port_type = softc->ctl_ports[
10455 	    ctl_port_idx(ctsio->io_hdr.nexus.targ_port)]->port_type;
10456 	if (port_type == CTL_PORT_IOCTL || port_type == CTL_PORT_INTERNAL)
10457 		port_type = CTL_PORT_SCSI;
10458 
10459 	lun = ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
10460 	cdb = (struct scsi_inquiry *)ctsio->cdb;
10461 	alloc_len = scsi_2btoul(cdb->length);
10462 
10463 	/*
10464 	 * We malloc the full inquiry data size here and fill it
10465 	 * in.  If the user only asks for less, we'll give him
10466 	 * that much.
10467 	 */
10468 	data_len = offsetof(struct scsi_inquiry_data, vendor_specific1);
10469 	ctsio->kern_data_ptr = malloc(data_len, M_CTL, M_WAITOK | M_ZERO);
10470 	inq_ptr = (struct scsi_inquiry_data *)ctsio->kern_data_ptr;
10471 	ctsio->kern_sg_entries = 0;
10472 	ctsio->kern_data_resid = 0;
10473 	ctsio->kern_rel_offset = 0;
10474 
10475 	if (data_len < alloc_len) {
10476 		ctsio->residual = alloc_len - data_len;
10477 		ctsio->kern_data_len = data_len;
10478 		ctsio->kern_total_len = data_len;
10479 	} else {
10480 		ctsio->residual = 0;
10481 		ctsio->kern_data_len = alloc_len;
10482 		ctsio->kern_total_len = alloc_len;
10483 	}
10484 
10485 	/*
10486 	 * If we have a LUN configured, report it as connected.  Otherwise,
10487 	 * report that it is offline or no device is supported, depending
10488 	 * on the value of inquiry_pq_no_lun.
10489 	 *
10490 	 * According to the spec (SPC-4 r34), the peripheral qualifier
10491 	 * SID_QUAL_LU_OFFLINE (001b) is used in the following scenario:
10492 	 *
10493 	 * "A peripheral device having the specified peripheral device type
10494 	 * is not connected to this logical unit. However, the device
10495 	 * server is capable of supporting the specified peripheral device
10496 	 * type on this logical unit."
10497 	 *
10498 	 * According to the same spec, the peripheral qualifier
10499 	 * SID_QUAL_BAD_LU (011b) is used in this scenario:
10500 	 *
10501 	 * "The device server is not capable of supporting a peripheral
10502 	 * device on this logical unit. For this peripheral qualifier the
10503 	 * peripheral device type shall be set to 1Fh. All other peripheral
10504 	 * device type values are reserved for this peripheral qualifier."
10505 	 *
10506 	 * Given the text, it would seem that we probably want to report that
10507 	 * the LUN is offline here.  There is no LUN connected, but we can
10508 	 * support a LUN at the given LUN number.
10509 	 *
10510 	 * In the real world, though, it sounds like things are a little
10511 	 * different:
10512 	 *
10513 	 * - Linux, when presented with a LUN with the offline peripheral
10514 	 *   qualifier, will create an sg driver instance for it.  So when
10515 	 *   you attach it to CTL, you wind up with a ton of sg driver
10516 	 *   instances.  (One for every LUN that Linux bothered to probe.)
10517 	 *   Linux does this despite the fact that it issues a REPORT LUNs
10518 	 *   to LUN 0 to get the inventory of supported LUNs.
10519 	 *
10520 	 * - There is other anecdotal evidence (from Emulex folks) about
10521 	 *   arrays that use the offline peripheral qualifier for LUNs that
10522 	 *   are on the "passive" path in an active/passive array.
10523 	 *
10524 	 * So the solution is provide a hopefully reasonable default
10525 	 * (return bad/no LUN) and allow the user to change the behavior
10526 	 * with a tunable/sysctl variable.
10527 	 */
10528 	if (lun != NULL)
10529 		inq_ptr->device = (SID_QUAL_LU_CONNECTED << 5) |
10530 				  lun->be_lun->lun_type;
10531 	else if (softc->inquiry_pq_no_lun == 0)
10532 		inq_ptr->device = (SID_QUAL_LU_OFFLINE << 5) | T_DIRECT;
10533 	else
10534 		inq_ptr->device = (SID_QUAL_BAD_LU << 5) | T_NODEVICE;
10535 
10536 	/* RMB in byte 2 is 0 */
10537 	inq_ptr->version = SCSI_REV_SPC4;
10538 
10539 	/*
10540 	 * According to SAM-3, even if a device only supports a single
10541 	 * level of LUN addressing, it should still set the HISUP bit:
10542 	 *
10543 	 * 4.9.1 Logical unit numbers overview
10544 	 *
10545 	 * All logical unit number formats described in this standard are
10546 	 * hierarchical in structure even when only a single level in that
10547 	 * hierarchy is used. The HISUP bit shall be set to one in the
10548 	 * standard INQUIRY data (see SPC-2) when any logical unit number
10549 	 * format described in this standard is used.  Non-hierarchical
10550 	 * formats are outside the scope of this standard.
10551 	 *
10552 	 * Therefore we set the HiSup bit here.
10553 	 *
10554 	 * The reponse format is 2, per SPC-3.
10555 	 */
10556 	inq_ptr->response_format = SID_HiSup | 2;
10557 
10558 	inq_ptr->additional_length = data_len -
10559 	    (offsetof(struct scsi_inquiry_data, additional_length) + 1);
10560 	CTL_DEBUG_PRINT(("additional_length = %d\n",
10561 			 inq_ptr->additional_length));
10562 
10563 	inq_ptr->spc3_flags = SPC3_SID_3PC | SPC3_SID_TPGS_IMPLICIT;
10564 	/* 16 bit addressing */
10565 	if (port_type == CTL_PORT_SCSI)
10566 		inq_ptr->spc2_flags = SPC2_SID_ADDR16;
10567 	/* XXX set the SID_MultiP bit here if we're actually going to
10568 	   respond on multiple ports */
10569 	inq_ptr->spc2_flags |= SPC2_SID_MultiP;
10570 
10571 	/* 16 bit data bus, synchronous transfers */
10572 	if (port_type == CTL_PORT_SCSI)
10573 		inq_ptr->flags = SID_WBus16 | SID_Sync;
10574 	/*
10575 	 * XXX KDM do we want to support tagged queueing on the control
10576 	 * device at all?
10577 	 */
10578 	if ((lun == NULL)
10579 	 || (lun->be_lun->lun_type != T_PROCESSOR))
10580 		inq_ptr->flags |= SID_CmdQue;
10581 	/*
10582 	 * Per SPC-3, unused bytes in ASCII strings are filled with spaces.
10583 	 * We have 8 bytes for the vendor name, and 16 bytes for the device
10584 	 * name and 4 bytes for the revision.
10585 	 */
10586 	if (lun == NULL || (val = ctl_get_opt(&lun->be_lun->options,
10587 	    "vendor")) == NULL) {
10588 		strncpy(inq_ptr->vendor, CTL_VENDOR, sizeof(inq_ptr->vendor));
10589 	} else {
10590 		memset(inq_ptr->vendor, ' ', sizeof(inq_ptr->vendor));
10591 		strncpy(inq_ptr->vendor, val,
10592 		    min(sizeof(inq_ptr->vendor), strlen(val)));
10593 	}
10594 	if (lun == NULL) {
10595 		strncpy(inq_ptr->product, CTL_DIRECT_PRODUCT,
10596 		    sizeof(inq_ptr->product));
10597 	} else if ((val = ctl_get_opt(&lun->be_lun->options, "product")) == NULL) {
10598 		switch (lun->be_lun->lun_type) {
10599 		case T_DIRECT:
10600 			strncpy(inq_ptr->product, CTL_DIRECT_PRODUCT,
10601 			    sizeof(inq_ptr->product));
10602 			break;
10603 		case T_PROCESSOR:
10604 			strncpy(inq_ptr->product, CTL_PROCESSOR_PRODUCT,
10605 			    sizeof(inq_ptr->product));
10606 			break;
10607 		default:
10608 			strncpy(inq_ptr->product, CTL_UNKNOWN_PRODUCT,
10609 			    sizeof(inq_ptr->product));
10610 			break;
10611 		}
10612 	} else {
10613 		memset(inq_ptr->product, ' ', sizeof(inq_ptr->product));
10614 		strncpy(inq_ptr->product, val,
10615 		    min(sizeof(inq_ptr->product), strlen(val)));
10616 	}
10617 
10618 	/*
10619 	 * XXX make this a macro somewhere so it automatically gets
10620 	 * incremented when we make changes.
10621 	 */
10622 	if (lun == NULL || (val = ctl_get_opt(&lun->be_lun->options,
10623 	    "revision")) == NULL) {
10624 		strncpy(inq_ptr->revision, "0001", sizeof(inq_ptr->revision));
10625 	} else {
10626 		memset(inq_ptr->revision, ' ', sizeof(inq_ptr->revision));
10627 		strncpy(inq_ptr->revision, val,
10628 		    min(sizeof(inq_ptr->revision), strlen(val)));
10629 	}
10630 
10631 	/*
10632 	 * For parallel SCSI, we support double transition and single
10633 	 * transition clocking.  We also support QAS (Quick Arbitration
10634 	 * and Selection) and Information Unit transfers on both the
10635 	 * control and array devices.
10636 	 */
10637 	if (port_type == CTL_PORT_SCSI)
10638 		inq_ptr->spi3data = SID_SPI_CLOCK_DT_ST | SID_SPI_QAS |
10639 				    SID_SPI_IUS;
10640 
10641 	/* SAM-5 (no version claimed) */
10642 	scsi_ulto2b(0x00A0, inq_ptr->version1);
10643 	/* SPC-4 (no version claimed) */
10644 	scsi_ulto2b(0x0460, inq_ptr->version2);
10645 	if (port_type == CTL_PORT_FC) {
10646 		/* FCP-2 ANSI INCITS.350:2003 */
10647 		scsi_ulto2b(0x0917, inq_ptr->version3);
10648 	} else if (port_type == CTL_PORT_SCSI) {
10649 		/* SPI-4 ANSI INCITS.362:200x */
10650 		scsi_ulto2b(0x0B56, inq_ptr->version3);
10651 	} else if (port_type == CTL_PORT_ISCSI) {
10652 		/* iSCSI (no version claimed) */
10653 		scsi_ulto2b(0x0960, inq_ptr->version3);
10654 	} else if (port_type == CTL_PORT_SAS) {
10655 		/* SAS (no version claimed) */
10656 		scsi_ulto2b(0x0BE0, inq_ptr->version3);
10657 	}
10658 
10659 	if (lun == NULL) {
10660 		/* SBC-4 (no version claimed) */
10661 		scsi_ulto2b(0x0600, inq_ptr->version4);
10662 	} else {
10663 		switch (lun->be_lun->lun_type) {
10664 		case T_DIRECT:
10665 			/* SBC-4 (no version claimed) */
10666 			scsi_ulto2b(0x0600, inq_ptr->version4);
10667 			break;
10668 		case T_PROCESSOR:
10669 		default:
10670 			break;
10671 		}
10672 	}
10673 
10674 	ctl_set_success(ctsio);
10675 	ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
10676 	ctsio->be_move_done = ctl_config_move_done;
10677 	ctl_datamove((union ctl_io *)ctsio);
10678 	return (CTL_RETVAL_COMPLETE);
10679 }
10680 
10681 int
10682 ctl_inquiry(struct ctl_scsiio *ctsio)
10683 {
10684 	struct scsi_inquiry *cdb;
10685 	int retval;
10686 
10687 	CTL_DEBUG_PRINT(("ctl_inquiry\n"));
10688 
10689 	cdb = (struct scsi_inquiry *)ctsio->cdb;
10690 	if (cdb->byte2 & SI_EVPD)
10691 		retval = ctl_inquiry_evpd(ctsio);
10692 	else if (cdb->page_code == 0)
10693 		retval = ctl_inquiry_std(ctsio);
10694 	else {
10695 		ctl_set_invalid_field(ctsio,
10696 				      /*sks_valid*/ 1,
10697 				      /*command*/ 1,
10698 				      /*field*/ 2,
10699 				      /*bit_valid*/ 0,
10700 				      /*bit*/ 0);
10701 		ctl_done((union ctl_io *)ctsio);
10702 		return (CTL_RETVAL_COMPLETE);
10703 	}
10704 
10705 	return (retval);
10706 }
10707 
10708 /*
10709  * For known CDB types, parse the LBA and length.
10710  */
10711 static int
10712 ctl_get_lba_len(union ctl_io *io, uint64_t *lba, uint64_t *len)
10713 {
10714 	if (io->io_hdr.io_type != CTL_IO_SCSI)
10715 		return (1);
10716 
10717 	switch (io->scsiio.cdb[0]) {
10718 	case COMPARE_AND_WRITE: {
10719 		struct scsi_compare_and_write *cdb;
10720 
10721 		cdb = (struct scsi_compare_and_write *)io->scsiio.cdb;
10722 
10723 		*lba = scsi_8btou64(cdb->addr);
10724 		*len = cdb->length;
10725 		break;
10726 	}
10727 	case READ_6:
10728 	case WRITE_6: {
10729 		struct scsi_rw_6 *cdb;
10730 
10731 		cdb = (struct scsi_rw_6 *)io->scsiio.cdb;
10732 
10733 		*lba = scsi_3btoul(cdb->addr);
10734 		/* only 5 bits are valid in the most significant address byte */
10735 		*lba &= 0x1fffff;
10736 		*len = cdb->length;
10737 		break;
10738 	}
10739 	case READ_10:
10740 	case WRITE_10: {
10741 		struct scsi_rw_10 *cdb;
10742 
10743 		cdb = (struct scsi_rw_10 *)io->scsiio.cdb;
10744 
10745 		*lba = scsi_4btoul(cdb->addr);
10746 		*len = scsi_2btoul(cdb->length);
10747 		break;
10748 	}
10749 	case WRITE_VERIFY_10: {
10750 		struct scsi_write_verify_10 *cdb;
10751 
10752 		cdb = (struct scsi_write_verify_10 *)io->scsiio.cdb;
10753 
10754 		*lba = scsi_4btoul(cdb->addr);
10755 		*len = scsi_2btoul(cdb->length);
10756 		break;
10757 	}
10758 	case READ_12:
10759 	case WRITE_12: {
10760 		struct scsi_rw_12 *cdb;
10761 
10762 		cdb = (struct scsi_rw_12 *)io->scsiio.cdb;
10763 
10764 		*lba = scsi_4btoul(cdb->addr);
10765 		*len = scsi_4btoul(cdb->length);
10766 		break;
10767 	}
10768 	case WRITE_VERIFY_12: {
10769 		struct scsi_write_verify_12 *cdb;
10770 
10771 		cdb = (struct scsi_write_verify_12 *)io->scsiio.cdb;
10772 
10773 		*lba = scsi_4btoul(cdb->addr);
10774 		*len = scsi_4btoul(cdb->length);
10775 		break;
10776 	}
10777 	case READ_16:
10778 	case WRITE_16:
10779 	case WRITE_ATOMIC_16: {
10780 		struct scsi_rw_16 *cdb;
10781 
10782 		cdb = (struct scsi_rw_16 *)io->scsiio.cdb;
10783 
10784 		*lba = scsi_8btou64(cdb->addr);
10785 		*len = scsi_4btoul(cdb->length);
10786 		break;
10787 	}
10788 	case WRITE_VERIFY_16: {
10789 		struct scsi_write_verify_16 *cdb;
10790 
10791 		cdb = (struct scsi_write_verify_16 *)io->scsiio.cdb;
10792 
10793 		*lba = scsi_8btou64(cdb->addr);
10794 		*len = scsi_4btoul(cdb->length);
10795 		break;
10796 	}
10797 	case WRITE_SAME_10: {
10798 		struct scsi_write_same_10 *cdb;
10799 
10800 		cdb = (struct scsi_write_same_10 *)io->scsiio.cdb;
10801 
10802 		*lba = scsi_4btoul(cdb->addr);
10803 		*len = scsi_2btoul(cdb->length);
10804 		break;
10805 	}
10806 	case WRITE_SAME_16: {
10807 		struct scsi_write_same_16 *cdb;
10808 
10809 		cdb = (struct scsi_write_same_16 *)io->scsiio.cdb;
10810 
10811 		*lba = scsi_8btou64(cdb->addr);
10812 		*len = scsi_4btoul(cdb->length);
10813 		break;
10814 	}
10815 	case VERIFY_10: {
10816 		struct scsi_verify_10 *cdb;
10817 
10818 		cdb = (struct scsi_verify_10 *)io->scsiio.cdb;
10819 
10820 		*lba = scsi_4btoul(cdb->addr);
10821 		*len = scsi_2btoul(cdb->length);
10822 		break;
10823 	}
10824 	case VERIFY_12: {
10825 		struct scsi_verify_12 *cdb;
10826 
10827 		cdb = (struct scsi_verify_12 *)io->scsiio.cdb;
10828 
10829 		*lba = scsi_4btoul(cdb->addr);
10830 		*len = scsi_4btoul(cdb->length);
10831 		break;
10832 	}
10833 	case VERIFY_16: {
10834 		struct scsi_verify_16 *cdb;
10835 
10836 		cdb = (struct scsi_verify_16 *)io->scsiio.cdb;
10837 
10838 		*lba = scsi_8btou64(cdb->addr);
10839 		*len = scsi_4btoul(cdb->length);
10840 		break;
10841 	}
10842 	case UNMAP: {
10843 		*lba = 0;
10844 		*len = UINT64_MAX;
10845 		break;
10846 	}
10847 	case SERVICE_ACTION_IN: {	/* GET LBA STATUS */
10848 		struct scsi_get_lba_status *cdb;
10849 
10850 		cdb = (struct scsi_get_lba_status *)io->scsiio.cdb;
10851 		*lba = scsi_8btou64(cdb->addr);
10852 		*len = UINT32_MAX;
10853 		break;
10854 	}
10855 	default:
10856 		return (1);
10857 		break; /* NOTREACHED */
10858 	}
10859 
10860 	return (0);
10861 }
10862 
10863 static ctl_action
10864 ctl_extent_check_lba(uint64_t lba1, uint64_t len1, uint64_t lba2, uint64_t len2,
10865     bool seq)
10866 {
10867 	uint64_t endlba1, endlba2;
10868 
10869 	endlba1 = lba1 + len1 - (seq ? 0 : 1);
10870 	endlba2 = lba2 + len2 - 1;
10871 
10872 	if ((endlba1 < lba2) || (endlba2 < lba1))
10873 		return (CTL_ACTION_PASS);
10874 	else
10875 		return (CTL_ACTION_BLOCK);
10876 }
10877 
10878 static int
10879 ctl_extent_check_unmap(union ctl_io *io, uint64_t lba2, uint64_t len2)
10880 {
10881 	struct ctl_ptr_len_flags *ptrlen;
10882 	struct scsi_unmap_desc *buf, *end, *range;
10883 	uint64_t lba;
10884 	uint32_t len;
10885 
10886 	/* If not UNMAP -- go other way. */
10887 	if (io->io_hdr.io_type != CTL_IO_SCSI ||
10888 	    io->scsiio.cdb[0] != UNMAP)
10889 		return (CTL_ACTION_ERROR);
10890 
10891 	/* If UNMAP without data -- block and wait for data. */
10892 	ptrlen = (struct ctl_ptr_len_flags *)
10893 	    &io->io_hdr.ctl_private[CTL_PRIV_LBA_LEN];
10894 	if ((io->io_hdr.flags & CTL_FLAG_ALLOCATED) == 0 ||
10895 	    ptrlen->ptr == NULL)
10896 		return (CTL_ACTION_BLOCK);
10897 
10898 	/* UNMAP with data -- check for collision. */
10899 	buf = (struct scsi_unmap_desc *)ptrlen->ptr;
10900 	end = buf + ptrlen->len / sizeof(*buf);
10901 	for (range = buf; range < end; range++) {
10902 		lba = scsi_8btou64(range->lba);
10903 		len = scsi_4btoul(range->length);
10904 		if ((lba < lba2 + len2) && (lba + len > lba2))
10905 			return (CTL_ACTION_BLOCK);
10906 	}
10907 	return (CTL_ACTION_PASS);
10908 }
10909 
10910 static ctl_action
10911 ctl_extent_check(union ctl_io *io1, union ctl_io *io2, bool seq)
10912 {
10913 	uint64_t lba1, lba2;
10914 	uint64_t len1, len2;
10915 	int retval;
10916 
10917 	if (ctl_get_lba_len(io2, &lba2, &len2) != 0)
10918 		return (CTL_ACTION_ERROR);
10919 
10920 	retval = ctl_extent_check_unmap(io1, lba2, len2);
10921 	if (retval != CTL_ACTION_ERROR)
10922 		return (retval);
10923 
10924 	if (ctl_get_lba_len(io1, &lba1, &len1) != 0)
10925 		return (CTL_ACTION_ERROR);
10926 
10927 	return (ctl_extent_check_lba(lba1, len1, lba2, len2, seq));
10928 }
10929 
10930 static ctl_action
10931 ctl_extent_check_seq(union ctl_io *io1, union ctl_io *io2)
10932 {
10933 	uint64_t lba1, lba2;
10934 	uint64_t len1, len2;
10935 
10936 	if (ctl_get_lba_len(io1, &lba1, &len1) != 0)
10937 		return (CTL_ACTION_ERROR);
10938 	if (ctl_get_lba_len(io2, &lba2, &len2) != 0)
10939 		return (CTL_ACTION_ERROR);
10940 
10941 	if (lba1 + len1 == lba2)
10942 		return (CTL_ACTION_BLOCK);
10943 	return (CTL_ACTION_PASS);
10944 }
10945 
10946 static ctl_action
10947 ctl_check_for_blockage(struct ctl_lun *lun, union ctl_io *pending_io,
10948     union ctl_io *ooa_io)
10949 {
10950 	const struct ctl_cmd_entry *pending_entry, *ooa_entry;
10951 	ctl_serialize_action *serialize_row;
10952 
10953 	/*
10954 	 * The initiator attempted multiple untagged commands at the same
10955 	 * time.  Can't do that.
10956 	 */
10957 	if ((pending_io->scsiio.tag_type == CTL_TAG_UNTAGGED)
10958 	 && (ooa_io->scsiio.tag_type == CTL_TAG_UNTAGGED)
10959 	 && ((pending_io->io_hdr.nexus.targ_port ==
10960 	      ooa_io->io_hdr.nexus.targ_port)
10961 	  && (pending_io->io_hdr.nexus.initid.id ==
10962 	      ooa_io->io_hdr.nexus.initid.id))
10963 	 && ((ooa_io->io_hdr.flags & (CTL_FLAG_ABORT |
10964 	      CTL_FLAG_STATUS_SENT)) == 0))
10965 		return (CTL_ACTION_OVERLAP);
10966 
10967 	/*
10968 	 * The initiator attempted to send multiple tagged commands with
10969 	 * the same ID.  (It's fine if different initiators have the same
10970 	 * tag ID.)
10971 	 *
10972 	 * Even if all of those conditions are true, we don't kill the I/O
10973 	 * if the command ahead of us has been aborted.  We won't end up
10974 	 * sending it to the FETD, and it's perfectly legal to resend a
10975 	 * command with the same tag number as long as the previous
10976 	 * instance of this tag number has been aborted somehow.
10977 	 */
10978 	if ((pending_io->scsiio.tag_type != CTL_TAG_UNTAGGED)
10979 	 && (ooa_io->scsiio.tag_type != CTL_TAG_UNTAGGED)
10980 	 && (pending_io->scsiio.tag_num == ooa_io->scsiio.tag_num)
10981 	 && ((pending_io->io_hdr.nexus.targ_port ==
10982 	      ooa_io->io_hdr.nexus.targ_port)
10983 	  && (pending_io->io_hdr.nexus.initid.id ==
10984 	      ooa_io->io_hdr.nexus.initid.id))
10985 	 && ((ooa_io->io_hdr.flags & (CTL_FLAG_ABORT |
10986 	      CTL_FLAG_STATUS_SENT)) == 0))
10987 		return (CTL_ACTION_OVERLAP_TAG);
10988 
10989 	/*
10990 	 * If we get a head of queue tag, SAM-3 says that we should
10991 	 * immediately execute it.
10992 	 *
10993 	 * What happens if this command would normally block for some other
10994 	 * reason?  e.g. a request sense with a head of queue tag
10995 	 * immediately after a write.  Normally that would block, but this
10996 	 * will result in its getting executed immediately...
10997 	 *
10998 	 * We currently return "pass" instead of "skip", so we'll end up
10999 	 * going through the rest of the queue to check for overlapped tags.
11000 	 *
11001 	 * XXX KDM check for other types of blockage first??
11002 	 */
11003 	if (pending_io->scsiio.tag_type == CTL_TAG_HEAD_OF_QUEUE)
11004 		return (CTL_ACTION_PASS);
11005 
11006 	/*
11007 	 * Ordered tags have to block until all items ahead of them
11008 	 * have completed.  If we get called with an ordered tag, we always
11009 	 * block, if something else is ahead of us in the queue.
11010 	 */
11011 	if (pending_io->scsiio.tag_type == CTL_TAG_ORDERED)
11012 		return (CTL_ACTION_BLOCK);
11013 
11014 	/*
11015 	 * Simple tags get blocked until all head of queue and ordered tags
11016 	 * ahead of them have completed.  I'm lumping untagged commands in
11017 	 * with simple tags here.  XXX KDM is that the right thing to do?
11018 	 */
11019 	if (((pending_io->scsiio.tag_type == CTL_TAG_UNTAGGED)
11020 	  || (pending_io->scsiio.tag_type == CTL_TAG_SIMPLE))
11021 	 && ((ooa_io->scsiio.tag_type == CTL_TAG_HEAD_OF_QUEUE)
11022 	  || (ooa_io->scsiio.tag_type == CTL_TAG_ORDERED)))
11023 		return (CTL_ACTION_BLOCK);
11024 
11025 	pending_entry = ctl_get_cmd_entry(&pending_io->scsiio, NULL);
11026 	ooa_entry = ctl_get_cmd_entry(&ooa_io->scsiio, NULL);
11027 
11028 	serialize_row = ctl_serialize_table[ooa_entry->seridx];
11029 
11030 	switch (serialize_row[pending_entry->seridx]) {
11031 	case CTL_SER_BLOCK:
11032 		return (CTL_ACTION_BLOCK);
11033 	case CTL_SER_EXTENT:
11034 		return (ctl_extent_check(ooa_io, pending_io,
11035 		    (lun->serseq == CTL_LUN_SERSEQ_ON)));
11036 	case CTL_SER_EXTENTOPT:
11037 		if ((lun->mode_pages.control_page[CTL_PAGE_CURRENT].queue_flags
11038 		    & SCP_QUEUE_ALG_MASK) != SCP_QUEUE_ALG_UNRESTRICTED)
11039 			return (ctl_extent_check(ooa_io, pending_io,
11040 			    (lun->serseq == CTL_LUN_SERSEQ_ON)));
11041 		return (CTL_ACTION_PASS);
11042 	case CTL_SER_EXTENTSEQ:
11043 		if (lun->serseq != CTL_LUN_SERSEQ_OFF)
11044 			return (ctl_extent_check_seq(ooa_io, pending_io));
11045 		return (CTL_ACTION_PASS);
11046 	case CTL_SER_PASS:
11047 		return (CTL_ACTION_PASS);
11048 	case CTL_SER_BLOCKOPT:
11049 		if ((lun->mode_pages.control_page[CTL_PAGE_CURRENT].queue_flags
11050 		    & SCP_QUEUE_ALG_MASK) != SCP_QUEUE_ALG_UNRESTRICTED)
11051 			return (CTL_ACTION_BLOCK);
11052 		return (CTL_ACTION_PASS);
11053 	case CTL_SER_SKIP:
11054 		return (CTL_ACTION_SKIP);
11055 	default:
11056 		panic("invalid serialization value %d",
11057 		      serialize_row[pending_entry->seridx]);
11058 	}
11059 
11060 	return (CTL_ACTION_ERROR);
11061 }
11062 
11063 /*
11064  * Check for blockage or overlaps against the OOA (Order Of Arrival) queue.
11065  * Assumptions:
11066  * - pending_io is generally either incoming, or on the blocked queue
11067  * - starting I/O is the I/O we want to start the check with.
11068  */
11069 static ctl_action
11070 ctl_check_ooa(struct ctl_lun *lun, union ctl_io *pending_io,
11071 	      union ctl_io *starting_io)
11072 {
11073 	union ctl_io *ooa_io;
11074 	ctl_action action;
11075 
11076 	mtx_assert(&lun->lun_lock, MA_OWNED);
11077 
11078 	/*
11079 	 * Run back along the OOA queue, starting with the current
11080 	 * blocked I/O and going through every I/O before it on the
11081 	 * queue.  If starting_io is NULL, we'll just end up returning
11082 	 * CTL_ACTION_PASS.
11083 	 */
11084 	for (ooa_io = starting_io; ooa_io != NULL;
11085 	     ooa_io = (union ctl_io *)TAILQ_PREV(&ooa_io->io_hdr, ctl_ooaq,
11086 	     ooa_links)){
11087 
11088 		/*
11089 		 * This routine just checks to see whether
11090 		 * cur_blocked is blocked by ooa_io, which is ahead
11091 		 * of it in the queue.  It doesn't queue/dequeue
11092 		 * cur_blocked.
11093 		 */
11094 		action = ctl_check_for_blockage(lun, pending_io, ooa_io);
11095 		switch (action) {
11096 		case CTL_ACTION_BLOCK:
11097 		case CTL_ACTION_OVERLAP:
11098 		case CTL_ACTION_OVERLAP_TAG:
11099 		case CTL_ACTION_SKIP:
11100 		case CTL_ACTION_ERROR:
11101 			return (action);
11102 			break; /* NOTREACHED */
11103 		case CTL_ACTION_PASS:
11104 			break;
11105 		default:
11106 			panic("invalid action %d", action);
11107 			break;  /* NOTREACHED */
11108 		}
11109 	}
11110 
11111 	return (CTL_ACTION_PASS);
11112 }
11113 
11114 /*
11115  * Assumptions:
11116  * - An I/O has just completed, and has been removed from the per-LUN OOA
11117  *   queue, so some items on the blocked queue may now be unblocked.
11118  */
11119 static int
11120 ctl_check_blocked(struct ctl_lun *lun)
11121 {
11122 	union ctl_io *cur_blocked, *next_blocked;
11123 
11124 	mtx_assert(&lun->lun_lock, MA_OWNED);
11125 
11126 	/*
11127 	 * Run forward from the head of the blocked queue, checking each
11128 	 * entry against the I/Os prior to it on the OOA queue to see if
11129 	 * there is still any blockage.
11130 	 *
11131 	 * We cannot use the TAILQ_FOREACH() macro, because it can't deal
11132 	 * with our removing a variable on it while it is traversing the
11133 	 * list.
11134 	 */
11135 	for (cur_blocked = (union ctl_io *)TAILQ_FIRST(&lun->blocked_queue);
11136 	     cur_blocked != NULL; cur_blocked = next_blocked) {
11137 		union ctl_io *prev_ooa;
11138 		ctl_action action;
11139 
11140 		next_blocked = (union ctl_io *)TAILQ_NEXT(&cur_blocked->io_hdr,
11141 							  blocked_links);
11142 
11143 		prev_ooa = (union ctl_io *)TAILQ_PREV(&cur_blocked->io_hdr,
11144 						      ctl_ooaq, ooa_links);
11145 
11146 		/*
11147 		 * If cur_blocked happens to be the first item in the OOA
11148 		 * queue now, prev_ooa will be NULL, and the action
11149 		 * returned will just be CTL_ACTION_PASS.
11150 		 */
11151 		action = ctl_check_ooa(lun, cur_blocked, prev_ooa);
11152 
11153 		switch (action) {
11154 		case CTL_ACTION_BLOCK:
11155 			/* Nothing to do here, still blocked */
11156 			break;
11157 		case CTL_ACTION_OVERLAP:
11158 		case CTL_ACTION_OVERLAP_TAG:
11159 			/*
11160 			 * This shouldn't happen!  In theory we've already
11161 			 * checked this command for overlap...
11162 			 */
11163 			break;
11164 		case CTL_ACTION_PASS:
11165 		case CTL_ACTION_SKIP: {
11166 			const struct ctl_cmd_entry *entry;
11167 			int isc_retval;
11168 
11169 			/*
11170 			 * The skip case shouldn't happen, this transaction
11171 			 * should have never made it onto the blocked queue.
11172 			 */
11173 			/*
11174 			 * This I/O is no longer blocked, we can remove it
11175 			 * from the blocked queue.  Since this is a TAILQ
11176 			 * (doubly linked list), we can do O(1) removals
11177 			 * from any place on the list.
11178 			 */
11179 			TAILQ_REMOVE(&lun->blocked_queue, &cur_blocked->io_hdr,
11180 				     blocked_links);
11181 			cur_blocked->io_hdr.flags &= ~CTL_FLAG_BLOCKED;
11182 
11183 			if (cur_blocked->io_hdr.flags & CTL_FLAG_FROM_OTHER_SC){
11184 				/*
11185 				 * Need to send IO back to original side to
11186 				 * run
11187 				 */
11188 				union ctl_ha_msg msg_info;
11189 
11190 				msg_info.hdr.original_sc =
11191 					cur_blocked->io_hdr.original_sc;
11192 				msg_info.hdr.serializing_sc = cur_blocked;
11193 				msg_info.hdr.msg_type = CTL_MSG_R2R;
11194 				if ((isc_retval=ctl_ha_msg_send(CTL_HA_CHAN_CTL,
11195 				     &msg_info, sizeof(msg_info), 0)) >
11196 				     CTL_HA_STATUS_SUCCESS) {
11197 					printf("CTL:Check Blocked error from "
11198 					       "ctl_ha_msg_send %d\n",
11199 					       isc_retval);
11200 				}
11201 				break;
11202 			}
11203 			entry = ctl_get_cmd_entry(&cur_blocked->scsiio, NULL);
11204 
11205 			/*
11206 			 * Check this I/O for LUN state changes that may
11207 			 * have happened while this command was blocked.
11208 			 * The LUN state may have been changed by a command
11209 			 * ahead of us in the queue, so we need to re-check
11210 			 * for any states that can be caused by SCSI
11211 			 * commands.
11212 			 */
11213 			if (ctl_scsiio_lun_check(lun, entry,
11214 						 &cur_blocked->scsiio) == 0) {
11215 				cur_blocked->io_hdr.flags |=
11216 				                      CTL_FLAG_IS_WAS_ON_RTR;
11217 				ctl_enqueue_rtr(cur_blocked);
11218 			} else
11219 				ctl_done(cur_blocked);
11220 			break;
11221 		}
11222 		default:
11223 			/*
11224 			 * This probably shouldn't happen -- we shouldn't
11225 			 * get CTL_ACTION_ERROR, or anything else.
11226 			 */
11227 			break;
11228 		}
11229 	}
11230 
11231 	return (CTL_RETVAL_COMPLETE);
11232 }
11233 
11234 /*
11235  * This routine (with one exception) checks LUN flags that can be set by
11236  * commands ahead of us in the OOA queue.  These flags have to be checked
11237  * when a command initially comes in, and when we pull a command off the
11238  * blocked queue and are preparing to execute it.  The reason we have to
11239  * check these flags for commands on the blocked queue is that the LUN
11240  * state may have been changed by a command ahead of us while we're on the
11241  * blocked queue.
11242  *
11243  * Ordering is somewhat important with these checks, so please pay
11244  * careful attention to the placement of any new checks.
11245  */
11246 static int
11247 ctl_scsiio_lun_check(struct ctl_lun *lun,
11248     const struct ctl_cmd_entry *entry, struct ctl_scsiio *ctsio)
11249 {
11250 	struct ctl_softc *softc = lun->ctl_softc;
11251 	int retval;
11252 	uint32_t residx;
11253 
11254 	retval = 0;
11255 
11256 	mtx_assert(&lun->lun_lock, MA_OWNED);
11257 
11258 	/*
11259 	 * If this shelf is a secondary shelf controller, we have to reject
11260 	 * any media access commands.
11261 	 */
11262 	if ((softc->flags & CTL_FLAG_ACTIVE_SHELF) == 0 &&
11263 	    (entry->flags & CTL_CMD_FLAG_OK_ON_SECONDARY) == 0) {
11264 		ctl_set_lun_standby(ctsio);
11265 		retval = 1;
11266 		goto bailout;
11267 	}
11268 
11269 	if (entry->pattern & CTL_LUN_PAT_WRITE) {
11270 		if (lun->flags & CTL_LUN_READONLY) {
11271 			ctl_set_sense(ctsio, /*current_error*/ 1,
11272 			    /*sense_key*/ SSD_KEY_DATA_PROTECT,
11273 			    /*asc*/ 0x27, /*ascq*/ 0x01, SSD_ELEM_NONE);
11274 			retval = 1;
11275 			goto bailout;
11276 		}
11277 		if ((lun->mode_pages.control_page[CTL_PAGE_CURRENT]
11278 		    .eca_and_aen & SCP_SWP) != 0) {
11279 			ctl_set_sense(ctsio, /*current_error*/ 1,
11280 			    /*sense_key*/ SSD_KEY_DATA_PROTECT,
11281 			    /*asc*/ 0x27, /*ascq*/ 0x02, SSD_ELEM_NONE);
11282 			retval = 1;
11283 			goto bailout;
11284 		}
11285 	}
11286 
11287 	/*
11288 	 * Check for a reservation conflict.  If this command isn't allowed
11289 	 * even on reserved LUNs, and if this initiator isn't the one who
11290 	 * reserved us, reject the command with a reservation conflict.
11291 	 */
11292 	residx = ctl_get_resindex(&ctsio->io_hdr.nexus);
11293 	if ((lun->flags & CTL_LUN_RESERVED)
11294 	 && ((entry->flags & CTL_CMD_FLAG_ALLOW_ON_RESV) == 0)) {
11295 		if (lun->res_idx != residx) {
11296 			ctl_set_reservation_conflict(ctsio);
11297 			retval = 1;
11298 			goto bailout;
11299 		}
11300 	}
11301 
11302 	if ((lun->flags & CTL_LUN_PR_RESERVED) == 0 ||
11303 	    (entry->flags & CTL_CMD_FLAG_ALLOW_ON_PR_RESV)) {
11304 		/* No reservation or command is allowed. */;
11305 	} else if ((entry->flags & CTL_CMD_FLAG_ALLOW_ON_PR_WRESV) &&
11306 	    (lun->res_type == SPR_TYPE_WR_EX ||
11307 	     lun->res_type == SPR_TYPE_WR_EX_RO ||
11308 	     lun->res_type == SPR_TYPE_WR_EX_AR)) {
11309 		/* The command is allowed for Write Exclusive resv. */;
11310 	} else {
11311 		/*
11312 		 * if we aren't registered or it's a res holder type
11313 		 * reservation and this isn't the res holder then set a
11314 		 * conflict.
11315 		 */
11316 		if (ctl_get_prkey(lun, residx) == 0
11317 		 || (residx != lun->pr_res_idx && lun->res_type < 4)) {
11318 			ctl_set_reservation_conflict(ctsio);
11319 			retval = 1;
11320 			goto bailout;
11321 		}
11322 
11323 	}
11324 
11325 	if ((lun->flags & CTL_LUN_OFFLINE)
11326 	 && ((entry->flags & CTL_CMD_FLAG_OK_ON_OFFLINE) == 0)) {
11327 		ctl_set_lun_not_ready(ctsio);
11328 		retval = 1;
11329 		goto bailout;
11330 	}
11331 
11332 	/*
11333 	 * If the LUN is stopped, see if this particular command is allowed
11334 	 * for a stopped lun.  Otherwise, reject it with 0x04,0x02.
11335 	 */
11336 	if ((lun->flags & CTL_LUN_STOPPED)
11337 	 && ((entry->flags & CTL_CMD_FLAG_OK_ON_STOPPED) == 0)) {
11338 		/* "Logical unit not ready, initializing cmd. required" */
11339 		ctl_set_lun_stopped(ctsio);
11340 		retval = 1;
11341 		goto bailout;
11342 	}
11343 
11344 	if ((lun->flags & CTL_LUN_INOPERABLE)
11345 	 && ((entry->flags & CTL_CMD_FLAG_OK_ON_INOPERABLE) == 0)) {
11346 		/* "Medium format corrupted" */
11347 		ctl_set_medium_format_corrupted(ctsio);
11348 		retval = 1;
11349 		goto bailout;
11350 	}
11351 
11352 bailout:
11353 	return (retval);
11354 
11355 }
11356 
11357 static void
11358 ctl_failover_io(union ctl_io *io, int have_lock)
11359 {
11360 	ctl_set_busy(&io->scsiio);
11361 	ctl_done(io);
11362 }
11363 
11364 static void
11365 ctl_failover(void)
11366 {
11367 	struct ctl_lun *lun;
11368 	struct ctl_softc *softc;
11369 	union ctl_io *next_io, *pending_io;
11370 	union ctl_io *io;
11371 	int lun_idx;
11372 
11373 	softc = control_softc;
11374 
11375 	mtx_lock(&softc->ctl_lock);
11376 	/*
11377 	 * Remove any cmds from the other SC from the rtr queue.  These
11378 	 * will obviously only be for LUNs for which we're the primary.
11379 	 * We can't send status or get/send data for these commands.
11380 	 * Since they haven't been executed yet, we can just remove them.
11381 	 * We'll either abort them or delete them below, depending on
11382 	 * which HA mode we're in.
11383 	 */
11384 #ifdef notyet
11385 	mtx_lock(&softc->queue_lock);
11386 	for (io = (union ctl_io *)STAILQ_FIRST(&softc->rtr_queue);
11387 	     io != NULL; io = next_io) {
11388 		next_io = (union ctl_io *)STAILQ_NEXT(&io->io_hdr, links);
11389 		if (io->io_hdr.flags & CTL_FLAG_FROM_OTHER_SC)
11390 			STAILQ_REMOVE(&softc->rtr_queue, &io->io_hdr,
11391 				      ctl_io_hdr, links);
11392 	}
11393 	mtx_unlock(&softc->queue_lock);
11394 #endif
11395 
11396 	for (lun_idx=0; lun_idx < softc->num_luns; lun_idx++) {
11397 		lun = softc->ctl_luns[lun_idx];
11398 		if (lun==NULL)
11399 			continue;
11400 
11401 		/*
11402 		 * Processor LUNs are primary on both sides.
11403 		 * XXX will this always be true?
11404 		 */
11405 		if (lun->be_lun->lun_type == T_PROCESSOR)
11406 			continue;
11407 
11408 		if ((lun->flags & CTL_LUN_PRIMARY_SC)
11409 		 && (softc->ha_mode == CTL_HA_MODE_SER_ONLY)) {
11410 			printf("FAILOVER: primary lun %d\n", lun_idx);
11411 		        /*
11412 			 * Remove all commands from the other SC. First from the
11413 			 * blocked queue then from the ooa queue. Once we have
11414 			 * removed them. Call ctl_check_blocked to see if there
11415 			 * is anything that can run.
11416 			 */
11417 			for (io = (union ctl_io *)TAILQ_FIRST(
11418 			     &lun->blocked_queue); io != NULL; io = next_io) {
11419 
11420 		        	next_io = (union ctl_io *)TAILQ_NEXT(
11421 				    &io->io_hdr, blocked_links);
11422 
11423 				if (io->io_hdr.flags & CTL_FLAG_FROM_OTHER_SC) {
11424 					TAILQ_REMOVE(&lun->blocked_queue,
11425 						     &io->io_hdr,blocked_links);
11426 					io->io_hdr.flags &= ~CTL_FLAG_BLOCKED;
11427 					TAILQ_REMOVE(&lun->ooa_queue,
11428 						     &io->io_hdr, ooa_links);
11429 
11430 					ctl_free_io(io);
11431 				}
11432 			}
11433 
11434 			for (io = (union ctl_io *)TAILQ_FIRST(&lun->ooa_queue);
11435 	     		     io != NULL; io = next_io) {
11436 
11437 		        	next_io = (union ctl_io *)TAILQ_NEXT(
11438 				    &io->io_hdr, ooa_links);
11439 
11440 				if (io->io_hdr.flags & CTL_FLAG_FROM_OTHER_SC) {
11441 
11442 					TAILQ_REMOVE(&lun->ooa_queue,
11443 						&io->io_hdr,
11444 					     	ooa_links);
11445 
11446 					ctl_free_io(io);
11447 				}
11448 			}
11449 			ctl_check_blocked(lun);
11450 		} else if ((lun->flags & CTL_LUN_PRIMARY_SC)
11451 			&& (softc->ha_mode == CTL_HA_MODE_XFER)) {
11452 
11453 			printf("FAILOVER: primary lun %d\n", lun_idx);
11454 			/*
11455 			 * Abort all commands from the other SC.  We can't
11456 			 * send status back for them now.  These should get
11457 			 * cleaned up when they are completed or come out
11458 			 * for a datamove operation.
11459 			 */
11460 			for (io = (union ctl_io *)TAILQ_FIRST(&lun->ooa_queue);
11461 	     		     io != NULL; io = next_io) {
11462 		        	next_io = (union ctl_io *)TAILQ_NEXT(
11463 					&io->io_hdr, ooa_links);
11464 
11465 				if (io->io_hdr.flags & CTL_FLAG_FROM_OTHER_SC)
11466 					io->io_hdr.flags |= CTL_FLAG_ABORT;
11467 			}
11468 		} else if (((lun->flags & CTL_LUN_PRIMARY_SC) == 0)
11469 			&& (softc->ha_mode == CTL_HA_MODE_XFER)) {
11470 
11471 			printf("FAILOVER: secondary lun %d\n", lun_idx);
11472 
11473 			lun->flags |= CTL_LUN_PRIMARY_SC;
11474 
11475 			/*
11476 			 * We send all I/O that was sent to this controller
11477 			 * and redirected to the other side back with
11478 			 * busy status, and have the initiator retry it.
11479 			 * Figuring out how much data has been transferred,
11480 			 * etc. and picking up where we left off would be
11481 			 * very tricky.
11482 			 *
11483 			 * XXX KDM need to remove I/O from the blocked
11484 			 * queue as well!
11485 			 */
11486 			for (pending_io = (union ctl_io *)TAILQ_FIRST(
11487 			     &lun->ooa_queue); pending_io != NULL;
11488 			     pending_io = next_io) {
11489 
11490 				next_io =  (union ctl_io *)TAILQ_NEXT(
11491 					&pending_io->io_hdr, ooa_links);
11492 
11493 				pending_io->io_hdr.flags &=
11494 					~CTL_FLAG_SENT_2OTHER_SC;
11495 
11496 				if (pending_io->io_hdr.flags &
11497 				    CTL_FLAG_IO_ACTIVE) {
11498 					pending_io->io_hdr.flags |=
11499 						CTL_FLAG_FAILOVER;
11500 				} else {
11501 					ctl_set_busy(&pending_io->scsiio);
11502 					ctl_done(pending_io);
11503 				}
11504 			}
11505 
11506 			ctl_est_ua_all(lun, -1, CTL_UA_ASYM_ACC_CHANGE);
11507 		} else if (((lun->flags & CTL_LUN_PRIMARY_SC) == 0)
11508 			&& (softc->ha_mode == CTL_HA_MODE_SER_ONLY)) {
11509 			printf("FAILOVER: secondary lun %d\n", lun_idx);
11510 			/*
11511 			 * if the first io on the OOA is not on the RtR queue
11512 			 * add it.
11513 			 */
11514 			lun->flags |= CTL_LUN_PRIMARY_SC;
11515 
11516 			pending_io = (union ctl_io *)TAILQ_FIRST(
11517 			    &lun->ooa_queue);
11518 			if (pending_io==NULL) {
11519 				printf("Nothing on OOA queue\n");
11520 				continue;
11521 			}
11522 
11523 			pending_io->io_hdr.flags &= ~CTL_FLAG_SENT_2OTHER_SC;
11524 			if ((pending_io->io_hdr.flags &
11525 			     CTL_FLAG_IS_WAS_ON_RTR) == 0) {
11526 				pending_io->io_hdr.flags |=
11527 				    CTL_FLAG_IS_WAS_ON_RTR;
11528 				ctl_enqueue_rtr(pending_io);
11529 			}
11530 #if 0
11531 			else
11532 			{
11533 				printf("Tag 0x%04x is running\n",
11534 				      pending_io->scsiio.tag_num);
11535 			}
11536 #endif
11537 
11538 			next_io = (union ctl_io *)TAILQ_NEXT(
11539 			    &pending_io->io_hdr, ooa_links);
11540 			for (pending_io=next_io; pending_io != NULL;
11541 			     pending_io = next_io) {
11542 				pending_io->io_hdr.flags &=
11543 				    ~CTL_FLAG_SENT_2OTHER_SC;
11544 				next_io = (union ctl_io *)TAILQ_NEXT(
11545 					&pending_io->io_hdr, ooa_links);
11546 				if (pending_io->io_hdr.flags &
11547 				    CTL_FLAG_IS_WAS_ON_RTR) {
11548 #if 0
11549 				        printf("Tag 0x%04x is running\n",
11550 				      		pending_io->scsiio.tag_num);
11551 #endif
11552 					continue;
11553 				}
11554 
11555 				switch (ctl_check_ooa(lun, pending_io,
11556 			            (union ctl_io *)TAILQ_PREV(
11557 				    &pending_io->io_hdr, ctl_ooaq,
11558 				    ooa_links))) {
11559 
11560 				case CTL_ACTION_BLOCK:
11561 					TAILQ_INSERT_TAIL(&lun->blocked_queue,
11562 							  &pending_io->io_hdr,
11563 							  blocked_links);
11564 					pending_io->io_hdr.flags |=
11565 					    CTL_FLAG_BLOCKED;
11566 					break;
11567 				case CTL_ACTION_PASS:
11568 				case CTL_ACTION_SKIP:
11569 					pending_io->io_hdr.flags |=
11570 					    CTL_FLAG_IS_WAS_ON_RTR;
11571 					ctl_enqueue_rtr(pending_io);
11572 					break;
11573 				case CTL_ACTION_OVERLAP:
11574 					ctl_set_overlapped_cmd(
11575 					    (struct ctl_scsiio *)pending_io);
11576 					ctl_done(pending_io);
11577 					break;
11578 				case CTL_ACTION_OVERLAP_TAG:
11579 					ctl_set_overlapped_tag(
11580 					    (struct ctl_scsiio *)pending_io,
11581 					    pending_io->scsiio.tag_num & 0xff);
11582 					ctl_done(pending_io);
11583 					break;
11584 				case CTL_ACTION_ERROR:
11585 				default:
11586 					ctl_set_internal_failure(
11587 						(struct ctl_scsiio *)pending_io,
11588 						0,  // sks_valid
11589 						0); //retry count
11590 					ctl_done(pending_io);
11591 					break;
11592 				}
11593 			}
11594 
11595 			ctl_est_ua_all(lun, -1, CTL_UA_ASYM_ACC_CHANGE);
11596 		} else {
11597 			panic("Unhandled HA mode failover, LUN flags = %#x, "
11598 			      "ha_mode = #%x", lun->flags, softc->ha_mode);
11599 		}
11600 	}
11601 	ctl_pause_rtr = 0;
11602 	mtx_unlock(&softc->ctl_lock);
11603 }
11604 
11605 static void
11606 ctl_clear_ua(struct ctl_softc *ctl_softc, uint32_t initidx,
11607 	     ctl_ua_type ua_type)
11608 {
11609 	struct ctl_lun *lun;
11610 	ctl_ua_type *pu;
11611 
11612 	mtx_assert(&ctl_softc->ctl_lock, MA_OWNED);
11613 
11614 	STAILQ_FOREACH(lun, &ctl_softc->lun_list, links) {
11615 		mtx_lock(&lun->lun_lock);
11616 		pu = lun->pending_ua[initidx / CTL_MAX_INIT_PER_PORT];
11617 		if (pu != NULL)
11618 			pu[initidx % CTL_MAX_INIT_PER_PORT] &= ~ua_type;
11619 		mtx_unlock(&lun->lun_lock);
11620 	}
11621 }
11622 
11623 static int
11624 ctl_scsiio_precheck(struct ctl_softc *softc, struct ctl_scsiio *ctsio)
11625 {
11626 	struct ctl_lun *lun;
11627 	const struct ctl_cmd_entry *entry;
11628 	uint32_t initidx, targ_lun;
11629 	int retval;
11630 
11631 	retval = 0;
11632 
11633 	lun = NULL;
11634 
11635 	targ_lun = ctsio->io_hdr.nexus.targ_mapped_lun;
11636 	if ((targ_lun < CTL_MAX_LUNS)
11637 	 && ((lun = softc->ctl_luns[targ_lun]) != NULL)) {
11638 		/*
11639 		 * If the LUN is invalid, pretend that it doesn't exist.
11640 		 * It will go away as soon as all pending I/O has been
11641 		 * completed.
11642 		 */
11643 		mtx_lock(&lun->lun_lock);
11644 		if (lun->flags & CTL_LUN_DISABLED) {
11645 			mtx_unlock(&lun->lun_lock);
11646 			lun = NULL;
11647 			ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr = NULL;
11648 			ctsio->io_hdr.ctl_private[CTL_PRIV_BACKEND_LUN].ptr = NULL;
11649 		} else {
11650 			ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr = lun;
11651 			ctsio->io_hdr.ctl_private[CTL_PRIV_BACKEND_LUN].ptr =
11652 				lun->be_lun;
11653 			if (lun->be_lun->lun_type == T_PROCESSOR) {
11654 				ctsio->io_hdr.flags |= CTL_FLAG_CONTROL_DEV;
11655 			}
11656 
11657 			/*
11658 			 * Every I/O goes into the OOA queue for a
11659 			 * particular LUN, and stays there until completion.
11660 			 */
11661 #ifdef CTL_TIME_IO
11662 			if (TAILQ_EMPTY(&lun->ooa_queue)) {
11663 				lun->idle_time += getsbinuptime() -
11664 				    lun->last_busy;
11665 			}
11666 #endif
11667 			TAILQ_INSERT_TAIL(&lun->ooa_queue, &ctsio->io_hdr,
11668 			    ooa_links);
11669 		}
11670 	} else {
11671 		ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr = NULL;
11672 		ctsio->io_hdr.ctl_private[CTL_PRIV_BACKEND_LUN].ptr = NULL;
11673 	}
11674 
11675 	/* Get command entry and return error if it is unsuppotyed. */
11676 	entry = ctl_validate_command(ctsio);
11677 	if (entry == NULL) {
11678 		if (lun)
11679 			mtx_unlock(&lun->lun_lock);
11680 		return (retval);
11681 	}
11682 
11683 	ctsio->io_hdr.flags &= ~CTL_FLAG_DATA_MASK;
11684 	ctsio->io_hdr.flags |= entry->flags & CTL_FLAG_DATA_MASK;
11685 
11686 	/*
11687 	 * Check to see whether we can send this command to LUNs that don't
11688 	 * exist.  This should pretty much only be the case for inquiry
11689 	 * and request sense.  Further checks, below, really require having
11690 	 * a LUN, so we can't really check the command anymore.  Just put
11691 	 * it on the rtr queue.
11692 	 */
11693 	if (lun == NULL) {
11694 		if (entry->flags & CTL_CMD_FLAG_OK_ON_ALL_LUNS) {
11695 			ctsio->io_hdr.flags |= CTL_FLAG_IS_WAS_ON_RTR;
11696 			ctl_enqueue_rtr((union ctl_io *)ctsio);
11697 			return (retval);
11698 		}
11699 
11700 		ctl_set_unsupported_lun(ctsio);
11701 		ctl_done((union ctl_io *)ctsio);
11702 		CTL_DEBUG_PRINT(("ctl_scsiio_precheck: bailing out due to invalid LUN\n"));
11703 		return (retval);
11704 	} else {
11705 		/*
11706 		 * Make sure we support this particular command on this LUN.
11707 		 * e.g., we don't support writes to the control LUN.
11708 		 */
11709 		if (!ctl_cmd_applicable(lun->be_lun->lun_type, entry)) {
11710 			mtx_unlock(&lun->lun_lock);
11711 			ctl_set_invalid_opcode(ctsio);
11712 			ctl_done((union ctl_io *)ctsio);
11713 			return (retval);
11714 		}
11715 	}
11716 
11717 	initidx = ctl_get_initindex(&ctsio->io_hdr.nexus);
11718 
11719 #ifdef CTL_WITH_CA
11720 	/*
11721 	 * If we've got a request sense, it'll clear the contingent
11722 	 * allegiance condition.  Otherwise, if we have a CA condition for
11723 	 * this initiator, clear it, because it sent down a command other
11724 	 * than request sense.
11725 	 */
11726 	if ((ctsio->cdb[0] != REQUEST_SENSE)
11727 	 && (ctl_is_set(lun->have_ca, initidx)))
11728 		ctl_clear_mask(lun->have_ca, initidx);
11729 #endif
11730 
11731 	/*
11732 	 * If the command has this flag set, it handles its own unit
11733 	 * attention reporting, we shouldn't do anything.  Otherwise we
11734 	 * check for any pending unit attentions, and send them back to the
11735 	 * initiator.  We only do this when a command initially comes in,
11736 	 * not when we pull it off the blocked queue.
11737 	 *
11738 	 * According to SAM-3, section 5.3.2, the order that things get
11739 	 * presented back to the host is basically unit attentions caused
11740 	 * by some sort of reset event, busy status, reservation conflicts
11741 	 * or task set full, and finally any other status.
11742 	 *
11743 	 * One issue here is that some of the unit attentions we report
11744 	 * don't fall into the "reset" category (e.g. "reported luns data
11745 	 * has changed").  So reporting it here, before the reservation
11746 	 * check, may be technically wrong.  I guess the only thing to do
11747 	 * would be to check for and report the reset events here, and then
11748 	 * check for the other unit attention types after we check for a
11749 	 * reservation conflict.
11750 	 *
11751 	 * XXX KDM need to fix this
11752 	 */
11753 	if ((entry->flags & CTL_CMD_FLAG_NO_SENSE) == 0) {
11754 		ctl_ua_type ua_type;
11755 		scsi_sense_data_type sense_format;
11756 
11757 		if (lun->flags & CTL_LUN_SENSE_DESC)
11758 			sense_format = SSD_TYPE_DESC;
11759 		else
11760 			sense_format = SSD_TYPE_FIXED;
11761 
11762 		ua_type = ctl_build_ua(lun, initidx, &ctsio->sense_data,
11763 		    sense_format);
11764 		if (ua_type != CTL_UA_NONE) {
11765 			mtx_unlock(&lun->lun_lock);
11766 			ctsio->scsi_status = SCSI_STATUS_CHECK_COND;
11767 			ctsio->io_hdr.status = CTL_SCSI_ERROR | CTL_AUTOSENSE;
11768 			ctsio->sense_len = SSD_FULL_SIZE;
11769 			ctl_done((union ctl_io *)ctsio);
11770 			return (retval);
11771 		}
11772 	}
11773 
11774 
11775 	if (ctl_scsiio_lun_check(lun, entry, ctsio) != 0) {
11776 		mtx_unlock(&lun->lun_lock);
11777 		ctl_done((union ctl_io *)ctsio);
11778 		return (retval);
11779 	}
11780 
11781 	/*
11782 	 * XXX CHD this is where we want to send IO to other side if
11783 	 * this LUN is secondary on this SC. We will need to make a copy
11784 	 * of the IO and flag the IO on this side as SENT_2OTHER and the flag
11785 	 * the copy we send as FROM_OTHER.
11786 	 * We also need to stuff the address of the original IO so we can
11787 	 * find it easily. Something similar will need be done on the other
11788 	 * side so when we are done we can find the copy.
11789 	 */
11790 	if ((lun->flags & CTL_LUN_PRIMARY_SC) == 0) {
11791 		union ctl_ha_msg msg_info;
11792 		int isc_retval;
11793 
11794 		ctsio->io_hdr.flags |= CTL_FLAG_SENT_2OTHER_SC;
11795 
11796 		msg_info.hdr.msg_type = CTL_MSG_SERIALIZE;
11797 		msg_info.hdr.original_sc = (union ctl_io *)ctsio;
11798 #if 0
11799 		printf("1. ctsio %p\n", ctsio);
11800 #endif
11801 		msg_info.hdr.serializing_sc = NULL;
11802 		msg_info.hdr.nexus = ctsio->io_hdr.nexus;
11803 		msg_info.scsi.tag_num = ctsio->tag_num;
11804 		msg_info.scsi.tag_type = ctsio->tag_type;
11805 		memcpy(msg_info.scsi.cdb, ctsio->cdb, CTL_MAX_CDBLEN);
11806 
11807 		ctsio->io_hdr.flags &= ~CTL_FLAG_IO_ACTIVE;
11808 
11809 		if ((isc_retval=ctl_ha_msg_send(CTL_HA_CHAN_CTL,
11810 		    (void *)&msg_info, sizeof(msg_info), 0)) >
11811 		    CTL_HA_STATUS_SUCCESS) {
11812 			printf("CTL:precheck, ctl_ha_msg_send returned %d\n",
11813 			       isc_retval);
11814 			printf("CTL:opcode is %x\n", ctsio->cdb[0]);
11815 		} else {
11816 #if 0
11817 			printf("CTL:Precheck sent msg, opcode is %x\n",opcode);
11818 #endif
11819 		}
11820 
11821 		/*
11822 		 * XXX KDM this I/O is off the incoming queue, but hasn't
11823 		 * been inserted on any other queue.  We may need to come
11824 		 * up with a holding queue while we wait for serialization
11825 		 * so that we have an idea of what we're waiting for from
11826 		 * the other side.
11827 		 */
11828 		mtx_unlock(&lun->lun_lock);
11829 		return (retval);
11830 	}
11831 
11832 	switch (ctl_check_ooa(lun, (union ctl_io *)ctsio,
11833 			      (union ctl_io *)TAILQ_PREV(&ctsio->io_hdr,
11834 			      ctl_ooaq, ooa_links))) {
11835 	case CTL_ACTION_BLOCK:
11836 		ctsio->io_hdr.flags |= CTL_FLAG_BLOCKED;
11837 		TAILQ_INSERT_TAIL(&lun->blocked_queue, &ctsio->io_hdr,
11838 				  blocked_links);
11839 		mtx_unlock(&lun->lun_lock);
11840 		return (retval);
11841 	case CTL_ACTION_PASS:
11842 	case CTL_ACTION_SKIP:
11843 		ctsio->io_hdr.flags |= CTL_FLAG_IS_WAS_ON_RTR;
11844 		mtx_unlock(&lun->lun_lock);
11845 		ctl_enqueue_rtr((union ctl_io *)ctsio);
11846 		break;
11847 	case CTL_ACTION_OVERLAP:
11848 		mtx_unlock(&lun->lun_lock);
11849 		ctl_set_overlapped_cmd(ctsio);
11850 		ctl_done((union ctl_io *)ctsio);
11851 		break;
11852 	case CTL_ACTION_OVERLAP_TAG:
11853 		mtx_unlock(&lun->lun_lock);
11854 		ctl_set_overlapped_tag(ctsio, ctsio->tag_num & 0xff);
11855 		ctl_done((union ctl_io *)ctsio);
11856 		break;
11857 	case CTL_ACTION_ERROR:
11858 	default:
11859 		mtx_unlock(&lun->lun_lock);
11860 		ctl_set_internal_failure(ctsio,
11861 					 /*sks_valid*/ 0,
11862 					 /*retry_count*/ 0);
11863 		ctl_done((union ctl_io *)ctsio);
11864 		break;
11865 	}
11866 	return (retval);
11867 }
11868 
11869 const struct ctl_cmd_entry *
11870 ctl_get_cmd_entry(struct ctl_scsiio *ctsio, int *sa)
11871 {
11872 	const struct ctl_cmd_entry *entry;
11873 	int service_action;
11874 
11875 	entry = &ctl_cmd_table[ctsio->cdb[0]];
11876 	if (sa)
11877 		*sa = ((entry->flags & CTL_CMD_FLAG_SA5) != 0);
11878 	if (entry->flags & CTL_CMD_FLAG_SA5) {
11879 		service_action = ctsio->cdb[1] & SERVICE_ACTION_MASK;
11880 		entry = &((const struct ctl_cmd_entry *)
11881 		    entry->execute)[service_action];
11882 	}
11883 	return (entry);
11884 }
11885 
11886 const struct ctl_cmd_entry *
11887 ctl_validate_command(struct ctl_scsiio *ctsio)
11888 {
11889 	const struct ctl_cmd_entry *entry;
11890 	int i, sa;
11891 	uint8_t diff;
11892 
11893 	entry = ctl_get_cmd_entry(ctsio, &sa);
11894 	if (entry->execute == NULL) {
11895 		if (sa)
11896 			ctl_set_invalid_field(ctsio,
11897 					      /*sks_valid*/ 1,
11898 					      /*command*/ 1,
11899 					      /*field*/ 1,
11900 					      /*bit_valid*/ 1,
11901 					      /*bit*/ 4);
11902 		else
11903 			ctl_set_invalid_opcode(ctsio);
11904 		ctl_done((union ctl_io *)ctsio);
11905 		return (NULL);
11906 	}
11907 	KASSERT(entry->length > 0,
11908 	    ("Not defined length for command 0x%02x/0x%02x",
11909 	     ctsio->cdb[0], ctsio->cdb[1]));
11910 	for (i = 1; i < entry->length; i++) {
11911 		diff = ctsio->cdb[i] & ~entry->usage[i - 1];
11912 		if (diff == 0)
11913 			continue;
11914 		ctl_set_invalid_field(ctsio,
11915 				      /*sks_valid*/ 1,
11916 				      /*command*/ 1,
11917 				      /*field*/ i,
11918 				      /*bit_valid*/ 1,
11919 				      /*bit*/ fls(diff) - 1);
11920 		ctl_done((union ctl_io *)ctsio);
11921 		return (NULL);
11922 	}
11923 	return (entry);
11924 }
11925 
11926 static int
11927 ctl_cmd_applicable(uint8_t lun_type, const struct ctl_cmd_entry *entry)
11928 {
11929 
11930 	switch (lun_type) {
11931 	case T_PROCESSOR:
11932 		if (((entry->flags & CTL_CMD_FLAG_OK_ON_PROC) == 0) &&
11933 		    ((entry->flags & CTL_CMD_FLAG_OK_ON_ALL_LUNS) == 0))
11934 			return (0);
11935 		break;
11936 	case T_DIRECT:
11937 		if (((entry->flags & CTL_CMD_FLAG_OK_ON_SLUN) == 0) &&
11938 		    ((entry->flags & CTL_CMD_FLAG_OK_ON_ALL_LUNS) == 0))
11939 			return (0);
11940 		break;
11941 	default:
11942 		return (0);
11943 	}
11944 	return (1);
11945 }
11946 
11947 static int
11948 ctl_scsiio(struct ctl_scsiio *ctsio)
11949 {
11950 	int retval;
11951 	const struct ctl_cmd_entry *entry;
11952 
11953 	retval = CTL_RETVAL_COMPLETE;
11954 
11955 	CTL_DEBUG_PRINT(("ctl_scsiio cdb[0]=%02X\n", ctsio->cdb[0]));
11956 
11957 	entry = ctl_get_cmd_entry(ctsio, NULL);
11958 
11959 	/*
11960 	 * If this I/O has been aborted, just send it straight to
11961 	 * ctl_done() without executing it.
11962 	 */
11963 	if (ctsio->io_hdr.flags & CTL_FLAG_ABORT) {
11964 		ctl_done((union ctl_io *)ctsio);
11965 		goto bailout;
11966 	}
11967 
11968 	/*
11969 	 * All the checks should have been handled by ctl_scsiio_precheck().
11970 	 * We should be clear now to just execute the I/O.
11971 	 */
11972 	retval = entry->execute(ctsio);
11973 
11974 bailout:
11975 	return (retval);
11976 }
11977 
11978 /*
11979  * Since we only implement one target right now, a bus reset simply resets
11980  * our single target.
11981  */
11982 static int
11983 ctl_bus_reset(struct ctl_softc *softc, union ctl_io *io)
11984 {
11985 	return(ctl_target_reset(softc, io, CTL_UA_BUS_RESET));
11986 }
11987 
11988 static int
11989 ctl_target_reset(struct ctl_softc *softc, union ctl_io *io,
11990 		 ctl_ua_type ua_type)
11991 {
11992 	struct ctl_lun *lun;
11993 	int retval;
11994 
11995 	if (!(io->io_hdr.flags & CTL_FLAG_FROM_OTHER_SC)) {
11996 		union ctl_ha_msg msg_info;
11997 
11998 		io->io_hdr.flags |= CTL_FLAG_SENT_2OTHER_SC;
11999 		msg_info.hdr.nexus = io->io_hdr.nexus;
12000 		if (ua_type==CTL_UA_TARG_RESET)
12001 			msg_info.task.task_action = CTL_TASK_TARGET_RESET;
12002 		else
12003 			msg_info.task.task_action = CTL_TASK_BUS_RESET;
12004 		msg_info.hdr.msg_type = CTL_MSG_MANAGE_TASKS;
12005 		msg_info.hdr.original_sc = NULL;
12006 		msg_info.hdr.serializing_sc = NULL;
12007 		if (CTL_HA_STATUS_SUCCESS != ctl_ha_msg_send(CTL_HA_CHAN_CTL,
12008 		    (void *)&msg_info, sizeof(msg_info), 0)) {
12009 		}
12010 	}
12011 	retval = 0;
12012 
12013 	mtx_lock(&softc->ctl_lock);
12014 	STAILQ_FOREACH(lun, &softc->lun_list, links)
12015 		retval += ctl_lun_reset(lun, io, ua_type);
12016 	mtx_unlock(&softc->ctl_lock);
12017 
12018 	return (retval);
12019 }
12020 
12021 /*
12022  * The LUN should always be set.  The I/O is optional, and is used to
12023  * distinguish between I/Os sent by this initiator, and by other
12024  * initiators.  We set unit attention for initiators other than this one.
12025  * SAM-3 is vague on this point.  It does say that a unit attention should
12026  * be established for other initiators when a LUN is reset (see section
12027  * 5.7.3), but it doesn't specifically say that the unit attention should
12028  * be established for this particular initiator when a LUN is reset.  Here
12029  * is the relevant text, from SAM-3 rev 8:
12030  *
12031  * 5.7.2 When a SCSI initiator port aborts its own tasks
12032  *
12033  * When a SCSI initiator port causes its own task(s) to be aborted, no
12034  * notification that the task(s) have been aborted shall be returned to
12035  * the SCSI initiator port other than the completion response for the
12036  * command or task management function action that caused the task(s) to
12037  * be aborted and notification(s) associated with related effects of the
12038  * action (e.g., a reset unit attention condition).
12039  *
12040  * XXX KDM for now, we're setting unit attention for all initiators.
12041  */
12042 static int
12043 ctl_lun_reset(struct ctl_lun *lun, union ctl_io *io, ctl_ua_type ua_type)
12044 {
12045 	union ctl_io *xio;
12046 #if 0
12047 	uint32_t initidx;
12048 #endif
12049 #ifdef CTL_WITH_CA
12050 	int i;
12051 #endif
12052 
12053 	mtx_lock(&lun->lun_lock);
12054 	/*
12055 	 * Run through the OOA queue and abort each I/O.
12056 	 */
12057 	for (xio = (union ctl_io *)TAILQ_FIRST(&lun->ooa_queue); xio != NULL;
12058 	     xio = (union ctl_io *)TAILQ_NEXT(&xio->io_hdr, ooa_links)) {
12059 		xio->io_hdr.flags |= CTL_FLAG_ABORT | CTL_FLAG_ABORT_STATUS;
12060 	}
12061 
12062 	/*
12063 	 * This version sets unit attention for every
12064 	 */
12065 #if 0
12066 	initidx = ctl_get_initindex(&io->io_hdr.nexus);
12067 	ctl_est_ua_all(lun, initidx, ua_type);
12068 #else
12069 	ctl_est_ua_all(lun, -1, ua_type);
12070 #endif
12071 
12072 	/*
12073 	 * A reset (any kind, really) clears reservations established with
12074 	 * RESERVE/RELEASE.  It does not clear reservations established
12075 	 * with PERSISTENT RESERVE OUT, but we don't support that at the
12076 	 * moment anyway.  See SPC-2, section 5.6.  SPC-3 doesn't address
12077 	 * reservations made with the RESERVE/RELEASE commands, because
12078 	 * those commands are obsolete in SPC-3.
12079 	 */
12080 	lun->flags &= ~CTL_LUN_RESERVED;
12081 
12082 #ifdef CTL_WITH_CA
12083 	for (i = 0; i < CTL_MAX_INITIATORS; i++)
12084 		ctl_clear_mask(lun->have_ca, i);
12085 #endif
12086 	mtx_unlock(&lun->lun_lock);
12087 
12088 	return (0);
12089 }
12090 
12091 static void
12092 ctl_abort_tasks_lun(struct ctl_lun *lun, uint32_t targ_port, uint32_t init_id,
12093     int other_sc)
12094 {
12095 	union ctl_io *xio;
12096 
12097 	mtx_assert(&lun->lun_lock, MA_OWNED);
12098 
12099 	/*
12100 	 * Run through the OOA queue and attempt to find the given I/O.
12101 	 * The target port, initiator ID, tag type and tag number have to
12102 	 * match the values that we got from the initiator.  If we have an
12103 	 * untagged command to abort, simply abort the first untagged command
12104 	 * we come to.  We only allow one untagged command at a time of course.
12105 	 */
12106 	for (xio = (union ctl_io *)TAILQ_FIRST(&lun->ooa_queue); xio != NULL;
12107 	     xio = (union ctl_io *)TAILQ_NEXT(&xio->io_hdr, ooa_links)) {
12108 
12109 		if ((targ_port == UINT32_MAX ||
12110 		     targ_port == xio->io_hdr.nexus.targ_port) &&
12111 		    (init_id == UINT32_MAX ||
12112 		     init_id == xio->io_hdr.nexus.initid.id)) {
12113 			if (targ_port != xio->io_hdr.nexus.targ_port ||
12114 			    init_id != xio->io_hdr.nexus.initid.id)
12115 				xio->io_hdr.flags |= CTL_FLAG_ABORT_STATUS;
12116 			xio->io_hdr.flags |= CTL_FLAG_ABORT;
12117 			if (!other_sc && !(lun->flags & CTL_LUN_PRIMARY_SC)) {
12118 				union ctl_ha_msg msg_info;
12119 
12120 				msg_info.hdr.nexus = xio->io_hdr.nexus;
12121 				msg_info.task.task_action = CTL_TASK_ABORT_TASK;
12122 				msg_info.task.tag_num = xio->scsiio.tag_num;
12123 				msg_info.task.tag_type = xio->scsiio.tag_type;
12124 				msg_info.hdr.msg_type = CTL_MSG_MANAGE_TASKS;
12125 				msg_info.hdr.original_sc = NULL;
12126 				msg_info.hdr.serializing_sc = NULL;
12127 				ctl_ha_msg_send(CTL_HA_CHAN_CTL,
12128 				    (void *)&msg_info, sizeof(msg_info), 0);
12129 			}
12130 		}
12131 	}
12132 }
12133 
12134 static int
12135 ctl_abort_task_set(union ctl_io *io)
12136 {
12137 	struct ctl_softc *softc = control_softc;
12138 	struct ctl_lun *lun;
12139 	uint32_t targ_lun;
12140 
12141 	/*
12142 	 * Look up the LUN.
12143 	 */
12144 	targ_lun = io->io_hdr.nexus.targ_mapped_lun;
12145 	mtx_lock(&softc->ctl_lock);
12146 	if ((targ_lun < CTL_MAX_LUNS) && (softc->ctl_luns[targ_lun] != NULL))
12147 		lun = softc->ctl_luns[targ_lun];
12148 	else {
12149 		mtx_unlock(&softc->ctl_lock);
12150 		return (1);
12151 	}
12152 
12153 	mtx_lock(&lun->lun_lock);
12154 	mtx_unlock(&softc->ctl_lock);
12155 	if (io->taskio.task_action == CTL_TASK_ABORT_TASK_SET) {
12156 		ctl_abort_tasks_lun(lun, io->io_hdr.nexus.targ_port,
12157 		    io->io_hdr.nexus.initid.id,
12158 		    (io->io_hdr.flags & CTL_FLAG_FROM_OTHER_SC) != 0);
12159 	} else { /* CTL_TASK_CLEAR_TASK_SET */
12160 		ctl_abort_tasks_lun(lun, UINT32_MAX, UINT32_MAX,
12161 		    (io->io_hdr.flags & CTL_FLAG_FROM_OTHER_SC) != 0);
12162 	}
12163 	mtx_unlock(&lun->lun_lock);
12164 	return (0);
12165 }
12166 
12167 static int
12168 ctl_i_t_nexus_reset(union ctl_io *io)
12169 {
12170 	struct ctl_softc *softc = control_softc;
12171 	struct ctl_lun *lun;
12172 	uint32_t initidx, residx;
12173 
12174 	initidx = ctl_get_initindex(&io->io_hdr.nexus);
12175 	residx = ctl_get_resindex(&io->io_hdr.nexus);
12176 	mtx_lock(&softc->ctl_lock);
12177 	STAILQ_FOREACH(lun, &softc->lun_list, links) {
12178 		mtx_lock(&lun->lun_lock);
12179 		ctl_abort_tasks_lun(lun, io->io_hdr.nexus.targ_port,
12180 		    io->io_hdr.nexus.initid.id,
12181 		    (io->io_hdr.flags & CTL_FLAG_FROM_OTHER_SC) != 0);
12182 #ifdef CTL_WITH_CA
12183 		ctl_clear_mask(lun->have_ca, initidx);
12184 #endif
12185 		if ((lun->flags & CTL_LUN_RESERVED) && (lun->res_idx == residx))
12186 			lun->flags &= ~CTL_LUN_RESERVED;
12187 		ctl_est_ua(lun, initidx, CTL_UA_I_T_NEXUS_LOSS);
12188 		mtx_unlock(&lun->lun_lock);
12189 	}
12190 	mtx_unlock(&softc->ctl_lock);
12191 	return (0);
12192 }
12193 
12194 static int
12195 ctl_abort_task(union ctl_io *io)
12196 {
12197 	union ctl_io *xio;
12198 	struct ctl_lun *lun;
12199 	struct ctl_softc *softc;
12200 #if 0
12201 	struct sbuf sb;
12202 	char printbuf[128];
12203 #endif
12204 	int found;
12205 	uint32_t targ_lun;
12206 
12207 	softc = control_softc;
12208 	found = 0;
12209 
12210 	/*
12211 	 * Look up the LUN.
12212 	 */
12213 	targ_lun = io->io_hdr.nexus.targ_mapped_lun;
12214 	mtx_lock(&softc->ctl_lock);
12215 	if ((targ_lun < CTL_MAX_LUNS)
12216 	 && (softc->ctl_luns[targ_lun] != NULL))
12217 		lun = softc->ctl_luns[targ_lun];
12218 	else {
12219 		mtx_unlock(&softc->ctl_lock);
12220 		return (1);
12221 	}
12222 
12223 #if 0
12224 	printf("ctl_abort_task: called for lun %lld, tag %d type %d\n",
12225 	       lun->lun, io->taskio.tag_num, io->taskio.tag_type);
12226 #endif
12227 
12228 	mtx_lock(&lun->lun_lock);
12229 	mtx_unlock(&softc->ctl_lock);
12230 	/*
12231 	 * Run through the OOA queue and attempt to find the given I/O.
12232 	 * The target port, initiator ID, tag type and tag number have to
12233 	 * match the values that we got from the initiator.  If we have an
12234 	 * untagged command to abort, simply abort the first untagged command
12235 	 * we come to.  We only allow one untagged command at a time of course.
12236 	 */
12237 	for (xio = (union ctl_io *)TAILQ_FIRST(&lun->ooa_queue); xio != NULL;
12238 	     xio = (union ctl_io *)TAILQ_NEXT(&xio->io_hdr, ooa_links)) {
12239 #if 0
12240 		sbuf_new(&sb, printbuf, sizeof(printbuf), SBUF_FIXEDLEN);
12241 
12242 		sbuf_printf(&sb, "LUN %lld tag %d type %d%s%s%s%s: ",
12243 			    lun->lun, xio->scsiio.tag_num,
12244 			    xio->scsiio.tag_type,
12245 			    (xio->io_hdr.blocked_links.tqe_prev
12246 			    == NULL) ? "" : " BLOCKED",
12247 			    (xio->io_hdr.flags &
12248 			    CTL_FLAG_DMA_INPROG) ? " DMA" : "",
12249 			    (xio->io_hdr.flags &
12250 			    CTL_FLAG_ABORT) ? " ABORT" : "",
12251 			    (xio->io_hdr.flags &
12252 			    CTL_FLAG_IS_WAS_ON_RTR ? " RTR" : ""));
12253 		ctl_scsi_command_string(&xio->scsiio, NULL, &sb);
12254 		sbuf_finish(&sb);
12255 		printf("%s\n", sbuf_data(&sb));
12256 #endif
12257 
12258 		if ((xio->io_hdr.nexus.targ_port != io->io_hdr.nexus.targ_port)
12259 		 || (xio->io_hdr.nexus.initid.id != io->io_hdr.nexus.initid.id)
12260 		 || (xio->io_hdr.flags & CTL_FLAG_ABORT))
12261 			continue;
12262 
12263 		/*
12264 		 * If the abort says that the task is untagged, the
12265 		 * task in the queue must be untagged.  Otherwise,
12266 		 * we just check to see whether the tag numbers
12267 		 * match.  This is because the QLogic firmware
12268 		 * doesn't pass back the tag type in an abort
12269 		 * request.
12270 		 */
12271 #if 0
12272 		if (((xio->scsiio.tag_type == CTL_TAG_UNTAGGED)
12273 		  && (io->taskio.tag_type == CTL_TAG_UNTAGGED))
12274 		 || (xio->scsiio.tag_num == io->taskio.tag_num))
12275 #endif
12276 		/*
12277 		 * XXX KDM we've got problems with FC, because it
12278 		 * doesn't send down a tag type with aborts.  So we
12279 		 * can only really go by the tag number...
12280 		 * This may cause problems with parallel SCSI.
12281 		 * Need to figure that out!!
12282 		 */
12283 		if (xio->scsiio.tag_num == io->taskio.tag_num) {
12284 			xio->io_hdr.flags |= CTL_FLAG_ABORT;
12285 			found = 1;
12286 			if ((io->io_hdr.flags & CTL_FLAG_FROM_OTHER_SC) == 0 &&
12287 			    !(lun->flags & CTL_LUN_PRIMARY_SC)) {
12288 				union ctl_ha_msg msg_info;
12289 
12290 				io->io_hdr.flags |= CTL_FLAG_SENT_2OTHER_SC;
12291 				msg_info.hdr.nexus = io->io_hdr.nexus;
12292 				msg_info.task.task_action = CTL_TASK_ABORT_TASK;
12293 				msg_info.task.tag_num = io->taskio.tag_num;
12294 				msg_info.task.tag_type = io->taskio.tag_type;
12295 				msg_info.hdr.msg_type = CTL_MSG_MANAGE_TASKS;
12296 				msg_info.hdr.original_sc = NULL;
12297 				msg_info.hdr.serializing_sc = NULL;
12298 #if 0
12299 				printf("Sent Abort to other side\n");
12300 #endif
12301 				if (ctl_ha_msg_send(CTL_HA_CHAN_CTL,
12302 				    (void *)&msg_info, sizeof(msg_info), 0) !=
12303 				    CTL_HA_STATUS_SUCCESS) {
12304 				}
12305 			}
12306 #if 0
12307 			printf("ctl_abort_task: found I/O to abort\n");
12308 #endif
12309 		}
12310 	}
12311 	mtx_unlock(&lun->lun_lock);
12312 
12313 	if (found == 0) {
12314 		/*
12315 		 * This isn't really an error.  It's entirely possible for
12316 		 * the abort and command completion to cross on the wire.
12317 		 * This is more of an informative/diagnostic error.
12318 		 */
12319 #if 0
12320 		printf("ctl_abort_task: ABORT sent for nonexistent I/O: "
12321 		       "%d:%d:%d:%d tag %d type %d\n",
12322 		       io->io_hdr.nexus.initid.id,
12323 		       io->io_hdr.nexus.targ_port,
12324 		       io->io_hdr.nexus.targ_target.id,
12325 		       io->io_hdr.nexus.targ_lun, io->taskio.tag_num,
12326 		       io->taskio.tag_type);
12327 #endif
12328 	}
12329 	return (0);
12330 }
12331 
12332 static void
12333 ctl_run_task(union ctl_io *io)
12334 {
12335 	struct ctl_softc *softc = control_softc;
12336 	int retval = 1;
12337 	const char *task_desc;
12338 
12339 	CTL_DEBUG_PRINT(("ctl_run_task\n"));
12340 
12341 	KASSERT(io->io_hdr.io_type == CTL_IO_TASK,
12342 	    ("ctl_run_task: Unextected io_type %d\n",
12343 	     io->io_hdr.io_type));
12344 
12345 	task_desc = ctl_scsi_task_string(&io->taskio);
12346 	if (task_desc != NULL) {
12347 #ifdef NEEDTOPORT
12348 		csevent_log(CSC_CTL | CSC_SHELF_SW |
12349 			    CTL_TASK_REPORT,
12350 			    csevent_LogType_Trace,
12351 			    csevent_Severity_Information,
12352 			    csevent_AlertLevel_Green,
12353 			    csevent_FRU_Firmware,
12354 			    csevent_FRU_Unknown,
12355 			    "CTL: received task: %s",task_desc);
12356 #endif
12357 	} else {
12358 #ifdef NEEDTOPORT
12359 		csevent_log(CSC_CTL | CSC_SHELF_SW |
12360 			    CTL_TASK_REPORT,
12361 			    csevent_LogType_Trace,
12362 			    csevent_Severity_Information,
12363 			    csevent_AlertLevel_Green,
12364 			    csevent_FRU_Firmware,
12365 			    csevent_FRU_Unknown,
12366 			    "CTL: received unknown task "
12367 			    "type: %d (%#x)",
12368 			    io->taskio.task_action,
12369 			    io->taskio.task_action);
12370 #endif
12371 	}
12372 	switch (io->taskio.task_action) {
12373 	case CTL_TASK_ABORT_TASK:
12374 		retval = ctl_abort_task(io);
12375 		break;
12376 	case CTL_TASK_ABORT_TASK_SET:
12377 	case CTL_TASK_CLEAR_TASK_SET:
12378 		retval = ctl_abort_task_set(io);
12379 		break;
12380 	case CTL_TASK_CLEAR_ACA:
12381 		break;
12382 	case CTL_TASK_I_T_NEXUS_RESET:
12383 		retval = ctl_i_t_nexus_reset(io);
12384 		break;
12385 	case CTL_TASK_LUN_RESET: {
12386 		struct ctl_lun *lun;
12387 		uint32_t targ_lun;
12388 
12389 		targ_lun = io->io_hdr.nexus.targ_mapped_lun;
12390 		mtx_lock(&softc->ctl_lock);
12391 		if ((targ_lun < CTL_MAX_LUNS)
12392 		 && (softc->ctl_luns[targ_lun] != NULL))
12393 			lun = softc->ctl_luns[targ_lun];
12394 		else {
12395 			mtx_unlock(&softc->ctl_lock);
12396 			retval = 1;
12397 			break;
12398 		}
12399 
12400 		if (!(io->io_hdr.flags &
12401 		    CTL_FLAG_FROM_OTHER_SC)) {
12402 			union ctl_ha_msg msg_info;
12403 
12404 			io->io_hdr.flags |=
12405 				CTL_FLAG_SENT_2OTHER_SC;
12406 			msg_info.hdr.msg_type =
12407 				CTL_MSG_MANAGE_TASKS;
12408 			msg_info.hdr.nexus = io->io_hdr.nexus;
12409 			msg_info.task.task_action =
12410 				CTL_TASK_LUN_RESET;
12411 			msg_info.hdr.original_sc = NULL;
12412 			msg_info.hdr.serializing_sc = NULL;
12413 			if (CTL_HA_STATUS_SUCCESS !=
12414 			    ctl_ha_msg_send(CTL_HA_CHAN_CTL,
12415 			    (void *)&msg_info,
12416 			    sizeof(msg_info), 0)) {
12417 			}
12418 		}
12419 
12420 		retval = ctl_lun_reset(lun, io,
12421 				       CTL_UA_LUN_RESET);
12422 		mtx_unlock(&softc->ctl_lock);
12423 		break;
12424 	}
12425 	case CTL_TASK_TARGET_RESET:
12426 		retval = ctl_target_reset(softc, io, CTL_UA_TARG_RESET);
12427 		break;
12428 	case CTL_TASK_BUS_RESET:
12429 		retval = ctl_bus_reset(softc, io);
12430 		break;
12431 	case CTL_TASK_PORT_LOGIN:
12432 		break;
12433 	case CTL_TASK_PORT_LOGOUT:
12434 		break;
12435 	default:
12436 		printf("ctl_run_task: got unknown task management event %d\n",
12437 		       io->taskio.task_action);
12438 		break;
12439 	}
12440 	if (retval == 0)
12441 		io->io_hdr.status = CTL_SUCCESS;
12442 	else
12443 		io->io_hdr.status = CTL_ERROR;
12444 	ctl_done(io);
12445 }
12446 
12447 /*
12448  * For HA operation.  Handle commands that come in from the other
12449  * controller.
12450  */
12451 static void
12452 ctl_handle_isc(union ctl_io *io)
12453 {
12454 	int free_io;
12455 	struct ctl_lun *lun;
12456 	struct ctl_softc *softc;
12457 	uint32_t targ_lun;
12458 
12459 	softc = control_softc;
12460 
12461 	targ_lun = io->io_hdr.nexus.targ_mapped_lun;
12462 	lun = softc->ctl_luns[targ_lun];
12463 
12464 	switch (io->io_hdr.msg_type) {
12465 	case CTL_MSG_SERIALIZE:
12466 		free_io = ctl_serialize_other_sc_cmd(&io->scsiio);
12467 		break;
12468 	case CTL_MSG_R2R: {
12469 		const struct ctl_cmd_entry *entry;
12470 
12471 		/*
12472 		 * This is only used in SER_ONLY mode.
12473 		 */
12474 		free_io = 0;
12475 		entry = ctl_get_cmd_entry(&io->scsiio, NULL);
12476 		mtx_lock(&lun->lun_lock);
12477 		if (ctl_scsiio_lun_check(lun,
12478 		    entry, (struct ctl_scsiio *)io) != 0) {
12479 			mtx_unlock(&lun->lun_lock);
12480 			ctl_done(io);
12481 			break;
12482 		}
12483 		io->io_hdr.flags |= CTL_FLAG_IS_WAS_ON_RTR;
12484 		mtx_unlock(&lun->lun_lock);
12485 		ctl_enqueue_rtr(io);
12486 		break;
12487 	}
12488 	case CTL_MSG_FINISH_IO:
12489 		if (softc->ha_mode == CTL_HA_MODE_XFER) {
12490 			free_io = 0;
12491 			ctl_done(io);
12492 		} else {
12493 			free_io = 1;
12494 			mtx_lock(&lun->lun_lock);
12495 			TAILQ_REMOVE(&lun->ooa_queue, &io->io_hdr,
12496 				     ooa_links);
12497 			ctl_check_blocked(lun);
12498 			mtx_unlock(&lun->lun_lock);
12499 		}
12500 		break;
12501 	case CTL_MSG_PERS_ACTION:
12502 		ctl_hndl_per_res_out_on_other_sc(
12503 			(union ctl_ha_msg *)&io->presio.pr_msg);
12504 		free_io = 1;
12505 		break;
12506 	case CTL_MSG_BAD_JUJU:
12507 		free_io = 0;
12508 		ctl_done(io);
12509 		break;
12510 	case CTL_MSG_DATAMOVE:
12511 		/* Only used in XFER mode */
12512 		free_io = 0;
12513 		ctl_datamove_remote(io);
12514 		break;
12515 	case CTL_MSG_DATAMOVE_DONE:
12516 		/* Only used in XFER mode */
12517 		free_io = 0;
12518 		io->scsiio.be_move_done(io);
12519 		break;
12520 	default:
12521 		free_io = 1;
12522 		printf("%s: Invalid message type %d\n",
12523 		       __func__, io->io_hdr.msg_type);
12524 		break;
12525 	}
12526 	if (free_io)
12527 		ctl_free_io(io);
12528 
12529 }
12530 
12531 
12532 /*
12533  * Returns the match type in the case of a match, or CTL_LUN_PAT_NONE if
12534  * there is no match.
12535  */
12536 static ctl_lun_error_pattern
12537 ctl_cmd_pattern_match(struct ctl_scsiio *ctsio, struct ctl_error_desc *desc)
12538 {
12539 	const struct ctl_cmd_entry *entry;
12540 	ctl_lun_error_pattern filtered_pattern, pattern;
12541 
12542 	pattern = desc->error_pattern;
12543 
12544 	/*
12545 	 * XXX KDM we need more data passed into this function to match a
12546 	 * custom pattern, and we actually need to implement custom pattern
12547 	 * matching.
12548 	 */
12549 	if (pattern & CTL_LUN_PAT_CMD)
12550 		return (CTL_LUN_PAT_CMD);
12551 
12552 	if ((pattern & CTL_LUN_PAT_MASK) == CTL_LUN_PAT_ANY)
12553 		return (CTL_LUN_PAT_ANY);
12554 
12555 	entry = ctl_get_cmd_entry(ctsio, NULL);
12556 
12557 	filtered_pattern = entry->pattern & pattern;
12558 
12559 	/*
12560 	 * If the user requested specific flags in the pattern (e.g.
12561 	 * CTL_LUN_PAT_RANGE), make sure the command supports all of those
12562 	 * flags.
12563 	 *
12564 	 * If the user did not specify any flags, it doesn't matter whether
12565 	 * or not the command supports the flags.
12566 	 */
12567 	if ((filtered_pattern & ~CTL_LUN_PAT_MASK) !=
12568 	     (pattern & ~CTL_LUN_PAT_MASK))
12569 		return (CTL_LUN_PAT_NONE);
12570 
12571 	/*
12572 	 * If the user asked for a range check, see if the requested LBA
12573 	 * range overlaps with this command's LBA range.
12574 	 */
12575 	if (filtered_pattern & CTL_LUN_PAT_RANGE) {
12576 		uint64_t lba1;
12577 		uint64_t len1;
12578 		ctl_action action;
12579 		int retval;
12580 
12581 		retval = ctl_get_lba_len((union ctl_io *)ctsio, &lba1, &len1);
12582 		if (retval != 0)
12583 			return (CTL_LUN_PAT_NONE);
12584 
12585 		action = ctl_extent_check_lba(lba1, len1, desc->lba_range.lba,
12586 					      desc->lba_range.len, FALSE);
12587 		/*
12588 		 * A "pass" means that the LBA ranges don't overlap, so
12589 		 * this doesn't match the user's range criteria.
12590 		 */
12591 		if (action == CTL_ACTION_PASS)
12592 			return (CTL_LUN_PAT_NONE);
12593 	}
12594 
12595 	return (filtered_pattern);
12596 }
12597 
12598 static void
12599 ctl_inject_error(struct ctl_lun *lun, union ctl_io *io)
12600 {
12601 	struct ctl_error_desc *desc, *desc2;
12602 
12603 	mtx_assert(&lun->lun_lock, MA_OWNED);
12604 
12605 	STAILQ_FOREACH_SAFE(desc, &lun->error_list, links, desc2) {
12606 		ctl_lun_error_pattern pattern;
12607 		/*
12608 		 * Check to see whether this particular command matches
12609 		 * the pattern in the descriptor.
12610 		 */
12611 		pattern = ctl_cmd_pattern_match(&io->scsiio, desc);
12612 		if ((pattern & CTL_LUN_PAT_MASK) == CTL_LUN_PAT_NONE)
12613 			continue;
12614 
12615 		switch (desc->lun_error & CTL_LUN_INJ_TYPE) {
12616 		case CTL_LUN_INJ_ABORTED:
12617 			ctl_set_aborted(&io->scsiio);
12618 			break;
12619 		case CTL_LUN_INJ_MEDIUM_ERR:
12620 			ctl_set_medium_error(&io->scsiio);
12621 			break;
12622 		case CTL_LUN_INJ_UA:
12623 			/* 29h/00h  POWER ON, RESET, OR BUS DEVICE RESET
12624 			 * OCCURRED */
12625 			ctl_set_ua(&io->scsiio, 0x29, 0x00);
12626 			break;
12627 		case CTL_LUN_INJ_CUSTOM:
12628 			/*
12629 			 * We're assuming the user knows what he is doing.
12630 			 * Just copy the sense information without doing
12631 			 * checks.
12632 			 */
12633 			bcopy(&desc->custom_sense, &io->scsiio.sense_data,
12634 			      MIN(sizeof(desc->custom_sense),
12635 				  sizeof(io->scsiio.sense_data)));
12636 			io->scsiio.scsi_status = SCSI_STATUS_CHECK_COND;
12637 			io->scsiio.sense_len = SSD_FULL_SIZE;
12638 			io->io_hdr.status = CTL_SCSI_ERROR | CTL_AUTOSENSE;
12639 			break;
12640 		case CTL_LUN_INJ_NONE:
12641 		default:
12642 			/*
12643 			 * If this is an error injection type we don't know
12644 			 * about, clear the continuous flag (if it is set)
12645 			 * so it will get deleted below.
12646 			 */
12647 			desc->lun_error &= ~CTL_LUN_INJ_CONTINUOUS;
12648 			break;
12649 		}
12650 		/*
12651 		 * By default, each error injection action is a one-shot
12652 		 */
12653 		if (desc->lun_error & CTL_LUN_INJ_CONTINUOUS)
12654 			continue;
12655 
12656 		STAILQ_REMOVE(&lun->error_list, desc, ctl_error_desc, links);
12657 
12658 		free(desc, M_CTL);
12659 	}
12660 }
12661 
12662 #ifdef CTL_IO_DELAY
12663 static void
12664 ctl_datamove_timer_wakeup(void *arg)
12665 {
12666 	union ctl_io *io;
12667 
12668 	io = (union ctl_io *)arg;
12669 
12670 	ctl_datamove(io);
12671 }
12672 #endif /* CTL_IO_DELAY */
12673 
12674 void
12675 ctl_datamove(union ctl_io *io)
12676 {
12677 	void (*fe_datamove)(union ctl_io *io);
12678 
12679 	mtx_assert(&control_softc->ctl_lock, MA_NOTOWNED);
12680 
12681 	CTL_DEBUG_PRINT(("ctl_datamove\n"));
12682 
12683 #ifdef CTL_TIME_IO
12684 	if ((time_uptime - io->io_hdr.start_time) > ctl_time_io_secs) {
12685 		char str[256];
12686 		char path_str[64];
12687 		struct sbuf sb;
12688 
12689 		ctl_scsi_path_string(io, path_str, sizeof(path_str));
12690 		sbuf_new(&sb, str, sizeof(str), SBUF_FIXEDLEN);
12691 
12692 		sbuf_cat(&sb, path_str);
12693 		switch (io->io_hdr.io_type) {
12694 		case CTL_IO_SCSI:
12695 			ctl_scsi_command_string(&io->scsiio, NULL, &sb);
12696 			sbuf_printf(&sb, "\n");
12697 			sbuf_cat(&sb, path_str);
12698 			sbuf_printf(&sb, "Tag: 0x%04x, type %d\n",
12699 				    io->scsiio.tag_num, io->scsiio.tag_type);
12700 			break;
12701 		case CTL_IO_TASK:
12702 			sbuf_printf(&sb, "Task I/O type: %d, Tag: 0x%04x, "
12703 				    "Tag Type: %d\n", io->taskio.task_action,
12704 				    io->taskio.tag_num, io->taskio.tag_type);
12705 			break;
12706 		default:
12707 			printf("Invalid CTL I/O type %d\n", io->io_hdr.io_type);
12708 			panic("Invalid CTL I/O type %d\n", io->io_hdr.io_type);
12709 			break;
12710 		}
12711 		sbuf_cat(&sb, path_str);
12712 		sbuf_printf(&sb, "ctl_datamove: %jd seconds\n",
12713 			    (intmax_t)time_uptime - io->io_hdr.start_time);
12714 		sbuf_finish(&sb);
12715 		printf("%s", sbuf_data(&sb));
12716 	}
12717 #endif /* CTL_TIME_IO */
12718 
12719 #ifdef CTL_IO_DELAY
12720 	if (io->io_hdr.flags & CTL_FLAG_DELAY_DONE) {
12721 		struct ctl_lun *lun;
12722 
12723 		lun =(struct ctl_lun *)io->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
12724 
12725 		io->io_hdr.flags &= ~CTL_FLAG_DELAY_DONE;
12726 	} else {
12727 		struct ctl_lun *lun;
12728 
12729 		lun =(struct ctl_lun *)io->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
12730 		if ((lun != NULL)
12731 		 && (lun->delay_info.datamove_delay > 0)) {
12732 			struct callout *callout;
12733 
12734 			callout = (struct callout *)&io->io_hdr.timer_bytes;
12735 			callout_init(callout, /*mpsafe*/ 1);
12736 			io->io_hdr.flags |= CTL_FLAG_DELAY_DONE;
12737 			callout_reset(callout,
12738 				      lun->delay_info.datamove_delay * hz,
12739 				      ctl_datamove_timer_wakeup, io);
12740 			if (lun->delay_info.datamove_type ==
12741 			    CTL_DELAY_TYPE_ONESHOT)
12742 				lun->delay_info.datamove_delay = 0;
12743 			return;
12744 		}
12745 	}
12746 #endif
12747 
12748 	/*
12749 	 * This command has been aborted.  Set the port status, so we fail
12750 	 * the data move.
12751 	 */
12752 	if (io->io_hdr.flags & CTL_FLAG_ABORT) {
12753 		printf("ctl_datamove: tag 0x%04x on (%ju:%d:%ju:%d) aborted\n",
12754 		       io->scsiio.tag_num,(uintmax_t)io->io_hdr.nexus.initid.id,
12755 		       io->io_hdr.nexus.targ_port,
12756 		       (uintmax_t)io->io_hdr.nexus.targ_target.id,
12757 		       io->io_hdr.nexus.targ_lun);
12758 		io->io_hdr.port_status = 31337;
12759 		/*
12760 		 * Note that the backend, in this case, will get the
12761 		 * callback in its context.  In other cases it may get
12762 		 * called in the frontend's interrupt thread context.
12763 		 */
12764 		io->scsiio.be_move_done(io);
12765 		return;
12766 	}
12767 
12768 	/* Don't confuse frontend with zero length data move. */
12769 	if (io->scsiio.kern_data_len == 0) {
12770 		io->scsiio.be_move_done(io);
12771 		return;
12772 	}
12773 
12774 	/*
12775 	 * If we're in XFER mode and this I/O is from the other shelf
12776 	 * controller, we need to send the DMA to the other side to
12777 	 * actually transfer the data to/from the host.  In serialize only
12778 	 * mode the transfer happens below CTL and ctl_datamove() is only
12779 	 * called on the machine that originally received the I/O.
12780 	 */
12781 	if ((control_softc->ha_mode == CTL_HA_MODE_XFER)
12782 	 && (io->io_hdr.flags & CTL_FLAG_FROM_OTHER_SC)) {
12783 		union ctl_ha_msg msg;
12784 		uint32_t sg_entries_sent;
12785 		int do_sg_copy;
12786 		int i;
12787 
12788 		memset(&msg, 0, sizeof(msg));
12789 		msg.hdr.msg_type = CTL_MSG_DATAMOVE;
12790 		msg.hdr.original_sc = io->io_hdr.original_sc;
12791 		msg.hdr.serializing_sc = io;
12792 		msg.hdr.nexus = io->io_hdr.nexus;
12793 		msg.dt.flags = io->io_hdr.flags;
12794 		/*
12795 		 * We convert everything into a S/G list here.  We can't
12796 		 * pass by reference, only by value between controllers.
12797 		 * So we can't pass a pointer to the S/G list, only as many
12798 		 * S/G entries as we can fit in here.  If it's possible for
12799 		 * us to get more than CTL_HA_MAX_SG_ENTRIES S/G entries,
12800 		 * then we need to break this up into multiple transfers.
12801 		 */
12802 		if (io->scsiio.kern_sg_entries == 0) {
12803 			msg.dt.kern_sg_entries = 1;
12804 			/*
12805 			 * If this is in cached memory, flush the cache
12806 			 * before we send the DMA request to the other
12807 			 * controller.  We want to do this in either the
12808 			 * read or the write case.  The read case is
12809 			 * straightforward.  In the write case, we want to
12810 			 * make sure nothing is in the local cache that
12811 			 * could overwrite the DMAed data.
12812 			 */
12813 			if ((io->io_hdr.flags & CTL_FLAG_NO_DATASYNC) == 0) {
12814 				/*
12815 				 * XXX KDM use bus_dmamap_sync() here.
12816 				 */
12817 			}
12818 
12819 			/*
12820 			 * Convert to a physical address if this is a
12821 			 * virtual address.
12822 			 */
12823 			if (io->io_hdr.flags & CTL_FLAG_BUS_ADDR) {
12824 				msg.dt.sg_list[0].addr =
12825 					io->scsiio.kern_data_ptr;
12826 			} else {
12827 				/*
12828 				 * XXX KDM use busdma here!
12829 				 */
12830 #if 0
12831 				msg.dt.sg_list[0].addr = (void *)
12832 					vtophys(io->scsiio.kern_data_ptr);
12833 #endif
12834 			}
12835 
12836 			msg.dt.sg_list[0].len = io->scsiio.kern_data_len;
12837 			do_sg_copy = 0;
12838 		} else {
12839 			struct ctl_sg_entry *sgl;
12840 
12841 			do_sg_copy = 1;
12842 			msg.dt.kern_sg_entries = io->scsiio.kern_sg_entries;
12843 			sgl = (struct ctl_sg_entry *)io->scsiio.kern_data_ptr;
12844 			if ((io->io_hdr.flags & CTL_FLAG_NO_DATASYNC) == 0) {
12845 				/*
12846 				 * XXX KDM use bus_dmamap_sync() here.
12847 				 */
12848 			}
12849 		}
12850 
12851 		msg.dt.kern_data_len = io->scsiio.kern_data_len;
12852 		msg.dt.kern_total_len = io->scsiio.kern_total_len;
12853 		msg.dt.kern_data_resid = io->scsiio.kern_data_resid;
12854 		msg.dt.kern_rel_offset = io->scsiio.kern_rel_offset;
12855 		msg.dt.sg_sequence = 0;
12856 
12857 		/*
12858 		 * Loop until we've sent all of the S/G entries.  On the
12859 		 * other end, we'll recompose these S/G entries into one
12860 		 * contiguous list before passing it to the
12861 		 */
12862 		for (sg_entries_sent = 0; sg_entries_sent <
12863 		     msg.dt.kern_sg_entries; msg.dt.sg_sequence++) {
12864 			msg.dt.cur_sg_entries = MIN((sizeof(msg.dt.sg_list)/
12865 				sizeof(msg.dt.sg_list[0])),
12866 				msg.dt.kern_sg_entries - sg_entries_sent);
12867 
12868 			if (do_sg_copy != 0) {
12869 				struct ctl_sg_entry *sgl;
12870 				int j;
12871 
12872 				sgl = (struct ctl_sg_entry *)
12873 					io->scsiio.kern_data_ptr;
12874 				/*
12875 				 * If this is in cached memory, flush the cache
12876 				 * before we send the DMA request to the other
12877 				 * controller.  We want to do this in either
12878 				 * the * read or the write case.  The read
12879 				 * case is straightforward.  In the write
12880 				 * case, we want to make sure nothing is
12881 				 * in the local cache that could overwrite
12882 				 * the DMAed data.
12883 				 */
12884 
12885 				for (i = sg_entries_sent, j = 0;
12886 				     i < msg.dt.cur_sg_entries; i++, j++) {
12887 					if ((io->io_hdr.flags &
12888 					     CTL_FLAG_NO_DATASYNC) == 0) {
12889 						/*
12890 						 * XXX KDM use bus_dmamap_sync()
12891 						 */
12892 					}
12893 					if ((io->io_hdr.flags &
12894 					     CTL_FLAG_BUS_ADDR) == 0) {
12895 						/*
12896 						 * XXX KDM use busdma.
12897 						 */
12898 #if 0
12899 						msg.dt.sg_list[j].addr =(void *)
12900 						       vtophys(sgl[i].addr);
12901 #endif
12902 					} else {
12903 						msg.dt.sg_list[j].addr =
12904 							sgl[i].addr;
12905 					}
12906 					msg.dt.sg_list[j].len = sgl[i].len;
12907 				}
12908 			}
12909 
12910 			sg_entries_sent += msg.dt.cur_sg_entries;
12911 			if (sg_entries_sent >= msg.dt.kern_sg_entries)
12912 				msg.dt.sg_last = 1;
12913 			else
12914 				msg.dt.sg_last = 0;
12915 
12916 			/*
12917 			 * XXX KDM drop and reacquire the lock here?
12918 			 */
12919 			if (ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg,
12920 			    sizeof(msg), 0) > CTL_HA_STATUS_SUCCESS) {
12921 				/*
12922 				 * XXX do something here.
12923 				 */
12924 			}
12925 
12926 			msg.dt.sent_sg_entries = sg_entries_sent;
12927 		}
12928 		io->io_hdr.flags &= ~CTL_FLAG_IO_ACTIVE;
12929 		if (io->io_hdr.flags & CTL_FLAG_FAILOVER)
12930 			ctl_failover_io(io, /*have_lock*/ 0);
12931 
12932 	} else {
12933 
12934 		/*
12935 		 * Lookup the fe_datamove() function for this particular
12936 		 * front end.
12937 		 */
12938 		fe_datamove =
12939 		    control_softc->ctl_ports[ctl_port_idx(io->io_hdr.nexus.targ_port)]->fe_datamove;
12940 
12941 		fe_datamove(io);
12942 	}
12943 }
12944 
12945 static void
12946 ctl_send_datamove_done(union ctl_io *io, int have_lock)
12947 {
12948 	union ctl_ha_msg msg;
12949 	int isc_status;
12950 
12951 	memset(&msg, 0, sizeof(msg));
12952 
12953 	msg.hdr.msg_type = CTL_MSG_DATAMOVE_DONE;
12954 	msg.hdr.original_sc = io;
12955 	msg.hdr.serializing_sc = io->io_hdr.serializing_sc;
12956 	msg.hdr.nexus = io->io_hdr.nexus;
12957 	msg.hdr.status = io->io_hdr.status;
12958 	msg.scsi.tag_num = io->scsiio.tag_num;
12959 	msg.scsi.tag_type = io->scsiio.tag_type;
12960 	msg.scsi.scsi_status = io->scsiio.scsi_status;
12961 	memcpy(&msg.scsi.sense_data, &io->scsiio.sense_data,
12962 	       sizeof(io->scsiio.sense_data));
12963 	msg.scsi.sense_len = io->scsiio.sense_len;
12964 	msg.scsi.sense_residual = io->scsiio.sense_residual;
12965 	msg.scsi.fetd_status = io->io_hdr.port_status;
12966 	msg.scsi.residual = io->scsiio.residual;
12967 	io->io_hdr.flags &= ~CTL_FLAG_IO_ACTIVE;
12968 
12969 	if (io->io_hdr.flags & CTL_FLAG_FAILOVER) {
12970 		ctl_failover_io(io, /*have_lock*/ have_lock);
12971 		return;
12972 	}
12973 
12974 	isc_status = ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg, sizeof(msg), 0);
12975 	if (isc_status > CTL_HA_STATUS_SUCCESS) {
12976 		/* XXX do something if this fails */
12977 	}
12978 
12979 }
12980 
12981 /*
12982  * The DMA to the remote side is done, now we need to tell the other side
12983  * we're done so it can continue with its data movement.
12984  */
12985 static void
12986 ctl_datamove_remote_write_cb(struct ctl_ha_dt_req *rq)
12987 {
12988 	union ctl_io *io;
12989 
12990 	io = rq->context;
12991 
12992 	if (rq->ret != CTL_HA_STATUS_SUCCESS) {
12993 		printf("%s: ISC DMA write failed with error %d", __func__,
12994 		       rq->ret);
12995 		ctl_set_internal_failure(&io->scsiio,
12996 					 /*sks_valid*/ 1,
12997 					 /*retry_count*/ rq->ret);
12998 	}
12999 
13000 	ctl_dt_req_free(rq);
13001 
13002 	/*
13003 	 * In this case, we had to malloc the memory locally.  Free it.
13004 	 */
13005 	if ((io->io_hdr.flags & CTL_FLAG_AUTO_MIRROR) == 0) {
13006 		int i;
13007 		for (i = 0; i < io->scsiio.kern_sg_entries; i++)
13008 			free(io->io_hdr.local_sglist[i].addr, M_CTL);
13009 	}
13010 	/*
13011 	 * The data is in local and remote memory, so now we need to send
13012 	 * status (good or back) back to the other side.
13013 	 */
13014 	ctl_send_datamove_done(io, /*have_lock*/ 0);
13015 }
13016 
13017 /*
13018  * We've moved the data from the host/controller into local memory.  Now we
13019  * need to push it over to the remote controller's memory.
13020  */
13021 static int
13022 ctl_datamove_remote_dm_write_cb(union ctl_io *io)
13023 {
13024 	int retval;
13025 
13026 	retval = 0;
13027 
13028 	retval = ctl_datamove_remote_xfer(io, CTL_HA_DT_CMD_WRITE,
13029 					  ctl_datamove_remote_write_cb);
13030 
13031 	return (retval);
13032 }
13033 
13034 static void
13035 ctl_datamove_remote_write(union ctl_io *io)
13036 {
13037 	int retval;
13038 	void (*fe_datamove)(union ctl_io *io);
13039 
13040 	/*
13041 	 * - Get the data from the host/HBA into local memory.
13042 	 * - DMA memory from the local controller to the remote controller.
13043 	 * - Send status back to the remote controller.
13044 	 */
13045 
13046 	retval = ctl_datamove_remote_sgl_setup(io);
13047 	if (retval != 0)
13048 		return;
13049 
13050 	/* Switch the pointer over so the FETD knows what to do */
13051 	io->scsiio.kern_data_ptr = (uint8_t *)io->io_hdr.local_sglist;
13052 
13053 	/*
13054 	 * Use a custom move done callback, since we need to send completion
13055 	 * back to the other controller, not to the backend on this side.
13056 	 */
13057 	io->scsiio.be_move_done = ctl_datamove_remote_dm_write_cb;
13058 
13059 	fe_datamove = control_softc->ctl_ports[ctl_port_idx(io->io_hdr.nexus.targ_port)]->fe_datamove;
13060 
13061 	fe_datamove(io);
13062 
13063 	return;
13064 
13065 }
13066 
13067 static int
13068 ctl_datamove_remote_dm_read_cb(union ctl_io *io)
13069 {
13070 #if 0
13071 	char str[256];
13072 	char path_str[64];
13073 	struct sbuf sb;
13074 #endif
13075 
13076 	/*
13077 	 * In this case, we had to malloc the memory locally.  Free it.
13078 	 */
13079 	if ((io->io_hdr.flags & CTL_FLAG_AUTO_MIRROR) == 0) {
13080 		int i;
13081 		for (i = 0; i < io->scsiio.kern_sg_entries; i++)
13082 			free(io->io_hdr.local_sglist[i].addr, M_CTL);
13083 	}
13084 
13085 #if 0
13086 	scsi_path_string(io, path_str, sizeof(path_str));
13087 	sbuf_new(&sb, str, sizeof(str), SBUF_FIXEDLEN);
13088 	sbuf_cat(&sb, path_str);
13089 	scsi_command_string(&io->scsiio, NULL, &sb);
13090 	sbuf_printf(&sb, "\n");
13091 	sbuf_cat(&sb, path_str);
13092 	sbuf_printf(&sb, "Tag: 0x%04x, type %d\n",
13093 		    io->scsiio.tag_num, io->scsiio.tag_type);
13094 	sbuf_cat(&sb, path_str);
13095 	sbuf_printf(&sb, "%s: flags %#x, status %#x\n", __func__,
13096 		    io->io_hdr.flags, io->io_hdr.status);
13097 	sbuf_finish(&sb);
13098 	printk("%s", sbuf_data(&sb));
13099 #endif
13100 
13101 
13102 	/*
13103 	 * The read is done, now we need to send status (good or bad) back
13104 	 * to the other side.
13105 	 */
13106 	ctl_send_datamove_done(io, /*have_lock*/ 0);
13107 
13108 	return (0);
13109 }
13110 
13111 static void
13112 ctl_datamove_remote_read_cb(struct ctl_ha_dt_req *rq)
13113 {
13114 	union ctl_io *io;
13115 	void (*fe_datamove)(union ctl_io *io);
13116 
13117 	io = rq->context;
13118 
13119 	if (rq->ret != CTL_HA_STATUS_SUCCESS) {
13120 		printf("%s: ISC DMA read failed with error %d", __func__,
13121 		       rq->ret);
13122 		ctl_set_internal_failure(&io->scsiio,
13123 					 /*sks_valid*/ 1,
13124 					 /*retry_count*/ rq->ret);
13125 	}
13126 
13127 	ctl_dt_req_free(rq);
13128 
13129 	/* Switch the pointer over so the FETD knows what to do */
13130 	io->scsiio.kern_data_ptr = (uint8_t *)io->io_hdr.local_sglist;
13131 
13132 	/*
13133 	 * Use a custom move done callback, since we need to send completion
13134 	 * back to the other controller, not to the backend on this side.
13135 	 */
13136 	io->scsiio.be_move_done = ctl_datamove_remote_dm_read_cb;
13137 
13138 	/* XXX KDM add checks like the ones in ctl_datamove? */
13139 
13140 	fe_datamove = control_softc->ctl_ports[ctl_port_idx(io->io_hdr.nexus.targ_port)]->fe_datamove;
13141 
13142 	fe_datamove(io);
13143 }
13144 
13145 static int
13146 ctl_datamove_remote_sgl_setup(union ctl_io *io)
13147 {
13148 	struct ctl_sg_entry *local_sglist, *remote_sglist;
13149 	struct ctl_sg_entry *local_dma_sglist, *remote_dma_sglist;
13150 	struct ctl_softc *softc;
13151 	int retval;
13152 	int i;
13153 
13154 	retval = 0;
13155 	softc = control_softc;
13156 
13157 	local_sglist = io->io_hdr.local_sglist;
13158 	local_dma_sglist = io->io_hdr.local_dma_sglist;
13159 	remote_sglist = io->io_hdr.remote_sglist;
13160 	remote_dma_sglist = io->io_hdr.remote_dma_sglist;
13161 
13162 	if (io->io_hdr.flags & CTL_FLAG_AUTO_MIRROR) {
13163 		for (i = 0; i < io->scsiio.kern_sg_entries; i++) {
13164 			local_sglist[i].len = remote_sglist[i].len;
13165 
13166 			/*
13167 			 * XXX Detect the situation where the RS-level I/O
13168 			 * redirector on the other side has already read the
13169 			 * data off of the AOR RS on this side, and
13170 			 * transferred it to remote (mirror) memory on the
13171 			 * other side.  Since we already have the data in
13172 			 * memory here, we just need to use it.
13173 			 *
13174 			 * XXX KDM this can probably be removed once we
13175 			 * get the cache device code in and take the
13176 			 * current AOR implementation out.
13177 			 */
13178 #ifdef NEEDTOPORT
13179 			if ((remote_sglist[i].addr >=
13180 			     (void *)vtophys(softc->mirr->addr))
13181 			 && (remote_sglist[i].addr <
13182 			     ((void *)vtophys(softc->mirr->addr) +
13183 			     CacheMirrorOffset))) {
13184 				local_sglist[i].addr = remote_sglist[i].addr -
13185 					CacheMirrorOffset;
13186 				if ((io->io_hdr.flags & CTL_FLAG_DATA_MASK) ==
13187 				     CTL_FLAG_DATA_IN)
13188 					io->io_hdr.flags |= CTL_FLAG_REDIR_DONE;
13189 			} else {
13190 				local_sglist[i].addr = remote_sglist[i].addr +
13191 					CacheMirrorOffset;
13192 			}
13193 #endif
13194 #if 0
13195 			printf("%s: local %p, remote %p, len %d\n",
13196 			       __func__, local_sglist[i].addr,
13197 			       remote_sglist[i].addr, local_sglist[i].len);
13198 #endif
13199 		}
13200 	} else {
13201 		uint32_t len_to_go;
13202 
13203 		/*
13204 		 * In this case, we don't have automatically allocated
13205 		 * memory for this I/O on this controller.  This typically
13206 		 * happens with internal CTL I/O -- e.g. inquiry, mode
13207 		 * sense, etc.  Anything coming from RAIDCore will have
13208 		 * a mirror area available.
13209 		 */
13210 		len_to_go = io->scsiio.kern_data_len;
13211 
13212 		/*
13213 		 * Clear the no datasync flag, we have to use malloced
13214 		 * buffers.
13215 		 */
13216 		io->io_hdr.flags &= ~CTL_FLAG_NO_DATASYNC;
13217 
13218 		/*
13219 		 * The difficult thing here is that the size of the various
13220 		 * S/G segments may be different than the size from the
13221 		 * remote controller.  That'll make it harder when DMAing
13222 		 * the data back to the other side.
13223 		 */
13224 		for (i = 0; (i < sizeof(io->io_hdr.remote_sglist) /
13225 		     sizeof(io->io_hdr.remote_sglist[0])) &&
13226 		     (len_to_go > 0); i++) {
13227 			local_sglist[i].len = MIN(len_to_go, 131072);
13228 			CTL_SIZE_8B(local_dma_sglist[i].len,
13229 				    local_sglist[i].len);
13230 			local_sglist[i].addr =
13231 				malloc(local_dma_sglist[i].len, M_CTL,M_WAITOK);
13232 
13233 			local_dma_sglist[i].addr = local_sglist[i].addr;
13234 
13235 			if (local_sglist[i].addr == NULL) {
13236 				int j;
13237 
13238 				printf("malloc failed for %zd bytes!",
13239 				       local_dma_sglist[i].len);
13240 				for (j = 0; j < i; j++) {
13241 					free(local_sglist[j].addr, M_CTL);
13242 				}
13243 				ctl_set_internal_failure(&io->scsiio,
13244 							 /*sks_valid*/ 1,
13245 							 /*retry_count*/ 4857);
13246 				retval = 1;
13247 				goto bailout_error;
13248 
13249 			}
13250 			/* XXX KDM do we need a sync here? */
13251 
13252 			len_to_go -= local_sglist[i].len;
13253 		}
13254 		/*
13255 		 * Reset the number of S/G entries accordingly.  The
13256 		 * original number of S/G entries is available in
13257 		 * rem_sg_entries.
13258 		 */
13259 		io->scsiio.kern_sg_entries = i;
13260 
13261 #if 0
13262 		printf("%s: kern_sg_entries = %d\n", __func__,
13263 		       io->scsiio.kern_sg_entries);
13264 		for (i = 0; i < io->scsiio.kern_sg_entries; i++)
13265 			printf("%s: sg[%d] = %p, %d (DMA: %d)\n", __func__, i,
13266 			       local_sglist[i].addr, local_sglist[i].len,
13267 			       local_dma_sglist[i].len);
13268 #endif
13269 	}
13270 
13271 
13272 	return (retval);
13273 
13274 bailout_error:
13275 
13276 	ctl_send_datamove_done(io, /*have_lock*/ 0);
13277 
13278 	return (retval);
13279 }
13280 
13281 static int
13282 ctl_datamove_remote_xfer(union ctl_io *io, unsigned command,
13283 			 ctl_ha_dt_cb callback)
13284 {
13285 	struct ctl_ha_dt_req *rq;
13286 	struct ctl_sg_entry *remote_sglist, *local_sglist;
13287 	struct ctl_sg_entry *remote_dma_sglist, *local_dma_sglist;
13288 	uint32_t local_used, remote_used, total_used;
13289 	int retval;
13290 	int i, j;
13291 
13292 	retval = 0;
13293 
13294 	rq = ctl_dt_req_alloc();
13295 
13296 	/*
13297 	 * If we failed to allocate the request, and if the DMA didn't fail
13298 	 * anyway, set busy status.  This is just a resource allocation
13299 	 * failure.
13300 	 */
13301 	if ((rq == NULL)
13302 	 && ((io->io_hdr.status & CTL_STATUS_MASK) != CTL_STATUS_NONE))
13303 		ctl_set_busy(&io->scsiio);
13304 
13305 	if ((io->io_hdr.status & CTL_STATUS_MASK) != CTL_STATUS_NONE) {
13306 
13307 		if (rq != NULL)
13308 			ctl_dt_req_free(rq);
13309 
13310 		/*
13311 		 * The data move failed.  We need to return status back
13312 		 * to the other controller.  No point in trying to DMA
13313 		 * data to the remote controller.
13314 		 */
13315 
13316 		ctl_send_datamove_done(io, /*have_lock*/ 0);
13317 
13318 		retval = 1;
13319 
13320 		goto bailout;
13321 	}
13322 
13323 	local_sglist = io->io_hdr.local_sglist;
13324 	local_dma_sglist = io->io_hdr.local_dma_sglist;
13325 	remote_sglist = io->io_hdr.remote_sglist;
13326 	remote_dma_sglist = io->io_hdr.remote_dma_sglist;
13327 	local_used = 0;
13328 	remote_used = 0;
13329 	total_used = 0;
13330 
13331 	if (io->io_hdr.flags & CTL_FLAG_REDIR_DONE) {
13332 		rq->ret = CTL_HA_STATUS_SUCCESS;
13333 		rq->context = io;
13334 		callback(rq);
13335 		goto bailout;
13336 	}
13337 
13338 	/*
13339 	 * Pull/push the data over the wire from/to the other controller.
13340 	 * This takes into account the possibility that the local and
13341 	 * remote sglists may not be identical in terms of the size of
13342 	 * the elements and the number of elements.
13343 	 *
13344 	 * One fundamental assumption here is that the length allocated for
13345 	 * both the local and remote sglists is identical.  Otherwise, we've
13346 	 * essentially got a coding error of some sort.
13347 	 */
13348 	for (i = 0, j = 0; total_used < io->scsiio.kern_data_len; ) {
13349 		int isc_ret;
13350 		uint32_t cur_len, dma_length;
13351 		uint8_t *tmp_ptr;
13352 
13353 		rq->id = CTL_HA_DATA_CTL;
13354 		rq->command = command;
13355 		rq->context = io;
13356 
13357 		/*
13358 		 * Both pointers should be aligned.  But it is possible
13359 		 * that the allocation length is not.  They should both
13360 		 * also have enough slack left over at the end, though,
13361 		 * to round up to the next 8 byte boundary.
13362 		 */
13363 		cur_len = MIN(local_sglist[i].len - local_used,
13364 			      remote_sglist[j].len - remote_used);
13365 
13366 		/*
13367 		 * In this case, we have a size issue and need to decrease
13368 		 * the size, except in the case where we actually have less
13369 		 * than 8 bytes left.  In that case, we need to increase
13370 		 * the DMA length to get the last bit.
13371 		 */
13372 		if ((cur_len & 0x7) != 0) {
13373 			if (cur_len > 0x7) {
13374 				cur_len = cur_len - (cur_len & 0x7);
13375 				dma_length = cur_len;
13376 			} else {
13377 				CTL_SIZE_8B(dma_length, cur_len);
13378 			}
13379 
13380 		} else
13381 			dma_length = cur_len;
13382 
13383 		/*
13384 		 * If we had to allocate memory for this I/O, instead of using
13385 		 * the non-cached mirror memory, we'll need to flush the cache
13386 		 * before trying to DMA to the other controller.
13387 		 *
13388 		 * We could end up doing this multiple times for the same
13389 		 * segment if we have a larger local segment than remote
13390 		 * segment.  That shouldn't be an issue.
13391 		 */
13392 		if ((io->io_hdr.flags & CTL_FLAG_NO_DATASYNC) == 0) {
13393 			/*
13394 			 * XXX KDM use bus_dmamap_sync() here.
13395 			 */
13396 		}
13397 
13398 		rq->size = dma_length;
13399 
13400 		tmp_ptr = (uint8_t *)local_sglist[i].addr;
13401 		tmp_ptr += local_used;
13402 
13403 		/* Use physical addresses when talking to ISC hardware */
13404 		if ((io->io_hdr.flags & CTL_FLAG_BUS_ADDR) == 0) {
13405 			/* XXX KDM use busdma */
13406 #if 0
13407 			rq->local = vtophys(tmp_ptr);
13408 #endif
13409 		} else
13410 			rq->local = tmp_ptr;
13411 
13412 		tmp_ptr = (uint8_t *)remote_sglist[j].addr;
13413 		tmp_ptr += remote_used;
13414 		rq->remote = tmp_ptr;
13415 
13416 		rq->callback = NULL;
13417 
13418 		local_used += cur_len;
13419 		if (local_used >= local_sglist[i].len) {
13420 			i++;
13421 			local_used = 0;
13422 		}
13423 
13424 		remote_used += cur_len;
13425 		if (remote_used >= remote_sglist[j].len) {
13426 			j++;
13427 			remote_used = 0;
13428 		}
13429 		total_used += cur_len;
13430 
13431 		if (total_used >= io->scsiio.kern_data_len)
13432 			rq->callback = callback;
13433 
13434 		if ((rq->size & 0x7) != 0) {
13435 			printf("%s: warning: size %d is not on 8b boundary\n",
13436 			       __func__, rq->size);
13437 		}
13438 		if (((uintptr_t)rq->local & 0x7) != 0) {
13439 			printf("%s: warning: local %p not on 8b boundary\n",
13440 			       __func__, rq->local);
13441 		}
13442 		if (((uintptr_t)rq->remote & 0x7) != 0) {
13443 			printf("%s: warning: remote %p not on 8b boundary\n",
13444 			       __func__, rq->local);
13445 		}
13446 #if 0
13447 		printf("%s: %s: local %#x remote %#x size %d\n", __func__,
13448 		       (command == CTL_HA_DT_CMD_WRITE) ? "WRITE" : "READ",
13449 		       rq->local, rq->remote, rq->size);
13450 #endif
13451 
13452 		isc_ret = ctl_dt_single(rq);
13453 		if (isc_ret == CTL_HA_STATUS_WAIT)
13454 			continue;
13455 
13456 		if (isc_ret == CTL_HA_STATUS_DISCONNECT) {
13457 			rq->ret = CTL_HA_STATUS_SUCCESS;
13458 		} else {
13459 			rq->ret = isc_ret;
13460 		}
13461 		callback(rq);
13462 		goto bailout;
13463 	}
13464 
13465 bailout:
13466 	return (retval);
13467 
13468 }
13469 
13470 static void
13471 ctl_datamove_remote_read(union ctl_io *io)
13472 {
13473 	int retval;
13474 	int i;
13475 
13476 	/*
13477 	 * This will send an error to the other controller in the case of a
13478 	 * failure.
13479 	 */
13480 	retval = ctl_datamove_remote_sgl_setup(io);
13481 	if (retval != 0)
13482 		return;
13483 
13484 	retval = ctl_datamove_remote_xfer(io, CTL_HA_DT_CMD_READ,
13485 					  ctl_datamove_remote_read_cb);
13486 	if ((retval != 0)
13487 	 && ((io->io_hdr.flags & CTL_FLAG_AUTO_MIRROR) == 0)) {
13488 		/*
13489 		 * Make sure we free memory if there was an error..  The
13490 		 * ctl_datamove_remote_xfer() function will send the
13491 		 * datamove done message, or call the callback with an
13492 		 * error if there is a problem.
13493 		 */
13494 		for (i = 0; i < io->scsiio.kern_sg_entries; i++)
13495 			free(io->io_hdr.local_sglist[i].addr, M_CTL);
13496 	}
13497 
13498 	return;
13499 }
13500 
13501 /*
13502  * Process a datamove request from the other controller.  This is used for
13503  * XFER mode only, not SER_ONLY mode.  For writes, we DMA into local memory
13504  * first.  Once that is complete, the data gets DMAed into the remote
13505  * controller's memory.  For reads, we DMA from the remote controller's
13506  * memory into our memory first, and then move it out to the FETD.
13507  */
13508 static void
13509 ctl_datamove_remote(union ctl_io *io)
13510 {
13511 	struct ctl_softc *softc;
13512 
13513 	softc = control_softc;
13514 
13515 	mtx_assert(&softc->ctl_lock, MA_NOTOWNED);
13516 
13517 	/*
13518 	 * Note that we look for an aborted I/O here, but don't do some of
13519 	 * the other checks that ctl_datamove() normally does.
13520 	 * We don't need to run the datamove delay code, since that should
13521 	 * have been done if need be on the other controller.
13522 	 */
13523 	if (io->io_hdr.flags & CTL_FLAG_ABORT) {
13524 		printf("%s: tag 0x%04x on (%d:%d:%d:%d) aborted\n", __func__,
13525 		       io->scsiio.tag_num, io->io_hdr.nexus.initid.id,
13526 		       io->io_hdr.nexus.targ_port,
13527 		       io->io_hdr.nexus.targ_target.id,
13528 		       io->io_hdr.nexus.targ_lun);
13529 		io->io_hdr.port_status = 31338;
13530 		ctl_send_datamove_done(io, /*have_lock*/ 0);
13531 		return;
13532 	}
13533 
13534 	if ((io->io_hdr.flags & CTL_FLAG_DATA_MASK) == CTL_FLAG_DATA_OUT) {
13535 		ctl_datamove_remote_write(io);
13536 	} else if ((io->io_hdr.flags & CTL_FLAG_DATA_MASK) == CTL_FLAG_DATA_IN){
13537 		ctl_datamove_remote_read(io);
13538 	} else {
13539 		union ctl_ha_msg msg;
13540 		struct scsi_sense_data *sense;
13541 		uint8_t sks[3];
13542 		int retry_count;
13543 
13544 		memset(&msg, 0, sizeof(msg));
13545 
13546 		msg.hdr.msg_type = CTL_MSG_BAD_JUJU;
13547 		msg.hdr.status = CTL_SCSI_ERROR;
13548 		msg.scsi.scsi_status = SCSI_STATUS_CHECK_COND;
13549 
13550 		retry_count = 4243;
13551 
13552 		sense = &msg.scsi.sense_data;
13553 		sks[0] = SSD_SCS_VALID;
13554 		sks[1] = (retry_count >> 8) & 0xff;
13555 		sks[2] = retry_count & 0xff;
13556 
13557 		/* "Internal target failure" */
13558 		scsi_set_sense_data(sense,
13559 				    /*sense_format*/ SSD_TYPE_NONE,
13560 				    /*current_error*/ 1,
13561 				    /*sense_key*/ SSD_KEY_HARDWARE_ERROR,
13562 				    /*asc*/ 0x44,
13563 				    /*ascq*/ 0x00,
13564 				    /*type*/ SSD_ELEM_SKS,
13565 				    /*size*/ sizeof(sks),
13566 				    /*data*/ sks,
13567 				    SSD_ELEM_NONE);
13568 
13569 		io->io_hdr.flags &= ~CTL_FLAG_IO_ACTIVE;
13570 		if (io->io_hdr.flags & CTL_FLAG_FAILOVER) {
13571 			ctl_failover_io(io, /*have_lock*/ 1);
13572 			return;
13573 		}
13574 
13575 		if (ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg, sizeof(msg), 0) >
13576 		    CTL_HA_STATUS_SUCCESS) {
13577 			/* XXX KDM what to do if this fails? */
13578 		}
13579 		return;
13580 	}
13581 
13582 }
13583 
13584 static int
13585 ctl_process_done(union ctl_io *io)
13586 {
13587 	struct ctl_lun *lun;
13588 	struct ctl_softc *softc = control_softc;
13589 	void (*fe_done)(union ctl_io *io);
13590 	uint32_t targ_port = ctl_port_idx(io->io_hdr.nexus.targ_port);
13591 
13592 	CTL_DEBUG_PRINT(("ctl_process_done\n"));
13593 
13594 	fe_done = softc->ctl_ports[targ_port]->fe_done;
13595 
13596 #ifdef CTL_TIME_IO
13597 	if ((time_uptime - io->io_hdr.start_time) > ctl_time_io_secs) {
13598 		char str[256];
13599 		char path_str[64];
13600 		struct sbuf sb;
13601 
13602 		ctl_scsi_path_string(io, path_str, sizeof(path_str));
13603 		sbuf_new(&sb, str, sizeof(str), SBUF_FIXEDLEN);
13604 
13605 		sbuf_cat(&sb, path_str);
13606 		switch (io->io_hdr.io_type) {
13607 		case CTL_IO_SCSI:
13608 			ctl_scsi_command_string(&io->scsiio, NULL, &sb);
13609 			sbuf_printf(&sb, "\n");
13610 			sbuf_cat(&sb, path_str);
13611 			sbuf_printf(&sb, "Tag: 0x%04x, type %d\n",
13612 				    io->scsiio.tag_num, io->scsiio.tag_type);
13613 			break;
13614 		case CTL_IO_TASK:
13615 			sbuf_printf(&sb, "Task I/O type: %d, Tag: 0x%04x, "
13616 				    "Tag Type: %d\n", io->taskio.task_action,
13617 				    io->taskio.tag_num, io->taskio.tag_type);
13618 			break;
13619 		default:
13620 			printf("Invalid CTL I/O type %d\n", io->io_hdr.io_type);
13621 			panic("Invalid CTL I/O type %d\n", io->io_hdr.io_type);
13622 			break;
13623 		}
13624 		sbuf_cat(&sb, path_str);
13625 		sbuf_printf(&sb, "ctl_process_done: %jd seconds\n",
13626 			    (intmax_t)time_uptime - io->io_hdr.start_time);
13627 		sbuf_finish(&sb);
13628 		printf("%s", sbuf_data(&sb));
13629 	}
13630 #endif /* CTL_TIME_IO */
13631 
13632 	switch (io->io_hdr.io_type) {
13633 	case CTL_IO_SCSI:
13634 		break;
13635 	case CTL_IO_TASK:
13636 		if (bootverbose || (ctl_debug & CTL_DEBUG_INFO))
13637 			ctl_io_error_print(io, NULL);
13638 		if (io->io_hdr.flags & CTL_FLAG_FROM_OTHER_SC)
13639 			ctl_free_io(io);
13640 		else
13641 			fe_done(io);
13642 		return (CTL_RETVAL_COMPLETE);
13643 	default:
13644 		panic("ctl_process_done: invalid io type %d\n",
13645 		      io->io_hdr.io_type);
13646 		break; /* NOTREACHED */
13647 	}
13648 
13649 	lun = (struct ctl_lun *)io->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
13650 	if (lun == NULL) {
13651 		CTL_DEBUG_PRINT(("NULL LUN for lun %d\n",
13652 				 io->io_hdr.nexus.targ_mapped_lun));
13653 		goto bailout;
13654 	}
13655 
13656 	mtx_lock(&lun->lun_lock);
13657 
13658 	/*
13659 	 * Check to see if we have any errors to inject here.  We only
13660 	 * inject errors for commands that don't already have errors set.
13661 	 */
13662 	if ((STAILQ_FIRST(&lun->error_list) != NULL) &&
13663 	    ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_SUCCESS) &&
13664 	    ((io->io_hdr.flags & CTL_FLAG_STATUS_SENT) == 0))
13665 		ctl_inject_error(lun, io);
13666 
13667 	/*
13668 	 * XXX KDM how do we treat commands that aren't completed
13669 	 * successfully?
13670 	 *
13671 	 * XXX KDM should we also track I/O latency?
13672 	 */
13673 	if ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_SUCCESS &&
13674 	    io->io_hdr.io_type == CTL_IO_SCSI) {
13675 #ifdef CTL_TIME_IO
13676 		struct bintime cur_bt;
13677 #endif
13678 		int type;
13679 
13680 		if ((io->io_hdr.flags & CTL_FLAG_DATA_MASK) ==
13681 		    CTL_FLAG_DATA_IN)
13682 			type = CTL_STATS_READ;
13683 		else if ((io->io_hdr.flags & CTL_FLAG_DATA_MASK) ==
13684 		    CTL_FLAG_DATA_OUT)
13685 			type = CTL_STATS_WRITE;
13686 		else
13687 			type = CTL_STATS_NO_IO;
13688 
13689 		lun->stats.ports[targ_port].bytes[type] +=
13690 		    io->scsiio.kern_total_len;
13691 		lun->stats.ports[targ_port].operations[type]++;
13692 #ifdef CTL_TIME_IO
13693 		bintime_add(&lun->stats.ports[targ_port].dma_time[type],
13694 		   &io->io_hdr.dma_bt);
13695 		lun->stats.ports[targ_port].num_dmas[type] +=
13696 		    io->io_hdr.num_dmas;
13697 		getbintime(&cur_bt);
13698 		bintime_sub(&cur_bt, &io->io_hdr.start_bt);
13699 		bintime_add(&lun->stats.ports[targ_port].time[type], &cur_bt);
13700 #endif
13701 	}
13702 
13703 	/*
13704 	 * Remove this from the OOA queue.
13705 	 */
13706 	TAILQ_REMOVE(&lun->ooa_queue, &io->io_hdr, ooa_links);
13707 #ifdef CTL_TIME_IO
13708 	if (TAILQ_EMPTY(&lun->ooa_queue))
13709 		lun->last_busy = getsbinuptime();
13710 #endif
13711 
13712 	/*
13713 	 * Run through the blocked queue on this LUN and see if anything
13714 	 * has become unblocked, now that this transaction is done.
13715 	 */
13716 	ctl_check_blocked(lun);
13717 
13718 	/*
13719 	 * If the LUN has been invalidated, free it if there is nothing
13720 	 * left on its OOA queue.
13721 	 */
13722 	if ((lun->flags & CTL_LUN_INVALID)
13723 	 && TAILQ_EMPTY(&lun->ooa_queue)) {
13724 		mtx_unlock(&lun->lun_lock);
13725 		mtx_lock(&softc->ctl_lock);
13726 		ctl_free_lun(lun);
13727 		mtx_unlock(&softc->ctl_lock);
13728 	} else
13729 		mtx_unlock(&lun->lun_lock);
13730 
13731 bailout:
13732 
13733 	/*
13734 	 * If this command has been aborted, make sure we set the status
13735 	 * properly.  The FETD is responsible for freeing the I/O and doing
13736 	 * whatever it needs to do to clean up its state.
13737 	 */
13738 	if (io->io_hdr.flags & CTL_FLAG_ABORT)
13739 		ctl_set_task_aborted(&io->scsiio);
13740 
13741 	/*
13742 	 * If enabled, print command error status.
13743 	 * We don't print UAs unless debugging was enabled explicitly.
13744 	 */
13745 	do {
13746 		if ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_SUCCESS)
13747 			break;
13748 		if (!bootverbose && (ctl_debug & CTL_DEBUG_INFO) == 0)
13749 			break;
13750 		if ((ctl_debug & CTL_DEBUG_INFO) == 0 &&
13751 		    ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_SCSI_ERROR) &&
13752 		     (io->scsiio.scsi_status == SCSI_STATUS_CHECK_COND)) {
13753 			int error_code, sense_key, asc, ascq;
13754 
13755 			scsi_extract_sense_len(&io->scsiio.sense_data,
13756 			    io->scsiio.sense_len, &error_code, &sense_key,
13757 			    &asc, &ascq, /*show_errors*/ 0);
13758 			if (sense_key == SSD_KEY_UNIT_ATTENTION)
13759 				break;
13760 		}
13761 
13762 		ctl_io_error_print(io, NULL);
13763 	} while (0);
13764 
13765 	/*
13766 	 * Tell the FETD or the other shelf controller we're done with this
13767 	 * command.  Note that only SCSI commands get to this point.  Task
13768 	 * management commands are completed above.
13769 	 *
13770 	 * We only send status to the other controller if we're in XFER
13771 	 * mode.  In SER_ONLY mode, the I/O is done on the controller that
13772 	 * received the I/O (from CTL's perspective), and so the status is
13773 	 * generated there.
13774 	 *
13775 	 * XXX KDM if we hold the lock here, we could cause a deadlock
13776 	 * if the frontend comes back in in this context to queue
13777 	 * something.
13778 	 */
13779 	if ((softc->ha_mode == CTL_HA_MODE_XFER)
13780 	 && (io->io_hdr.flags & CTL_FLAG_FROM_OTHER_SC)) {
13781 		union ctl_ha_msg msg;
13782 
13783 		memset(&msg, 0, sizeof(msg));
13784 		msg.hdr.msg_type = CTL_MSG_FINISH_IO;
13785 		msg.hdr.original_sc = io->io_hdr.original_sc;
13786 		msg.hdr.nexus = io->io_hdr.nexus;
13787 		msg.hdr.status = io->io_hdr.status;
13788 		msg.scsi.scsi_status = io->scsiio.scsi_status;
13789 		msg.scsi.tag_num = io->scsiio.tag_num;
13790 		msg.scsi.tag_type = io->scsiio.tag_type;
13791 		msg.scsi.sense_len = io->scsiio.sense_len;
13792 		msg.scsi.sense_residual = io->scsiio.sense_residual;
13793 		msg.scsi.residual = io->scsiio.residual;
13794 		memcpy(&msg.scsi.sense_data, &io->scsiio.sense_data,
13795 		       sizeof(io->scsiio.sense_data));
13796 		/*
13797 		 * We copy this whether or not this is an I/O-related
13798 		 * command.  Otherwise, we'd have to go and check to see
13799 		 * whether it's a read/write command, and it really isn't
13800 		 * worth it.
13801 		 */
13802 		memcpy(&msg.scsi.lbalen,
13803 		       &io->io_hdr.ctl_private[CTL_PRIV_LBA_LEN].bytes,
13804 		       sizeof(msg.scsi.lbalen));
13805 
13806 		if (ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg,
13807 				sizeof(msg), 0) > CTL_HA_STATUS_SUCCESS) {
13808 			/* XXX do something here */
13809 		}
13810 
13811 		ctl_free_io(io);
13812 	} else
13813 		fe_done(io);
13814 
13815 	return (CTL_RETVAL_COMPLETE);
13816 }
13817 
13818 #ifdef CTL_WITH_CA
13819 /*
13820  * Front end should call this if it doesn't do autosense.  When the request
13821  * sense comes back in from the initiator, we'll dequeue this and send it.
13822  */
13823 int
13824 ctl_queue_sense(union ctl_io *io)
13825 {
13826 	struct ctl_lun *lun;
13827 	struct ctl_port *port;
13828 	struct ctl_softc *softc;
13829 	uint32_t initidx, targ_lun;
13830 
13831 	softc = control_softc;
13832 
13833 	CTL_DEBUG_PRINT(("ctl_queue_sense\n"));
13834 
13835 	/*
13836 	 * LUN lookup will likely move to the ctl_work_thread() once we
13837 	 * have our new queueing infrastructure (that doesn't put things on
13838 	 * a per-LUN queue initially).  That is so that we can handle
13839 	 * things like an INQUIRY to a LUN that we don't have enabled.  We
13840 	 * can't deal with that right now.
13841 	 */
13842 	mtx_lock(&softc->ctl_lock);
13843 
13844 	/*
13845 	 * If we don't have a LUN for this, just toss the sense
13846 	 * information.
13847 	 */
13848 	port = ctl_io_port(&ctsio->io_hdr);
13849 	targ_lun = ctl_lun_map_from_port(port, io->io_hdr.nexus.targ_lun);
13850 	if ((targ_lun < CTL_MAX_LUNS)
13851 	 && (softc->ctl_luns[targ_lun] != NULL))
13852 		lun = softc->ctl_luns[targ_lun];
13853 	else
13854 		goto bailout;
13855 
13856 	initidx = ctl_get_initindex(&io->io_hdr.nexus);
13857 
13858 	mtx_lock(&lun->lun_lock);
13859 	/*
13860 	 * Already have CA set for this LUN...toss the sense information.
13861 	 */
13862 	if (ctl_is_set(lun->have_ca, initidx)) {
13863 		mtx_unlock(&lun->lun_lock);
13864 		goto bailout;
13865 	}
13866 
13867 	memcpy(&lun->pending_sense[initidx], &io->scsiio.sense_data,
13868 	       MIN(sizeof(lun->pending_sense[initidx]),
13869 	       sizeof(io->scsiio.sense_data)));
13870 	ctl_set_mask(lun->have_ca, initidx);
13871 	mtx_unlock(&lun->lun_lock);
13872 
13873 bailout:
13874 	mtx_unlock(&softc->ctl_lock);
13875 
13876 	ctl_free_io(io);
13877 
13878 	return (CTL_RETVAL_COMPLETE);
13879 }
13880 #endif
13881 
13882 /*
13883  * Primary command inlet from frontend ports.  All SCSI and task I/O
13884  * requests must go through this function.
13885  */
13886 int
13887 ctl_queue(union ctl_io *io)
13888 {
13889 	struct ctl_port *port;
13890 
13891 	CTL_DEBUG_PRINT(("ctl_queue cdb[0]=%02X\n", io->scsiio.cdb[0]));
13892 
13893 #ifdef CTL_TIME_IO
13894 	io->io_hdr.start_time = time_uptime;
13895 	getbintime(&io->io_hdr.start_bt);
13896 #endif /* CTL_TIME_IO */
13897 
13898 	/* Map FE-specific LUN ID into global one. */
13899 	port = ctl_io_port(&io->io_hdr);
13900 	io->io_hdr.nexus.targ_mapped_lun =
13901 	    ctl_lun_map_from_port(port, io->io_hdr.nexus.targ_lun);
13902 
13903 	switch (io->io_hdr.io_type) {
13904 	case CTL_IO_SCSI:
13905 	case CTL_IO_TASK:
13906 		if (ctl_debug & CTL_DEBUG_CDB)
13907 			ctl_io_print(io);
13908 		ctl_enqueue_incoming(io);
13909 		break;
13910 	default:
13911 		printf("ctl_queue: unknown I/O type %d\n", io->io_hdr.io_type);
13912 		return (EINVAL);
13913 	}
13914 
13915 	return (CTL_RETVAL_COMPLETE);
13916 }
13917 
13918 #ifdef CTL_IO_DELAY
13919 static void
13920 ctl_done_timer_wakeup(void *arg)
13921 {
13922 	union ctl_io *io;
13923 
13924 	io = (union ctl_io *)arg;
13925 	ctl_done(io);
13926 }
13927 #endif /* CTL_IO_DELAY */
13928 
13929 void
13930 ctl_done(union ctl_io *io)
13931 {
13932 
13933 	/*
13934 	 * Enable this to catch duplicate completion issues.
13935 	 */
13936 #if 0
13937 	if (io->io_hdr.flags & CTL_FLAG_ALREADY_DONE) {
13938 		printf("%s: type %d msg %d cdb %x iptl: "
13939 		       "%d:%d:%d:%d tag 0x%04x "
13940 		       "flag %#x status %x\n",
13941 			__func__,
13942 			io->io_hdr.io_type,
13943 			io->io_hdr.msg_type,
13944 			io->scsiio.cdb[0],
13945 			io->io_hdr.nexus.initid.id,
13946 			io->io_hdr.nexus.targ_port,
13947 			io->io_hdr.nexus.targ_target.id,
13948 			io->io_hdr.nexus.targ_lun,
13949 			(io->io_hdr.io_type ==
13950 			CTL_IO_TASK) ?
13951 			io->taskio.tag_num :
13952 			io->scsiio.tag_num,
13953 		        io->io_hdr.flags,
13954 			io->io_hdr.status);
13955 	} else
13956 		io->io_hdr.flags |= CTL_FLAG_ALREADY_DONE;
13957 #endif
13958 
13959 	/*
13960 	 * This is an internal copy of an I/O, and should not go through
13961 	 * the normal done processing logic.
13962 	 */
13963 	if (io->io_hdr.flags & CTL_FLAG_INT_COPY)
13964 		return;
13965 
13966 	/*
13967 	 * We need to send a msg to the serializing shelf to finish the IO
13968 	 * as well.  We don't send a finish message to the other shelf if
13969 	 * this is a task management command.  Task management commands
13970 	 * aren't serialized in the OOA queue, but rather just executed on
13971 	 * both shelf controllers for commands that originated on that
13972 	 * controller.
13973 	 */
13974 	if ((io->io_hdr.flags & CTL_FLAG_SENT_2OTHER_SC)
13975 	 && (io->io_hdr.io_type != CTL_IO_TASK)) {
13976 		union ctl_ha_msg msg_io;
13977 
13978 		msg_io.hdr.msg_type = CTL_MSG_FINISH_IO;
13979 		msg_io.hdr.serializing_sc = io->io_hdr.serializing_sc;
13980 		if (ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg_io,
13981 		    sizeof(msg_io), 0 ) != CTL_HA_STATUS_SUCCESS) {
13982 		}
13983 		/* continue on to finish IO */
13984 	}
13985 #ifdef CTL_IO_DELAY
13986 	if (io->io_hdr.flags & CTL_FLAG_DELAY_DONE) {
13987 		struct ctl_lun *lun;
13988 
13989 		lun =(struct ctl_lun *)io->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
13990 
13991 		io->io_hdr.flags &= ~CTL_FLAG_DELAY_DONE;
13992 	} else {
13993 		struct ctl_lun *lun;
13994 
13995 		lun =(struct ctl_lun *)io->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
13996 
13997 		if ((lun != NULL)
13998 		 && (lun->delay_info.done_delay > 0)) {
13999 			struct callout *callout;
14000 
14001 			callout = (struct callout *)&io->io_hdr.timer_bytes;
14002 			callout_init(callout, /*mpsafe*/ 1);
14003 			io->io_hdr.flags |= CTL_FLAG_DELAY_DONE;
14004 			callout_reset(callout,
14005 				      lun->delay_info.done_delay * hz,
14006 				      ctl_done_timer_wakeup, io);
14007 			if (lun->delay_info.done_type == CTL_DELAY_TYPE_ONESHOT)
14008 				lun->delay_info.done_delay = 0;
14009 			return;
14010 		}
14011 	}
14012 #endif /* CTL_IO_DELAY */
14013 
14014 	ctl_enqueue_done(io);
14015 }
14016 
14017 int
14018 ctl_isc(struct ctl_scsiio *ctsio)
14019 {
14020 	struct ctl_lun *lun;
14021 	int retval;
14022 
14023 	lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
14024 
14025 	CTL_DEBUG_PRINT(("ctl_isc: command: %02x\n", ctsio->cdb[0]));
14026 
14027 	CTL_DEBUG_PRINT(("ctl_isc: calling data_submit()\n"));
14028 
14029 	retval = lun->backend->data_submit((union ctl_io *)ctsio);
14030 
14031 	return (retval);
14032 }
14033 
14034 
14035 static void
14036 ctl_work_thread(void *arg)
14037 {
14038 	struct ctl_thread *thr = (struct ctl_thread *)arg;
14039 	struct ctl_softc *softc = thr->ctl_softc;
14040 	union ctl_io *io;
14041 	int retval;
14042 
14043 	CTL_DEBUG_PRINT(("ctl_work_thread starting\n"));
14044 
14045 	for (;;) {
14046 		retval = 0;
14047 
14048 		/*
14049 		 * We handle the queues in this order:
14050 		 * - ISC
14051 		 * - done queue (to free up resources, unblock other commands)
14052 		 * - RtR queue
14053 		 * - incoming queue
14054 		 *
14055 		 * If those queues are empty, we break out of the loop and
14056 		 * go to sleep.
14057 		 */
14058 		mtx_lock(&thr->queue_lock);
14059 		io = (union ctl_io *)STAILQ_FIRST(&thr->isc_queue);
14060 		if (io != NULL) {
14061 			STAILQ_REMOVE_HEAD(&thr->isc_queue, links);
14062 			mtx_unlock(&thr->queue_lock);
14063 			ctl_handle_isc(io);
14064 			continue;
14065 		}
14066 		io = (union ctl_io *)STAILQ_FIRST(&thr->done_queue);
14067 		if (io != NULL) {
14068 			STAILQ_REMOVE_HEAD(&thr->done_queue, links);
14069 			/* clear any blocked commands, call fe_done */
14070 			mtx_unlock(&thr->queue_lock);
14071 			retval = ctl_process_done(io);
14072 			continue;
14073 		}
14074 		io = (union ctl_io *)STAILQ_FIRST(&thr->incoming_queue);
14075 		if (io != NULL) {
14076 			STAILQ_REMOVE_HEAD(&thr->incoming_queue, links);
14077 			mtx_unlock(&thr->queue_lock);
14078 			if (io->io_hdr.io_type == CTL_IO_TASK)
14079 				ctl_run_task(io);
14080 			else
14081 				ctl_scsiio_precheck(softc, &io->scsiio);
14082 			continue;
14083 		}
14084 		if (!ctl_pause_rtr) {
14085 			io = (union ctl_io *)STAILQ_FIRST(&thr->rtr_queue);
14086 			if (io != NULL) {
14087 				STAILQ_REMOVE_HEAD(&thr->rtr_queue, links);
14088 				mtx_unlock(&thr->queue_lock);
14089 				retval = ctl_scsiio(&io->scsiio);
14090 				if (retval != CTL_RETVAL_COMPLETE)
14091 					CTL_DEBUG_PRINT(("ctl_scsiio failed\n"));
14092 				continue;
14093 			}
14094 		}
14095 
14096 		/* Sleep until we have something to do. */
14097 		mtx_sleep(thr, &thr->queue_lock, PDROP | PRIBIO, "-", 0);
14098 	}
14099 }
14100 
14101 static void
14102 ctl_lun_thread(void *arg)
14103 {
14104 	struct ctl_softc *softc = (struct ctl_softc *)arg;
14105 	struct ctl_be_lun *be_lun;
14106 	int retval;
14107 
14108 	CTL_DEBUG_PRINT(("ctl_lun_thread starting\n"));
14109 
14110 	for (;;) {
14111 		retval = 0;
14112 		mtx_lock(&softc->ctl_lock);
14113 		be_lun = STAILQ_FIRST(&softc->pending_lun_queue);
14114 		if (be_lun != NULL) {
14115 			STAILQ_REMOVE_HEAD(&softc->pending_lun_queue, links);
14116 			mtx_unlock(&softc->ctl_lock);
14117 			ctl_create_lun(be_lun);
14118 			continue;
14119 		}
14120 
14121 		/* Sleep until we have something to do. */
14122 		mtx_sleep(&softc->pending_lun_queue, &softc->ctl_lock,
14123 		    PDROP | PRIBIO, "-", 0);
14124 	}
14125 }
14126 
14127 static void
14128 ctl_thresh_thread(void *arg)
14129 {
14130 	struct ctl_softc *softc = (struct ctl_softc *)arg;
14131 	struct ctl_lun *lun;
14132 	struct ctl_be_lun *be_lun;
14133 	struct scsi_da_rw_recovery_page *rwpage;
14134 	struct ctl_logical_block_provisioning_page *page;
14135 	const char *attr;
14136 	uint64_t thres, val;
14137 	int i, e;
14138 
14139 	CTL_DEBUG_PRINT(("ctl_thresh_thread starting\n"));
14140 
14141 	for (;;) {
14142 		mtx_lock(&softc->ctl_lock);
14143 		STAILQ_FOREACH(lun, &softc->lun_list, links) {
14144 			be_lun = lun->be_lun;
14145 			if ((lun->flags & CTL_LUN_DISABLED) ||
14146 			    (lun->flags & CTL_LUN_OFFLINE) ||
14147 			    lun->backend->lun_attr == NULL)
14148 				continue;
14149 			rwpage = &lun->mode_pages.rw_er_page[CTL_PAGE_CURRENT];
14150 			if ((rwpage->byte8 & SMS_RWER_LBPERE) == 0)
14151 				continue;
14152 			e = 0;
14153 			page = &lun->mode_pages.lbp_page[CTL_PAGE_CURRENT];
14154 			for (i = 0; i < CTL_NUM_LBP_THRESH; i++) {
14155 				if ((page->descr[i].flags & SLBPPD_ENABLED) == 0)
14156 					continue;
14157 				thres = scsi_4btoul(page->descr[i].count);
14158 				thres <<= CTL_LBP_EXPONENT;
14159 				switch (page->descr[i].resource) {
14160 				case 0x01:
14161 					attr = "blocksavail";
14162 					break;
14163 				case 0x02:
14164 					attr = "blocksused";
14165 					break;
14166 				case 0xf1:
14167 					attr = "poolblocksavail";
14168 					break;
14169 				case 0xf2:
14170 					attr = "poolblocksused";
14171 					break;
14172 				default:
14173 					continue;
14174 				}
14175 				mtx_unlock(&softc->ctl_lock); // XXX
14176 				val = lun->backend->lun_attr(
14177 				    lun->be_lun->be_lun, attr);
14178 				mtx_lock(&softc->ctl_lock);
14179 				if (val == UINT64_MAX)
14180 					continue;
14181 				if ((page->descr[i].flags & SLBPPD_ARMING_MASK)
14182 				    == SLBPPD_ARMING_INC)
14183 					e |= (val >= thres);
14184 				else
14185 					e |= (val <= thres);
14186 			}
14187 			mtx_lock(&lun->lun_lock);
14188 			if (e) {
14189 				if (lun->lasttpt == 0 ||
14190 				    time_uptime - lun->lasttpt >= CTL_LBP_UA_PERIOD) {
14191 					lun->lasttpt = time_uptime;
14192 					ctl_est_ua_all(lun, -1, CTL_UA_THIN_PROV_THRES);
14193 				}
14194 			} else {
14195 				lun->lasttpt = 0;
14196 				ctl_clr_ua_all(lun, -1, CTL_UA_THIN_PROV_THRES);
14197 			}
14198 			mtx_unlock(&lun->lun_lock);
14199 		}
14200 		mtx_unlock(&softc->ctl_lock);
14201 		pause("-", CTL_LBP_PERIOD * hz);
14202 	}
14203 }
14204 
14205 static void
14206 ctl_enqueue_incoming(union ctl_io *io)
14207 {
14208 	struct ctl_softc *softc = control_softc;
14209 	struct ctl_thread *thr;
14210 	u_int idx;
14211 
14212 	idx = (io->io_hdr.nexus.targ_port * 127 +
14213 	       io->io_hdr.nexus.initid.id) % worker_threads;
14214 	thr = &softc->threads[idx];
14215 	mtx_lock(&thr->queue_lock);
14216 	STAILQ_INSERT_TAIL(&thr->incoming_queue, &io->io_hdr, links);
14217 	mtx_unlock(&thr->queue_lock);
14218 	wakeup(thr);
14219 }
14220 
14221 static void
14222 ctl_enqueue_rtr(union ctl_io *io)
14223 {
14224 	struct ctl_softc *softc = control_softc;
14225 	struct ctl_thread *thr;
14226 
14227 	thr = &softc->threads[io->io_hdr.nexus.targ_mapped_lun % worker_threads];
14228 	mtx_lock(&thr->queue_lock);
14229 	STAILQ_INSERT_TAIL(&thr->rtr_queue, &io->io_hdr, links);
14230 	mtx_unlock(&thr->queue_lock);
14231 	wakeup(thr);
14232 }
14233 
14234 static void
14235 ctl_enqueue_done(union ctl_io *io)
14236 {
14237 	struct ctl_softc *softc = control_softc;
14238 	struct ctl_thread *thr;
14239 
14240 	thr = &softc->threads[io->io_hdr.nexus.targ_mapped_lun % worker_threads];
14241 	mtx_lock(&thr->queue_lock);
14242 	STAILQ_INSERT_TAIL(&thr->done_queue, &io->io_hdr, links);
14243 	mtx_unlock(&thr->queue_lock);
14244 	wakeup(thr);
14245 }
14246 
14247 static void
14248 ctl_enqueue_isc(union ctl_io *io)
14249 {
14250 	struct ctl_softc *softc = control_softc;
14251 	struct ctl_thread *thr;
14252 
14253 	thr = &softc->threads[io->io_hdr.nexus.targ_mapped_lun % worker_threads];
14254 	mtx_lock(&thr->queue_lock);
14255 	STAILQ_INSERT_TAIL(&thr->isc_queue, &io->io_hdr, links);
14256 	mtx_unlock(&thr->queue_lock);
14257 	wakeup(thr);
14258 }
14259 
14260 /* Initialization and failover */
14261 
14262 void
14263 ctl_init_isc_msg(void)
14264 {
14265 	printf("CTL: Still calling this thing\n");
14266 }
14267 
14268 /*
14269  * Init component
14270  * 	Initializes component into configuration defined by bootMode
14271  *	(see hasc-sv.c)
14272  *  	returns hasc_Status:
14273  * 		OK
14274  *		ERROR - fatal error
14275  */
14276 static ctl_ha_comp_status
14277 ctl_isc_init(struct ctl_ha_component *c)
14278 {
14279 	ctl_ha_comp_status ret = CTL_HA_COMP_STATUS_OK;
14280 
14281 	c->status = ret;
14282 	return ret;
14283 }
14284 
14285 /* Start component
14286  * 	Starts component in state requested. If component starts successfully,
14287  *	it must set its own state to the requestrd state
14288  *	When requested state is HASC_STATE_HA, the component may refine it
14289  * 	by adding _SLAVE or _MASTER flags.
14290  *	Currently allowed state transitions are:
14291  *	UNKNOWN->HA		- initial startup
14292  *	UNKNOWN->SINGLE - initial startup when no parter detected
14293  *	HA->SINGLE		- failover
14294  * returns ctl_ha_comp_status:
14295  * 		OK	- component successfully started in requested state
14296  *		FAILED  - could not start the requested state, failover may
14297  * 			  be possible
14298  *		ERROR	- fatal error detected, no future startup possible
14299  */
14300 static ctl_ha_comp_status
14301 ctl_isc_start(struct ctl_ha_component *c, ctl_ha_state state)
14302 {
14303 	ctl_ha_comp_status ret = CTL_HA_COMP_STATUS_OK;
14304 
14305 	printf("%s: go\n", __func__);
14306 
14307 	// UNKNOWN->HA or UNKNOWN->SINGLE (bootstrap)
14308 	if (c->state == CTL_HA_STATE_UNKNOWN ) {
14309 		control_softc->is_single = 0;
14310 		if (ctl_ha_msg_create(CTL_HA_CHAN_CTL, ctl_isc_event_handler)
14311 		    != CTL_HA_STATUS_SUCCESS) {
14312 			printf("ctl_isc_start: ctl_ha_msg_create failed.\n");
14313 			ret = CTL_HA_COMP_STATUS_ERROR;
14314 		}
14315 	} else if (CTL_HA_STATE_IS_HA(c->state)
14316 		&& CTL_HA_STATE_IS_SINGLE(state)){
14317 		// HA->SINGLE transition
14318 	        ctl_failover();
14319 		control_softc->is_single = 1;
14320 	} else {
14321 		printf("ctl_isc_start:Invalid state transition %X->%X\n",
14322 		       c->state, state);
14323 		ret = CTL_HA_COMP_STATUS_ERROR;
14324 	}
14325 	if (CTL_HA_STATE_IS_SINGLE(state))
14326 		control_softc->is_single = 1;
14327 
14328 	c->state = state;
14329 	c->status = ret;
14330 	return ret;
14331 }
14332 
14333 /*
14334  * Quiesce component
14335  * The component must clear any error conditions (set status to OK) and
14336  * prepare itself to another Start call
14337  * returns ctl_ha_comp_status:
14338  * 	OK
14339  *	ERROR
14340  */
14341 static ctl_ha_comp_status
14342 ctl_isc_quiesce(struct ctl_ha_component *c)
14343 {
14344 	int ret = CTL_HA_COMP_STATUS_OK;
14345 
14346 	ctl_pause_rtr = 1;
14347 	c->status = ret;
14348 	return ret;
14349 }
14350 
14351 struct ctl_ha_component ctl_ha_component_ctlisc =
14352 {
14353 	.name = "CTL ISC",
14354 	.state = CTL_HA_STATE_UNKNOWN,
14355 	.init = ctl_isc_init,
14356 	.start = ctl_isc_start,
14357 	.quiesce = ctl_isc_quiesce
14358 };
14359 
14360 /*
14361  *  vim: ts=8
14362  */
14363