xref: /freebsd/sys/cam/ctl/ctl_frontend_iscsi.c (revision bbb29a3c0f2c4565eff6fda70426807b6ed97f8b)
1 /*-
2  * Copyright (c) 2012 The FreeBSD Foundation
3  * All rights reserved.
4  *
5  * This software was developed by Edward Tomasz Napierala under sponsorship
6  * from the FreeBSD Foundation.
7  *
8  * Redistribution and use in source and binary forms, with or without
9  * modification, are permitted provided that the following conditions
10  * are met:
11  * 1. Redistributions of source code must retain the above copyright
12  *    notice, this list of conditions and the following disclaimer.
13  * 2. Redistributions in binary form must reproduce the above copyright
14  *    notice, this list of conditions and the following disclaimer in the
15  *    documentation and/or other materials provided with the distribution.
16  *
17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
21  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27  * SUCH DAMAGE.
28  *
29  * $FreeBSD$
30  */
31 
32 /*
33  * CTL frontend for the iSCSI protocol.
34  */
35 
36 #include <sys/cdefs.h>
37 __FBSDID("$FreeBSD$");
38 
39 #include <sys/param.h>
40 #include <sys/capsicum.h>
41 #include <sys/condvar.h>
42 #include <sys/file.h>
43 #include <sys/kernel.h>
44 #include <sys/kthread.h>
45 #include <sys/lock.h>
46 #include <sys/malloc.h>
47 #include <sys/module.h>
48 #include <sys/mutex.h>
49 #include <sys/queue.h>
50 #include <sys/sbuf.h>
51 #include <sys/sysctl.h>
52 #include <sys/systm.h>
53 #include <sys/uio.h>
54 #include <sys/unistd.h>
55 #include <vm/uma.h>
56 
57 #include <cam/scsi/scsi_all.h>
58 #include <cam/scsi/scsi_da.h>
59 #include <cam/ctl/ctl_io.h>
60 #include <cam/ctl/ctl.h>
61 #include <cam/ctl/ctl_backend.h>
62 #include <cam/ctl/ctl_error.h>
63 #include <cam/ctl/ctl_frontend.h>
64 #include <cam/ctl/ctl_frontend_internal.h>
65 #include <cam/ctl/ctl_debug.h>
66 #include <cam/ctl/ctl_ha.h>
67 #include <cam/ctl/ctl_ioctl.h>
68 #include <cam/ctl/ctl_private.h>
69 
70 #include <dev/iscsi/icl.h>
71 #include <dev/iscsi/iscsi_proto.h>
72 #include <cam/ctl/ctl_frontend_iscsi.h>
73 
74 #ifdef ICL_KERNEL_PROXY
75 #include <sys/socketvar.h>
76 #endif
77 
78 #ifdef ICL_KERNEL_PROXY
79 FEATURE(cfiscsi_kernel_proxy, "iSCSI target built with ICL_KERNEL_PROXY");
80 #endif
81 
82 static MALLOC_DEFINE(M_CFISCSI, "cfiscsi", "Memory used for CTL iSCSI frontend");
83 static uma_zone_t cfiscsi_data_wait_zone;
84 
85 SYSCTL_NODE(_kern_cam_ctl, OID_AUTO, iscsi, CTLFLAG_RD, 0,
86     "CAM Target Layer iSCSI Frontend");
87 static int debug = 1;
88 SYSCTL_INT(_kern_cam_ctl_iscsi, OID_AUTO, debug, CTLFLAG_RWTUN,
89     &debug, 1, "Enable debug messages");
90 static int ping_timeout = 5;
91 SYSCTL_INT(_kern_cam_ctl_iscsi, OID_AUTO, ping_timeout, CTLFLAG_RWTUN,
92     &ping_timeout, 5, "Interval between ping (NOP-Out) requests, in seconds");
93 static int login_timeout = 60;
94 SYSCTL_INT(_kern_cam_ctl_iscsi, OID_AUTO, login_timeout, CTLFLAG_RWTUN,
95     &login_timeout, 60, "Time to wait for ctld(8) to finish Login Phase, in seconds");
96 static int maxcmdsn_delta = 256;
97 SYSCTL_INT(_kern_cam_ctl_iscsi, OID_AUTO, maxcmdsn_delta, CTLFLAG_RWTUN,
98     &maxcmdsn_delta, 256, "Number of commands the initiator can send "
99     "without confirmation");
100 
101 #define	CFISCSI_DEBUG(X, ...)						\
102 	do {								\
103 		if (debug > 1) {					\
104 			printf("%s: " X "\n",				\
105 			    __func__, ## __VA_ARGS__);			\
106 		}							\
107 	} while (0)
108 
109 #define	CFISCSI_WARN(X, ...)						\
110 	do {								\
111 		if (debug > 0) {					\
112 			printf("WARNING: %s: " X "\n",			\
113 			    __func__, ## __VA_ARGS__);			\
114 		}							\
115 	} while (0)
116 
117 #define	CFISCSI_SESSION_DEBUG(S, X, ...)				\
118 	do {								\
119 		if (debug > 1) {					\
120 			printf("%s: %s (%s): " X "\n",			\
121 			    __func__, S->cs_initiator_addr,		\
122 			    S->cs_initiator_name, ## __VA_ARGS__);	\
123 		}							\
124 	} while (0)
125 
126 #define	CFISCSI_SESSION_WARN(S, X, ...)					\
127 	do  {								\
128 		if (debug > 0) {					\
129 			printf("WARNING: %s (%s): " X "\n",		\
130 			    S->cs_initiator_addr,			\
131 			    S->cs_initiator_name, ## __VA_ARGS__);	\
132 		}							\
133 	} while (0)
134 
135 #define CFISCSI_SESSION_LOCK(X)		mtx_lock(&X->cs_lock)
136 #define CFISCSI_SESSION_UNLOCK(X)	mtx_unlock(&X->cs_lock)
137 #define CFISCSI_SESSION_LOCK_ASSERT(X)	mtx_assert(&X->cs_lock, MA_OWNED)
138 
139 #define	CONN_SESSION(X)			((struct cfiscsi_session *)(X)->ic_prv0)
140 #define	PDU_SESSION(X)			CONN_SESSION((X)->ip_conn)
141 #define	PDU_EXPDATASN(X)		(X)->ip_prv0
142 #define	PDU_TOTAL_TRANSFER_LEN(X)	(X)->ip_prv1
143 #define	PDU_R2TSN(X)			(X)->ip_prv2
144 
145 int		cfiscsi_init(void);
146 static void	cfiscsi_online(void *arg);
147 static void	cfiscsi_offline(void *arg);
148 static int	cfiscsi_info(void *arg, struct sbuf *sb);
149 static int	cfiscsi_lun_enable(void *arg,
150 		    struct ctl_id target_id, int lun_id);
151 static int	cfiscsi_lun_disable(void *arg,
152 		    struct ctl_id target_id, int lun_id);
153 static uint32_t	cfiscsi_lun_map(void *arg, uint32_t lun);
154 static int	cfiscsi_ioctl(struct cdev *dev,
155 		    u_long cmd, caddr_t addr, int flag, struct thread *td);
156 static void	cfiscsi_datamove(union ctl_io *io);
157 static void	cfiscsi_datamove_in(union ctl_io *io);
158 static void	cfiscsi_datamove_out(union ctl_io *io);
159 static void	cfiscsi_done(union ctl_io *io);
160 static bool	cfiscsi_pdu_update_cmdsn(const struct icl_pdu *request);
161 static void	cfiscsi_pdu_handle_nop_out(struct icl_pdu *request);
162 static void	cfiscsi_pdu_handle_scsi_command(struct icl_pdu *request);
163 static void	cfiscsi_pdu_handle_task_request(struct icl_pdu *request);
164 static void	cfiscsi_pdu_handle_data_out(struct icl_pdu *request);
165 static void	cfiscsi_pdu_handle_logout_request(struct icl_pdu *request);
166 static void	cfiscsi_session_terminate(struct cfiscsi_session *cs);
167 static struct cfiscsi_target	*cfiscsi_target_find(struct cfiscsi_softc
168 		    *softc, const char *name);
169 static struct cfiscsi_target	*cfiscsi_target_find_or_create(
170     struct cfiscsi_softc *softc, const char *name, const char *alias);
171 static void	cfiscsi_target_release(struct cfiscsi_target *ct);
172 static void	cfiscsi_session_delete(struct cfiscsi_session *cs);
173 
174 static struct cfiscsi_softc cfiscsi_softc;
175 extern struct ctl_softc *control_softc;
176 
177 static struct ctl_frontend cfiscsi_frontend =
178 {
179 	.name = "iscsi",
180 	.init = cfiscsi_init,
181 	.ioctl = cfiscsi_ioctl,
182 };
183 CTL_FRONTEND_DECLARE(ctlcfiscsi, cfiscsi_frontend);
184 MODULE_DEPEND(ctlcfiscsi, icl, 1, 1, 1);
185 
186 static struct icl_pdu *
187 cfiscsi_pdu_new_response(struct icl_pdu *request, int flags)
188 {
189 
190 	return (icl_pdu_new(request->ip_conn, flags));
191 }
192 
193 static bool
194 cfiscsi_pdu_update_cmdsn(const struct icl_pdu *request)
195 {
196 	const struct iscsi_bhs_scsi_command *bhssc;
197 	struct cfiscsi_session *cs;
198 	uint32_t cmdsn, expstatsn;
199 
200 	cs = PDU_SESSION(request);
201 
202 	/*
203 	 * Every incoming PDU - not just NOP-Out - resets the ping timer.
204 	 * The purpose of the timeout is to reset the connection when it stalls;
205 	 * we don't want this to happen when NOP-In or NOP-Out ends up delayed
206 	 * in some queue.
207 	 *
208 	 * XXX: Locking?
209 	 */
210 	cs->cs_timeout = 0;
211 
212 	/*
213 	 * Data-Out PDUs don't contain CmdSN.
214 	 */
215 	if ((request->ip_bhs->bhs_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
216 	    ISCSI_BHS_OPCODE_SCSI_DATA_OUT)
217 		return (false);
218 
219 	/*
220 	 * We're only using fields common for all the request
221 	 * (initiator -> target) PDUs.
222 	 */
223 	bhssc = (const struct iscsi_bhs_scsi_command *)request->ip_bhs;
224 	cmdsn = ntohl(bhssc->bhssc_cmdsn);
225 	expstatsn = ntohl(bhssc->bhssc_expstatsn);
226 
227 	CFISCSI_SESSION_LOCK(cs);
228 #if 0
229 	if (expstatsn != cs->cs_statsn) {
230 		CFISCSI_SESSION_DEBUG(cs, "received PDU with ExpStatSN %d, "
231 		    "while current StatSN is %d", expstatsn,
232 		    cs->cs_statsn);
233 	}
234 #endif
235 
236 	/*
237 	 * The target MUST silently ignore any non-immediate command outside
238 	 * of this range.
239 	 */
240 	if (cmdsn < cs->cs_cmdsn || cmdsn > cs->cs_cmdsn + maxcmdsn_delta) {
241 		CFISCSI_SESSION_UNLOCK(cs);
242 		CFISCSI_SESSION_WARN(cs, "received PDU with CmdSN %d, "
243 		    "while expected CmdSN was %d", cmdsn, cs->cs_cmdsn);
244 		return (true);
245 	}
246 
247 	if ((request->ip_bhs->bhs_opcode & ISCSI_BHS_OPCODE_IMMEDIATE) == 0)
248 		cs->cs_cmdsn++;
249 
250 	CFISCSI_SESSION_UNLOCK(cs);
251 
252 	return (false);
253 }
254 
255 static void
256 cfiscsi_pdu_handle(struct icl_pdu *request)
257 {
258 	struct cfiscsi_session *cs;
259 	bool ignore;
260 
261 	cs = PDU_SESSION(request);
262 
263 	ignore = cfiscsi_pdu_update_cmdsn(request);
264 	if (ignore) {
265 		icl_pdu_free(request);
266 		return;
267 	}
268 
269 	/*
270 	 * Handle the PDU; this includes e.g. receiving the remaining
271 	 * part of PDU and submitting the SCSI command to CTL
272 	 * or queueing a reply.  The handling routine is responsible
273 	 * for freeing the PDU when it's no longer needed.
274 	 */
275 	switch (request->ip_bhs->bhs_opcode &
276 	    ~ISCSI_BHS_OPCODE_IMMEDIATE) {
277 	case ISCSI_BHS_OPCODE_NOP_OUT:
278 		cfiscsi_pdu_handle_nop_out(request);
279 		break;
280 	case ISCSI_BHS_OPCODE_SCSI_COMMAND:
281 		cfiscsi_pdu_handle_scsi_command(request);
282 		break;
283 	case ISCSI_BHS_OPCODE_TASK_REQUEST:
284 		cfiscsi_pdu_handle_task_request(request);
285 		break;
286 	case ISCSI_BHS_OPCODE_SCSI_DATA_OUT:
287 		cfiscsi_pdu_handle_data_out(request);
288 		break;
289 	case ISCSI_BHS_OPCODE_LOGOUT_REQUEST:
290 		cfiscsi_pdu_handle_logout_request(request);
291 		break;
292 	default:
293 		CFISCSI_SESSION_WARN(cs, "received PDU with unsupported "
294 		    "opcode 0x%x; dropping connection",
295 		    request->ip_bhs->bhs_opcode);
296 		icl_pdu_free(request);
297 		cfiscsi_session_terminate(cs);
298 	}
299 
300 }
301 
302 static void
303 cfiscsi_receive_callback(struct icl_pdu *request)
304 {
305 	struct cfiscsi_session *cs;
306 
307 	cs = PDU_SESSION(request);
308 
309 #ifdef ICL_KERNEL_PROXY
310 	if (cs->cs_waiting_for_ctld || cs->cs_login_phase) {
311 		if (cs->cs_login_pdu == NULL)
312 			cs->cs_login_pdu = request;
313 		else
314 			icl_pdu_free(request);
315 		cv_signal(&cs->cs_login_cv);
316 		return;
317 	}
318 #endif
319 
320 	cfiscsi_pdu_handle(request);
321 }
322 
323 static void
324 cfiscsi_error_callback(struct icl_conn *ic)
325 {
326 	struct cfiscsi_session *cs;
327 
328 	cs = CONN_SESSION(ic);
329 
330 	CFISCSI_SESSION_WARN(cs, "connection error; dropping connection");
331 	cfiscsi_session_terminate(cs);
332 }
333 
334 static int
335 cfiscsi_pdu_prepare(struct icl_pdu *response)
336 {
337 	struct cfiscsi_session *cs;
338 	struct iscsi_bhs_scsi_response *bhssr;
339 	bool advance_statsn = true;
340 
341 	cs = PDU_SESSION(response);
342 
343 	CFISCSI_SESSION_LOCK_ASSERT(cs);
344 
345 	/*
346 	 * We're only using fields common for all the response
347 	 * (target -> initiator) PDUs.
348 	 */
349 	bhssr = (struct iscsi_bhs_scsi_response *)response->ip_bhs;
350 
351 	/*
352 	 * 10.8.3: "The StatSN for this connection is not advanced
353 	 * after this PDU is sent."
354 	 */
355 	if (bhssr->bhssr_opcode == ISCSI_BHS_OPCODE_R2T)
356 		advance_statsn = false;
357 
358 	/*
359 	 * 10.19.2: "However, when the Initiator Task Tag is set to 0xffffffff,
360 	 * StatSN for the connection is not advanced after this PDU is sent."
361 	 */
362 	if (bhssr->bhssr_opcode == ISCSI_BHS_OPCODE_NOP_IN &&
363 	    bhssr->bhssr_initiator_task_tag == 0xffffffff)
364 		advance_statsn = false;
365 
366 	/*
367 	 * See the comment below - StatSN is not meaningful and must
368 	 * not be advanced.
369 	 */
370 	if (bhssr->bhssr_opcode == ISCSI_BHS_OPCODE_SCSI_DATA_IN &&
371 	    (bhssr->bhssr_flags & BHSDI_FLAGS_S) == 0)
372 		advance_statsn = false;
373 
374 	/*
375 	 * 10.7.3: "The fields StatSN, Status, and Residual Count
376 	 * only have meaningful content if the S bit is set to 1."
377 	 */
378 	if (bhssr->bhssr_opcode != ISCSI_BHS_OPCODE_SCSI_DATA_IN ||
379 	    (bhssr->bhssr_flags & BHSDI_FLAGS_S))
380 		bhssr->bhssr_statsn = htonl(cs->cs_statsn);
381 	bhssr->bhssr_expcmdsn = htonl(cs->cs_cmdsn);
382 	bhssr->bhssr_maxcmdsn = htonl(cs->cs_cmdsn + maxcmdsn_delta);
383 
384 	if (advance_statsn)
385 		cs->cs_statsn++;
386 
387 	return (0);
388 }
389 
390 static void
391 cfiscsi_pdu_queue(struct icl_pdu *response)
392 {
393 	struct cfiscsi_session *cs;
394 
395 	cs = PDU_SESSION(response);
396 
397 	CFISCSI_SESSION_LOCK(cs);
398 	cfiscsi_pdu_prepare(response);
399 	icl_pdu_queue(response);
400 	CFISCSI_SESSION_UNLOCK(cs);
401 }
402 
403 static uint32_t
404 cfiscsi_decode_lun(uint64_t encoded)
405 {
406 	uint8_t lun[8];
407 	uint32_t result;
408 
409 	/*
410 	 * The LUN field in iSCSI PDUs may look like an ordinary 64 bit number,
411 	 * but is in fact an evil, multidimensional structure defined
412 	 * in SCSI Architecture Model 5 (SAM-5), section 4.6.
413 	 */
414 	memcpy(lun, &encoded, sizeof(lun));
415 	switch (lun[0] & 0xC0) {
416 	case 0x00:
417 		if ((lun[0] & 0x3f) != 0 || lun[2] != 0 || lun[3] != 0 ||
418 		    lun[4] != 0 || lun[5] != 0 || lun[6] != 0 || lun[7] != 0) {
419 			CFISCSI_WARN("malformed LUN "
420 			    "(peripheral device addressing method): 0x%jx",
421 			    (uintmax_t)encoded);
422 			result = 0xffffffff;
423 			break;
424 		}
425 		result = lun[1];
426 		break;
427 	case 0x40:
428 		if (lun[2] != 0 || lun[3] != 0 || lun[4] != 0 || lun[5] != 0 ||
429 		    lun[6] != 0 || lun[7] != 0) {
430 			CFISCSI_WARN("malformed LUN "
431 			    "(flat address space addressing method): 0x%jx",
432 			    (uintmax_t)encoded);
433 			result = 0xffffffff;
434 			break;
435 		}
436 		result = ((lun[0] & 0x3f) << 8) + lun[1];
437 		break;
438 	case 0xC0:
439 		if (lun[0] != 0xD2 || lun[4] != 0 || lun[5] != 0 ||
440 		    lun[6] != 0 || lun[7] != 0) {
441 			CFISCSI_WARN("malformed LUN (extended flat "
442 			    "address space addressing method): 0x%jx",
443 			    (uintmax_t)encoded);
444 			result = 0xffffffff;
445 			break;
446 		}
447 		result = (lun[1] << 16) + (lun[2] << 8) + lun[3];
448 	default:
449 		CFISCSI_WARN("unsupported LUN format 0x%jx",
450 		    (uintmax_t)encoded);
451 		result = 0xffffffff;
452 		break;
453 	}
454 
455 	return (result);
456 }
457 
458 static void
459 cfiscsi_pdu_handle_nop_out(struct icl_pdu *request)
460 {
461 	struct cfiscsi_session *cs;
462 	struct iscsi_bhs_nop_out *bhsno;
463 	struct iscsi_bhs_nop_in *bhsni;
464 	struct icl_pdu *response;
465 	void *data = NULL;
466 	size_t datasize;
467 	int error;
468 
469 	cs = PDU_SESSION(request);
470 	bhsno = (struct iscsi_bhs_nop_out *)request->ip_bhs;
471 
472 	if (bhsno->bhsno_initiator_task_tag == 0xffffffff) {
473 		/*
474 		 * Nothing to do, iscsi_pdu_update_statsn() already
475 		 * zeroed the timeout.
476 		 */
477 		icl_pdu_free(request);
478 		return;
479 	}
480 
481 	datasize = icl_pdu_data_segment_length(request);
482 	if (datasize > 0) {
483 		data = malloc(datasize, M_CFISCSI, M_NOWAIT | M_ZERO);
484 		if (data == NULL) {
485 			CFISCSI_SESSION_WARN(cs, "failed to allocate memory; "
486 			    "dropping connection");
487 			icl_pdu_free(request);
488 			cfiscsi_session_terminate(cs);
489 			return;
490 		}
491 		icl_pdu_get_data(request, 0, data, datasize);
492 	}
493 
494 	response = cfiscsi_pdu_new_response(request, M_NOWAIT);
495 	if (response == NULL) {
496 		CFISCSI_SESSION_WARN(cs, "failed to allocate memory; "
497 		    "droppping connection");
498 		free(data, M_CFISCSI);
499 		icl_pdu_free(request);
500 		cfiscsi_session_terminate(cs);
501 		return;
502 	}
503 	bhsni = (struct iscsi_bhs_nop_in *)response->ip_bhs;
504 	bhsni->bhsni_opcode = ISCSI_BHS_OPCODE_NOP_IN;
505 	bhsni->bhsni_flags = 0x80;
506 	bhsni->bhsni_initiator_task_tag = bhsno->bhsno_initiator_task_tag;
507 	bhsni->bhsni_target_transfer_tag = 0xffffffff;
508 	if (datasize > 0) {
509 		error = icl_pdu_append_data(response, data, datasize, M_NOWAIT);
510 		if (error != 0) {
511 			CFISCSI_SESSION_WARN(cs, "failed to allocate memory; "
512 			    "dropping connection");
513 			free(data, M_CFISCSI);
514 			icl_pdu_free(request);
515 			icl_pdu_free(response);
516 			cfiscsi_session_terminate(cs);
517 			return;
518 		}
519 		free(data, M_CFISCSI);
520 	}
521 
522 	icl_pdu_free(request);
523 	cfiscsi_pdu_queue(response);
524 }
525 
526 static void
527 cfiscsi_pdu_handle_scsi_command(struct icl_pdu *request)
528 {
529 	struct iscsi_bhs_scsi_command *bhssc;
530 	struct cfiscsi_session *cs;
531 	union ctl_io *io;
532 	int error;
533 
534 	cs = PDU_SESSION(request);
535 	bhssc = (struct iscsi_bhs_scsi_command *)request->ip_bhs;
536 	//CFISCSI_SESSION_DEBUG(cs, "initiator task tag 0x%x",
537 	//    bhssc->bhssc_initiator_task_tag);
538 
539 	if (request->ip_data_len > 0 && cs->cs_immediate_data == false) {
540 		CFISCSI_SESSION_WARN(cs, "unsolicited data with "
541 		    "ImmediateData=No; dropping connection");
542 		icl_pdu_free(request);
543 		cfiscsi_session_terminate(cs);
544 		return;
545 	}
546 	io = ctl_alloc_io(cs->cs_target->ct_port.ctl_pool_ref);
547 	ctl_zero_io(io);
548 	io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr = request;
549 	io->io_hdr.io_type = CTL_IO_SCSI;
550 	io->io_hdr.nexus.initid.id = cs->cs_ctl_initid;
551 	io->io_hdr.nexus.targ_port = cs->cs_target->ct_port.targ_port;
552 	io->io_hdr.nexus.targ_target.id = 0;
553 	io->io_hdr.nexus.targ_lun = cfiscsi_decode_lun(bhssc->bhssc_lun);
554 	io->scsiio.tag_num = bhssc->bhssc_initiator_task_tag;
555 	switch ((bhssc->bhssc_flags & BHSSC_FLAGS_ATTR)) {
556 	case BHSSC_FLAGS_ATTR_UNTAGGED:
557 		io->scsiio.tag_type = CTL_TAG_UNTAGGED;
558 		break;
559 	case BHSSC_FLAGS_ATTR_SIMPLE:
560 		io->scsiio.tag_type = CTL_TAG_SIMPLE;
561 		break;
562 	case BHSSC_FLAGS_ATTR_ORDERED:
563         	io->scsiio.tag_type = CTL_TAG_ORDERED;
564 		break;
565 	case BHSSC_FLAGS_ATTR_HOQ:
566         	io->scsiio.tag_type = CTL_TAG_HEAD_OF_QUEUE;
567 		break;
568 	case BHSSC_FLAGS_ATTR_ACA:
569 		io->scsiio.tag_type = CTL_TAG_ACA;
570 		break;
571 	default:
572 		io->scsiio.tag_type = CTL_TAG_UNTAGGED;
573 		CFISCSI_SESSION_WARN(cs, "unhandled tag type %d",
574 		    bhssc->bhssc_flags & BHSSC_FLAGS_ATTR);
575 		break;
576 	}
577 	io->scsiio.cdb_len = sizeof(bhssc->bhssc_cdb); /* Which is 16. */
578 	memcpy(io->scsiio.cdb, bhssc->bhssc_cdb, sizeof(bhssc->bhssc_cdb));
579 	refcount_acquire(&cs->cs_outstanding_ctl_pdus);
580 	error = ctl_queue(io);
581 	if (error != CTL_RETVAL_COMPLETE) {
582 		CFISCSI_SESSION_WARN(cs, "ctl_queue() failed; error %d; "
583 		    "dropping connection", error);
584 		ctl_free_io(io);
585 		refcount_release(&cs->cs_outstanding_ctl_pdus);
586 		icl_pdu_free(request);
587 		cfiscsi_session_terminate(cs);
588 	}
589 }
590 
591 static void
592 cfiscsi_pdu_handle_task_request(struct icl_pdu *request)
593 {
594 	struct iscsi_bhs_task_management_request *bhstmr;
595 	struct iscsi_bhs_task_management_response *bhstmr2;
596 	struct icl_pdu *response;
597 	struct cfiscsi_session *cs;
598 	union ctl_io *io;
599 	int error;
600 
601 	cs = PDU_SESSION(request);
602 	bhstmr = (struct iscsi_bhs_task_management_request *)request->ip_bhs;
603 	io = ctl_alloc_io(cs->cs_target->ct_port.ctl_pool_ref);
604 	ctl_zero_io(io);
605 	io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr = request;
606 	io->io_hdr.io_type = CTL_IO_TASK;
607 	io->io_hdr.nexus.initid.id = cs->cs_ctl_initid;
608 	io->io_hdr.nexus.targ_port = cs->cs_target->ct_port.targ_port;
609 	io->io_hdr.nexus.targ_target.id = 0;
610 	io->io_hdr.nexus.targ_lun = cfiscsi_decode_lun(bhstmr->bhstmr_lun);
611 	io->taskio.tag_type = CTL_TAG_SIMPLE; /* XXX */
612 
613 	switch (bhstmr->bhstmr_function & ~0x80) {
614 	case BHSTMR_FUNCTION_ABORT_TASK:
615 #if 0
616 		CFISCSI_SESSION_DEBUG(cs, "BHSTMR_FUNCTION_ABORT_TASK");
617 #endif
618 		io->taskio.task_action = CTL_TASK_ABORT_TASK;
619 		io->taskio.tag_num = bhstmr->bhstmr_referenced_task_tag;
620 		break;
621 	case BHSTMR_FUNCTION_ABORT_TASK_SET:
622 #if 0
623 		CFISCSI_SESSION_DEBUG(cs, "BHSTMR_FUNCTION_ABORT_TASK_SET");
624 #endif
625 		io->taskio.task_action = CTL_TASK_ABORT_TASK_SET;
626 		break;
627 	case BHSTMR_FUNCTION_LOGICAL_UNIT_RESET:
628 #if 0
629 		CFISCSI_SESSION_DEBUG(cs, "BHSTMR_FUNCTION_LOGICAL_UNIT_RESET");
630 #endif
631 		io->taskio.task_action = CTL_TASK_LUN_RESET;
632 		break;
633 	case BHSTMR_FUNCTION_TARGET_WARM_RESET:
634 #if 0
635 		CFISCSI_SESSION_DEBUG(cs, "BHSTMR_FUNCTION_TARGET_WARM_RESET");
636 #endif
637 		io->taskio.task_action = CTL_TASK_TARGET_RESET;
638 		break;
639 	default:
640 		CFISCSI_SESSION_DEBUG(cs, "unsupported function 0x%x",
641 		    bhstmr->bhstmr_function & ~0x80);
642 		ctl_free_io(io);
643 
644 		response = cfiscsi_pdu_new_response(request, M_NOWAIT);
645 		if (response == NULL) {
646 			CFISCSI_SESSION_WARN(cs, "failed to allocate memory; "
647 			    "dropping connection");
648 			icl_pdu_free(request);
649 			cfiscsi_session_terminate(cs);
650 			return;
651 		}
652 		bhstmr2 = (struct iscsi_bhs_task_management_response *)
653 		    response->ip_bhs;
654 		bhstmr2->bhstmr_opcode = ISCSI_BHS_OPCODE_TASK_RESPONSE;
655 		bhstmr2->bhstmr_flags = 0x80;
656 		bhstmr2->bhstmr_response =
657 		    BHSTMR_RESPONSE_FUNCTION_NOT_SUPPORTED;
658 		bhstmr2->bhstmr_initiator_task_tag =
659 		    bhstmr->bhstmr_initiator_task_tag;
660 		icl_pdu_free(request);
661 		cfiscsi_pdu_queue(response);
662 		return;
663 	}
664 
665 	refcount_acquire(&cs->cs_outstanding_ctl_pdus);
666 	error = ctl_queue(io);
667 	if (error != CTL_RETVAL_COMPLETE) {
668 		CFISCSI_SESSION_WARN(cs, "ctl_queue() failed; error %d; "
669 		    "dropping connection", error);
670 		ctl_free_io(io);
671 		refcount_release(&cs->cs_outstanding_ctl_pdus);
672 		icl_pdu_free(request);
673 		cfiscsi_session_terminate(cs);
674 	}
675 }
676 
677 static bool
678 cfiscsi_handle_data_segment(struct icl_pdu *request, struct cfiscsi_data_wait *cdw)
679 {
680 	struct iscsi_bhs_data_out *bhsdo;
681 	struct cfiscsi_session *cs;
682 	struct ctl_sg_entry ctl_sg_entry, *ctl_sglist;
683 	size_t copy_len, len, off, buffer_offset;
684 	int ctl_sg_count;
685 	union ctl_io *io;
686 
687 	cs = PDU_SESSION(request);
688 
689 	KASSERT((request->ip_bhs->bhs_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
690 	    ISCSI_BHS_OPCODE_SCSI_DATA_OUT ||
691 	    (request->ip_bhs->bhs_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
692 	    ISCSI_BHS_OPCODE_SCSI_COMMAND,
693 	    ("bad opcode 0x%x", request->ip_bhs->bhs_opcode));
694 
695 	/*
696 	 * We're only using fields common for Data-Out and SCSI Command PDUs.
697 	 */
698 	bhsdo = (struct iscsi_bhs_data_out *)request->ip_bhs;
699 
700 	io = cdw->cdw_ctl_io;
701 	KASSERT((io->io_hdr.flags & CTL_FLAG_DATA_MASK) != CTL_FLAG_DATA_IN,
702 	    ("CTL_FLAG_DATA_IN"));
703 
704 #if 0
705 	CFISCSI_SESSION_DEBUG(cs, "received %zd bytes out of %d",
706 	    request->ip_data_len, io->scsiio.kern_total_len);
707 #endif
708 
709 	if (io->scsiio.kern_sg_entries > 0) {
710 		ctl_sglist = (struct ctl_sg_entry *)io->scsiio.kern_data_ptr;
711 		ctl_sg_count = io->scsiio.kern_sg_entries;
712 	} else {
713 		ctl_sglist = &ctl_sg_entry;
714 		ctl_sglist->addr = io->scsiio.kern_data_ptr;
715 		ctl_sglist->len = io->scsiio.kern_data_len;
716 		ctl_sg_count = 1;
717 	}
718 
719 	if ((request->ip_bhs->bhs_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
720 	    ISCSI_BHS_OPCODE_SCSI_DATA_OUT)
721 		buffer_offset = ntohl(bhsdo->bhsdo_buffer_offset);
722 	else
723 		buffer_offset = 0;
724 	len = icl_pdu_data_segment_length(request);
725 
726 	/*
727 	 * Make sure the offset, as sent by the initiator, matches the offset
728 	 * we're supposed to be at in the scatter-gather list.
729 	 */
730 	if (buffer_offset >
731 	    io->scsiio.kern_rel_offset + io->scsiio.ext_data_filled ||
732 	    buffer_offset + len <=
733 	    io->scsiio.kern_rel_offset + io->scsiio.ext_data_filled) {
734 		CFISCSI_SESSION_WARN(cs, "received bad buffer offset %zd, "
735 		    "expected %zd; dropping connection", buffer_offset,
736 		    (size_t)io->scsiio.kern_rel_offset +
737 		    (size_t)io->scsiio.ext_data_filled);
738 		ctl_set_data_phase_error(&io->scsiio);
739 		cfiscsi_session_terminate(cs);
740 		return (true);
741 	}
742 
743 	/*
744 	 * This is the offset within the PDU data segment, as opposed
745 	 * to buffer_offset, which is the offset within the task (SCSI
746 	 * command).
747 	 */
748 	off = io->scsiio.kern_rel_offset + io->scsiio.ext_data_filled -
749 	    buffer_offset;
750 
751 	/*
752 	 * Iterate over the scatter/gather segments, filling them with data
753 	 * from the PDU data segment.  Note that this can get called multiple
754 	 * times for one SCSI command; the cdw structure holds state for the
755 	 * scatter/gather list.
756 	 */
757 	for (;;) {
758 		KASSERT(cdw->cdw_sg_index < ctl_sg_count,
759 		    ("cdw->cdw_sg_index >= ctl_sg_count"));
760 		if (cdw->cdw_sg_len == 0) {
761 			cdw->cdw_sg_addr = ctl_sglist[cdw->cdw_sg_index].addr;
762 			cdw->cdw_sg_len = ctl_sglist[cdw->cdw_sg_index].len;
763 		}
764 		KASSERT(off <= len, ("len > off"));
765 		copy_len = len - off;
766 		if (copy_len > cdw->cdw_sg_len)
767 			copy_len = cdw->cdw_sg_len;
768 
769 		icl_pdu_get_data(request, off, cdw->cdw_sg_addr, copy_len);
770 		cdw->cdw_sg_addr += copy_len;
771 		cdw->cdw_sg_len -= copy_len;
772 		off += copy_len;
773 		io->scsiio.ext_data_filled += copy_len;
774 
775 		if (cdw->cdw_sg_len == 0) {
776 			/*
777 			 * End of current segment.
778 			 */
779 			if (cdw->cdw_sg_index == ctl_sg_count - 1) {
780 				/*
781 				 * Last segment in scatter/gather list.
782 				 */
783 				break;
784 			}
785 			cdw->cdw_sg_index++;
786 		}
787 
788 		if (off == len) {
789 			/*
790 			 * End of PDU payload.
791 			 */
792 			break;
793 		}
794 	}
795 
796 	if (len > off) {
797 		/*
798 		 * In case of unsolicited data, it's possible that the buffer
799 		 * provided by CTL is smaller than negotiated FirstBurstLength.
800 		 * Just ignore the superfluous data; will ask for them with R2T
801 		 * on next call to cfiscsi_datamove().
802 		 *
803 		 * This obviously can only happen with SCSI Command PDU.
804 		 */
805 		if ((request->ip_bhs->bhs_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
806 		    ISCSI_BHS_OPCODE_SCSI_COMMAND)
807 			return (true);
808 
809 		CFISCSI_SESSION_WARN(cs, "received too much data: got %zd bytes, "
810 		    "expected %zd; dropping connection",
811 		    icl_pdu_data_segment_length(request), off);
812 		ctl_set_data_phase_error(&io->scsiio);
813 		cfiscsi_session_terminate(cs);
814 		return (true);
815 	}
816 
817 	if (io->scsiio.ext_data_filled == cdw->cdw_r2t_end &&
818 	    (bhsdo->bhsdo_flags & BHSDO_FLAGS_F) == 0) {
819 		CFISCSI_SESSION_WARN(cs, "got the final packet without "
820 		    "the F flag; flags = 0x%x; dropping connection",
821 		    bhsdo->bhsdo_flags);
822 		ctl_set_data_phase_error(&io->scsiio);
823 		cfiscsi_session_terminate(cs);
824 		return (true);
825 	}
826 
827 	if (io->scsiio.ext_data_filled != cdw->cdw_r2t_end &&
828 	    (bhsdo->bhsdo_flags & BHSDO_FLAGS_F) != 0) {
829 		if ((request->ip_bhs->bhs_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
830 		    ISCSI_BHS_OPCODE_SCSI_DATA_OUT) {
831 			CFISCSI_SESSION_WARN(cs, "got the final packet, but the "
832 			    "transmitted size was %zd bytes instead of %d; "
833 			    "dropping connection",
834 			    (size_t)io->scsiio.ext_data_filled,
835 			    cdw->cdw_r2t_end);
836 			ctl_set_data_phase_error(&io->scsiio);
837 			cfiscsi_session_terminate(cs);
838 			return (true);
839 		} else {
840 			/*
841 			 * For SCSI Command PDU, this just means we need to
842 			 * solicit more data by sending R2T.
843 			 */
844 			return (false);
845 		}
846 	}
847 
848 	if (io->scsiio.ext_data_filled == cdw->cdw_r2t_end) {
849 #if 0
850 		CFISCSI_SESSION_DEBUG(cs, "no longer expecting Data-Out with target "
851 		    "transfer tag 0x%x", cdw->cdw_target_transfer_tag);
852 #endif
853 
854 		return (true);
855 	}
856 
857 	return (false);
858 }
859 
860 static void
861 cfiscsi_pdu_handle_data_out(struct icl_pdu *request)
862 {
863 	struct iscsi_bhs_data_out *bhsdo;
864 	struct cfiscsi_session *cs;
865 	struct cfiscsi_data_wait *cdw = NULL;
866 	union ctl_io *io;
867 	bool done;
868 
869 	cs = PDU_SESSION(request);
870 	bhsdo = (struct iscsi_bhs_data_out *)request->ip_bhs;
871 
872 	CFISCSI_SESSION_LOCK(cs);
873 	TAILQ_FOREACH(cdw, &cs->cs_waiting_for_data_out, cdw_next) {
874 #if 0
875 		CFISCSI_SESSION_DEBUG(cs, "have ttt 0x%x, itt 0x%x; looking for "
876 		    "ttt 0x%x, itt 0x%x",
877 		    bhsdo->bhsdo_target_transfer_tag,
878 		    bhsdo->bhsdo_initiator_task_tag,
879 		    cdw->cdw_target_transfer_tag, cdw->cdw_initiator_task_tag));
880 #endif
881 		if (bhsdo->bhsdo_target_transfer_tag ==
882 		    cdw->cdw_target_transfer_tag)
883 			break;
884 	}
885 	CFISCSI_SESSION_UNLOCK(cs);
886 	if (cdw == NULL) {
887 		CFISCSI_SESSION_WARN(cs, "data transfer tag 0x%x, initiator task tag "
888 		    "0x%x, not found; dropping connection",
889 		    bhsdo->bhsdo_target_transfer_tag, bhsdo->bhsdo_initiator_task_tag);
890 		icl_pdu_free(request);
891 		cfiscsi_session_terminate(cs);
892 		return;
893 	}
894 
895 	io = cdw->cdw_ctl_io;
896 	KASSERT((io->io_hdr.flags & CTL_FLAG_DATA_MASK) != CTL_FLAG_DATA_IN,
897 	    ("CTL_FLAG_DATA_IN"));
898 
899 	done = cfiscsi_handle_data_segment(request, cdw);
900 	if (done) {
901 		CFISCSI_SESSION_LOCK(cs);
902 		TAILQ_REMOVE(&cs->cs_waiting_for_data_out, cdw, cdw_next);
903 		CFISCSI_SESSION_UNLOCK(cs);
904 		done = (io->scsiio.ext_data_filled != cdw->cdw_r2t_end ||
905 		    io->scsiio.ext_data_filled == io->scsiio.kern_data_len);
906 		uma_zfree(cfiscsi_data_wait_zone, cdw);
907 		if (done)
908 			io->scsiio.be_move_done(io);
909 		else
910 			cfiscsi_datamove_out(io);
911 	}
912 
913 	icl_pdu_free(request);
914 }
915 
916 static void
917 cfiscsi_pdu_handle_logout_request(struct icl_pdu *request)
918 {
919 	struct iscsi_bhs_logout_request *bhslr;
920 	struct iscsi_bhs_logout_response *bhslr2;
921 	struct icl_pdu *response;
922 	struct cfiscsi_session *cs;
923 
924 	cs = PDU_SESSION(request);
925 	bhslr = (struct iscsi_bhs_logout_request *)request->ip_bhs;
926 	switch (bhslr->bhslr_reason & 0x7f) {
927 	case BHSLR_REASON_CLOSE_SESSION:
928 	case BHSLR_REASON_CLOSE_CONNECTION:
929 		response = cfiscsi_pdu_new_response(request, M_NOWAIT);
930 		if (response == NULL) {
931 			CFISCSI_SESSION_DEBUG(cs, "failed to allocate memory");
932 			icl_pdu_free(request);
933 			cfiscsi_session_terminate(cs);
934 			return;
935 		}
936 		bhslr2 = (struct iscsi_bhs_logout_response *)response->ip_bhs;
937 		bhslr2->bhslr_opcode = ISCSI_BHS_OPCODE_LOGOUT_RESPONSE;
938 		bhslr2->bhslr_flags = 0x80;
939 		bhslr2->bhslr_response = BHSLR_RESPONSE_CLOSED_SUCCESSFULLY;
940 		bhslr2->bhslr_initiator_task_tag =
941 		    bhslr->bhslr_initiator_task_tag;
942 		icl_pdu_free(request);
943 		cfiscsi_pdu_queue(response);
944 		cfiscsi_session_terminate(cs);
945 		break;
946 	case BHSLR_REASON_REMOVE_FOR_RECOVERY:
947 		response = cfiscsi_pdu_new_response(request, M_NOWAIT);
948 		if (response == NULL) {
949 			CFISCSI_SESSION_WARN(cs,
950 			    "failed to allocate memory; dropping connection");
951 			icl_pdu_free(request);
952 			cfiscsi_session_terminate(cs);
953 			return;
954 		}
955 		bhslr2 = (struct iscsi_bhs_logout_response *)response->ip_bhs;
956 		bhslr2->bhslr_opcode = ISCSI_BHS_OPCODE_LOGOUT_RESPONSE;
957 		bhslr2->bhslr_flags = 0x80;
958 		bhslr2->bhslr_response = BHSLR_RESPONSE_RECOVERY_NOT_SUPPORTED;
959 		bhslr2->bhslr_initiator_task_tag =
960 		    bhslr->bhslr_initiator_task_tag;
961 		icl_pdu_free(request);
962 		cfiscsi_pdu_queue(response);
963 		break;
964 	default:
965 		CFISCSI_SESSION_WARN(cs, "invalid reason 0%x; dropping connection",
966 		    bhslr->bhslr_reason);
967 		icl_pdu_free(request);
968 		cfiscsi_session_terminate(cs);
969 		break;
970 	}
971 }
972 
973 static void
974 cfiscsi_callout(void *context)
975 {
976 	struct icl_pdu *cp;
977 	struct iscsi_bhs_nop_in *bhsni;
978 	struct cfiscsi_session *cs;
979 
980 	cs = context;
981 
982 	if (cs->cs_terminating)
983 		return;
984 
985 	callout_schedule(&cs->cs_callout, 1 * hz);
986 
987 	atomic_add_int(&cs->cs_timeout, 1);
988 
989 #ifdef ICL_KERNEL_PROXY
990 	if (cs->cs_waiting_for_ctld || cs->cs_login_phase) {
991 		if (login_timeout > 0 && cs->cs_timeout > login_timeout) {
992 			CFISCSI_SESSION_WARN(cs, "login timed out after "
993 			    "%d seconds; dropping connection", cs->cs_timeout);
994 			cfiscsi_session_terminate(cs);
995 		}
996 		return;
997 	}
998 #endif
999 
1000 	if (ping_timeout <= 0) {
1001 		/*
1002 		 * Pings are disabled.  Don't send NOP-In in this case;
1003 		 * user might have disabled pings to work around problems
1004 		 * with certain initiators that can't properly handle
1005 		 * NOP-In, such as iPXE.  Reset the timeout, to avoid
1006 		 * triggering reconnection, should the user decide to
1007 		 * reenable them.
1008 		 */
1009 		cs->cs_timeout = 0;
1010 		return;
1011 	}
1012 
1013 	if (cs->cs_timeout >= ping_timeout) {
1014 		CFISCSI_SESSION_WARN(cs, "no ping reply (NOP-Out) after %d seconds; "
1015 		    "dropping connection",  ping_timeout);
1016 		cfiscsi_session_terminate(cs);
1017 		return;
1018 	}
1019 
1020 	/*
1021 	 * If the ping was reset less than one second ago - which means
1022 	 * that we've received some PDU during the last second - assume
1023 	 * the traffic flows correctly and don't bother sending a NOP-Out.
1024 	 *
1025 	 * (It's 2 - one for one second, and one for incrementing is_timeout
1026 	 * earlier in this routine.)
1027 	 */
1028 	if (cs->cs_timeout < 2)
1029 		return;
1030 
1031 	cp = icl_pdu_new(cs->cs_conn, M_NOWAIT);
1032 	if (cp == NULL) {
1033 		CFISCSI_SESSION_WARN(cs, "failed to allocate memory");
1034 		return;
1035 	}
1036 	bhsni = (struct iscsi_bhs_nop_in *)cp->ip_bhs;
1037 	bhsni->bhsni_opcode = ISCSI_BHS_OPCODE_NOP_IN;
1038 	bhsni->bhsni_flags = 0x80;
1039 	bhsni->bhsni_initiator_task_tag = 0xffffffff;
1040 
1041 	cfiscsi_pdu_queue(cp);
1042 }
1043 
1044 static void
1045 cfiscsi_session_terminate_tasks(struct cfiscsi_session *cs)
1046 {
1047 	struct cfiscsi_data_wait *cdw;
1048 	union ctl_io *io;
1049 	int error, last, wait;
1050 
1051 	if (cs->cs_target == NULL)
1052 		return;		/* No target yet, so nothing to do. */
1053 	io = ctl_alloc_io(cs->cs_target->ct_port.ctl_pool_ref);
1054 	ctl_zero_io(io);
1055 	io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr = cs;
1056 	io->io_hdr.io_type = CTL_IO_TASK;
1057 	io->io_hdr.nexus.initid.id = cs->cs_ctl_initid;
1058 	io->io_hdr.nexus.targ_port = cs->cs_target->ct_port.targ_port;
1059 	io->io_hdr.nexus.targ_target.id = 0;
1060 	io->io_hdr.nexus.targ_lun = 0;
1061 	io->taskio.tag_type = CTL_TAG_SIMPLE; /* XXX */
1062 	io->taskio.task_action = CTL_TASK_I_T_NEXUS_RESET;
1063 	wait = cs->cs_outstanding_ctl_pdus;
1064 	refcount_acquire(&cs->cs_outstanding_ctl_pdus);
1065 	error = ctl_queue(io);
1066 	if (error != CTL_RETVAL_COMPLETE) {
1067 		CFISCSI_SESSION_WARN(cs, "ctl_queue() failed; error %d", error);
1068 		refcount_release(&cs->cs_outstanding_ctl_pdus);
1069 		ctl_free_io(io);
1070 	}
1071 
1072 	CFISCSI_SESSION_LOCK(cs);
1073 	while ((cdw = TAILQ_FIRST(&cs->cs_waiting_for_data_out)) != NULL) {
1074 		TAILQ_REMOVE(&cs->cs_waiting_for_data_out, cdw, cdw_next);
1075 		CFISCSI_SESSION_UNLOCK(cs);
1076 		/*
1077 		 * Set nonzero port status; this prevents backends from
1078 		 * assuming that the data transfer actually succeeded
1079 		 * and writing uninitialized data to disk.
1080 		 */
1081 		cdw->cdw_ctl_io->scsiio.io_hdr.port_status = 42;
1082 		cdw->cdw_ctl_io->scsiio.be_move_done(cdw->cdw_ctl_io);
1083 		uma_zfree(cfiscsi_data_wait_zone, cdw);
1084 		CFISCSI_SESSION_LOCK(cs);
1085 	}
1086 	CFISCSI_SESSION_UNLOCK(cs);
1087 
1088 	/*
1089 	 * Wait for CTL to terminate all the tasks.
1090 	 */
1091 	if (wait > 0)
1092 		CFISCSI_SESSION_WARN(cs,
1093 		    "waiting for CTL to terminate %d tasks", wait);
1094 	for (;;) {
1095 		refcount_acquire(&cs->cs_outstanding_ctl_pdus);
1096 		last = refcount_release(&cs->cs_outstanding_ctl_pdus);
1097 		if (last != 0)
1098 			break;
1099 		tsleep(__DEVOLATILE(void *, &cs->cs_outstanding_ctl_pdus),
1100 		    0, "cfiscsi_terminate", hz / 100);
1101 	}
1102 	if (wait > 0)
1103 		CFISCSI_SESSION_WARN(cs, "tasks terminated");
1104 }
1105 
1106 static void
1107 cfiscsi_maintenance_thread(void *arg)
1108 {
1109 	struct cfiscsi_session *cs;
1110 
1111 	cs = arg;
1112 
1113 	for (;;) {
1114 		CFISCSI_SESSION_LOCK(cs);
1115 		if (cs->cs_terminating == false)
1116 			cv_wait(&cs->cs_maintenance_cv, &cs->cs_lock);
1117 		CFISCSI_SESSION_UNLOCK(cs);
1118 
1119 		if (cs->cs_terminating) {
1120 
1121 			/*
1122 			 * We used to wait up to 30 seconds to deliver queued
1123 			 * PDUs to the initiator.  We also tried hard to deliver
1124 			 * SCSI Responses for the aborted PDUs.  We don't do
1125 			 * that anymore.  We might need to revisit that.
1126 			 */
1127 			callout_drain(&cs->cs_callout);
1128 			icl_conn_close(cs->cs_conn);
1129 
1130 			/*
1131 			 * At this point ICL receive thread is no longer
1132 			 * running; no new tasks can be queued.
1133 			 */
1134 			cfiscsi_session_terminate_tasks(cs);
1135 			cfiscsi_session_delete(cs);
1136 			kthread_exit();
1137 			return;
1138 		}
1139 		CFISCSI_SESSION_DEBUG(cs, "nothing to do");
1140 	}
1141 }
1142 
1143 static void
1144 cfiscsi_session_terminate(struct cfiscsi_session *cs)
1145 {
1146 
1147 	if (cs->cs_terminating)
1148 		return;
1149 	cs->cs_terminating = true;
1150 	cv_signal(&cs->cs_maintenance_cv);
1151 #ifdef ICL_KERNEL_PROXY
1152 	cv_signal(&cs->cs_login_cv);
1153 #endif
1154 }
1155 
1156 static int
1157 cfiscsi_session_register_initiator(struct cfiscsi_session *cs)
1158 {
1159 	struct cfiscsi_target *ct;
1160 	char *name;
1161 	int i;
1162 
1163 	KASSERT(cs->cs_ctl_initid == -1, ("already registered"));
1164 
1165 	ct = cs->cs_target;
1166 	name = strdup(cs->cs_initiator_id, M_CTL);
1167 	i = ctl_add_initiator(&ct->ct_port, -1, 0, name);
1168 	if (i < 0) {
1169 		CFISCSI_SESSION_WARN(cs, "ctl_add_initiator failed with error %d",
1170 		    i);
1171 		cs->cs_ctl_initid = -1;
1172 		return (1);
1173 	}
1174 	cs->cs_ctl_initid = i;
1175 #if 0
1176 	CFISCSI_SESSION_DEBUG(cs, "added initiator id %d", i);
1177 #endif
1178 
1179 	return (0);
1180 }
1181 
1182 static void
1183 cfiscsi_session_unregister_initiator(struct cfiscsi_session *cs)
1184 {
1185 	int error;
1186 
1187 	if (cs->cs_ctl_initid == -1)
1188 		return;
1189 
1190 	error = ctl_remove_initiator(&cs->cs_target->ct_port, cs->cs_ctl_initid);
1191 	if (error != 0) {
1192 		CFISCSI_SESSION_WARN(cs, "ctl_remove_initiator failed with error %d",
1193 		    error);
1194 	}
1195 	cs->cs_ctl_initid = -1;
1196 }
1197 
1198 static struct cfiscsi_session *
1199 cfiscsi_session_new(struct cfiscsi_softc *softc)
1200 {
1201 	struct cfiscsi_session *cs;
1202 	int error;
1203 
1204 	cs = malloc(sizeof(*cs), M_CFISCSI, M_NOWAIT | M_ZERO);
1205 	if (cs == NULL) {
1206 		CFISCSI_WARN("malloc failed");
1207 		return (NULL);
1208 	}
1209 	cs->cs_ctl_initid = -1;
1210 
1211 	refcount_init(&cs->cs_outstanding_ctl_pdus, 0);
1212 	TAILQ_INIT(&cs->cs_waiting_for_data_out);
1213 	mtx_init(&cs->cs_lock, "cfiscsi_lock", NULL, MTX_DEF);
1214 	cv_init(&cs->cs_maintenance_cv, "cfiscsi_mt");
1215 #ifdef ICL_KERNEL_PROXY
1216 	cv_init(&cs->cs_login_cv, "cfiscsi_login");
1217 #endif
1218 
1219 	cs->cs_conn = icl_conn_new("cfiscsi", &cs->cs_lock);
1220 	cs->cs_conn->ic_receive = cfiscsi_receive_callback;
1221 	cs->cs_conn->ic_error = cfiscsi_error_callback;
1222 	cs->cs_conn->ic_prv0 = cs;
1223 
1224 	error = kthread_add(cfiscsi_maintenance_thread, cs, NULL, NULL, 0, 0, "cfiscsimt");
1225 	if (error != 0) {
1226 		CFISCSI_SESSION_WARN(cs, "kthread_add(9) failed with error %d", error);
1227 		free(cs, M_CFISCSI);
1228 		return (NULL);
1229 	}
1230 
1231 	mtx_lock(&softc->lock);
1232 	cs->cs_id = ++softc->last_session_id;
1233 	TAILQ_INSERT_TAIL(&softc->sessions, cs, cs_next);
1234 	mtx_unlock(&softc->lock);
1235 
1236 	/*
1237 	 * Start pinging the initiator.
1238 	 */
1239 	callout_init(&cs->cs_callout, 1);
1240 	callout_reset(&cs->cs_callout, 1 * hz, cfiscsi_callout, cs);
1241 
1242 	return (cs);
1243 }
1244 
1245 static void
1246 cfiscsi_session_delete(struct cfiscsi_session *cs)
1247 {
1248 	struct cfiscsi_softc *softc;
1249 
1250 	softc = &cfiscsi_softc;
1251 
1252 	KASSERT(cs->cs_outstanding_ctl_pdus == 0,
1253 	    ("destroying session with outstanding CTL pdus"));
1254 	KASSERT(TAILQ_EMPTY(&cs->cs_waiting_for_data_out),
1255 	    ("destroying session with non-empty queue"));
1256 
1257 	cfiscsi_session_unregister_initiator(cs);
1258 	if (cs->cs_target != NULL)
1259 		cfiscsi_target_release(cs->cs_target);
1260 	icl_conn_close(cs->cs_conn);
1261 	icl_conn_free(cs->cs_conn);
1262 
1263 	mtx_lock(&softc->lock);
1264 	TAILQ_REMOVE(&softc->sessions, cs, cs_next);
1265 	cv_signal(&softc->sessions_cv);
1266 	mtx_unlock(&softc->lock);
1267 
1268 	free(cs, M_CFISCSI);
1269 }
1270 
1271 int
1272 cfiscsi_init(void)
1273 {
1274 	struct cfiscsi_softc *softc;
1275 	int retval;
1276 
1277 	softc = &cfiscsi_softc;
1278 	retval = 0;
1279 	bzero(softc, sizeof(*softc));
1280 	mtx_init(&softc->lock, "cfiscsi", NULL, MTX_DEF);
1281 
1282 	cv_init(&softc->sessions_cv, "cfiscsi_sessions");
1283 #ifdef ICL_KERNEL_PROXY
1284 	cv_init(&softc->accept_cv, "cfiscsi_accept");
1285 #endif
1286 	TAILQ_INIT(&softc->sessions);
1287 	TAILQ_INIT(&softc->targets);
1288 
1289 	cfiscsi_data_wait_zone = uma_zcreate("cfiscsi_data_wait",
1290 	    sizeof(struct cfiscsi_data_wait), NULL, NULL, NULL, NULL,
1291 	    UMA_ALIGN_PTR, 0);
1292 
1293 	return (0);
1294 }
1295 
1296 #ifdef ICL_KERNEL_PROXY
1297 static void
1298 cfiscsi_accept(struct socket *so, struct sockaddr *sa, int portal_id)
1299 {
1300 	struct cfiscsi_session *cs;
1301 
1302 	cs = cfiscsi_session_new(&cfiscsi_softc);
1303 	if (cs == NULL) {
1304 		CFISCSI_WARN("failed to create session");
1305 		return;
1306 	}
1307 
1308 	icl_conn_handoff_sock(cs->cs_conn, so);
1309 	cs->cs_initiator_sa = sa;
1310 	cs->cs_portal_id = portal_id;
1311 	cs->cs_waiting_for_ctld = true;
1312 	cv_signal(&cfiscsi_softc.accept_cv);
1313 }
1314 #endif
1315 
1316 static void
1317 cfiscsi_online(void *arg)
1318 {
1319 	struct cfiscsi_softc *softc;
1320 	struct cfiscsi_target *ct;
1321 	int online;
1322 
1323 	ct = (struct cfiscsi_target *)arg;
1324 	softc = ct->ct_softc;
1325 
1326 	mtx_lock(&softc->lock);
1327 	if (ct->ct_online) {
1328 		mtx_unlock(&softc->lock);
1329 		return;
1330 	}
1331 	ct->ct_online = 1;
1332 	online = softc->online++;
1333 	mtx_unlock(&softc->lock);
1334 	if (online > 0)
1335 		return;
1336 
1337 #ifdef ICL_KERNEL_PROXY
1338 	if (softc->listener != NULL)
1339 		icl_listen_free(softc->listener);
1340 	softc->listener = icl_listen_new(cfiscsi_accept);
1341 #endif
1342 }
1343 
1344 static void
1345 cfiscsi_offline(void *arg)
1346 {
1347 	struct cfiscsi_softc *softc;
1348 	struct cfiscsi_target *ct;
1349 	struct cfiscsi_session *cs;
1350 	int online;
1351 
1352 	ct = (struct cfiscsi_target *)arg;
1353 	softc = ct->ct_softc;
1354 
1355 	mtx_lock(&softc->lock);
1356 	if (!ct->ct_online) {
1357 		mtx_unlock(&softc->lock);
1358 		return;
1359 	}
1360 	ct->ct_online = 0;
1361 	online = --softc->online;
1362 
1363 	TAILQ_FOREACH(cs, &softc->sessions, cs_next) {
1364 		if (cs->cs_target == ct)
1365 			cfiscsi_session_terminate(cs);
1366 	}
1367 	do {
1368 		TAILQ_FOREACH(cs, &softc->sessions, cs_next) {
1369 			if (cs->cs_target == ct)
1370 				break;
1371 		}
1372 		if (cs != NULL)
1373 			cv_wait(&softc->sessions_cv, &softc->lock);
1374 	} while (cs != NULL && ct->ct_online == 0);
1375 	mtx_unlock(&softc->lock);
1376 	if (online > 0)
1377 		return;
1378 
1379 #ifdef ICL_KERNEL_PROXY
1380 	icl_listen_free(softc->listener);
1381 	softc->listener = NULL;
1382 #endif
1383 }
1384 
1385 static int
1386 cfiscsi_info(void *arg, struct sbuf *sb)
1387 {
1388 	struct cfiscsi_target *ct = (struct cfiscsi_target *)arg;
1389 	int retval;
1390 
1391 	retval = sbuf_printf(sb, "\t<cfiscsi_state>%d</cfiscsi_state>\n",
1392 	    ct->ct_state);
1393 	return (retval);
1394 }
1395 
1396 static void
1397 cfiscsi_ioctl_handoff(struct ctl_iscsi *ci)
1398 {
1399 	struct cfiscsi_softc *softc;
1400 	struct cfiscsi_session *cs, *cs2;
1401 	struct cfiscsi_target *ct;
1402 	struct ctl_iscsi_handoff_params *cihp;
1403 	int error;
1404 
1405 	cihp = (struct ctl_iscsi_handoff_params *)&(ci->data);
1406 	softc = &cfiscsi_softc;
1407 
1408 	CFISCSI_DEBUG("new connection from %s (%s) to %s",
1409 	    cihp->initiator_name, cihp->initiator_addr,
1410 	    cihp->target_name);
1411 
1412 	ct = cfiscsi_target_find(softc, cihp->target_name);
1413 	if (ct == NULL) {
1414 		ci->status = CTL_ISCSI_ERROR;
1415 		snprintf(ci->error_str, sizeof(ci->error_str),
1416 		    "%s: target not found", __func__);
1417 		return;
1418 	}
1419 
1420 #ifdef ICL_KERNEL_PROXY
1421 	if (cihp->socket > 0 && cihp->connection_id > 0) {
1422 		snprintf(ci->error_str, sizeof(ci->error_str),
1423 		    "both socket and connection_id set");
1424 		ci->status = CTL_ISCSI_ERROR;
1425 		cfiscsi_target_release(ct);
1426 		return;
1427 	}
1428 	if (cihp->socket == 0) {
1429 		mtx_lock(&cfiscsi_softc.lock);
1430 		TAILQ_FOREACH(cs, &cfiscsi_softc.sessions, cs_next) {
1431 			if (cs->cs_id == cihp->connection_id)
1432 				break;
1433 		}
1434 		if (cs == NULL) {
1435 			mtx_unlock(&cfiscsi_softc.lock);
1436 			snprintf(ci->error_str, sizeof(ci->error_str),
1437 			    "connection not found");
1438 			ci->status = CTL_ISCSI_ERROR;
1439 			cfiscsi_target_release(ct);
1440 			return;
1441 		}
1442 		mtx_unlock(&cfiscsi_softc.lock);
1443 	} else {
1444 #endif
1445 		cs = cfiscsi_session_new(softc);
1446 		if (cs == NULL) {
1447 			ci->status = CTL_ISCSI_ERROR;
1448 			snprintf(ci->error_str, sizeof(ci->error_str),
1449 			    "%s: cfiscsi_session_new failed", __func__);
1450 			cfiscsi_target_release(ct);
1451 			return;
1452 		}
1453 #ifdef ICL_KERNEL_PROXY
1454 	}
1455 #endif
1456 
1457 	/*
1458 	 * First PDU of Full Feature phase has the same CmdSN as the last
1459 	 * PDU from the Login Phase received from the initiator.  Thus,
1460 	 * the -1 below.
1461 	 */
1462 	cs->cs_portal_group_tag = cihp->portal_group_tag;
1463 	cs->cs_cmdsn = cihp->cmdsn;
1464 	cs->cs_statsn = cihp->statsn;
1465 	cs->cs_max_data_segment_length = cihp->max_recv_data_segment_length;
1466 	cs->cs_max_burst_length = cihp->max_burst_length;
1467 	cs->cs_immediate_data = !!cihp->immediate_data;
1468 	if (cihp->header_digest == CTL_ISCSI_DIGEST_CRC32C)
1469 		cs->cs_conn->ic_header_crc32c = true;
1470 	if (cihp->data_digest == CTL_ISCSI_DIGEST_CRC32C)
1471 		cs->cs_conn->ic_data_crc32c = true;
1472 
1473 	strlcpy(cs->cs_initiator_name,
1474 	    cihp->initiator_name, sizeof(cs->cs_initiator_name));
1475 	strlcpy(cs->cs_initiator_addr,
1476 	    cihp->initiator_addr, sizeof(cs->cs_initiator_addr));
1477 	strlcpy(cs->cs_initiator_alias,
1478 	    cihp->initiator_alias, sizeof(cs->cs_initiator_alias));
1479 	memcpy(cs->cs_initiator_isid,
1480 	    cihp->initiator_isid, sizeof(cs->cs_initiator_isid));
1481 	snprintf(cs->cs_initiator_id, sizeof(cs->cs_initiator_id),
1482 	    "%s,i,0x%02x%02x%02x%02x%02x%02x", cs->cs_initiator_name,
1483 	    cihp->initiator_isid[0], cihp->initiator_isid[1],
1484 	    cihp->initiator_isid[2], cihp->initiator_isid[3],
1485 	    cihp->initiator_isid[4], cihp->initiator_isid[5]);
1486 
1487 	mtx_lock(&softc->lock);
1488 	if (ct->ct_online == 0) {
1489 		mtx_unlock(&softc->lock);
1490 		cfiscsi_session_terminate(cs);
1491 		cfiscsi_target_release(ct);
1492 		ci->status = CTL_ISCSI_ERROR;
1493 		snprintf(ci->error_str, sizeof(ci->error_str),
1494 		    "%s: port offline", __func__);
1495 		return;
1496 	}
1497 	cs->cs_target = ct;
1498 	mtx_unlock(&softc->lock);
1499 
1500 	refcount_acquire(&cs->cs_outstanding_ctl_pdus);
1501 restart:
1502 	if (!cs->cs_terminating) {
1503 		mtx_lock(&softc->lock);
1504 		TAILQ_FOREACH(cs2, &softc->sessions, cs_next) {
1505 			if (cs2 != cs && cs2->cs_tasks_aborted == false &&
1506 			    cs->cs_target == cs2->cs_target &&
1507 			    cs->cs_portal_group_tag == cs2->cs_portal_group_tag &&
1508 			    strcmp(cs->cs_initiator_id, cs2->cs_initiator_id) == 0) {
1509 				cfiscsi_session_terminate(cs2);
1510 				mtx_unlock(&softc->lock);
1511 				pause("cfiscsi_reinstate", 1);
1512 				goto restart;
1513 			}
1514 		}
1515 		mtx_unlock(&softc->lock);
1516 	}
1517 
1518 	/*
1519 	 * Register initiator with CTL.
1520 	 */
1521 	cfiscsi_session_register_initiator(cs);
1522 
1523 #ifdef ICL_KERNEL_PROXY
1524 	if (cihp->socket > 0) {
1525 #endif
1526 		error = icl_conn_handoff(cs->cs_conn, cihp->socket);
1527 		if (error != 0) {
1528 			cfiscsi_session_terminate(cs);
1529 			refcount_release(&cs->cs_outstanding_ctl_pdus);
1530 			ci->status = CTL_ISCSI_ERROR;
1531 			snprintf(ci->error_str, sizeof(ci->error_str),
1532 			    "%s: icl_conn_handoff failed with error %d",
1533 			    __func__, error);
1534 			return;
1535 		}
1536 #ifdef ICL_KERNEL_PROXY
1537 	}
1538 #endif
1539 
1540 #ifdef ICL_KERNEL_PROXY
1541 	cs->cs_login_phase = false;
1542 
1543 	/*
1544 	 * First PDU of the Full Feature phase has likely already arrived.
1545 	 * We have to pick it up and execute properly.
1546 	 */
1547 	if (cs->cs_login_pdu != NULL) {
1548 		CFISCSI_SESSION_DEBUG(cs, "picking up first PDU");
1549 		cfiscsi_pdu_handle(cs->cs_login_pdu);
1550 		cs->cs_login_pdu = NULL;
1551 	}
1552 #endif
1553 
1554 	refcount_release(&cs->cs_outstanding_ctl_pdus);
1555 	ci->status = CTL_ISCSI_OK;
1556 }
1557 
1558 static void
1559 cfiscsi_ioctl_list(struct ctl_iscsi *ci)
1560 {
1561 	struct ctl_iscsi_list_params *cilp;
1562 	struct cfiscsi_session *cs;
1563 	struct cfiscsi_softc *softc;
1564 	struct sbuf *sb;
1565 	int error;
1566 
1567 	cilp = (struct ctl_iscsi_list_params *)&(ci->data);
1568 	softc = &cfiscsi_softc;
1569 
1570 	sb = sbuf_new(NULL, NULL, cilp->alloc_len, SBUF_FIXEDLEN);
1571 	if (sb == NULL) {
1572 		ci->status = CTL_ISCSI_ERROR;
1573 		snprintf(ci->error_str, sizeof(ci->error_str),
1574 		    "Unable to allocate %d bytes for iSCSI session list",
1575 		    cilp->alloc_len);
1576 		return;
1577 	}
1578 
1579 	sbuf_printf(sb, "<ctlislist>\n");
1580 	mtx_lock(&softc->lock);
1581 	TAILQ_FOREACH(cs, &softc->sessions, cs_next) {
1582 #ifdef ICL_KERNEL_PROXY
1583 		if (cs->cs_target == NULL)
1584 			continue;
1585 #endif
1586 		error = sbuf_printf(sb, "<connection id=\"%d\">"
1587 		    "<initiator>%s</initiator>"
1588 		    "<initiator_addr>%s</initiator_addr>"
1589 		    "<initiator_alias>%s</initiator_alias>"
1590 		    "<target>%s</target>"
1591 		    "<target_alias>%s</target_alias>"
1592 		    "<header_digest>%s</header_digest>"
1593 		    "<data_digest>%s</data_digest>"
1594 		    "<max_data_segment_length>%zd</max_data_segment_length>"
1595 		    "<immediate_data>%d</immediate_data>"
1596 		    "<iser>%d</iser>"
1597 		    "</connection>\n",
1598 		    cs->cs_id,
1599 		    cs->cs_initiator_name, cs->cs_initiator_addr, cs->cs_initiator_alias,
1600 		    cs->cs_target->ct_name, cs->cs_target->ct_alias,
1601 		    cs->cs_conn->ic_header_crc32c ? "CRC32C" : "None",
1602 		    cs->cs_conn->ic_data_crc32c ? "CRC32C" : "None",
1603 		    cs->cs_max_data_segment_length,
1604 		    cs->cs_immediate_data,
1605 		    cs->cs_conn->ic_iser);
1606 		if (error != 0)
1607 			break;
1608 	}
1609 	mtx_unlock(&softc->lock);
1610 	error = sbuf_printf(sb, "</ctlislist>\n");
1611 	if (error != 0) {
1612 		sbuf_delete(sb);
1613 		ci->status = CTL_ISCSI_LIST_NEED_MORE_SPACE;
1614 		snprintf(ci->error_str, sizeof(ci->error_str),
1615 		    "Out of space, %d bytes is too small", cilp->alloc_len);
1616 		return;
1617 	}
1618 	sbuf_finish(sb);
1619 
1620 	error = copyout(sbuf_data(sb), cilp->conn_xml, sbuf_len(sb) + 1);
1621 	cilp->fill_len = sbuf_len(sb) + 1;
1622 	ci->status = CTL_ISCSI_OK;
1623 	sbuf_delete(sb);
1624 }
1625 
1626 static void
1627 cfiscsi_ioctl_terminate(struct ctl_iscsi *ci)
1628 {
1629 	struct icl_pdu *response;
1630 	struct iscsi_bhs_asynchronous_message *bhsam;
1631 	struct ctl_iscsi_terminate_params *citp;
1632 	struct cfiscsi_session *cs;
1633 	struct cfiscsi_softc *softc;
1634 	int found = 0;
1635 
1636 	citp = (struct ctl_iscsi_terminate_params *)&(ci->data);
1637 	softc = &cfiscsi_softc;
1638 
1639 	mtx_lock(&softc->lock);
1640 	TAILQ_FOREACH(cs, &softc->sessions, cs_next) {
1641 		if (citp->all == 0 && cs->cs_id != citp->connection_id &&
1642 		    strcmp(cs->cs_initiator_name, citp->initiator_name) != 0 &&
1643 		    strcmp(cs->cs_initiator_addr, citp->initiator_addr) != 0)
1644 			continue;
1645 
1646 		response = icl_pdu_new(cs->cs_conn, M_NOWAIT);
1647 		if (response == NULL) {
1648 			/*
1649 			 * Oh well.  Just terminate the connection.
1650 			 */
1651 		} else {
1652 			bhsam = (struct iscsi_bhs_asynchronous_message *)
1653 			    response->ip_bhs;
1654 			bhsam->bhsam_opcode = ISCSI_BHS_OPCODE_ASYNC_MESSAGE;
1655 			bhsam->bhsam_flags = 0x80;
1656 			bhsam->bhsam_0xffffffff = 0xffffffff;
1657 			bhsam->bhsam_async_event =
1658 			    BHSAM_EVENT_TARGET_TERMINATES_SESSION;
1659 			cfiscsi_pdu_queue(response);
1660 		}
1661 		cfiscsi_session_terminate(cs);
1662 		found++;
1663 	}
1664 	mtx_unlock(&softc->lock);
1665 
1666 	if (found == 0) {
1667 		ci->status = CTL_ISCSI_SESSION_NOT_FOUND;
1668 		snprintf(ci->error_str, sizeof(ci->error_str),
1669 		    "No matching connections found");
1670 		return;
1671 	}
1672 
1673 	ci->status = CTL_ISCSI_OK;
1674 }
1675 
1676 static void
1677 cfiscsi_ioctl_logout(struct ctl_iscsi *ci)
1678 {
1679 	struct icl_pdu *response;
1680 	struct iscsi_bhs_asynchronous_message *bhsam;
1681 	struct ctl_iscsi_logout_params *cilp;
1682 	struct cfiscsi_session *cs;
1683 	struct cfiscsi_softc *softc;
1684 	int found = 0;
1685 
1686 	cilp = (struct ctl_iscsi_logout_params *)&(ci->data);
1687 	softc = &cfiscsi_softc;
1688 
1689 	mtx_lock(&softc->lock);
1690 	TAILQ_FOREACH(cs, &softc->sessions, cs_next) {
1691 		if (cilp->all == 0 && cs->cs_id != cilp->connection_id &&
1692 		    strcmp(cs->cs_initiator_name, cilp->initiator_name) != 0 &&
1693 		    strcmp(cs->cs_initiator_addr, cilp->initiator_addr) != 0)
1694 			continue;
1695 
1696 		response = icl_pdu_new(cs->cs_conn, M_NOWAIT);
1697 		if (response == NULL) {
1698 			ci->status = CTL_ISCSI_ERROR;
1699 			snprintf(ci->error_str, sizeof(ci->error_str),
1700 			    "Unable to allocate memory");
1701 			mtx_unlock(&softc->lock);
1702 			return;
1703 		}
1704 		bhsam =
1705 		    (struct iscsi_bhs_asynchronous_message *)response->ip_bhs;
1706 		bhsam->bhsam_opcode = ISCSI_BHS_OPCODE_ASYNC_MESSAGE;
1707 		bhsam->bhsam_flags = 0x80;
1708 		bhsam->bhsam_async_event = BHSAM_EVENT_TARGET_REQUESTS_LOGOUT;
1709 		bhsam->bhsam_parameter3 = htons(10);
1710 		cfiscsi_pdu_queue(response);
1711 		found++;
1712 	}
1713 	mtx_unlock(&softc->lock);
1714 
1715 	if (found == 0) {
1716 		ci->status = CTL_ISCSI_SESSION_NOT_FOUND;
1717 		snprintf(ci->error_str, sizeof(ci->error_str),
1718 		    "No matching connections found");
1719 		return;
1720 	}
1721 
1722 	ci->status = CTL_ISCSI_OK;
1723 }
1724 
1725 #ifdef ICL_KERNEL_PROXY
1726 static void
1727 cfiscsi_ioctl_listen(struct ctl_iscsi *ci)
1728 {
1729 	struct ctl_iscsi_listen_params *cilp;
1730 	struct sockaddr *sa;
1731 	int error;
1732 
1733 	cilp = (struct ctl_iscsi_listen_params *)&(ci->data);
1734 
1735 	if (cfiscsi_softc.listener == NULL) {
1736 		CFISCSI_DEBUG("no listener");
1737 		snprintf(ci->error_str, sizeof(ci->error_str), "no listener");
1738 		ci->status = CTL_ISCSI_ERROR;
1739 		return;
1740 	}
1741 
1742 	error = getsockaddr(&sa, (void *)cilp->addr, cilp->addrlen);
1743 	if (error != 0) {
1744 		CFISCSI_DEBUG("getsockaddr, error %d", error);
1745 		snprintf(ci->error_str, sizeof(ci->error_str), "getsockaddr failed");
1746 		ci->status = CTL_ISCSI_ERROR;
1747 		return;
1748 	}
1749 
1750 	error = icl_listen_add(cfiscsi_softc.listener, cilp->iser, cilp->domain,
1751 	    cilp->socktype, cilp->protocol, sa, cilp->portal_id);
1752 	if (error != 0) {
1753 		free(sa, M_SONAME);
1754 		CFISCSI_DEBUG("icl_listen_add, error %d", error);
1755 		snprintf(ci->error_str, sizeof(ci->error_str),
1756 		    "icl_listen_add failed, error %d", error);
1757 		ci->status = CTL_ISCSI_ERROR;
1758 		return;
1759 	}
1760 
1761 	ci->status = CTL_ISCSI_OK;
1762 }
1763 
1764 static void
1765 cfiscsi_ioctl_accept(struct ctl_iscsi *ci)
1766 {
1767 	struct ctl_iscsi_accept_params *ciap;
1768 	struct cfiscsi_session *cs;
1769 	int error;
1770 
1771 	ciap = (struct ctl_iscsi_accept_params *)&(ci->data);
1772 
1773 	mtx_lock(&cfiscsi_softc.lock);
1774 	for (;;) {
1775 		TAILQ_FOREACH(cs, &cfiscsi_softc.sessions, cs_next) {
1776 			if (cs->cs_waiting_for_ctld)
1777 				break;
1778 		}
1779 		if (cs != NULL)
1780 			break;
1781 		error = cv_wait_sig(&cfiscsi_softc.accept_cv, &cfiscsi_softc.lock);
1782 		if (error != 0) {
1783 			mtx_unlock(&cfiscsi_softc.lock);
1784 			snprintf(ci->error_str, sizeof(ci->error_str), "interrupted");
1785 			ci->status = CTL_ISCSI_ERROR;
1786 			return;
1787 		}
1788 	}
1789 	mtx_unlock(&cfiscsi_softc.lock);
1790 
1791 	cs->cs_waiting_for_ctld = false;
1792 	cs->cs_login_phase = true;
1793 
1794 	ciap->connection_id = cs->cs_id;
1795 	ciap->portal_id = cs->cs_portal_id;
1796 	ciap->initiator_addrlen = cs->cs_initiator_sa->sa_len;
1797 	error = copyout(cs->cs_initiator_sa, ciap->initiator_addr,
1798 	    cs->cs_initiator_sa->sa_len);
1799 	if (error != 0) {
1800 		snprintf(ci->error_str, sizeof(ci->error_str),
1801 		    "copyout failed with error %d", error);
1802 		ci->status = CTL_ISCSI_ERROR;
1803 		return;
1804 	}
1805 
1806 	ci->status = CTL_ISCSI_OK;
1807 }
1808 
1809 static void
1810 cfiscsi_ioctl_send(struct ctl_iscsi *ci)
1811 {
1812 	struct ctl_iscsi_send_params *cisp;
1813 	struct cfiscsi_session *cs;
1814 	struct icl_pdu *ip;
1815 	size_t datalen;
1816 	void *data;
1817 	int error;
1818 
1819 	cisp = (struct ctl_iscsi_send_params *)&(ci->data);
1820 
1821 	mtx_lock(&cfiscsi_softc.lock);
1822 	TAILQ_FOREACH(cs, &cfiscsi_softc.sessions, cs_next) {
1823 		if (cs->cs_id == cisp->connection_id)
1824 			break;
1825 	}
1826 	if (cs == NULL) {
1827 		mtx_unlock(&cfiscsi_softc.lock);
1828 		snprintf(ci->error_str, sizeof(ci->error_str), "connection not found");
1829 		ci->status = CTL_ISCSI_ERROR;
1830 		return;
1831 	}
1832 	mtx_unlock(&cfiscsi_softc.lock);
1833 
1834 #if 0
1835 	if (cs->cs_login_phase == false)
1836 		return (EBUSY);
1837 #endif
1838 
1839 	if (cs->cs_terminating) {
1840 		snprintf(ci->error_str, sizeof(ci->error_str), "connection is terminating");
1841 		ci->status = CTL_ISCSI_ERROR;
1842 		return;
1843 	}
1844 
1845 	datalen = cisp->data_segment_len;
1846 	/*
1847 	 * XXX
1848 	 */
1849 	//if (datalen > CFISCSI_MAX_DATA_SEGMENT_LENGTH) {
1850 	if (datalen > 65535) {
1851 		snprintf(ci->error_str, sizeof(ci->error_str), "data segment too big");
1852 		ci->status = CTL_ISCSI_ERROR;
1853 		return;
1854 	}
1855 	if (datalen > 0) {
1856 		data = malloc(datalen, M_CFISCSI, M_WAITOK);
1857 		error = copyin(cisp->data_segment, data, datalen);
1858 		if (error != 0) {
1859 			free(data, M_CFISCSI);
1860 			snprintf(ci->error_str, sizeof(ci->error_str), "copyin error %d", error);
1861 			ci->status = CTL_ISCSI_ERROR;
1862 			return;
1863 		}
1864 	}
1865 
1866 	ip = icl_pdu_new(cs->cs_conn, M_WAITOK);
1867 	memcpy(ip->ip_bhs, cisp->bhs, sizeof(*ip->ip_bhs));
1868 	if (datalen > 0) {
1869 		icl_pdu_append_data(ip, data, datalen, M_WAITOK);
1870 		free(data, M_CFISCSI);
1871 	}
1872 	CFISCSI_SESSION_LOCK(cs);
1873 	icl_pdu_queue(ip);
1874 	CFISCSI_SESSION_UNLOCK(cs);
1875 	ci->status = CTL_ISCSI_OK;
1876 }
1877 
1878 static void
1879 cfiscsi_ioctl_receive(struct ctl_iscsi *ci)
1880 {
1881 	struct ctl_iscsi_receive_params *cirp;
1882 	struct cfiscsi_session *cs;
1883 	struct icl_pdu *ip;
1884 	void *data;
1885 	int error;
1886 
1887 	cirp = (struct ctl_iscsi_receive_params *)&(ci->data);
1888 
1889 	mtx_lock(&cfiscsi_softc.lock);
1890 	TAILQ_FOREACH(cs, &cfiscsi_softc.sessions, cs_next) {
1891 		if (cs->cs_id == cirp->connection_id)
1892 			break;
1893 	}
1894 	if (cs == NULL) {
1895 		mtx_unlock(&cfiscsi_softc.lock);
1896 		snprintf(ci->error_str, sizeof(ci->error_str),
1897 		    "connection not found");
1898 		ci->status = CTL_ISCSI_ERROR;
1899 		return;
1900 	}
1901 	mtx_unlock(&cfiscsi_softc.lock);
1902 
1903 #if 0
1904 	if (is->is_login_phase == false)
1905 		return (EBUSY);
1906 #endif
1907 
1908 	CFISCSI_SESSION_LOCK(cs);
1909 	while (cs->cs_login_pdu == NULL && cs->cs_terminating == false) {
1910 		error = cv_wait_sig(&cs->cs_login_cv, &cs->cs_lock);
1911 		if (error != 0) {
1912 			CFISCSI_SESSION_UNLOCK(cs);
1913 			snprintf(ci->error_str, sizeof(ci->error_str),
1914 			    "interrupted by signal");
1915 			ci->status = CTL_ISCSI_ERROR;
1916 			return;
1917 		}
1918 	}
1919 
1920 	if (cs->cs_terminating) {
1921 		CFISCSI_SESSION_UNLOCK(cs);
1922 		snprintf(ci->error_str, sizeof(ci->error_str),
1923 		    "connection terminating");
1924 		ci->status = CTL_ISCSI_ERROR;
1925 		return;
1926 	}
1927 	ip = cs->cs_login_pdu;
1928 	cs->cs_login_pdu = NULL;
1929 	CFISCSI_SESSION_UNLOCK(cs);
1930 
1931 	if (ip->ip_data_len > cirp->data_segment_len) {
1932 		icl_pdu_free(ip);
1933 		snprintf(ci->error_str, sizeof(ci->error_str),
1934 		    "data segment too big");
1935 		ci->status = CTL_ISCSI_ERROR;
1936 		return;
1937 	}
1938 
1939 	copyout(ip->ip_bhs, cirp->bhs, sizeof(*ip->ip_bhs));
1940 	if (ip->ip_data_len > 0) {
1941 		data = malloc(ip->ip_data_len, M_CFISCSI, M_WAITOK);
1942 		icl_pdu_get_data(ip, 0, data, ip->ip_data_len);
1943 		copyout(data, cirp->data_segment, ip->ip_data_len);
1944 		free(data, M_CFISCSI);
1945 	}
1946 
1947 	icl_pdu_free(ip);
1948 	ci->status = CTL_ISCSI_OK;
1949 }
1950 
1951 #endif /* !ICL_KERNEL_PROXY */
1952 
1953 static void
1954 cfiscsi_ioctl_port_create(struct ctl_req *req)
1955 {
1956 	struct cfiscsi_target *ct;
1957 	struct ctl_port *port;
1958 	const char *target, *alias, *tag;
1959 	struct scsi_vpd_id_descriptor *desc;
1960 	ctl_options_t opts;
1961 	int retval, len, idlen;
1962 
1963 	ctl_init_opts(&opts, req->num_args, req->kern_args);
1964 	target = ctl_get_opt(&opts, "cfiscsi_target");
1965 	alias = ctl_get_opt(&opts, "cfiscsi_target_alias");
1966 	tag = ctl_get_opt(&opts, "cfiscsi_portal_group_tag");
1967 	if (target == NULL || tag == NULL) {
1968 		req->status = CTL_LUN_ERROR;
1969 		snprintf(req->error_str, sizeof(req->error_str),
1970 		    "Missing required argument");
1971 		ctl_free_opts(&opts);
1972 		return;
1973 	}
1974 	ct = cfiscsi_target_find_or_create(&cfiscsi_softc, target, alias);
1975 	if (ct == NULL) {
1976 		req->status = CTL_LUN_ERROR;
1977 		snprintf(req->error_str, sizeof(req->error_str),
1978 		    "failed to create target \"%s\"", target);
1979 		ctl_free_opts(&opts);
1980 		return;
1981 	}
1982 	if (ct->ct_state == CFISCSI_TARGET_STATE_ACTIVE) {
1983 		req->status = CTL_LUN_ERROR;
1984 		snprintf(req->error_str, sizeof(req->error_str),
1985 		    "target \"%s\" already exists", target);
1986 		cfiscsi_target_release(ct);
1987 		ctl_free_opts(&opts);
1988 		return;
1989 	}
1990 	port = &ct->ct_port;
1991 	if (ct->ct_state == CFISCSI_TARGET_STATE_DYING)
1992 		goto done;
1993 
1994 	port->frontend = &cfiscsi_frontend;
1995 	port->port_type = CTL_PORT_ISCSI;
1996 	/* XXX KDM what should the real number be here? */
1997 	port->num_requested_ctl_io = 4096;
1998 	port->port_name = "iscsi";
1999 	port->physical_port = strtoul(tag, NULL, 0);
2000 	port->virtual_port = ct->ct_target_id;
2001 	port->port_online = cfiscsi_online;
2002 	port->port_offline = cfiscsi_offline;
2003 	port->port_info = cfiscsi_info;
2004 	port->onoff_arg = ct;
2005 	port->lun_enable = cfiscsi_lun_enable;
2006 	port->lun_disable = cfiscsi_lun_disable;
2007 	port->lun_map = cfiscsi_lun_map;
2008 	port->targ_lun_arg = ct;
2009 	port->fe_datamove = cfiscsi_datamove;
2010 	port->fe_done = cfiscsi_done;
2011 
2012 	/* XXX KDM what should we report here? */
2013 	/* XXX These should probably be fetched from CTL. */
2014 	port->max_targets = 1;
2015 	port->max_target_id = 15;
2016 
2017 	port->options = opts;
2018 	STAILQ_INIT(&opts);
2019 
2020 	/* Generate Port ID. */
2021 	idlen = strlen(target) + strlen(",t,0x0001") + 1;
2022 	idlen = roundup2(idlen, 4);
2023 	len = sizeof(struct scsi_vpd_device_id) + idlen;
2024 	port->port_devid = malloc(sizeof(struct ctl_devid) + len,
2025 	    M_CTL, M_WAITOK | M_ZERO);
2026 	port->port_devid->len = len;
2027 	desc = (struct scsi_vpd_id_descriptor *)port->port_devid->data;
2028 	desc->proto_codeset = (SCSI_PROTO_ISCSI << 4) | SVPD_ID_CODESET_UTF8;
2029 	desc->id_type = SVPD_ID_PIV | SVPD_ID_ASSOC_PORT |
2030 	    SVPD_ID_TYPE_SCSI_NAME;
2031 	desc->length = idlen;
2032 	snprintf(desc->identifier, idlen, "%s,t,0x%4.4x",
2033 	    target, port->physical_port);
2034 
2035 	/* Generate Target ID. */
2036 	idlen = strlen(target) + 1;
2037 	idlen = roundup2(idlen, 4);
2038 	len = sizeof(struct scsi_vpd_device_id) + idlen;
2039 	port->target_devid = malloc(sizeof(struct ctl_devid) + len,
2040 	    M_CTL, M_WAITOK | M_ZERO);
2041 	port->target_devid->len = len;
2042 	desc = (struct scsi_vpd_id_descriptor *)port->target_devid->data;
2043 	desc->proto_codeset = (SCSI_PROTO_ISCSI << 4) | SVPD_ID_CODESET_UTF8;
2044 	desc->id_type = SVPD_ID_PIV | SVPD_ID_ASSOC_TARGET |
2045 	    SVPD_ID_TYPE_SCSI_NAME;
2046 	desc->length = idlen;
2047 	strlcpy(desc->identifier, target, idlen);
2048 
2049 	retval = ctl_port_register(port);
2050 	if (retval != 0) {
2051 		ctl_free_opts(&port->options);
2052 		cfiscsi_target_release(ct);
2053 		free(port->port_devid, M_CFISCSI);
2054 		free(port->target_devid, M_CFISCSI);
2055 		req->status = CTL_LUN_ERROR;
2056 		snprintf(req->error_str, sizeof(req->error_str),
2057 		    "ctl_frontend_register() failed with error %d", retval);
2058 		return;
2059 	}
2060 done:
2061 	ct->ct_state = CFISCSI_TARGET_STATE_ACTIVE;
2062 	req->status = CTL_LUN_OK;
2063 	memcpy(req->kern_args[0].kvalue, &port->targ_port,
2064 	    sizeof(port->targ_port)); //XXX
2065 }
2066 
2067 static void
2068 cfiscsi_ioctl_port_remove(struct ctl_req *req)
2069 {
2070 	struct cfiscsi_target *ct;
2071 	const char *target;
2072 	ctl_options_t opts;
2073 
2074 	ctl_init_opts(&opts, req->num_args, req->kern_args);
2075 	target = ctl_get_opt(&opts, "cfiscsi_target");
2076 	if (target == NULL) {
2077 		ctl_free_opts(&opts);
2078 		req->status = CTL_LUN_ERROR;
2079 		snprintf(req->error_str, sizeof(req->error_str),
2080 		    "Missing required argument");
2081 		return;
2082 	}
2083 	ct = cfiscsi_target_find(&cfiscsi_softc, target);
2084 	if (ct == NULL) {
2085 		ctl_free_opts(&opts);
2086 		req->status = CTL_LUN_ERROR;
2087 		snprintf(req->error_str, sizeof(req->error_str),
2088 		    "can't find target \"%s\"", target);
2089 		return;
2090 	}
2091 	if (ct->ct_state != CFISCSI_TARGET_STATE_ACTIVE) {
2092 		ctl_free_opts(&opts);
2093 		req->status = CTL_LUN_ERROR;
2094 		snprintf(req->error_str, sizeof(req->error_str),
2095 		    "target \"%s\" is already dying", target);
2096 		return;
2097 	}
2098 	ctl_free_opts(&opts);
2099 
2100 	ct->ct_state = CFISCSI_TARGET_STATE_DYING;
2101 	ctl_port_offline(&ct->ct_port);
2102 	cfiscsi_target_release(ct);
2103 	cfiscsi_target_release(ct);
2104 }
2105 
2106 static int
2107 cfiscsi_ioctl(struct cdev *dev,
2108     u_long cmd, caddr_t addr, int flag, struct thread *td)
2109 {
2110 	struct ctl_iscsi *ci;
2111 	struct ctl_req *req;
2112 
2113 	if (cmd == CTL_PORT_REQ) {
2114 		req = (struct ctl_req *)addr;
2115 		switch (req->reqtype) {
2116 		case CTL_REQ_CREATE:
2117 			cfiscsi_ioctl_port_create(req);
2118 			break;
2119 		case CTL_REQ_REMOVE:
2120 			cfiscsi_ioctl_port_remove(req);
2121 			break;
2122 		default:
2123 			req->status = CTL_LUN_ERROR;
2124 			snprintf(req->error_str, sizeof(req->error_str),
2125 			    "Unsupported request type %d", req->reqtype);
2126 		}
2127 		return (0);
2128 	}
2129 
2130 	if (cmd != CTL_ISCSI)
2131 		return (ENOTTY);
2132 
2133 	ci = (struct ctl_iscsi *)addr;
2134 	switch (ci->type) {
2135 	case CTL_ISCSI_HANDOFF:
2136 		cfiscsi_ioctl_handoff(ci);
2137 		break;
2138 	case CTL_ISCSI_LIST:
2139 		cfiscsi_ioctl_list(ci);
2140 		break;
2141 	case CTL_ISCSI_TERMINATE:
2142 		cfiscsi_ioctl_terminate(ci);
2143 		break;
2144 	case CTL_ISCSI_LOGOUT:
2145 		cfiscsi_ioctl_logout(ci);
2146 		break;
2147 #ifdef ICL_KERNEL_PROXY
2148 	case CTL_ISCSI_LISTEN:
2149 		cfiscsi_ioctl_listen(ci);
2150 		break;
2151 	case CTL_ISCSI_ACCEPT:
2152 		cfiscsi_ioctl_accept(ci);
2153 		break;
2154 	case CTL_ISCSI_SEND:
2155 		cfiscsi_ioctl_send(ci);
2156 		break;
2157 	case CTL_ISCSI_RECEIVE:
2158 		cfiscsi_ioctl_receive(ci);
2159 		break;
2160 #else
2161 	case CTL_ISCSI_LISTEN:
2162 	case CTL_ISCSI_ACCEPT:
2163 	case CTL_ISCSI_SEND:
2164 	case CTL_ISCSI_RECEIVE:
2165 		ci->status = CTL_ISCSI_ERROR;
2166 		snprintf(ci->error_str, sizeof(ci->error_str),
2167 		    "%s: CTL compiled without ICL_KERNEL_PROXY",
2168 		    __func__);
2169 		break;
2170 #endif /* !ICL_KERNEL_PROXY */
2171 	default:
2172 		ci->status = CTL_ISCSI_ERROR;
2173 		snprintf(ci->error_str, sizeof(ci->error_str),
2174 		    "%s: invalid iSCSI request type %d", __func__, ci->type);
2175 		break;
2176 	}
2177 
2178 	return (0);
2179 }
2180 
2181 static void
2182 cfiscsi_target_hold(struct cfiscsi_target *ct)
2183 {
2184 
2185 	refcount_acquire(&ct->ct_refcount);
2186 }
2187 
2188 static void
2189 cfiscsi_target_release(struct cfiscsi_target *ct)
2190 {
2191 	struct cfiscsi_softc *softc;
2192 
2193 	softc = ct->ct_softc;
2194 	mtx_lock(&softc->lock);
2195 	if (refcount_release(&ct->ct_refcount)) {
2196 		TAILQ_REMOVE(&softc->targets, ct, ct_next);
2197 		mtx_unlock(&softc->lock);
2198 		if (ct->ct_state != CFISCSI_TARGET_STATE_INVALID) {
2199 			ct->ct_state = CFISCSI_TARGET_STATE_INVALID;
2200 			if (ctl_port_deregister(&ct->ct_port) != 0)
2201 				printf("%s: ctl_port_deregister() failed\n",
2202 				    __func__);
2203 		}
2204 		free(ct, M_CFISCSI);
2205 
2206 		return;
2207 	}
2208 	mtx_unlock(&softc->lock);
2209 }
2210 
2211 static struct cfiscsi_target *
2212 cfiscsi_target_find(struct cfiscsi_softc *softc, const char *name)
2213 {
2214 	struct cfiscsi_target *ct;
2215 
2216 	mtx_lock(&softc->lock);
2217 	TAILQ_FOREACH(ct, &softc->targets, ct_next) {
2218 		if (strcmp(name, ct->ct_name) != 0 ||
2219 		    ct->ct_state != CFISCSI_TARGET_STATE_ACTIVE)
2220 			continue;
2221 		cfiscsi_target_hold(ct);
2222 		mtx_unlock(&softc->lock);
2223 		return (ct);
2224 	}
2225 	mtx_unlock(&softc->lock);
2226 
2227 	return (NULL);
2228 }
2229 
2230 static struct cfiscsi_target *
2231 cfiscsi_target_find_or_create(struct cfiscsi_softc *softc, const char *name,
2232     const char *alias)
2233 {
2234 	struct cfiscsi_target *ct, *newct;
2235 	int i;
2236 
2237 	if (name[0] == '\0' || strlen(name) >= CTL_ISCSI_NAME_LEN)
2238 		return (NULL);
2239 
2240 	newct = malloc(sizeof(*newct), M_CFISCSI, M_WAITOK | M_ZERO);
2241 
2242 	mtx_lock(&softc->lock);
2243 	TAILQ_FOREACH(ct, &softc->targets, ct_next) {
2244 		if (strcmp(name, ct->ct_name) != 0 ||
2245 		    ct->ct_state == CFISCSI_TARGET_STATE_INVALID)
2246 			continue;
2247 		cfiscsi_target_hold(ct);
2248 		mtx_unlock(&softc->lock);
2249 		free(newct, M_CFISCSI);
2250 		return (ct);
2251 	}
2252 
2253 	for (i = 0; i < CTL_MAX_LUNS; i++)
2254 		newct->ct_luns[i] = UINT32_MAX;
2255 
2256 	strlcpy(newct->ct_name, name, sizeof(newct->ct_name));
2257 	if (alias != NULL)
2258 		strlcpy(newct->ct_alias, alias, sizeof(newct->ct_alias));
2259 	refcount_init(&newct->ct_refcount, 1);
2260 	newct->ct_softc = softc;
2261 	if (TAILQ_EMPTY(&softc->targets))
2262 		softc->last_target_id = 0;
2263 	newct->ct_target_id = ++softc->last_target_id;
2264 	TAILQ_INSERT_TAIL(&softc->targets, newct, ct_next);
2265 	mtx_unlock(&softc->lock);
2266 
2267 	return (newct);
2268 }
2269 
2270 /*
2271  * Takes LUN from the target space and returns LUN from the CTL space.
2272  */
2273 static uint32_t
2274 cfiscsi_lun_map(void *arg, uint32_t lun)
2275 {
2276 	struct cfiscsi_target *ct = arg;
2277 
2278 	if (lun >= CTL_MAX_LUNS) {
2279 		CFISCSI_DEBUG("requested lun number %d is higher "
2280 		    "than maximum %d", lun, CTL_MAX_LUNS - 1);
2281 		return (UINT32_MAX);
2282 	}
2283 	return (ct->ct_luns[lun]);
2284 }
2285 
2286 static int
2287 cfiscsi_target_set_lun(struct cfiscsi_target *ct,
2288     unsigned long lun_id, unsigned long ctl_lun_id)
2289 {
2290 
2291 	if (lun_id >= CTL_MAX_LUNS) {
2292 		CFISCSI_WARN("requested lun number %ld is higher "
2293 		    "than maximum %d", lun_id, CTL_MAX_LUNS - 1);
2294 		return (-1);
2295 	}
2296 
2297 	if (ct->ct_luns[lun_id] < CTL_MAX_LUNS) {
2298 		/*
2299 		 * CTL calls cfiscsi_lun_enable() twice for each LUN - once
2300 		 * when the LUN is created, and a second time just before
2301 		 * the port is brought online; don't emit warnings
2302 		 * for that case.
2303 		 */
2304 		if (ct->ct_luns[lun_id] == ctl_lun_id)
2305 			return (0);
2306 		CFISCSI_WARN("lun %ld already allocated", lun_id);
2307 		return (-1);
2308 	}
2309 
2310 #if 0
2311 	CFISCSI_DEBUG("adding mapping for lun %ld, target %s "
2312 	    "to ctl lun %ld", lun_id, ct->ct_name, ctl_lun_id);
2313 #endif
2314 
2315 	ct->ct_luns[lun_id] = ctl_lun_id;
2316 
2317 	return (0);
2318 }
2319 
2320 static int
2321 cfiscsi_lun_enable(void *arg, struct ctl_id target_id, int lun_id)
2322 {
2323 	struct cfiscsi_softc *softc;
2324 	struct cfiscsi_target *ct;
2325 	const char *target = NULL;
2326 	const char *lun = NULL;
2327 	unsigned long tmp;
2328 
2329 	ct = (struct cfiscsi_target *)arg;
2330 	softc = ct->ct_softc;
2331 
2332 	target = ctl_get_opt(&control_softc->ctl_luns[lun_id]->be_lun->options,
2333 	    "cfiscsi_target");
2334 	lun = ctl_get_opt(&control_softc->ctl_luns[lun_id]->be_lun->options,
2335 	    "cfiscsi_lun");
2336 
2337 	if (target == NULL && lun == NULL)
2338 		return (0);
2339 
2340 	if (target == NULL || lun == NULL) {
2341 		CFISCSI_WARN("lun added with cfiscsi_target, but without "
2342 		    "cfiscsi_lun, or the other way around; ignoring");
2343 		return (0);
2344 	}
2345 
2346 	if (strcmp(target, ct->ct_name) != 0)
2347 		return (0);
2348 
2349 	tmp = strtoul(lun, NULL, 10);
2350 	cfiscsi_target_set_lun(ct, tmp, lun_id);
2351 	return (0);
2352 }
2353 
2354 static int
2355 cfiscsi_lun_disable(void *arg, struct ctl_id target_id, int lun_id)
2356 {
2357 	struct cfiscsi_softc *softc;
2358 	struct cfiscsi_target *ct;
2359 	int i;
2360 
2361 	ct = (struct cfiscsi_target *)arg;
2362 	softc = ct->ct_softc;
2363 
2364 	mtx_lock(&softc->lock);
2365 	for (i = 0; i < CTL_MAX_LUNS; i++) {
2366 		if (ct->ct_luns[i] != lun_id)
2367 			continue;
2368 		ct->ct_luns[i] = UINT32_MAX;
2369 		break;
2370 	}
2371 	mtx_unlock(&softc->lock);
2372 	return (0);
2373 }
2374 
2375 static void
2376 cfiscsi_datamove_in(union ctl_io *io)
2377 {
2378 	struct cfiscsi_session *cs;
2379 	struct icl_pdu *request, *response;
2380 	const struct iscsi_bhs_scsi_command *bhssc;
2381 	struct iscsi_bhs_data_in *bhsdi;
2382 	struct ctl_sg_entry ctl_sg_entry, *ctl_sglist;
2383 	size_t len, expected_len, sg_len, buffer_offset;
2384 	const char *sg_addr;
2385 	int ctl_sg_count, error, i;
2386 
2387 	request = io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr;
2388 	cs = PDU_SESSION(request);
2389 
2390 	bhssc = (const struct iscsi_bhs_scsi_command *)request->ip_bhs;
2391 	KASSERT((bhssc->bhssc_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
2392 	    ISCSI_BHS_OPCODE_SCSI_COMMAND,
2393 	    ("bhssc->bhssc_opcode != ISCSI_BHS_OPCODE_SCSI_COMMAND"));
2394 
2395 	if (io->scsiio.kern_sg_entries > 0) {
2396 		ctl_sglist = (struct ctl_sg_entry *)io->scsiio.kern_data_ptr;
2397 		ctl_sg_count = io->scsiio.kern_sg_entries;
2398 	} else {
2399 		ctl_sglist = &ctl_sg_entry;
2400 		ctl_sglist->addr = io->scsiio.kern_data_ptr;
2401 		ctl_sglist->len = io->scsiio.kern_data_len;
2402 		ctl_sg_count = 1;
2403 	}
2404 
2405 	/*
2406 	 * This is the total amount of data to be transferred within the current
2407 	 * SCSI command.  We need to record it so that we can properly report
2408 	 * underflow/underflow.
2409 	 */
2410 	PDU_TOTAL_TRANSFER_LEN(request) = io->scsiio.kern_total_len;
2411 
2412 	/*
2413 	 * This is the offset within the current SCSI command; for the first
2414 	 * call to cfiscsi_datamove() it will be 0, and for subsequent ones
2415 	 * it will be the sum of lengths of previous ones.
2416 	 */
2417 	buffer_offset = io->scsiio.kern_rel_offset;
2418 
2419 	/*
2420 	 * This is the transfer length expected by the initiator.  In theory,
2421 	 * it could be different from the correct amount of data from the SCSI
2422 	 * point of view, even if that doesn't make any sense.
2423 	 */
2424 	expected_len = ntohl(bhssc->bhssc_expected_data_transfer_length);
2425 #if 0
2426 	if (expected_len != io->scsiio.kern_total_len) {
2427 		CFISCSI_SESSION_DEBUG(cs, "expected transfer length %zd, "
2428 		    "actual length %zd", expected_len,
2429 		    (size_t)io->scsiio.kern_total_len);
2430 	}
2431 #endif
2432 
2433 	if (buffer_offset >= expected_len) {
2434 #if 0
2435 		CFISCSI_SESSION_DEBUG(cs, "buffer_offset = %zd, "
2436 		    "already sent the expected len", buffer_offset);
2437 #endif
2438 		io->scsiio.be_move_done(io);
2439 		return;
2440 	}
2441 
2442 	i = 0;
2443 	sg_addr = NULL;
2444 	sg_len = 0;
2445 	response = NULL;
2446 	bhsdi = NULL;
2447 	for (;;) {
2448 		if (response == NULL) {
2449 			response = cfiscsi_pdu_new_response(request, M_NOWAIT);
2450 			if (response == NULL) {
2451 				CFISCSI_SESSION_WARN(cs, "failed to "
2452 				    "allocate memory; dropping connection");
2453 				ctl_set_busy(&io->scsiio);
2454 				io->scsiio.be_move_done(io);
2455 				cfiscsi_session_terminate(cs);
2456 				return;
2457 			}
2458 			bhsdi = (struct iscsi_bhs_data_in *)response->ip_bhs;
2459 			bhsdi->bhsdi_opcode = ISCSI_BHS_OPCODE_SCSI_DATA_IN;
2460 			bhsdi->bhsdi_initiator_task_tag =
2461 			    bhssc->bhssc_initiator_task_tag;
2462 			bhsdi->bhsdi_datasn = htonl(PDU_EXPDATASN(request));
2463 			PDU_EXPDATASN(request)++;
2464 			bhsdi->bhsdi_buffer_offset = htonl(buffer_offset);
2465 		}
2466 
2467 		KASSERT(i < ctl_sg_count, ("i >= ctl_sg_count"));
2468 		if (sg_len == 0) {
2469 			sg_addr = ctl_sglist[i].addr;
2470 			sg_len = ctl_sglist[i].len;
2471 			KASSERT(sg_len > 0, ("sg_len <= 0"));
2472 		}
2473 
2474 		len = sg_len;
2475 
2476 		/*
2477 		 * Truncate to maximum data segment length.
2478 		 */
2479 		KASSERT(response->ip_data_len < cs->cs_max_data_segment_length,
2480 		    ("ip_data_len %zd >= max_data_segment_length %zd",
2481 		    response->ip_data_len, cs->cs_max_data_segment_length));
2482 		if (response->ip_data_len + len >
2483 		    cs->cs_max_data_segment_length) {
2484 			len = cs->cs_max_data_segment_length -
2485 			    response->ip_data_len;
2486 			KASSERT(len <= sg_len, ("len %zd > sg_len %zd",
2487 			    len, sg_len));
2488 		}
2489 
2490 		/*
2491 		 * Truncate to expected data transfer length.
2492 		 */
2493 		KASSERT(buffer_offset + response->ip_data_len < expected_len,
2494 		    ("buffer_offset %zd + ip_data_len %zd >= expected_len %zd",
2495 		    buffer_offset, response->ip_data_len, expected_len));
2496 		if (buffer_offset + response->ip_data_len + len > expected_len) {
2497 			CFISCSI_SESSION_DEBUG(cs, "truncating from %zd "
2498 			    "to expected data transfer length %zd",
2499 			    buffer_offset + response->ip_data_len + len, expected_len);
2500 			len = expected_len - (buffer_offset + response->ip_data_len);
2501 			KASSERT(len <= sg_len, ("len %zd > sg_len %zd",
2502 			    len, sg_len));
2503 		}
2504 
2505 		error = icl_pdu_append_data(response, sg_addr, len, M_NOWAIT);
2506 		if (error != 0) {
2507 			CFISCSI_SESSION_WARN(cs, "failed to "
2508 			    "allocate memory; dropping connection");
2509 			icl_pdu_free(response);
2510 			ctl_set_busy(&io->scsiio);
2511 			io->scsiio.be_move_done(io);
2512 			cfiscsi_session_terminate(cs);
2513 			return;
2514 		}
2515 		sg_addr += len;
2516 		sg_len -= len;
2517 
2518 		KASSERT(buffer_offset + response->ip_data_len <= expected_len,
2519 		    ("buffer_offset %zd + ip_data_len %zd > expected_len %zd",
2520 		    buffer_offset, response->ip_data_len, expected_len));
2521 		if (buffer_offset + response->ip_data_len == expected_len) {
2522 			/*
2523 			 * Already have the amount of data the initiator wanted.
2524 			 */
2525 			break;
2526 		}
2527 
2528 		if (sg_len == 0) {
2529 			/*
2530 			 * End of scatter-gather segment;
2531 			 * proceed to the next one...
2532 			 */
2533 			if (i == ctl_sg_count - 1) {
2534 				/*
2535 				 * ... unless this was the last one.
2536 				 */
2537 				break;
2538 			}
2539 			i++;
2540 		}
2541 
2542 		if (response->ip_data_len == cs->cs_max_data_segment_length) {
2543 			/*
2544 			 * Can't stuff more data into the current PDU;
2545 			 * queue it.  Note that's not enough to check
2546 			 * for kern_data_resid == 0 instead; there
2547 			 * may be several Data-In PDUs for the final
2548 			 * call to cfiscsi_datamove(), and we want
2549 			 * to set the F flag only on the last of them.
2550 			 */
2551 			buffer_offset += response->ip_data_len;
2552 			if (buffer_offset == io->scsiio.kern_total_len ||
2553 			    buffer_offset == expected_len) {
2554 				buffer_offset -= response->ip_data_len;
2555 				break;
2556 			}
2557 			cfiscsi_pdu_queue(response);
2558 			response = NULL;
2559 			bhsdi = NULL;
2560 		}
2561 	}
2562 	if (response != NULL) {
2563 		buffer_offset += response->ip_data_len;
2564 		if (buffer_offset == io->scsiio.kern_total_len ||
2565 		    buffer_offset == expected_len) {
2566 			bhsdi->bhsdi_flags |= BHSDI_FLAGS_F;
2567 			if (io->io_hdr.status == CTL_SUCCESS) {
2568 				bhsdi->bhsdi_flags |= BHSDI_FLAGS_S;
2569 				if (PDU_TOTAL_TRANSFER_LEN(request) <
2570 				    ntohl(bhssc->bhssc_expected_data_transfer_length)) {
2571 					bhsdi->bhsdi_flags |= BHSSR_FLAGS_RESIDUAL_UNDERFLOW;
2572 					bhsdi->bhsdi_residual_count =
2573 					    htonl(ntohl(bhssc->bhssc_expected_data_transfer_length) -
2574 					    PDU_TOTAL_TRANSFER_LEN(request));
2575 				} else if (PDU_TOTAL_TRANSFER_LEN(request) >
2576 				    ntohl(bhssc->bhssc_expected_data_transfer_length)) {
2577 					bhsdi->bhsdi_flags |= BHSSR_FLAGS_RESIDUAL_OVERFLOW;
2578 					bhsdi->bhsdi_residual_count =
2579 					    htonl(PDU_TOTAL_TRANSFER_LEN(request) -
2580 					    ntohl(bhssc->bhssc_expected_data_transfer_length));
2581 				}
2582 				bhsdi->bhsdi_status = io->scsiio.scsi_status;
2583 				io->io_hdr.flags |= CTL_FLAG_STATUS_SENT;
2584 			}
2585 		}
2586 		KASSERT(response->ip_data_len > 0, ("sending empty Data-In"));
2587 		cfiscsi_pdu_queue(response);
2588 	}
2589 
2590 	io->scsiio.be_move_done(io);
2591 }
2592 
2593 static void
2594 cfiscsi_datamove_out(union ctl_io *io)
2595 {
2596 	struct cfiscsi_session *cs;
2597 	struct icl_pdu *request, *response;
2598 	const struct iscsi_bhs_scsi_command *bhssc;
2599 	struct iscsi_bhs_r2t *bhsr2t;
2600 	struct cfiscsi_data_wait *cdw;
2601 	struct ctl_sg_entry ctl_sg_entry, *ctl_sglist;
2602 	uint32_t expected_len, r2t_off, r2t_len;
2603 	uint32_t target_transfer_tag;
2604 	bool done;
2605 
2606 	request = io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr;
2607 	cs = PDU_SESSION(request);
2608 
2609 	bhssc = (const struct iscsi_bhs_scsi_command *)request->ip_bhs;
2610 	KASSERT((bhssc->bhssc_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
2611 	    ISCSI_BHS_OPCODE_SCSI_COMMAND,
2612 	    ("bhssc->bhssc_opcode != ISCSI_BHS_OPCODE_SCSI_COMMAND"));
2613 
2614 	/*
2615 	 * We need to record it so that we can properly report
2616 	 * underflow/underflow.
2617 	 */
2618 	PDU_TOTAL_TRANSFER_LEN(request) = io->scsiio.kern_total_len;
2619 
2620 	/*
2621 	 * Report write underflow as error since CTL and backends don't
2622 	 * really support it, and SCSI does not tell how to do it right.
2623 	 */
2624 	expected_len = ntohl(bhssc->bhssc_expected_data_transfer_length);
2625 	if (io->scsiio.kern_rel_offset + io->scsiio.kern_data_len >
2626 	    expected_len) {
2627 		io->scsiio.io_hdr.port_status = 43;
2628 		io->scsiio.be_move_done(io);
2629 		return;
2630 	}
2631 
2632 	target_transfer_tag =
2633 	    atomic_fetchadd_32(&cs->cs_target_transfer_tag, 1);
2634 
2635 #if 0
2636 	CFISCSI_SESSION_DEBUG(cs, "expecting Data-Out with initiator "
2637 	    "task tag 0x%x, target transfer tag 0x%x",
2638 	    bhssc->bhssc_initiator_task_tag, target_transfer_tag);
2639 #endif
2640 	cdw = uma_zalloc(cfiscsi_data_wait_zone, M_NOWAIT | M_ZERO);
2641 	if (cdw == NULL) {
2642 		CFISCSI_SESSION_WARN(cs, "failed to "
2643 		    "allocate memory; dropping connection");
2644 		ctl_set_busy(&io->scsiio);
2645 		io->scsiio.be_move_done(io);
2646 		cfiscsi_session_terminate(cs);
2647 		return;
2648 	}
2649 	cdw->cdw_ctl_io = io;
2650 	cdw->cdw_target_transfer_tag = target_transfer_tag;
2651 	cdw->cdw_initiator_task_tag = bhssc->bhssc_initiator_task_tag;
2652 	cdw->cdw_r2t_end = io->scsiio.kern_data_len;
2653 
2654 	/* Set initial data pointer for the CDW respecting ext_data_filled. */
2655 	if (io->scsiio.kern_sg_entries > 0) {
2656 		ctl_sglist = (struct ctl_sg_entry *)io->scsiio.kern_data_ptr;
2657 	} else {
2658 		ctl_sglist = &ctl_sg_entry;
2659 		ctl_sglist->addr = io->scsiio.kern_data_ptr;
2660 		ctl_sglist->len = io->scsiio.kern_data_len;
2661 	}
2662 	cdw->cdw_sg_index = 0;
2663 	cdw->cdw_sg_addr = ctl_sglist[cdw->cdw_sg_index].addr;
2664 	cdw->cdw_sg_len = ctl_sglist[cdw->cdw_sg_index].len;
2665 	r2t_off = io->scsiio.ext_data_filled;
2666 	while (r2t_off > 0) {
2667 		if (r2t_off >= cdw->cdw_sg_len) {
2668 			r2t_off -= cdw->cdw_sg_len;
2669 			cdw->cdw_sg_index++;
2670 			cdw->cdw_sg_addr = ctl_sglist[cdw->cdw_sg_index].addr;
2671 			cdw->cdw_sg_len = ctl_sglist[cdw->cdw_sg_index].len;
2672 			continue;
2673 		}
2674 		cdw->cdw_sg_addr += r2t_off;
2675 		cdw->cdw_sg_len -= r2t_off;
2676 		r2t_off = 0;
2677 	}
2678 
2679 	if (cs->cs_immediate_data &&
2680 	    io->scsiio.kern_rel_offset + io->scsiio.ext_data_filled <
2681 	    icl_pdu_data_segment_length(request)) {
2682 		done = cfiscsi_handle_data_segment(request, cdw);
2683 		if (done) {
2684 			uma_zfree(cfiscsi_data_wait_zone, cdw);
2685 			io->scsiio.be_move_done(io);
2686 			return;
2687 		}
2688 	}
2689 
2690 	r2t_off = io->scsiio.kern_rel_offset + io->scsiio.ext_data_filled;
2691 	r2t_len = MIN(io->scsiio.kern_data_len - io->scsiio.ext_data_filled,
2692 	    cs->cs_max_burst_length);
2693 	cdw->cdw_r2t_end = io->scsiio.ext_data_filled + r2t_len;
2694 
2695 	CFISCSI_SESSION_LOCK(cs);
2696 	TAILQ_INSERT_TAIL(&cs->cs_waiting_for_data_out, cdw, cdw_next);
2697 	CFISCSI_SESSION_UNLOCK(cs);
2698 
2699 	/*
2700 	 * XXX: We should limit the number of outstanding R2T PDUs
2701 	 * 	per task to MaxOutstandingR2T.
2702 	 */
2703 	response = cfiscsi_pdu_new_response(request, M_NOWAIT);
2704 	if (response == NULL) {
2705 		CFISCSI_SESSION_WARN(cs, "failed to "
2706 		    "allocate memory; dropping connection");
2707 		ctl_set_busy(&io->scsiio);
2708 		io->scsiio.be_move_done(io);
2709 		cfiscsi_session_terminate(cs);
2710 		return;
2711 	}
2712 	bhsr2t = (struct iscsi_bhs_r2t *)response->ip_bhs;
2713 	bhsr2t->bhsr2t_opcode = ISCSI_BHS_OPCODE_R2T;
2714 	bhsr2t->bhsr2t_flags = 0x80;
2715 	bhsr2t->bhsr2t_lun = bhssc->bhssc_lun;
2716 	bhsr2t->bhsr2t_initiator_task_tag = bhssc->bhssc_initiator_task_tag;
2717 	bhsr2t->bhsr2t_target_transfer_tag = target_transfer_tag;
2718 	/*
2719 	 * XXX: Here we assume that cfiscsi_datamove() won't ever
2720 	 *	be running concurrently on several CPUs for a given
2721 	 *	command.
2722 	 */
2723 	bhsr2t->bhsr2t_r2tsn = htonl(PDU_R2TSN(request));
2724 	PDU_R2TSN(request)++;
2725 	/*
2726 	 * This is the offset within the current SCSI command;
2727 	 * i.e. for the first call of datamove(), it will be 0,
2728 	 * and for subsequent ones it will be the sum of lengths
2729 	 * of previous ones.
2730 	 *
2731 	 * The ext_data_filled is to account for unsolicited
2732 	 * (immediate) data that might have already arrived.
2733 	 */
2734 	bhsr2t->bhsr2t_buffer_offset = htonl(r2t_off);
2735 	/*
2736 	 * This is the total length (sum of S/G lengths) this call
2737 	 * to cfiscsi_datamove() is supposed to handle, limited by
2738 	 * MaxBurstLength.
2739 	 */
2740 	bhsr2t->bhsr2t_desired_data_transfer_length = htonl(r2t_len);
2741 	cfiscsi_pdu_queue(response);
2742 }
2743 
2744 static void
2745 cfiscsi_datamove(union ctl_io *io)
2746 {
2747 
2748 	if ((io->io_hdr.flags & CTL_FLAG_DATA_MASK) == CTL_FLAG_DATA_IN)
2749 		cfiscsi_datamove_in(io);
2750 	else {
2751 		/* We hadn't received anything during this datamove yet. */
2752 		io->scsiio.ext_data_filled = 0;
2753 		cfiscsi_datamove_out(io);
2754 	}
2755 }
2756 
2757 static void
2758 cfiscsi_scsi_command_done(union ctl_io *io)
2759 {
2760 	struct icl_pdu *request, *response;
2761 	struct iscsi_bhs_scsi_command *bhssc;
2762 	struct iscsi_bhs_scsi_response *bhssr;
2763 #ifdef DIAGNOSTIC
2764 	struct cfiscsi_data_wait *cdw;
2765 #endif
2766 	struct cfiscsi_session *cs;
2767 	uint16_t sense_length;
2768 
2769 	request = io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr;
2770 	cs = PDU_SESSION(request);
2771 	bhssc = (struct iscsi_bhs_scsi_command *)request->ip_bhs;
2772 	KASSERT((bhssc->bhssc_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
2773 	    ISCSI_BHS_OPCODE_SCSI_COMMAND,
2774 	    ("replying to wrong opcode 0x%x", bhssc->bhssc_opcode));
2775 
2776 	//CFISCSI_SESSION_DEBUG(cs, "initiator task tag 0x%x",
2777 	//    bhssc->bhssc_initiator_task_tag);
2778 
2779 #ifdef DIAGNOSTIC
2780 	CFISCSI_SESSION_LOCK(cs);
2781 	TAILQ_FOREACH(cdw, &cs->cs_waiting_for_data_out, cdw_next)
2782 		KASSERT(bhssc->bhssc_initiator_task_tag !=
2783 		    cdw->cdw_initiator_task_tag, ("dangling cdw"));
2784 	CFISCSI_SESSION_UNLOCK(cs);
2785 #endif
2786 
2787 	/*
2788 	 * Do not return status for aborted commands.
2789 	 * There are exceptions, but none supported by CTL yet.
2790 	 */
2791 	if (((io->io_hdr.flags & CTL_FLAG_ABORT) &&
2792 	     (io->io_hdr.flags & CTL_FLAG_ABORT_STATUS) == 0) ||
2793 	    (io->io_hdr.flags & CTL_FLAG_STATUS_SENT)) {
2794 		ctl_free_io(io);
2795 		icl_pdu_free(request);
2796 		return;
2797 	}
2798 
2799 	response = cfiscsi_pdu_new_response(request, M_WAITOK);
2800 	bhssr = (struct iscsi_bhs_scsi_response *)response->ip_bhs;
2801 	bhssr->bhssr_opcode = ISCSI_BHS_OPCODE_SCSI_RESPONSE;
2802 	bhssr->bhssr_flags = 0x80;
2803 	/*
2804 	 * XXX: We don't deal with bidirectional under/overflows;
2805 	 *	does anything actually support those?
2806 	 */
2807 	if (PDU_TOTAL_TRANSFER_LEN(request) <
2808 	    ntohl(bhssc->bhssc_expected_data_transfer_length)) {
2809 		bhssr->bhssr_flags |= BHSSR_FLAGS_RESIDUAL_UNDERFLOW;
2810 		bhssr->bhssr_residual_count =
2811 		    htonl(ntohl(bhssc->bhssc_expected_data_transfer_length) -
2812 		    PDU_TOTAL_TRANSFER_LEN(request));
2813 		//CFISCSI_SESSION_DEBUG(cs, "underflow; residual count %d",
2814 		//    ntohl(bhssr->bhssr_residual_count));
2815 	} else if (PDU_TOTAL_TRANSFER_LEN(request) >
2816 	    ntohl(bhssc->bhssc_expected_data_transfer_length)) {
2817 		bhssr->bhssr_flags |= BHSSR_FLAGS_RESIDUAL_OVERFLOW;
2818 		bhssr->bhssr_residual_count =
2819 		    htonl(PDU_TOTAL_TRANSFER_LEN(request) -
2820 		    ntohl(bhssc->bhssc_expected_data_transfer_length));
2821 		//CFISCSI_SESSION_DEBUG(cs, "overflow; residual count %d",
2822 		//    ntohl(bhssr->bhssr_residual_count));
2823 	}
2824 	bhssr->bhssr_response = BHSSR_RESPONSE_COMMAND_COMPLETED;
2825 	bhssr->bhssr_status = io->scsiio.scsi_status;
2826 	bhssr->bhssr_initiator_task_tag = bhssc->bhssc_initiator_task_tag;
2827 	bhssr->bhssr_expdatasn = htonl(PDU_EXPDATASN(request));
2828 
2829 	if (io->scsiio.sense_len > 0) {
2830 #if 0
2831 		CFISCSI_SESSION_DEBUG(cs, "returning %d bytes of sense data",
2832 		    io->scsiio.sense_len);
2833 #endif
2834 		sense_length = htons(io->scsiio.sense_len);
2835 		icl_pdu_append_data(response,
2836 		    &sense_length, sizeof(sense_length), M_WAITOK);
2837 		icl_pdu_append_data(response,
2838 		    &io->scsiio.sense_data, io->scsiio.sense_len, M_WAITOK);
2839 	}
2840 
2841 	ctl_free_io(io);
2842 	icl_pdu_free(request);
2843 	cfiscsi_pdu_queue(response);
2844 }
2845 
2846 static void
2847 cfiscsi_task_management_done(union ctl_io *io)
2848 {
2849 	struct icl_pdu *request, *response;
2850 	struct iscsi_bhs_task_management_request *bhstmr;
2851 	struct iscsi_bhs_task_management_response *bhstmr2;
2852 	struct cfiscsi_data_wait *cdw, *tmpcdw;
2853 	struct cfiscsi_session *cs;
2854 
2855 	request = io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr;
2856 	cs = PDU_SESSION(request);
2857 	bhstmr = (struct iscsi_bhs_task_management_request *)request->ip_bhs;
2858 	KASSERT((bhstmr->bhstmr_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
2859 	    ISCSI_BHS_OPCODE_TASK_REQUEST,
2860 	    ("replying to wrong opcode 0x%x", bhstmr->bhstmr_opcode));
2861 
2862 #if 0
2863 	CFISCSI_SESSION_DEBUG(cs, "initiator task tag 0x%x; referenced task tag 0x%x",
2864 	    bhstmr->bhstmr_initiator_task_tag,
2865 	    bhstmr->bhstmr_referenced_task_tag);
2866 #endif
2867 
2868 	if ((bhstmr->bhstmr_function & ~0x80) ==
2869 	    BHSTMR_FUNCTION_ABORT_TASK) {
2870 		/*
2871 		 * Make sure we no longer wait for Data-Out for this command.
2872 		 */
2873 		CFISCSI_SESSION_LOCK(cs);
2874 		TAILQ_FOREACH_SAFE(cdw,
2875 		    &cs->cs_waiting_for_data_out, cdw_next, tmpcdw) {
2876 			if (bhstmr->bhstmr_referenced_task_tag !=
2877 			    cdw->cdw_initiator_task_tag)
2878 				continue;
2879 
2880 #if 0
2881 			CFISCSI_SESSION_DEBUG(cs, "removing csw for initiator task "
2882 			    "tag 0x%x", bhstmr->bhstmr_initiator_task_tag);
2883 #endif
2884 			TAILQ_REMOVE(&cs->cs_waiting_for_data_out,
2885 			    cdw, cdw_next);
2886 			cdw->cdw_ctl_io->scsiio.be_move_done(cdw->cdw_ctl_io);
2887 			uma_zfree(cfiscsi_data_wait_zone, cdw);
2888 		}
2889 		CFISCSI_SESSION_UNLOCK(cs);
2890 	}
2891 
2892 	response = cfiscsi_pdu_new_response(request, M_WAITOK);
2893 	bhstmr2 = (struct iscsi_bhs_task_management_response *)
2894 	    response->ip_bhs;
2895 	bhstmr2->bhstmr_opcode = ISCSI_BHS_OPCODE_TASK_RESPONSE;
2896 	bhstmr2->bhstmr_flags = 0x80;
2897 	if (io->io_hdr.status == CTL_SUCCESS) {
2898 		bhstmr2->bhstmr_response = BHSTMR_RESPONSE_FUNCTION_COMPLETE;
2899 	} else {
2900 		/*
2901 		 * XXX: How to figure out what exactly went wrong?  iSCSI spec
2902 		 * 	expects us to provide detailed error, e.g. "Task does
2903 		 * 	not exist" or "LUN does not exist".
2904 		 */
2905 		CFISCSI_SESSION_DEBUG(cs, "BHSTMR_RESPONSE_FUNCTION_NOT_SUPPORTED");
2906 		bhstmr2->bhstmr_response =
2907 		    BHSTMR_RESPONSE_FUNCTION_NOT_SUPPORTED;
2908 	}
2909 	bhstmr2->bhstmr_initiator_task_tag = bhstmr->bhstmr_initiator_task_tag;
2910 
2911 	ctl_free_io(io);
2912 	icl_pdu_free(request);
2913 	cfiscsi_pdu_queue(response);
2914 }
2915 
2916 static void
2917 cfiscsi_done(union ctl_io *io)
2918 {
2919 	struct icl_pdu *request;
2920 	struct cfiscsi_session *cs;
2921 
2922 	KASSERT(((io->io_hdr.status & CTL_STATUS_MASK) != CTL_STATUS_NONE),
2923 		("invalid CTL status %#x", io->io_hdr.status));
2924 
2925 	if (io->io_hdr.io_type == CTL_IO_TASK &&
2926 	    io->taskio.task_action == CTL_TASK_I_T_NEXUS_RESET) {
2927 		/*
2928 		 * Implicit task termination has just completed; nothing to do.
2929 		 */
2930 		cs = io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr;
2931 		cs->cs_tasks_aborted = true;
2932 		refcount_release(&cs->cs_outstanding_ctl_pdus);
2933 		wakeup(__DEVOLATILE(void *, &cs->cs_outstanding_ctl_pdus));
2934 		ctl_free_io(io);
2935 		return;
2936 	}
2937 
2938 	request = io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr;
2939 	cs = PDU_SESSION(request);
2940 	refcount_release(&cs->cs_outstanding_ctl_pdus);
2941 
2942 	switch (request->ip_bhs->bhs_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) {
2943 	case ISCSI_BHS_OPCODE_SCSI_COMMAND:
2944 		cfiscsi_scsi_command_done(io);
2945 		break;
2946 	case ISCSI_BHS_OPCODE_TASK_REQUEST:
2947 		cfiscsi_task_management_done(io);
2948 		break;
2949 	default:
2950 		panic("cfiscsi_done called with wrong opcode 0x%x",
2951 		    request->ip_bhs->bhs_opcode);
2952 	}
2953 }
2954