xref: /freebsd/sys/cam/ctl/ctl_frontend_iscsi.c (revision 076ad2f836d5f49dc1375f1677335a48fe0d4b82)
1 /*-
2  * Copyright (c) 2012 The FreeBSD Foundation
3  * All rights reserved.
4  *
5  * This software was developed by Edward Tomasz Napierala under sponsorship
6  * from the FreeBSD Foundation.
7  *
8  * Redistribution and use in source and binary forms, with or without
9  * modification, are permitted provided that the following conditions
10  * are met:
11  * 1. Redistributions of source code must retain the above copyright
12  *    notice, this list of conditions and the following disclaimer.
13  * 2. Redistributions in binary form must reproduce the above copyright
14  *    notice, this list of conditions and the following disclaimer in the
15  *    documentation and/or other materials provided with the distribution.
16  *
17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
21  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27  * SUCH DAMAGE.
28  *
29  * $FreeBSD$
30  */
31 
32 /*
33  * CTL frontend for the iSCSI protocol.
34  */
35 
36 #include <sys/cdefs.h>
37 __FBSDID("$FreeBSD$");
38 
39 #include <sys/param.h>
40 #include <sys/capsicum.h>
41 #include <sys/condvar.h>
42 #include <sys/endian.h>
43 #include <sys/file.h>
44 #include <sys/kernel.h>
45 #include <sys/kthread.h>
46 #include <sys/lock.h>
47 #include <sys/malloc.h>
48 #include <sys/module.h>
49 #include <sys/mutex.h>
50 #include <sys/queue.h>
51 #include <sys/sbuf.h>
52 #include <sys/socket.h>
53 #include <sys/sysctl.h>
54 #include <sys/systm.h>
55 #include <sys/uio.h>
56 #include <sys/unistd.h>
57 #include <vm/uma.h>
58 
59 #include <cam/scsi/scsi_all.h>
60 #include <cam/scsi/scsi_da.h>
61 #include <cam/ctl/ctl_io.h>
62 #include <cam/ctl/ctl.h>
63 #include <cam/ctl/ctl_backend.h>
64 #include <cam/ctl/ctl_error.h>
65 #include <cam/ctl/ctl_frontend.h>
66 #include <cam/ctl/ctl_debug.h>
67 #include <cam/ctl/ctl_ha.h>
68 #include <cam/ctl/ctl_ioctl.h>
69 #include <cam/ctl/ctl_private.h>
70 
71 #include <dev/iscsi/icl.h>
72 #include <dev/iscsi/icl_wrappers.h>
73 #include <dev/iscsi/iscsi_proto.h>
74 #include <cam/ctl/ctl_frontend_iscsi.h>
75 
76 #ifdef ICL_KERNEL_PROXY
77 #include <sys/socketvar.h>
78 #endif
79 
80 #ifdef ICL_KERNEL_PROXY
81 FEATURE(cfiscsi_kernel_proxy, "iSCSI target built with ICL_KERNEL_PROXY");
82 #endif
83 
84 static MALLOC_DEFINE(M_CFISCSI, "cfiscsi", "Memory used for CTL iSCSI frontend");
85 static uma_zone_t cfiscsi_data_wait_zone;
86 
87 SYSCTL_NODE(_kern_cam_ctl, OID_AUTO, iscsi, CTLFLAG_RD, 0,
88     "CAM Target Layer iSCSI Frontend");
89 static int debug = 1;
90 SYSCTL_INT(_kern_cam_ctl_iscsi, OID_AUTO, debug, CTLFLAG_RWTUN,
91     &debug, 1, "Enable debug messages");
92 static int ping_timeout = 5;
93 SYSCTL_INT(_kern_cam_ctl_iscsi, OID_AUTO, ping_timeout, CTLFLAG_RWTUN,
94     &ping_timeout, 5, "Interval between ping (NOP-Out) requests, in seconds");
95 static int login_timeout = 60;
96 SYSCTL_INT(_kern_cam_ctl_iscsi, OID_AUTO, login_timeout, CTLFLAG_RWTUN,
97     &login_timeout, 60, "Time to wait for ctld(8) to finish Login Phase, in seconds");
98 static int maxtags = 256;
99 SYSCTL_INT(_kern_cam_ctl_iscsi, OID_AUTO, maxtags, CTLFLAG_RWTUN,
100     &maxtags, 0, "Max number of requests queued by initiator");
101 
102 #define	CFISCSI_DEBUG(X, ...)						\
103 	do {								\
104 		if (debug > 1) {					\
105 			printf("%s: " X "\n",				\
106 			    __func__, ## __VA_ARGS__);			\
107 		}							\
108 	} while (0)
109 
110 #define	CFISCSI_WARN(X, ...)						\
111 	do {								\
112 		if (debug > 0) {					\
113 			printf("WARNING: %s: " X "\n",			\
114 			    __func__, ## __VA_ARGS__);			\
115 		}							\
116 	} while (0)
117 
118 #define	CFISCSI_SESSION_DEBUG(S, X, ...)				\
119 	do {								\
120 		if (debug > 1) {					\
121 			printf("%s: %s (%s): " X "\n",			\
122 			    __func__, S->cs_initiator_addr,		\
123 			    S->cs_initiator_name, ## __VA_ARGS__);	\
124 		}							\
125 	} while (0)
126 
127 #define	CFISCSI_SESSION_WARN(S, X, ...)					\
128 	do  {								\
129 		if (debug > 0) {					\
130 			printf("WARNING: %s (%s): " X "\n",		\
131 			    S->cs_initiator_addr,			\
132 			    S->cs_initiator_name, ## __VA_ARGS__);	\
133 		}							\
134 	} while (0)
135 
136 #define CFISCSI_SESSION_LOCK(X)		mtx_lock(&X->cs_lock)
137 #define CFISCSI_SESSION_UNLOCK(X)	mtx_unlock(&X->cs_lock)
138 #define CFISCSI_SESSION_LOCK_ASSERT(X)	mtx_assert(&X->cs_lock, MA_OWNED)
139 
140 #define	CONN_SESSION(X)			((struct cfiscsi_session *)(X)->ic_prv0)
141 #define	PDU_SESSION(X)			CONN_SESSION((X)->ip_conn)
142 #define	PDU_EXPDATASN(X)		(X)->ip_prv0
143 #define	PDU_TOTAL_TRANSFER_LEN(X)	(X)->ip_prv1
144 #define	PDU_R2TSN(X)			(X)->ip_prv2
145 
146 static int	cfiscsi_init(void);
147 static int	cfiscsi_shutdown(void);
148 static void	cfiscsi_online(void *arg);
149 static void	cfiscsi_offline(void *arg);
150 static int	cfiscsi_info(void *arg, struct sbuf *sb);
151 static int	cfiscsi_ioctl(struct cdev *dev,
152 		    u_long cmd, caddr_t addr, int flag, struct thread *td);
153 static void	cfiscsi_datamove(union ctl_io *io);
154 static void	cfiscsi_datamove_in(union ctl_io *io);
155 static void	cfiscsi_datamove_out(union ctl_io *io);
156 static void	cfiscsi_done(union ctl_io *io);
157 static bool	cfiscsi_pdu_update_cmdsn(const struct icl_pdu *request);
158 static void	cfiscsi_pdu_handle_nop_out(struct icl_pdu *request);
159 static void	cfiscsi_pdu_handle_scsi_command(struct icl_pdu *request);
160 static void	cfiscsi_pdu_handle_task_request(struct icl_pdu *request);
161 static void	cfiscsi_pdu_handle_data_out(struct icl_pdu *request);
162 static void	cfiscsi_pdu_handle_logout_request(struct icl_pdu *request);
163 static void	cfiscsi_session_terminate(struct cfiscsi_session *cs);
164 static struct cfiscsi_data_wait	*cfiscsi_data_wait_new(
165 		    struct cfiscsi_session *cs, union ctl_io *io,
166 		    uint32_t initiator_task_tag,
167 		    uint32_t *target_transfer_tagp);
168 static void	cfiscsi_data_wait_free(struct cfiscsi_session *cs,
169 		    struct cfiscsi_data_wait *cdw);
170 static struct cfiscsi_target	*cfiscsi_target_find(struct cfiscsi_softc
171 		    *softc, const char *name, uint16_t tag);
172 static struct cfiscsi_target	*cfiscsi_target_find_or_create(
173     struct cfiscsi_softc *softc, const char *name, const char *alias,
174     uint16_t tag);
175 static void	cfiscsi_target_release(struct cfiscsi_target *ct);
176 static void	cfiscsi_session_delete(struct cfiscsi_session *cs);
177 
178 static struct cfiscsi_softc cfiscsi_softc;
179 
180 static struct ctl_frontend cfiscsi_frontend =
181 {
182 	.name = "iscsi",
183 	.init = cfiscsi_init,
184 	.ioctl = cfiscsi_ioctl,
185 	.shutdown = cfiscsi_shutdown,
186 };
187 CTL_FRONTEND_DECLARE(ctlcfiscsi, cfiscsi_frontend);
188 MODULE_DEPEND(ctlcfiscsi, icl, 1, 1, 1);
189 
190 static struct icl_pdu *
191 cfiscsi_pdu_new_response(struct icl_pdu *request, int flags)
192 {
193 
194 	return (icl_pdu_new(request->ip_conn, flags));
195 }
196 
197 static bool
198 cfiscsi_pdu_update_cmdsn(const struct icl_pdu *request)
199 {
200 	const struct iscsi_bhs_scsi_command *bhssc;
201 	struct cfiscsi_session *cs;
202 	uint32_t cmdsn, expstatsn;
203 
204 	cs = PDU_SESSION(request);
205 
206 	/*
207 	 * Every incoming PDU - not just NOP-Out - resets the ping timer.
208 	 * The purpose of the timeout is to reset the connection when it stalls;
209 	 * we don't want this to happen when NOP-In or NOP-Out ends up delayed
210 	 * in some queue.
211 	 *
212 	 * XXX: Locking?
213 	 */
214 	cs->cs_timeout = 0;
215 
216 	/*
217 	 * Data-Out PDUs don't contain CmdSN.
218 	 */
219 	if ((request->ip_bhs->bhs_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
220 	    ISCSI_BHS_OPCODE_SCSI_DATA_OUT)
221 		return (false);
222 
223 	/*
224 	 * We're only using fields common for all the request
225 	 * (initiator -> target) PDUs.
226 	 */
227 	bhssc = (const struct iscsi_bhs_scsi_command *)request->ip_bhs;
228 	cmdsn = ntohl(bhssc->bhssc_cmdsn);
229 	expstatsn = ntohl(bhssc->bhssc_expstatsn);
230 
231 	CFISCSI_SESSION_LOCK(cs);
232 #if 0
233 	if (expstatsn != cs->cs_statsn) {
234 		CFISCSI_SESSION_DEBUG(cs, "received PDU with ExpStatSN %d, "
235 		    "while current StatSN is %d", expstatsn,
236 		    cs->cs_statsn);
237 	}
238 #endif
239 
240 	if ((request->ip_bhs->bhs_opcode & ISCSI_BHS_OPCODE_IMMEDIATE) == 0) {
241 		/*
242 		 * The target MUST silently ignore any non-immediate command
243 		 * outside of this range.
244 		 */
245 		if (ISCSI_SNLT(cmdsn, cs->cs_cmdsn) ||
246 		    ISCSI_SNGT(cmdsn, cs->cs_cmdsn - 1 + maxtags)) {
247 			CFISCSI_SESSION_UNLOCK(cs);
248 			CFISCSI_SESSION_WARN(cs, "received PDU with CmdSN %u, "
249 			    "while expected %u", cmdsn, cs->cs_cmdsn);
250 			return (true);
251 		}
252 
253 		/*
254 		 * We don't support multiple connections now, so any
255 		 * discontinuity in CmdSN means lost PDUs.  Since we don't
256 		 * support PDU retransmission -- terminate the connection.
257 		 */
258 		if (cmdsn != cs->cs_cmdsn) {
259 			CFISCSI_SESSION_UNLOCK(cs);
260 			CFISCSI_SESSION_WARN(cs, "received PDU with CmdSN %u, "
261 			    "while expected %u; dropping connection",
262 			    cmdsn, cs->cs_cmdsn);
263 			cfiscsi_session_terminate(cs);
264 			return (true);
265 		}
266 		cs->cs_cmdsn++;
267 	}
268 
269 	CFISCSI_SESSION_UNLOCK(cs);
270 
271 	return (false);
272 }
273 
274 static void
275 cfiscsi_pdu_handle(struct icl_pdu *request)
276 {
277 	struct cfiscsi_session *cs;
278 	bool ignore;
279 
280 	cs = PDU_SESSION(request);
281 
282 	ignore = cfiscsi_pdu_update_cmdsn(request);
283 	if (ignore) {
284 		icl_pdu_free(request);
285 		return;
286 	}
287 
288 	/*
289 	 * Handle the PDU; this includes e.g. receiving the remaining
290 	 * part of PDU and submitting the SCSI command to CTL
291 	 * or queueing a reply.  The handling routine is responsible
292 	 * for freeing the PDU when it's no longer needed.
293 	 */
294 	switch (request->ip_bhs->bhs_opcode &
295 	    ~ISCSI_BHS_OPCODE_IMMEDIATE) {
296 	case ISCSI_BHS_OPCODE_NOP_OUT:
297 		cfiscsi_pdu_handle_nop_out(request);
298 		break;
299 	case ISCSI_BHS_OPCODE_SCSI_COMMAND:
300 		cfiscsi_pdu_handle_scsi_command(request);
301 		break;
302 	case ISCSI_BHS_OPCODE_TASK_REQUEST:
303 		cfiscsi_pdu_handle_task_request(request);
304 		break;
305 	case ISCSI_BHS_OPCODE_SCSI_DATA_OUT:
306 		cfiscsi_pdu_handle_data_out(request);
307 		break;
308 	case ISCSI_BHS_OPCODE_LOGOUT_REQUEST:
309 		cfiscsi_pdu_handle_logout_request(request);
310 		break;
311 	default:
312 		CFISCSI_SESSION_WARN(cs, "received PDU with unsupported "
313 		    "opcode 0x%x; dropping connection",
314 		    request->ip_bhs->bhs_opcode);
315 		icl_pdu_free(request);
316 		cfiscsi_session_terminate(cs);
317 	}
318 
319 }
320 
321 static void
322 cfiscsi_receive_callback(struct icl_pdu *request)
323 {
324 	struct cfiscsi_session *cs;
325 
326 	cs = PDU_SESSION(request);
327 
328 #ifdef ICL_KERNEL_PROXY
329 	if (cs->cs_waiting_for_ctld || cs->cs_login_phase) {
330 		if (cs->cs_login_pdu == NULL)
331 			cs->cs_login_pdu = request;
332 		else
333 			icl_pdu_free(request);
334 		cv_signal(&cs->cs_login_cv);
335 		return;
336 	}
337 #endif
338 
339 	cfiscsi_pdu_handle(request);
340 }
341 
342 static void
343 cfiscsi_error_callback(struct icl_conn *ic)
344 {
345 	struct cfiscsi_session *cs;
346 
347 	cs = CONN_SESSION(ic);
348 
349 	CFISCSI_SESSION_WARN(cs, "connection error; dropping connection");
350 	cfiscsi_session_terminate(cs);
351 }
352 
353 static int
354 cfiscsi_pdu_prepare(struct icl_pdu *response)
355 {
356 	struct cfiscsi_session *cs;
357 	struct iscsi_bhs_scsi_response *bhssr;
358 	bool advance_statsn = true;
359 
360 	cs = PDU_SESSION(response);
361 
362 	CFISCSI_SESSION_LOCK_ASSERT(cs);
363 
364 	/*
365 	 * We're only using fields common for all the response
366 	 * (target -> initiator) PDUs.
367 	 */
368 	bhssr = (struct iscsi_bhs_scsi_response *)response->ip_bhs;
369 
370 	/*
371 	 * 10.8.3: "The StatSN for this connection is not advanced
372 	 * after this PDU is sent."
373 	 */
374 	if (bhssr->bhssr_opcode == ISCSI_BHS_OPCODE_R2T)
375 		advance_statsn = false;
376 
377 	/*
378 	 * 10.19.2: "However, when the Initiator Task Tag is set to 0xffffffff,
379 	 * StatSN for the connection is not advanced after this PDU is sent."
380 	 */
381 	if (bhssr->bhssr_opcode == ISCSI_BHS_OPCODE_NOP_IN &&
382 	    bhssr->bhssr_initiator_task_tag == 0xffffffff)
383 		advance_statsn = false;
384 
385 	/*
386 	 * See the comment below - StatSN is not meaningful and must
387 	 * not be advanced.
388 	 */
389 	if (bhssr->bhssr_opcode == ISCSI_BHS_OPCODE_SCSI_DATA_IN &&
390 	    (bhssr->bhssr_flags & BHSDI_FLAGS_S) == 0)
391 		advance_statsn = false;
392 
393 	/*
394 	 * 10.7.3: "The fields StatSN, Status, and Residual Count
395 	 * only have meaningful content if the S bit is set to 1."
396 	 */
397 	if (bhssr->bhssr_opcode != ISCSI_BHS_OPCODE_SCSI_DATA_IN ||
398 	    (bhssr->bhssr_flags & BHSDI_FLAGS_S))
399 		bhssr->bhssr_statsn = htonl(cs->cs_statsn);
400 	bhssr->bhssr_expcmdsn = htonl(cs->cs_cmdsn);
401 	bhssr->bhssr_maxcmdsn = htonl(cs->cs_cmdsn - 1 +
402 	    imax(0, maxtags - cs->cs_outstanding_ctl_pdus));
403 
404 	if (advance_statsn)
405 		cs->cs_statsn++;
406 
407 	return (0);
408 }
409 
410 static void
411 cfiscsi_pdu_queue(struct icl_pdu *response)
412 {
413 	struct cfiscsi_session *cs;
414 
415 	cs = PDU_SESSION(response);
416 
417 	CFISCSI_SESSION_LOCK(cs);
418 	cfiscsi_pdu_prepare(response);
419 	icl_pdu_queue(response);
420 	CFISCSI_SESSION_UNLOCK(cs);
421 }
422 
423 static void
424 cfiscsi_pdu_handle_nop_out(struct icl_pdu *request)
425 {
426 	struct cfiscsi_session *cs;
427 	struct iscsi_bhs_nop_out *bhsno;
428 	struct iscsi_bhs_nop_in *bhsni;
429 	struct icl_pdu *response;
430 	void *data = NULL;
431 	size_t datasize;
432 	int error;
433 
434 	cs = PDU_SESSION(request);
435 	bhsno = (struct iscsi_bhs_nop_out *)request->ip_bhs;
436 
437 	if (bhsno->bhsno_initiator_task_tag == 0xffffffff) {
438 		/*
439 		 * Nothing to do, iscsi_pdu_update_statsn() already
440 		 * zeroed the timeout.
441 		 */
442 		icl_pdu_free(request);
443 		return;
444 	}
445 
446 	datasize = icl_pdu_data_segment_length(request);
447 	if (datasize > 0) {
448 		data = malloc(datasize, M_CFISCSI, M_NOWAIT | M_ZERO);
449 		if (data == NULL) {
450 			CFISCSI_SESSION_WARN(cs, "failed to allocate memory; "
451 			    "dropping connection");
452 			icl_pdu_free(request);
453 			cfiscsi_session_terminate(cs);
454 			return;
455 		}
456 		icl_pdu_get_data(request, 0, data, datasize);
457 	}
458 
459 	response = cfiscsi_pdu_new_response(request, M_NOWAIT);
460 	if (response == NULL) {
461 		CFISCSI_SESSION_WARN(cs, "failed to allocate memory; "
462 		    "droppping connection");
463 		free(data, M_CFISCSI);
464 		icl_pdu_free(request);
465 		cfiscsi_session_terminate(cs);
466 		return;
467 	}
468 	bhsni = (struct iscsi_bhs_nop_in *)response->ip_bhs;
469 	bhsni->bhsni_opcode = ISCSI_BHS_OPCODE_NOP_IN;
470 	bhsni->bhsni_flags = 0x80;
471 	bhsni->bhsni_initiator_task_tag = bhsno->bhsno_initiator_task_tag;
472 	bhsni->bhsni_target_transfer_tag = 0xffffffff;
473 	if (datasize > 0) {
474 		error = icl_pdu_append_data(response, data, datasize, M_NOWAIT);
475 		if (error != 0) {
476 			CFISCSI_SESSION_WARN(cs, "failed to allocate memory; "
477 			    "dropping connection");
478 			free(data, M_CFISCSI);
479 			icl_pdu_free(request);
480 			icl_pdu_free(response);
481 			cfiscsi_session_terminate(cs);
482 			return;
483 		}
484 		free(data, M_CFISCSI);
485 	}
486 
487 	icl_pdu_free(request);
488 	cfiscsi_pdu_queue(response);
489 }
490 
491 static void
492 cfiscsi_pdu_handle_scsi_command(struct icl_pdu *request)
493 {
494 	struct iscsi_bhs_scsi_command *bhssc;
495 	struct cfiscsi_session *cs;
496 	union ctl_io *io;
497 	int error;
498 
499 	cs = PDU_SESSION(request);
500 	bhssc = (struct iscsi_bhs_scsi_command *)request->ip_bhs;
501 	//CFISCSI_SESSION_DEBUG(cs, "initiator task tag 0x%x",
502 	//    bhssc->bhssc_initiator_task_tag);
503 
504 	if (request->ip_data_len > 0 && cs->cs_immediate_data == false) {
505 		CFISCSI_SESSION_WARN(cs, "unsolicited data with "
506 		    "ImmediateData=No; dropping connection");
507 		icl_pdu_free(request);
508 		cfiscsi_session_terminate(cs);
509 		return;
510 	}
511 	io = ctl_alloc_io(cs->cs_target->ct_port.ctl_pool_ref);
512 	ctl_zero_io(io);
513 	io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr = request;
514 	io->io_hdr.io_type = CTL_IO_SCSI;
515 	io->io_hdr.nexus.initid = cs->cs_ctl_initid;
516 	io->io_hdr.nexus.targ_port = cs->cs_target->ct_port.targ_port;
517 	io->io_hdr.nexus.targ_lun = ctl_decode_lun(be64toh(bhssc->bhssc_lun));
518 	io->scsiio.tag_num = bhssc->bhssc_initiator_task_tag;
519 	switch ((bhssc->bhssc_flags & BHSSC_FLAGS_ATTR)) {
520 	case BHSSC_FLAGS_ATTR_UNTAGGED:
521 		io->scsiio.tag_type = CTL_TAG_UNTAGGED;
522 		break;
523 	case BHSSC_FLAGS_ATTR_SIMPLE:
524 		io->scsiio.tag_type = CTL_TAG_SIMPLE;
525 		break;
526 	case BHSSC_FLAGS_ATTR_ORDERED:
527         	io->scsiio.tag_type = CTL_TAG_ORDERED;
528 		break;
529 	case BHSSC_FLAGS_ATTR_HOQ:
530         	io->scsiio.tag_type = CTL_TAG_HEAD_OF_QUEUE;
531 		break;
532 	case BHSSC_FLAGS_ATTR_ACA:
533 		io->scsiio.tag_type = CTL_TAG_ACA;
534 		break;
535 	default:
536 		io->scsiio.tag_type = CTL_TAG_UNTAGGED;
537 		CFISCSI_SESSION_WARN(cs, "unhandled tag type %d",
538 		    bhssc->bhssc_flags & BHSSC_FLAGS_ATTR);
539 		break;
540 	}
541 	io->scsiio.cdb_len = sizeof(bhssc->bhssc_cdb); /* Which is 16. */
542 	memcpy(io->scsiio.cdb, bhssc->bhssc_cdb, sizeof(bhssc->bhssc_cdb));
543 	refcount_acquire(&cs->cs_outstanding_ctl_pdus);
544 	error = ctl_queue(io);
545 	if (error != CTL_RETVAL_COMPLETE) {
546 		CFISCSI_SESSION_WARN(cs, "ctl_queue() failed; error %d; "
547 		    "dropping connection", error);
548 		ctl_free_io(io);
549 		refcount_release(&cs->cs_outstanding_ctl_pdus);
550 		icl_pdu_free(request);
551 		cfiscsi_session_terminate(cs);
552 	}
553 }
554 
555 static void
556 cfiscsi_pdu_handle_task_request(struct icl_pdu *request)
557 {
558 	struct iscsi_bhs_task_management_request *bhstmr;
559 	struct iscsi_bhs_task_management_response *bhstmr2;
560 	struct icl_pdu *response;
561 	struct cfiscsi_session *cs;
562 	union ctl_io *io;
563 	int error;
564 
565 	cs = PDU_SESSION(request);
566 	bhstmr = (struct iscsi_bhs_task_management_request *)request->ip_bhs;
567 	io = ctl_alloc_io(cs->cs_target->ct_port.ctl_pool_ref);
568 	ctl_zero_io(io);
569 	io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr = request;
570 	io->io_hdr.io_type = CTL_IO_TASK;
571 	io->io_hdr.nexus.initid = cs->cs_ctl_initid;
572 	io->io_hdr.nexus.targ_port = cs->cs_target->ct_port.targ_port;
573 	io->io_hdr.nexus.targ_lun = ctl_decode_lun(be64toh(bhstmr->bhstmr_lun));
574 	io->taskio.tag_type = CTL_TAG_SIMPLE; /* XXX */
575 
576 	switch (bhstmr->bhstmr_function & ~0x80) {
577 	case BHSTMR_FUNCTION_ABORT_TASK:
578 #if 0
579 		CFISCSI_SESSION_DEBUG(cs, "BHSTMR_FUNCTION_ABORT_TASK");
580 #endif
581 		io->taskio.task_action = CTL_TASK_ABORT_TASK;
582 		io->taskio.tag_num = bhstmr->bhstmr_referenced_task_tag;
583 		break;
584 	case BHSTMR_FUNCTION_ABORT_TASK_SET:
585 #if 0
586 		CFISCSI_SESSION_DEBUG(cs, "BHSTMR_FUNCTION_ABORT_TASK_SET");
587 #endif
588 		io->taskio.task_action = CTL_TASK_ABORT_TASK_SET;
589 		break;
590 	case BHSTMR_FUNCTION_CLEAR_TASK_SET:
591 #if 0
592 		CFISCSI_SESSION_DEBUG(cs, "BHSTMR_FUNCTION_CLEAR_TASK_SET");
593 #endif
594 		io->taskio.task_action = CTL_TASK_CLEAR_TASK_SET;
595 		break;
596 	case BHSTMR_FUNCTION_LOGICAL_UNIT_RESET:
597 #if 0
598 		CFISCSI_SESSION_DEBUG(cs, "BHSTMR_FUNCTION_LOGICAL_UNIT_RESET");
599 #endif
600 		io->taskio.task_action = CTL_TASK_LUN_RESET;
601 		break;
602 	case BHSTMR_FUNCTION_TARGET_WARM_RESET:
603 #if 0
604 		CFISCSI_SESSION_DEBUG(cs, "BHSTMR_FUNCTION_TARGET_WARM_RESET");
605 #endif
606 		io->taskio.task_action = CTL_TASK_TARGET_RESET;
607 		break;
608 	case BHSTMR_FUNCTION_TARGET_COLD_RESET:
609 #if 0
610 		CFISCSI_SESSION_DEBUG(cs, "BHSTMR_FUNCTION_TARGET_COLD_RESET");
611 #endif
612 		io->taskio.task_action = CTL_TASK_TARGET_RESET;
613 		break;
614 	case BHSTMR_FUNCTION_QUERY_TASK:
615 #if 0
616 		CFISCSI_SESSION_DEBUG(cs, "BHSTMR_FUNCTION_QUERY_TASK");
617 #endif
618 		io->taskio.task_action = CTL_TASK_QUERY_TASK;
619 		io->taskio.tag_num = bhstmr->bhstmr_referenced_task_tag;
620 		break;
621 	case BHSTMR_FUNCTION_QUERY_TASK_SET:
622 #if 0
623 		CFISCSI_SESSION_DEBUG(cs, "BHSTMR_FUNCTION_QUERY_TASK_SET");
624 #endif
625 		io->taskio.task_action = CTL_TASK_QUERY_TASK_SET;
626 		break;
627 	case BHSTMR_FUNCTION_I_T_NEXUS_RESET:
628 #if 0
629 		CFISCSI_SESSION_DEBUG(cs, "BHSTMR_FUNCTION_I_T_NEXUS_RESET");
630 #endif
631 		io->taskio.task_action = CTL_TASK_I_T_NEXUS_RESET;
632 		break;
633 	case BHSTMR_FUNCTION_QUERY_ASYNC_EVENT:
634 #if 0
635 		CFISCSI_SESSION_DEBUG(cs, "BHSTMR_FUNCTION_QUERY_ASYNC_EVENT");
636 #endif
637 		io->taskio.task_action = CTL_TASK_QUERY_ASYNC_EVENT;
638 		break;
639 	default:
640 		CFISCSI_SESSION_DEBUG(cs, "unsupported function 0x%x",
641 		    bhstmr->bhstmr_function & ~0x80);
642 		ctl_free_io(io);
643 
644 		response = cfiscsi_pdu_new_response(request, M_NOWAIT);
645 		if (response == NULL) {
646 			CFISCSI_SESSION_WARN(cs, "failed to allocate memory; "
647 			    "dropping connection");
648 			icl_pdu_free(request);
649 			cfiscsi_session_terminate(cs);
650 			return;
651 		}
652 		bhstmr2 = (struct iscsi_bhs_task_management_response *)
653 		    response->ip_bhs;
654 		bhstmr2->bhstmr_opcode = ISCSI_BHS_OPCODE_TASK_RESPONSE;
655 		bhstmr2->bhstmr_flags = 0x80;
656 		bhstmr2->bhstmr_response =
657 		    BHSTMR_RESPONSE_FUNCTION_NOT_SUPPORTED;
658 		bhstmr2->bhstmr_initiator_task_tag =
659 		    bhstmr->bhstmr_initiator_task_tag;
660 		icl_pdu_free(request);
661 		cfiscsi_pdu_queue(response);
662 		return;
663 	}
664 
665 	refcount_acquire(&cs->cs_outstanding_ctl_pdus);
666 	error = ctl_queue(io);
667 	if (error != CTL_RETVAL_COMPLETE) {
668 		CFISCSI_SESSION_WARN(cs, "ctl_queue() failed; error %d; "
669 		    "dropping connection", error);
670 		ctl_free_io(io);
671 		refcount_release(&cs->cs_outstanding_ctl_pdus);
672 		icl_pdu_free(request);
673 		cfiscsi_session_terminate(cs);
674 	}
675 }
676 
677 static bool
678 cfiscsi_handle_data_segment(struct icl_pdu *request, struct cfiscsi_data_wait *cdw)
679 {
680 	struct iscsi_bhs_data_out *bhsdo;
681 	struct cfiscsi_session *cs;
682 	struct ctl_sg_entry ctl_sg_entry, *ctl_sglist;
683 	size_t copy_len, len, off, buffer_offset;
684 	int ctl_sg_count;
685 	union ctl_io *io;
686 
687 	cs = PDU_SESSION(request);
688 
689 	KASSERT((request->ip_bhs->bhs_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
690 	    ISCSI_BHS_OPCODE_SCSI_DATA_OUT ||
691 	    (request->ip_bhs->bhs_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
692 	    ISCSI_BHS_OPCODE_SCSI_COMMAND,
693 	    ("bad opcode 0x%x", request->ip_bhs->bhs_opcode));
694 
695 	/*
696 	 * We're only using fields common for Data-Out and SCSI Command PDUs.
697 	 */
698 	bhsdo = (struct iscsi_bhs_data_out *)request->ip_bhs;
699 
700 	io = cdw->cdw_ctl_io;
701 	KASSERT((io->io_hdr.flags & CTL_FLAG_DATA_MASK) != CTL_FLAG_DATA_IN,
702 	    ("CTL_FLAG_DATA_IN"));
703 
704 #if 0
705 	CFISCSI_SESSION_DEBUG(cs, "received %zd bytes out of %d",
706 	    request->ip_data_len, io->scsiio.kern_total_len);
707 #endif
708 
709 	if (io->scsiio.kern_sg_entries > 0) {
710 		ctl_sglist = (struct ctl_sg_entry *)io->scsiio.kern_data_ptr;
711 		ctl_sg_count = io->scsiio.kern_sg_entries;
712 	} else {
713 		ctl_sglist = &ctl_sg_entry;
714 		ctl_sglist->addr = io->scsiio.kern_data_ptr;
715 		ctl_sglist->len = io->scsiio.kern_data_len;
716 		ctl_sg_count = 1;
717 	}
718 
719 	if ((request->ip_bhs->bhs_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
720 	    ISCSI_BHS_OPCODE_SCSI_DATA_OUT)
721 		buffer_offset = ntohl(bhsdo->bhsdo_buffer_offset);
722 	else
723 		buffer_offset = 0;
724 	len = icl_pdu_data_segment_length(request);
725 
726 	/*
727 	 * Make sure the offset, as sent by the initiator, matches the offset
728 	 * we're supposed to be at in the scatter-gather list.
729 	 */
730 	if (buffer_offset >
731 	    io->scsiio.kern_rel_offset + io->scsiio.ext_data_filled ||
732 	    buffer_offset + len <=
733 	    io->scsiio.kern_rel_offset + io->scsiio.ext_data_filled) {
734 		CFISCSI_SESSION_WARN(cs, "received bad buffer offset %zd, "
735 		    "expected %zd; dropping connection", buffer_offset,
736 		    (size_t)io->scsiio.kern_rel_offset +
737 		    (size_t)io->scsiio.ext_data_filled);
738 		ctl_set_data_phase_error(&io->scsiio);
739 		cfiscsi_session_terminate(cs);
740 		return (true);
741 	}
742 
743 	/*
744 	 * This is the offset within the PDU data segment, as opposed
745 	 * to buffer_offset, which is the offset within the task (SCSI
746 	 * command).
747 	 */
748 	off = io->scsiio.kern_rel_offset + io->scsiio.ext_data_filled -
749 	    buffer_offset;
750 
751 	/*
752 	 * Iterate over the scatter/gather segments, filling them with data
753 	 * from the PDU data segment.  Note that this can get called multiple
754 	 * times for one SCSI command; the cdw structure holds state for the
755 	 * scatter/gather list.
756 	 */
757 	for (;;) {
758 		KASSERT(cdw->cdw_sg_index < ctl_sg_count,
759 		    ("cdw->cdw_sg_index >= ctl_sg_count"));
760 		if (cdw->cdw_sg_len == 0) {
761 			cdw->cdw_sg_addr = ctl_sglist[cdw->cdw_sg_index].addr;
762 			cdw->cdw_sg_len = ctl_sglist[cdw->cdw_sg_index].len;
763 		}
764 		KASSERT(off <= len, ("len > off"));
765 		copy_len = len - off;
766 		if (copy_len > cdw->cdw_sg_len)
767 			copy_len = cdw->cdw_sg_len;
768 
769 		icl_pdu_get_data(request, off, cdw->cdw_sg_addr, copy_len);
770 		cdw->cdw_sg_addr += copy_len;
771 		cdw->cdw_sg_len -= copy_len;
772 		off += copy_len;
773 		io->scsiio.ext_data_filled += copy_len;
774 		io->scsiio.kern_data_resid -= copy_len;
775 
776 		if (cdw->cdw_sg_len == 0) {
777 			/*
778 			 * End of current segment.
779 			 */
780 			if (cdw->cdw_sg_index == ctl_sg_count - 1) {
781 				/*
782 				 * Last segment in scatter/gather list.
783 				 */
784 				break;
785 			}
786 			cdw->cdw_sg_index++;
787 		}
788 
789 		if (off == len) {
790 			/*
791 			 * End of PDU payload.
792 			 */
793 			break;
794 		}
795 	}
796 
797 	if (len > off) {
798 		/*
799 		 * In case of unsolicited data, it's possible that the buffer
800 		 * provided by CTL is smaller than negotiated FirstBurstLength.
801 		 * Just ignore the superfluous data; will ask for them with R2T
802 		 * on next call to cfiscsi_datamove().
803 		 *
804 		 * This obviously can only happen with SCSI Command PDU.
805 		 */
806 		if ((request->ip_bhs->bhs_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
807 		    ISCSI_BHS_OPCODE_SCSI_COMMAND)
808 			return (true);
809 
810 		CFISCSI_SESSION_WARN(cs, "received too much data: got %zd bytes, "
811 		    "expected %zd; dropping connection",
812 		    icl_pdu_data_segment_length(request), off);
813 		ctl_set_data_phase_error(&io->scsiio);
814 		cfiscsi_session_terminate(cs);
815 		return (true);
816 	}
817 
818 	if (io->scsiio.ext_data_filled == cdw->cdw_r2t_end &&
819 	    (bhsdo->bhsdo_flags & BHSDO_FLAGS_F) == 0) {
820 		CFISCSI_SESSION_WARN(cs, "got the final packet without "
821 		    "the F flag; flags = 0x%x; dropping connection",
822 		    bhsdo->bhsdo_flags);
823 		ctl_set_data_phase_error(&io->scsiio);
824 		cfiscsi_session_terminate(cs);
825 		return (true);
826 	}
827 
828 	if (io->scsiio.ext_data_filled != cdw->cdw_r2t_end &&
829 	    (bhsdo->bhsdo_flags & BHSDO_FLAGS_F) != 0) {
830 		if ((request->ip_bhs->bhs_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
831 		    ISCSI_BHS_OPCODE_SCSI_DATA_OUT) {
832 			CFISCSI_SESSION_WARN(cs, "got the final packet, but the "
833 			    "transmitted size was %zd bytes instead of %d; "
834 			    "dropping connection",
835 			    (size_t)io->scsiio.ext_data_filled,
836 			    cdw->cdw_r2t_end);
837 			ctl_set_data_phase_error(&io->scsiio);
838 			cfiscsi_session_terminate(cs);
839 			return (true);
840 		} else {
841 			/*
842 			 * For SCSI Command PDU, this just means we need to
843 			 * solicit more data by sending R2T.
844 			 */
845 			return (false);
846 		}
847 	}
848 
849 	if (io->scsiio.ext_data_filled == cdw->cdw_r2t_end) {
850 #if 0
851 		CFISCSI_SESSION_DEBUG(cs, "no longer expecting Data-Out with target "
852 		    "transfer tag 0x%x", cdw->cdw_target_transfer_tag);
853 #endif
854 
855 		return (true);
856 	}
857 
858 	return (false);
859 }
860 
861 static void
862 cfiscsi_pdu_handle_data_out(struct icl_pdu *request)
863 {
864 	struct iscsi_bhs_data_out *bhsdo;
865 	struct cfiscsi_session *cs;
866 	struct cfiscsi_data_wait *cdw = NULL;
867 	union ctl_io *io;
868 	bool done;
869 
870 	cs = PDU_SESSION(request);
871 	bhsdo = (struct iscsi_bhs_data_out *)request->ip_bhs;
872 
873 	CFISCSI_SESSION_LOCK(cs);
874 	TAILQ_FOREACH(cdw, &cs->cs_waiting_for_data_out, cdw_next) {
875 #if 0
876 		CFISCSI_SESSION_DEBUG(cs, "have ttt 0x%x, itt 0x%x; looking for "
877 		    "ttt 0x%x, itt 0x%x",
878 		    bhsdo->bhsdo_target_transfer_tag,
879 		    bhsdo->bhsdo_initiator_task_tag,
880 		    cdw->cdw_target_transfer_tag, cdw->cdw_initiator_task_tag));
881 #endif
882 		if (bhsdo->bhsdo_target_transfer_tag ==
883 		    cdw->cdw_target_transfer_tag)
884 			break;
885 	}
886 	CFISCSI_SESSION_UNLOCK(cs);
887 	if (cdw == NULL) {
888 		CFISCSI_SESSION_WARN(cs, "data transfer tag 0x%x, initiator task tag "
889 		    "0x%x, not found; dropping connection",
890 		    bhsdo->bhsdo_target_transfer_tag, bhsdo->bhsdo_initiator_task_tag);
891 		icl_pdu_free(request);
892 		cfiscsi_session_terminate(cs);
893 		return;
894 	}
895 
896 	if (cdw->cdw_datasn != ntohl(bhsdo->bhsdo_datasn)) {
897 		CFISCSI_SESSION_WARN(cs, "received Data-Out PDU with "
898 		    "DataSN %u, while expected %u; dropping connection",
899 		    ntohl(bhsdo->bhsdo_datasn), cdw->cdw_datasn);
900 		icl_pdu_free(request);
901 		cfiscsi_session_terminate(cs);
902 		return;
903 	}
904 	cdw->cdw_datasn++;
905 
906 	io = cdw->cdw_ctl_io;
907 	KASSERT((io->io_hdr.flags & CTL_FLAG_DATA_MASK) != CTL_FLAG_DATA_IN,
908 	    ("CTL_FLAG_DATA_IN"));
909 
910 	done = cfiscsi_handle_data_segment(request, cdw);
911 	if (done) {
912 		CFISCSI_SESSION_LOCK(cs);
913 		TAILQ_REMOVE(&cs->cs_waiting_for_data_out, cdw, cdw_next);
914 		CFISCSI_SESSION_UNLOCK(cs);
915 		done = (io->scsiio.ext_data_filled != cdw->cdw_r2t_end ||
916 		    io->scsiio.ext_data_filled == io->scsiio.kern_data_len);
917 		cfiscsi_data_wait_free(cs, cdw);
918 		io->io_hdr.flags &= ~CTL_FLAG_DMA_INPROG;
919 		if (done)
920 			io->scsiio.be_move_done(io);
921 		else
922 			cfiscsi_datamove_out(io);
923 	}
924 
925 	icl_pdu_free(request);
926 }
927 
928 static void
929 cfiscsi_pdu_handle_logout_request(struct icl_pdu *request)
930 {
931 	struct iscsi_bhs_logout_request *bhslr;
932 	struct iscsi_bhs_logout_response *bhslr2;
933 	struct icl_pdu *response;
934 	struct cfiscsi_session *cs;
935 
936 	cs = PDU_SESSION(request);
937 	bhslr = (struct iscsi_bhs_logout_request *)request->ip_bhs;
938 	switch (bhslr->bhslr_reason & 0x7f) {
939 	case BHSLR_REASON_CLOSE_SESSION:
940 	case BHSLR_REASON_CLOSE_CONNECTION:
941 		response = cfiscsi_pdu_new_response(request, M_NOWAIT);
942 		if (response == NULL) {
943 			CFISCSI_SESSION_DEBUG(cs, "failed to allocate memory");
944 			icl_pdu_free(request);
945 			cfiscsi_session_terminate(cs);
946 			return;
947 		}
948 		bhslr2 = (struct iscsi_bhs_logout_response *)response->ip_bhs;
949 		bhslr2->bhslr_opcode = ISCSI_BHS_OPCODE_LOGOUT_RESPONSE;
950 		bhslr2->bhslr_flags = 0x80;
951 		bhslr2->bhslr_response = BHSLR_RESPONSE_CLOSED_SUCCESSFULLY;
952 		bhslr2->bhslr_initiator_task_tag =
953 		    bhslr->bhslr_initiator_task_tag;
954 		icl_pdu_free(request);
955 		cfiscsi_pdu_queue(response);
956 		cfiscsi_session_terminate(cs);
957 		break;
958 	case BHSLR_REASON_REMOVE_FOR_RECOVERY:
959 		response = cfiscsi_pdu_new_response(request, M_NOWAIT);
960 		if (response == NULL) {
961 			CFISCSI_SESSION_WARN(cs,
962 			    "failed to allocate memory; dropping connection");
963 			icl_pdu_free(request);
964 			cfiscsi_session_terminate(cs);
965 			return;
966 		}
967 		bhslr2 = (struct iscsi_bhs_logout_response *)response->ip_bhs;
968 		bhslr2->bhslr_opcode = ISCSI_BHS_OPCODE_LOGOUT_RESPONSE;
969 		bhslr2->bhslr_flags = 0x80;
970 		bhslr2->bhslr_response = BHSLR_RESPONSE_RECOVERY_NOT_SUPPORTED;
971 		bhslr2->bhslr_initiator_task_tag =
972 		    bhslr->bhslr_initiator_task_tag;
973 		icl_pdu_free(request);
974 		cfiscsi_pdu_queue(response);
975 		break;
976 	default:
977 		CFISCSI_SESSION_WARN(cs, "invalid reason 0%x; dropping connection",
978 		    bhslr->bhslr_reason);
979 		icl_pdu_free(request);
980 		cfiscsi_session_terminate(cs);
981 		break;
982 	}
983 }
984 
985 static void
986 cfiscsi_callout(void *context)
987 {
988 	struct icl_pdu *cp;
989 	struct iscsi_bhs_nop_in *bhsni;
990 	struct cfiscsi_session *cs;
991 
992 	cs = context;
993 
994 	if (cs->cs_terminating)
995 		return;
996 
997 	callout_schedule(&cs->cs_callout, 1 * hz);
998 
999 	atomic_add_int(&cs->cs_timeout, 1);
1000 
1001 #ifdef ICL_KERNEL_PROXY
1002 	if (cs->cs_waiting_for_ctld || cs->cs_login_phase) {
1003 		if (login_timeout > 0 && cs->cs_timeout > login_timeout) {
1004 			CFISCSI_SESSION_WARN(cs, "login timed out after "
1005 			    "%d seconds; dropping connection", cs->cs_timeout);
1006 			cfiscsi_session_terminate(cs);
1007 		}
1008 		return;
1009 	}
1010 #endif
1011 
1012 	if (ping_timeout <= 0) {
1013 		/*
1014 		 * Pings are disabled.  Don't send NOP-In in this case;
1015 		 * user might have disabled pings to work around problems
1016 		 * with certain initiators that can't properly handle
1017 		 * NOP-In, such as iPXE.  Reset the timeout, to avoid
1018 		 * triggering reconnection, should the user decide to
1019 		 * reenable them.
1020 		 */
1021 		cs->cs_timeout = 0;
1022 		return;
1023 	}
1024 
1025 	if (cs->cs_timeout >= ping_timeout) {
1026 		CFISCSI_SESSION_WARN(cs, "no ping reply (NOP-Out) after %d seconds; "
1027 		    "dropping connection",  ping_timeout);
1028 		cfiscsi_session_terminate(cs);
1029 		return;
1030 	}
1031 
1032 	/*
1033 	 * If the ping was reset less than one second ago - which means
1034 	 * that we've received some PDU during the last second - assume
1035 	 * the traffic flows correctly and don't bother sending a NOP-Out.
1036 	 *
1037 	 * (It's 2 - one for one second, and one for incrementing is_timeout
1038 	 * earlier in this routine.)
1039 	 */
1040 	if (cs->cs_timeout < 2)
1041 		return;
1042 
1043 	cp = icl_pdu_new(cs->cs_conn, M_NOWAIT);
1044 	if (cp == NULL) {
1045 		CFISCSI_SESSION_WARN(cs, "failed to allocate memory");
1046 		return;
1047 	}
1048 	bhsni = (struct iscsi_bhs_nop_in *)cp->ip_bhs;
1049 	bhsni->bhsni_opcode = ISCSI_BHS_OPCODE_NOP_IN;
1050 	bhsni->bhsni_flags = 0x80;
1051 	bhsni->bhsni_initiator_task_tag = 0xffffffff;
1052 
1053 	cfiscsi_pdu_queue(cp);
1054 }
1055 
1056 static struct cfiscsi_data_wait *
1057 cfiscsi_data_wait_new(struct cfiscsi_session *cs, union ctl_io *io,
1058     uint32_t initiator_task_tag, uint32_t *target_transfer_tagp)
1059 {
1060 	struct cfiscsi_data_wait *cdw;
1061 	int error;
1062 
1063 	cdw = uma_zalloc(cfiscsi_data_wait_zone, M_NOWAIT | M_ZERO);
1064 	if (cdw == NULL) {
1065 		CFISCSI_SESSION_WARN(cs,
1066 		    "failed to allocate %zd bytes", sizeof(*cdw));
1067 		return (NULL);
1068 	}
1069 
1070 	error = icl_conn_transfer_setup(cs->cs_conn, io, target_transfer_tagp,
1071 	    &cdw->cdw_icl_prv);
1072 	if (error != 0) {
1073 		CFISCSI_SESSION_WARN(cs,
1074 		    "icl_conn_transfer_setup() failed with error %d", error);
1075 		uma_zfree(cfiscsi_data_wait_zone, cdw);
1076 		return (NULL);
1077 	}
1078 
1079 	cdw->cdw_ctl_io = io;
1080 	cdw->cdw_target_transfer_tag = *target_transfer_tagp;
1081 	cdw->cdw_initiator_task_tag = initiator_task_tag;
1082 
1083 	return (cdw);
1084 }
1085 
1086 static void
1087 cfiscsi_data_wait_free(struct cfiscsi_session *cs,
1088     struct cfiscsi_data_wait *cdw)
1089 {
1090 
1091 	icl_conn_transfer_done(cs->cs_conn, cdw->cdw_icl_prv);
1092 	uma_zfree(cfiscsi_data_wait_zone, cdw);
1093 }
1094 
1095 static void
1096 cfiscsi_session_terminate_tasks(struct cfiscsi_session *cs)
1097 {
1098 	struct cfiscsi_data_wait *cdw;
1099 	union ctl_io *io;
1100 	int error, last, wait;
1101 
1102 	if (cs->cs_target == NULL)
1103 		return;		/* No target yet, so nothing to do. */
1104 	io = ctl_alloc_io(cs->cs_target->ct_port.ctl_pool_ref);
1105 	ctl_zero_io(io);
1106 	io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr = cs;
1107 	io->io_hdr.io_type = CTL_IO_TASK;
1108 	io->io_hdr.nexus.initid = cs->cs_ctl_initid;
1109 	io->io_hdr.nexus.targ_port = cs->cs_target->ct_port.targ_port;
1110 	io->io_hdr.nexus.targ_lun = 0;
1111 	io->taskio.tag_type = CTL_TAG_SIMPLE; /* XXX */
1112 	io->taskio.task_action = CTL_TASK_I_T_NEXUS_RESET;
1113 	wait = cs->cs_outstanding_ctl_pdus;
1114 	refcount_acquire(&cs->cs_outstanding_ctl_pdus);
1115 	error = ctl_queue(io);
1116 	if (error != CTL_RETVAL_COMPLETE) {
1117 		CFISCSI_SESSION_WARN(cs, "ctl_queue() failed; error %d", error);
1118 		refcount_release(&cs->cs_outstanding_ctl_pdus);
1119 		ctl_free_io(io);
1120 	}
1121 
1122 	CFISCSI_SESSION_LOCK(cs);
1123 	while ((cdw = TAILQ_FIRST(&cs->cs_waiting_for_data_out)) != NULL) {
1124 		TAILQ_REMOVE(&cs->cs_waiting_for_data_out, cdw, cdw_next);
1125 		CFISCSI_SESSION_UNLOCK(cs);
1126 		/*
1127 		 * Set nonzero port status; this prevents backends from
1128 		 * assuming that the data transfer actually succeeded
1129 		 * and writing uninitialized data to disk.
1130 		 */
1131 		cdw->cdw_ctl_io->io_hdr.flags &= ~CTL_FLAG_DMA_INPROG;
1132 		cdw->cdw_ctl_io->scsiio.io_hdr.port_status = 42;
1133 		cdw->cdw_ctl_io->scsiio.be_move_done(cdw->cdw_ctl_io);
1134 		cfiscsi_data_wait_free(cs, cdw);
1135 		CFISCSI_SESSION_LOCK(cs);
1136 	}
1137 	CFISCSI_SESSION_UNLOCK(cs);
1138 
1139 	/*
1140 	 * Wait for CTL to terminate all the tasks.
1141 	 */
1142 	if (wait > 0)
1143 		CFISCSI_SESSION_WARN(cs,
1144 		    "waiting for CTL to terminate %d tasks", wait);
1145 	for (;;) {
1146 		refcount_acquire(&cs->cs_outstanding_ctl_pdus);
1147 		last = refcount_release(&cs->cs_outstanding_ctl_pdus);
1148 		if (last != 0)
1149 			break;
1150 		tsleep(__DEVOLATILE(void *, &cs->cs_outstanding_ctl_pdus),
1151 		    0, "cfiscsi_terminate", hz / 100);
1152 	}
1153 	if (wait > 0)
1154 		CFISCSI_SESSION_WARN(cs, "tasks terminated");
1155 }
1156 
1157 static void
1158 cfiscsi_maintenance_thread(void *arg)
1159 {
1160 	struct cfiscsi_session *cs;
1161 
1162 	cs = arg;
1163 
1164 	for (;;) {
1165 		CFISCSI_SESSION_LOCK(cs);
1166 		if (cs->cs_terminating == false)
1167 			cv_wait(&cs->cs_maintenance_cv, &cs->cs_lock);
1168 		CFISCSI_SESSION_UNLOCK(cs);
1169 
1170 		if (cs->cs_terminating) {
1171 
1172 			/*
1173 			 * We used to wait up to 30 seconds to deliver queued
1174 			 * PDUs to the initiator.  We also tried hard to deliver
1175 			 * SCSI Responses for the aborted PDUs.  We don't do
1176 			 * that anymore.  We might need to revisit that.
1177 			 */
1178 			callout_drain(&cs->cs_callout);
1179 			icl_conn_close(cs->cs_conn);
1180 
1181 			/*
1182 			 * At this point ICL receive thread is no longer
1183 			 * running; no new tasks can be queued.
1184 			 */
1185 			cfiscsi_session_terminate_tasks(cs);
1186 			cfiscsi_session_delete(cs);
1187 			kthread_exit();
1188 			return;
1189 		}
1190 		CFISCSI_SESSION_DEBUG(cs, "nothing to do");
1191 	}
1192 }
1193 
1194 static void
1195 cfiscsi_session_terminate(struct cfiscsi_session *cs)
1196 {
1197 
1198 	if (cs->cs_terminating)
1199 		return;
1200 	cs->cs_terminating = true;
1201 	cv_signal(&cs->cs_maintenance_cv);
1202 #ifdef ICL_KERNEL_PROXY
1203 	cv_signal(&cs->cs_login_cv);
1204 #endif
1205 }
1206 
1207 static int
1208 cfiscsi_session_register_initiator(struct cfiscsi_session *cs)
1209 {
1210 	struct cfiscsi_target *ct;
1211 	char *name;
1212 	int i;
1213 
1214 	KASSERT(cs->cs_ctl_initid == -1, ("already registered"));
1215 
1216 	ct = cs->cs_target;
1217 	name = strdup(cs->cs_initiator_id, M_CTL);
1218 	i = ctl_add_initiator(&ct->ct_port, -1, 0, name);
1219 	if (i < 0) {
1220 		CFISCSI_SESSION_WARN(cs, "ctl_add_initiator failed with error %d",
1221 		    i);
1222 		cs->cs_ctl_initid = -1;
1223 		return (1);
1224 	}
1225 	cs->cs_ctl_initid = i;
1226 #if 0
1227 	CFISCSI_SESSION_DEBUG(cs, "added initiator id %d", i);
1228 #endif
1229 
1230 	return (0);
1231 }
1232 
1233 static void
1234 cfiscsi_session_unregister_initiator(struct cfiscsi_session *cs)
1235 {
1236 	int error;
1237 
1238 	if (cs->cs_ctl_initid == -1)
1239 		return;
1240 
1241 	error = ctl_remove_initiator(&cs->cs_target->ct_port, cs->cs_ctl_initid);
1242 	if (error != 0) {
1243 		CFISCSI_SESSION_WARN(cs, "ctl_remove_initiator failed with error %d",
1244 		    error);
1245 	}
1246 	cs->cs_ctl_initid = -1;
1247 }
1248 
1249 static struct cfiscsi_session *
1250 cfiscsi_session_new(struct cfiscsi_softc *softc, const char *offload)
1251 {
1252 	struct cfiscsi_session *cs;
1253 	int error;
1254 
1255 	cs = malloc(sizeof(*cs), M_CFISCSI, M_NOWAIT | M_ZERO);
1256 	if (cs == NULL) {
1257 		CFISCSI_WARN("malloc failed");
1258 		return (NULL);
1259 	}
1260 	cs->cs_ctl_initid = -1;
1261 
1262 	refcount_init(&cs->cs_outstanding_ctl_pdus, 0);
1263 	TAILQ_INIT(&cs->cs_waiting_for_data_out);
1264 	mtx_init(&cs->cs_lock, "cfiscsi_lock", NULL, MTX_DEF);
1265 	cv_init(&cs->cs_maintenance_cv, "cfiscsi_mt");
1266 #ifdef ICL_KERNEL_PROXY
1267 	cv_init(&cs->cs_login_cv, "cfiscsi_login");
1268 #endif
1269 
1270 	cs->cs_conn = icl_new_conn(offload, false, "cfiscsi", &cs->cs_lock);
1271 	if (cs->cs_conn == NULL) {
1272 		free(cs, M_CFISCSI);
1273 		return (NULL);
1274 	}
1275 	cs->cs_conn->ic_receive = cfiscsi_receive_callback;
1276 	cs->cs_conn->ic_error = cfiscsi_error_callback;
1277 	cs->cs_conn->ic_prv0 = cs;
1278 
1279 	error = kthread_add(cfiscsi_maintenance_thread, cs, NULL, NULL, 0, 0, "cfiscsimt");
1280 	if (error != 0) {
1281 		CFISCSI_SESSION_WARN(cs, "kthread_add(9) failed with error %d", error);
1282 		free(cs, M_CFISCSI);
1283 		return (NULL);
1284 	}
1285 
1286 	mtx_lock(&softc->lock);
1287 	cs->cs_id = ++softc->last_session_id;
1288 	TAILQ_INSERT_TAIL(&softc->sessions, cs, cs_next);
1289 	mtx_unlock(&softc->lock);
1290 
1291 	/*
1292 	 * Start pinging the initiator.
1293 	 */
1294 	callout_init(&cs->cs_callout, 1);
1295 	callout_reset(&cs->cs_callout, 1 * hz, cfiscsi_callout, cs);
1296 
1297 	return (cs);
1298 }
1299 
1300 static void
1301 cfiscsi_session_delete(struct cfiscsi_session *cs)
1302 {
1303 	struct cfiscsi_softc *softc;
1304 
1305 	softc = &cfiscsi_softc;
1306 
1307 	KASSERT(cs->cs_outstanding_ctl_pdus == 0,
1308 	    ("destroying session with outstanding CTL pdus"));
1309 	KASSERT(TAILQ_EMPTY(&cs->cs_waiting_for_data_out),
1310 	    ("destroying session with non-empty queue"));
1311 
1312 	cfiscsi_session_unregister_initiator(cs);
1313 	if (cs->cs_target != NULL)
1314 		cfiscsi_target_release(cs->cs_target);
1315 	icl_conn_close(cs->cs_conn);
1316 	icl_conn_free(cs->cs_conn);
1317 
1318 	mtx_lock(&softc->lock);
1319 	TAILQ_REMOVE(&softc->sessions, cs, cs_next);
1320 	cv_signal(&softc->sessions_cv);
1321 	mtx_unlock(&softc->lock);
1322 
1323 	free(cs, M_CFISCSI);
1324 }
1325 
1326 static int
1327 cfiscsi_init(void)
1328 {
1329 	struct cfiscsi_softc *softc;
1330 
1331 	softc = &cfiscsi_softc;
1332 	bzero(softc, sizeof(*softc));
1333 	mtx_init(&softc->lock, "cfiscsi", NULL, MTX_DEF);
1334 
1335 	cv_init(&softc->sessions_cv, "cfiscsi_sessions");
1336 #ifdef ICL_KERNEL_PROXY
1337 	cv_init(&softc->accept_cv, "cfiscsi_accept");
1338 #endif
1339 	TAILQ_INIT(&softc->sessions);
1340 	TAILQ_INIT(&softc->targets);
1341 
1342 	cfiscsi_data_wait_zone = uma_zcreate("cfiscsi_data_wait",
1343 	    sizeof(struct cfiscsi_data_wait), NULL, NULL, NULL, NULL,
1344 	    UMA_ALIGN_PTR, 0);
1345 
1346 	return (0);
1347 }
1348 
1349 static int
1350 cfiscsi_shutdown(void)
1351 {
1352 	struct cfiscsi_softc *softc = &cfiscsi_softc;
1353 
1354 	if (!TAILQ_EMPTY(&softc->sessions) || !TAILQ_EMPTY(&softc->targets))
1355 		return (EBUSY);
1356 
1357 	uma_zdestroy(cfiscsi_data_wait_zone);
1358 #ifdef ICL_KERNEL_PROXY
1359 	cv_destroy(&softc->accept_cv);
1360 #endif
1361 	cv_destroy(&softc->sessions_cv);
1362 	mtx_destroy(&softc->lock);
1363 	return (0);
1364 }
1365 
1366 #ifdef ICL_KERNEL_PROXY
1367 static void
1368 cfiscsi_accept(struct socket *so, struct sockaddr *sa, int portal_id)
1369 {
1370 	struct cfiscsi_session *cs;
1371 
1372 	cs = cfiscsi_session_new(&cfiscsi_softc, NULL);
1373 	if (cs == NULL) {
1374 		CFISCSI_WARN("failed to create session");
1375 		return;
1376 	}
1377 
1378 	icl_conn_handoff_sock(cs->cs_conn, so);
1379 	cs->cs_initiator_sa = sa;
1380 	cs->cs_portal_id = portal_id;
1381 	cs->cs_waiting_for_ctld = true;
1382 	cv_signal(&cfiscsi_softc.accept_cv);
1383 }
1384 #endif
1385 
1386 static void
1387 cfiscsi_online(void *arg)
1388 {
1389 	struct cfiscsi_softc *softc;
1390 	struct cfiscsi_target *ct;
1391 	int online;
1392 
1393 	ct = (struct cfiscsi_target *)arg;
1394 	softc = ct->ct_softc;
1395 
1396 	mtx_lock(&softc->lock);
1397 	if (ct->ct_online) {
1398 		mtx_unlock(&softc->lock);
1399 		return;
1400 	}
1401 	ct->ct_online = 1;
1402 	online = softc->online++;
1403 	mtx_unlock(&softc->lock);
1404 	if (online > 0)
1405 		return;
1406 
1407 #ifdef ICL_KERNEL_PROXY
1408 	if (softc->listener != NULL)
1409 		icl_listen_free(softc->listener);
1410 	softc->listener = icl_listen_new(cfiscsi_accept);
1411 #endif
1412 }
1413 
1414 static void
1415 cfiscsi_offline(void *arg)
1416 {
1417 	struct cfiscsi_softc *softc;
1418 	struct cfiscsi_target *ct;
1419 	struct cfiscsi_session *cs;
1420 	int online;
1421 
1422 	ct = (struct cfiscsi_target *)arg;
1423 	softc = ct->ct_softc;
1424 
1425 	mtx_lock(&softc->lock);
1426 	if (!ct->ct_online) {
1427 		mtx_unlock(&softc->lock);
1428 		return;
1429 	}
1430 	ct->ct_online = 0;
1431 	online = --softc->online;
1432 
1433 	TAILQ_FOREACH(cs, &softc->sessions, cs_next) {
1434 		if (cs->cs_target == ct)
1435 			cfiscsi_session_terminate(cs);
1436 	}
1437 	do {
1438 		TAILQ_FOREACH(cs, &softc->sessions, cs_next) {
1439 			if (cs->cs_target == ct)
1440 				break;
1441 		}
1442 		if (cs != NULL)
1443 			cv_wait(&softc->sessions_cv, &softc->lock);
1444 	} while (cs != NULL && ct->ct_online == 0);
1445 	mtx_unlock(&softc->lock);
1446 	if (online > 0)
1447 		return;
1448 
1449 #ifdef ICL_KERNEL_PROXY
1450 	icl_listen_free(softc->listener);
1451 	softc->listener = NULL;
1452 #endif
1453 }
1454 
1455 static int
1456 cfiscsi_info(void *arg, struct sbuf *sb)
1457 {
1458 	struct cfiscsi_target *ct = (struct cfiscsi_target *)arg;
1459 	int retval;
1460 
1461 	retval = sbuf_printf(sb, "\t<cfiscsi_state>%d</cfiscsi_state>\n",
1462 	    ct->ct_state);
1463 	return (retval);
1464 }
1465 
1466 static void
1467 cfiscsi_ioctl_handoff(struct ctl_iscsi *ci)
1468 {
1469 	struct cfiscsi_softc *softc;
1470 	struct cfiscsi_session *cs, *cs2;
1471 	struct cfiscsi_target *ct;
1472 	struct ctl_iscsi_handoff_params *cihp;
1473 	int error;
1474 
1475 	cihp = (struct ctl_iscsi_handoff_params *)&(ci->data);
1476 	softc = &cfiscsi_softc;
1477 
1478 	CFISCSI_DEBUG("new connection from %s (%s) to %s",
1479 	    cihp->initiator_name, cihp->initiator_addr,
1480 	    cihp->target_name);
1481 
1482 	ct = cfiscsi_target_find(softc, cihp->target_name,
1483 	    cihp->portal_group_tag);
1484 	if (ct == NULL) {
1485 		ci->status = CTL_ISCSI_ERROR;
1486 		snprintf(ci->error_str, sizeof(ci->error_str),
1487 		    "%s: target not found", __func__);
1488 		return;
1489 	}
1490 
1491 #ifdef ICL_KERNEL_PROXY
1492 	if (cihp->socket > 0 && cihp->connection_id > 0) {
1493 		snprintf(ci->error_str, sizeof(ci->error_str),
1494 		    "both socket and connection_id set");
1495 		ci->status = CTL_ISCSI_ERROR;
1496 		cfiscsi_target_release(ct);
1497 		return;
1498 	}
1499 	if (cihp->socket == 0) {
1500 		mtx_lock(&cfiscsi_softc.lock);
1501 		TAILQ_FOREACH(cs, &cfiscsi_softc.sessions, cs_next) {
1502 			if (cs->cs_id == cihp->connection_id)
1503 				break;
1504 		}
1505 		if (cs == NULL) {
1506 			mtx_unlock(&cfiscsi_softc.lock);
1507 			snprintf(ci->error_str, sizeof(ci->error_str),
1508 			    "connection not found");
1509 			ci->status = CTL_ISCSI_ERROR;
1510 			cfiscsi_target_release(ct);
1511 			return;
1512 		}
1513 		mtx_unlock(&cfiscsi_softc.lock);
1514 	} else {
1515 #endif
1516 		cs = cfiscsi_session_new(softc, cihp->offload);
1517 		if (cs == NULL) {
1518 			ci->status = CTL_ISCSI_ERROR;
1519 			snprintf(ci->error_str, sizeof(ci->error_str),
1520 			    "%s: cfiscsi_session_new failed", __func__);
1521 			cfiscsi_target_release(ct);
1522 			return;
1523 		}
1524 #ifdef ICL_KERNEL_PROXY
1525 	}
1526 #endif
1527 
1528 	/*
1529 	 * First PDU of Full Feature phase has the same CmdSN as the last
1530 	 * PDU from the Login Phase received from the initiator.  Thus,
1531 	 * the -1 below.
1532 	 */
1533 	cs->cs_cmdsn = cihp->cmdsn;
1534 	cs->cs_statsn = cihp->statsn;
1535 	cs->cs_max_recv_data_segment_length = cihp->max_recv_data_segment_length;
1536 	cs->cs_max_send_data_segment_length = cihp->max_send_data_segment_length;
1537 	cs->cs_max_burst_length = cihp->max_burst_length;
1538 	cs->cs_first_burst_length = cihp->first_burst_length;
1539 	cs->cs_immediate_data = !!cihp->immediate_data;
1540 	if (cihp->header_digest == CTL_ISCSI_DIGEST_CRC32C)
1541 		cs->cs_conn->ic_header_crc32c = true;
1542 	if (cihp->data_digest == CTL_ISCSI_DIGEST_CRC32C)
1543 		cs->cs_conn->ic_data_crc32c = true;
1544 
1545 	strlcpy(cs->cs_initiator_name,
1546 	    cihp->initiator_name, sizeof(cs->cs_initiator_name));
1547 	strlcpy(cs->cs_initiator_addr,
1548 	    cihp->initiator_addr, sizeof(cs->cs_initiator_addr));
1549 	strlcpy(cs->cs_initiator_alias,
1550 	    cihp->initiator_alias, sizeof(cs->cs_initiator_alias));
1551 	memcpy(cs->cs_initiator_isid,
1552 	    cihp->initiator_isid, sizeof(cs->cs_initiator_isid));
1553 	snprintf(cs->cs_initiator_id, sizeof(cs->cs_initiator_id),
1554 	    "%s,i,0x%02x%02x%02x%02x%02x%02x", cs->cs_initiator_name,
1555 	    cihp->initiator_isid[0], cihp->initiator_isid[1],
1556 	    cihp->initiator_isid[2], cihp->initiator_isid[3],
1557 	    cihp->initiator_isid[4], cihp->initiator_isid[5]);
1558 
1559 	mtx_lock(&softc->lock);
1560 	if (ct->ct_online == 0) {
1561 		mtx_unlock(&softc->lock);
1562 		cfiscsi_session_terminate(cs);
1563 		cfiscsi_target_release(ct);
1564 		ci->status = CTL_ISCSI_ERROR;
1565 		snprintf(ci->error_str, sizeof(ci->error_str),
1566 		    "%s: port offline", __func__);
1567 		return;
1568 	}
1569 	cs->cs_target = ct;
1570 	mtx_unlock(&softc->lock);
1571 
1572 	refcount_acquire(&cs->cs_outstanding_ctl_pdus);
1573 restart:
1574 	if (!cs->cs_terminating) {
1575 		mtx_lock(&softc->lock);
1576 		TAILQ_FOREACH(cs2, &softc->sessions, cs_next) {
1577 			if (cs2 != cs && cs2->cs_tasks_aborted == false &&
1578 			    cs->cs_target == cs2->cs_target &&
1579 			    strcmp(cs->cs_initiator_id, cs2->cs_initiator_id) == 0) {
1580 				if (strcmp(cs->cs_initiator_addr,
1581 				    cs2->cs_initiator_addr) != 0) {
1582 					CFISCSI_SESSION_WARN(cs2,
1583 					    "session reinstatement from "
1584 					    "different address %s",
1585 					    cs->cs_initiator_addr);
1586 				} else {
1587 					CFISCSI_SESSION_DEBUG(cs2,
1588 					    "session reinstatement");
1589 				}
1590 				cfiscsi_session_terminate(cs2);
1591 				mtx_unlock(&softc->lock);
1592 				pause("cfiscsi_reinstate", 1);
1593 				goto restart;
1594 			}
1595 		}
1596 		mtx_unlock(&softc->lock);
1597 	}
1598 
1599 	/*
1600 	 * Register initiator with CTL.
1601 	 */
1602 	cfiscsi_session_register_initiator(cs);
1603 
1604 #ifdef ICL_KERNEL_PROXY
1605 	if (cihp->socket > 0) {
1606 #endif
1607 		error = icl_conn_handoff(cs->cs_conn, cihp->socket);
1608 		if (error != 0) {
1609 			cfiscsi_session_terminate(cs);
1610 			refcount_release(&cs->cs_outstanding_ctl_pdus);
1611 			ci->status = CTL_ISCSI_ERROR;
1612 			snprintf(ci->error_str, sizeof(ci->error_str),
1613 			    "%s: icl_conn_handoff failed with error %d",
1614 			    __func__, error);
1615 			return;
1616 		}
1617 #ifdef ICL_KERNEL_PROXY
1618 	}
1619 #endif
1620 
1621 #ifdef ICL_KERNEL_PROXY
1622 	cs->cs_login_phase = false;
1623 
1624 	/*
1625 	 * First PDU of the Full Feature phase has likely already arrived.
1626 	 * We have to pick it up and execute properly.
1627 	 */
1628 	if (cs->cs_login_pdu != NULL) {
1629 		CFISCSI_SESSION_DEBUG(cs, "picking up first PDU");
1630 		cfiscsi_pdu_handle(cs->cs_login_pdu);
1631 		cs->cs_login_pdu = NULL;
1632 	}
1633 #endif
1634 
1635 	refcount_release(&cs->cs_outstanding_ctl_pdus);
1636 	ci->status = CTL_ISCSI_OK;
1637 }
1638 
1639 static void
1640 cfiscsi_ioctl_list(struct ctl_iscsi *ci)
1641 {
1642 	struct ctl_iscsi_list_params *cilp;
1643 	struct cfiscsi_session *cs;
1644 	struct cfiscsi_softc *softc;
1645 	struct sbuf *sb;
1646 	int error;
1647 
1648 	cilp = (struct ctl_iscsi_list_params *)&(ci->data);
1649 	softc = &cfiscsi_softc;
1650 
1651 	sb = sbuf_new(NULL, NULL, cilp->alloc_len, SBUF_FIXEDLEN);
1652 	if (sb == NULL) {
1653 		ci->status = CTL_ISCSI_ERROR;
1654 		snprintf(ci->error_str, sizeof(ci->error_str),
1655 		    "Unable to allocate %d bytes for iSCSI session list",
1656 		    cilp->alloc_len);
1657 		return;
1658 	}
1659 
1660 	sbuf_printf(sb, "<ctlislist>\n");
1661 	mtx_lock(&softc->lock);
1662 	TAILQ_FOREACH(cs, &softc->sessions, cs_next) {
1663 #ifdef ICL_KERNEL_PROXY
1664 		if (cs->cs_target == NULL)
1665 			continue;
1666 #endif
1667 		error = sbuf_printf(sb, "<connection id=\"%d\">"
1668 		    "<initiator>%s</initiator>"
1669 		    "<initiator_addr>%s</initiator_addr>"
1670 		    "<initiator_alias>%s</initiator_alias>"
1671 		    "<target>%s</target>"
1672 		    "<target_alias>%s</target_alias>"
1673 		    "<target_portal_group_tag>%u</target_portal_group_tag>"
1674 		    "<header_digest>%s</header_digest>"
1675 		    "<data_digest>%s</data_digest>"
1676 		    "<max_recv_data_segment_length>%d</max_recv_data_segment_length>"
1677 		    "<max_send_data_segment_length>%d</max_send_data_segment_length>"
1678 		    "<max_burst_length>%d</max_burst_length>"
1679 		    "<first_burst_length>%d</first_burst_length>"
1680 		    "<immediate_data>%d</immediate_data>"
1681 		    "<iser>%d</iser>"
1682 		    "<offload>%s</offload>"
1683 		    "</connection>\n",
1684 		    cs->cs_id,
1685 		    cs->cs_initiator_name, cs->cs_initiator_addr, cs->cs_initiator_alias,
1686 		    cs->cs_target->ct_name, cs->cs_target->ct_alias,
1687 		    cs->cs_target->ct_tag,
1688 		    cs->cs_conn->ic_header_crc32c ? "CRC32C" : "None",
1689 		    cs->cs_conn->ic_data_crc32c ? "CRC32C" : "None",
1690 		    cs->cs_max_recv_data_segment_length,
1691 		    cs->cs_max_send_data_segment_length,
1692 		    cs->cs_max_burst_length,
1693 		    cs->cs_first_burst_length,
1694 		    cs->cs_immediate_data,
1695 		    cs->cs_conn->ic_iser,
1696 		    cs->cs_conn->ic_offload);
1697 		if (error != 0)
1698 			break;
1699 	}
1700 	mtx_unlock(&softc->lock);
1701 	error = sbuf_printf(sb, "</ctlislist>\n");
1702 	if (error != 0) {
1703 		sbuf_delete(sb);
1704 		ci->status = CTL_ISCSI_LIST_NEED_MORE_SPACE;
1705 		snprintf(ci->error_str, sizeof(ci->error_str),
1706 		    "Out of space, %d bytes is too small", cilp->alloc_len);
1707 		return;
1708 	}
1709 	sbuf_finish(sb);
1710 
1711 	error = copyout(sbuf_data(sb), cilp->conn_xml, sbuf_len(sb) + 1);
1712 	cilp->fill_len = sbuf_len(sb) + 1;
1713 	ci->status = CTL_ISCSI_OK;
1714 	sbuf_delete(sb);
1715 }
1716 
1717 static void
1718 cfiscsi_ioctl_logout(struct ctl_iscsi *ci)
1719 {
1720 	struct icl_pdu *response;
1721 	struct iscsi_bhs_asynchronous_message *bhsam;
1722 	struct ctl_iscsi_logout_params *cilp;
1723 	struct cfiscsi_session *cs;
1724 	struct cfiscsi_softc *softc;
1725 	int found = 0;
1726 
1727 	cilp = (struct ctl_iscsi_logout_params *)&(ci->data);
1728 	softc = &cfiscsi_softc;
1729 
1730 	mtx_lock(&softc->lock);
1731 	TAILQ_FOREACH(cs, &softc->sessions, cs_next) {
1732 		if (cilp->all == 0 && cs->cs_id != cilp->connection_id &&
1733 		    strcmp(cs->cs_initiator_name, cilp->initiator_name) != 0 &&
1734 		    strcmp(cs->cs_initiator_addr, cilp->initiator_addr) != 0)
1735 			continue;
1736 
1737 		response = icl_pdu_new(cs->cs_conn, M_NOWAIT);
1738 		if (response == NULL) {
1739 			ci->status = CTL_ISCSI_ERROR;
1740 			snprintf(ci->error_str, sizeof(ci->error_str),
1741 			    "Unable to allocate memory");
1742 			mtx_unlock(&softc->lock);
1743 			return;
1744 		}
1745 		bhsam =
1746 		    (struct iscsi_bhs_asynchronous_message *)response->ip_bhs;
1747 		bhsam->bhsam_opcode = ISCSI_BHS_OPCODE_ASYNC_MESSAGE;
1748 		bhsam->bhsam_flags = 0x80;
1749 		bhsam->bhsam_async_event = BHSAM_EVENT_TARGET_REQUESTS_LOGOUT;
1750 		bhsam->bhsam_parameter3 = htons(10);
1751 		cfiscsi_pdu_queue(response);
1752 		found++;
1753 	}
1754 	mtx_unlock(&softc->lock);
1755 
1756 	if (found == 0) {
1757 		ci->status = CTL_ISCSI_SESSION_NOT_FOUND;
1758 		snprintf(ci->error_str, sizeof(ci->error_str),
1759 		    "No matching connections found");
1760 		return;
1761 	}
1762 
1763 	ci->status = CTL_ISCSI_OK;
1764 }
1765 
1766 static void
1767 cfiscsi_ioctl_terminate(struct ctl_iscsi *ci)
1768 {
1769 	struct icl_pdu *response;
1770 	struct iscsi_bhs_asynchronous_message *bhsam;
1771 	struct ctl_iscsi_terminate_params *citp;
1772 	struct cfiscsi_session *cs;
1773 	struct cfiscsi_softc *softc;
1774 	int found = 0;
1775 
1776 	citp = (struct ctl_iscsi_terminate_params *)&(ci->data);
1777 	softc = &cfiscsi_softc;
1778 
1779 	mtx_lock(&softc->lock);
1780 	TAILQ_FOREACH(cs, &softc->sessions, cs_next) {
1781 		if (citp->all == 0 && cs->cs_id != citp->connection_id &&
1782 		    strcmp(cs->cs_initiator_name, citp->initiator_name) != 0 &&
1783 		    strcmp(cs->cs_initiator_addr, citp->initiator_addr) != 0)
1784 			continue;
1785 
1786 		response = icl_pdu_new(cs->cs_conn, M_NOWAIT);
1787 		if (response == NULL) {
1788 			/*
1789 			 * Oh well.  Just terminate the connection.
1790 			 */
1791 		} else {
1792 			bhsam = (struct iscsi_bhs_asynchronous_message *)
1793 			    response->ip_bhs;
1794 			bhsam->bhsam_opcode = ISCSI_BHS_OPCODE_ASYNC_MESSAGE;
1795 			bhsam->bhsam_flags = 0x80;
1796 			bhsam->bhsam_0xffffffff = 0xffffffff;
1797 			bhsam->bhsam_async_event =
1798 			    BHSAM_EVENT_TARGET_TERMINATES_SESSION;
1799 			cfiscsi_pdu_queue(response);
1800 		}
1801 		cfiscsi_session_terminate(cs);
1802 		found++;
1803 	}
1804 	mtx_unlock(&softc->lock);
1805 
1806 	if (found == 0) {
1807 		ci->status = CTL_ISCSI_SESSION_NOT_FOUND;
1808 		snprintf(ci->error_str, sizeof(ci->error_str),
1809 		    "No matching connections found");
1810 		return;
1811 	}
1812 
1813 	ci->status = CTL_ISCSI_OK;
1814 }
1815 
1816 static void
1817 cfiscsi_ioctl_limits(struct ctl_iscsi *ci)
1818 {
1819 	struct ctl_iscsi_limits_params *cilp;
1820 	struct icl_drv_limits idl;
1821 	int error;
1822 
1823 	cilp = (struct ctl_iscsi_limits_params *)&(ci->data);
1824 
1825 	error = icl_limits(cilp->offload, false, &idl);
1826 	if (error != 0) {
1827 		ci->status = CTL_ISCSI_ERROR;
1828 		snprintf(ci->error_str, sizeof(ci->error_str),
1829 			"%s: icl_limits failed with error %d",
1830 			__func__, error);
1831 		return;
1832 	}
1833 
1834 	cilp->max_recv_data_segment_length =
1835 	    idl.idl_max_recv_data_segment_length;
1836 	cilp->max_send_data_segment_length =
1837 	    idl.idl_max_send_data_segment_length;
1838 	cilp->max_burst_length = idl.idl_max_burst_length;
1839 	cilp->first_burst_length = idl.idl_first_burst_length;
1840 
1841 	ci->status = CTL_ISCSI_OK;
1842 }
1843 
1844 #ifdef ICL_KERNEL_PROXY
1845 static void
1846 cfiscsi_ioctl_listen(struct ctl_iscsi *ci)
1847 {
1848 	struct ctl_iscsi_listen_params *cilp;
1849 	struct sockaddr *sa;
1850 	int error;
1851 
1852 	cilp = (struct ctl_iscsi_listen_params *)&(ci->data);
1853 
1854 	if (cfiscsi_softc.listener == NULL) {
1855 		CFISCSI_DEBUG("no listener");
1856 		snprintf(ci->error_str, sizeof(ci->error_str), "no listener");
1857 		ci->status = CTL_ISCSI_ERROR;
1858 		return;
1859 	}
1860 
1861 	error = getsockaddr(&sa, (void *)cilp->addr, cilp->addrlen);
1862 	if (error != 0) {
1863 		CFISCSI_DEBUG("getsockaddr, error %d", error);
1864 		snprintf(ci->error_str, sizeof(ci->error_str), "getsockaddr failed");
1865 		ci->status = CTL_ISCSI_ERROR;
1866 		return;
1867 	}
1868 
1869 	error = icl_listen_add(cfiscsi_softc.listener, cilp->iser, cilp->domain,
1870 	    cilp->socktype, cilp->protocol, sa, cilp->portal_id);
1871 	if (error != 0) {
1872 		free(sa, M_SONAME);
1873 		CFISCSI_DEBUG("icl_listen_add, error %d", error);
1874 		snprintf(ci->error_str, sizeof(ci->error_str),
1875 		    "icl_listen_add failed, error %d", error);
1876 		ci->status = CTL_ISCSI_ERROR;
1877 		return;
1878 	}
1879 
1880 	ci->status = CTL_ISCSI_OK;
1881 }
1882 
1883 static void
1884 cfiscsi_ioctl_accept(struct ctl_iscsi *ci)
1885 {
1886 	struct ctl_iscsi_accept_params *ciap;
1887 	struct cfiscsi_session *cs;
1888 	int error;
1889 
1890 	ciap = (struct ctl_iscsi_accept_params *)&(ci->data);
1891 
1892 	mtx_lock(&cfiscsi_softc.lock);
1893 	for (;;) {
1894 		TAILQ_FOREACH(cs, &cfiscsi_softc.sessions, cs_next) {
1895 			if (cs->cs_waiting_for_ctld)
1896 				break;
1897 		}
1898 		if (cs != NULL)
1899 			break;
1900 		error = cv_wait_sig(&cfiscsi_softc.accept_cv, &cfiscsi_softc.lock);
1901 		if (error != 0) {
1902 			mtx_unlock(&cfiscsi_softc.lock);
1903 			snprintf(ci->error_str, sizeof(ci->error_str), "interrupted");
1904 			ci->status = CTL_ISCSI_ERROR;
1905 			return;
1906 		}
1907 	}
1908 	mtx_unlock(&cfiscsi_softc.lock);
1909 
1910 	cs->cs_waiting_for_ctld = false;
1911 	cs->cs_login_phase = true;
1912 
1913 	ciap->connection_id = cs->cs_id;
1914 	ciap->portal_id = cs->cs_portal_id;
1915 	ciap->initiator_addrlen = cs->cs_initiator_sa->sa_len;
1916 	error = copyout(cs->cs_initiator_sa, ciap->initiator_addr,
1917 	    cs->cs_initiator_sa->sa_len);
1918 	if (error != 0) {
1919 		snprintf(ci->error_str, sizeof(ci->error_str),
1920 		    "copyout failed with error %d", error);
1921 		ci->status = CTL_ISCSI_ERROR;
1922 		return;
1923 	}
1924 
1925 	ci->status = CTL_ISCSI_OK;
1926 }
1927 
1928 static void
1929 cfiscsi_ioctl_send(struct ctl_iscsi *ci)
1930 {
1931 	struct ctl_iscsi_send_params *cisp;
1932 	struct cfiscsi_session *cs;
1933 	struct icl_pdu *ip;
1934 	size_t datalen;
1935 	void *data;
1936 	int error;
1937 
1938 	cisp = (struct ctl_iscsi_send_params *)&(ci->data);
1939 
1940 	mtx_lock(&cfiscsi_softc.lock);
1941 	TAILQ_FOREACH(cs, &cfiscsi_softc.sessions, cs_next) {
1942 		if (cs->cs_id == cisp->connection_id)
1943 			break;
1944 	}
1945 	if (cs == NULL) {
1946 		mtx_unlock(&cfiscsi_softc.lock);
1947 		snprintf(ci->error_str, sizeof(ci->error_str), "connection not found");
1948 		ci->status = CTL_ISCSI_ERROR;
1949 		return;
1950 	}
1951 	mtx_unlock(&cfiscsi_softc.lock);
1952 
1953 #if 0
1954 	if (cs->cs_login_phase == false)
1955 		return (EBUSY);
1956 #endif
1957 
1958 	if (cs->cs_terminating) {
1959 		snprintf(ci->error_str, sizeof(ci->error_str), "connection is terminating");
1960 		ci->status = CTL_ISCSI_ERROR;
1961 		return;
1962 	}
1963 
1964 	datalen = cisp->data_segment_len;
1965 	/*
1966 	 * XXX
1967 	 */
1968 	//if (datalen > CFISCSI_MAX_DATA_SEGMENT_LENGTH) {
1969 	if (datalen > 65535) {
1970 		snprintf(ci->error_str, sizeof(ci->error_str), "data segment too big");
1971 		ci->status = CTL_ISCSI_ERROR;
1972 		return;
1973 	}
1974 	if (datalen > 0) {
1975 		data = malloc(datalen, M_CFISCSI, M_WAITOK);
1976 		error = copyin(cisp->data_segment, data, datalen);
1977 		if (error != 0) {
1978 			free(data, M_CFISCSI);
1979 			snprintf(ci->error_str, sizeof(ci->error_str), "copyin error %d", error);
1980 			ci->status = CTL_ISCSI_ERROR;
1981 			return;
1982 		}
1983 	}
1984 
1985 	ip = icl_pdu_new(cs->cs_conn, M_WAITOK);
1986 	memcpy(ip->ip_bhs, cisp->bhs, sizeof(*ip->ip_bhs));
1987 	if (datalen > 0) {
1988 		icl_pdu_append_data(ip, data, datalen, M_WAITOK);
1989 		free(data, M_CFISCSI);
1990 	}
1991 	CFISCSI_SESSION_LOCK(cs);
1992 	icl_pdu_queue(ip);
1993 	CFISCSI_SESSION_UNLOCK(cs);
1994 	ci->status = CTL_ISCSI_OK;
1995 }
1996 
1997 static void
1998 cfiscsi_ioctl_receive(struct ctl_iscsi *ci)
1999 {
2000 	struct ctl_iscsi_receive_params *cirp;
2001 	struct cfiscsi_session *cs;
2002 	struct icl_pdu *ip;
2003 	void *data;
2004 	int error;
2005 
2006 	cirp = (struct ctl_iscsi_receive_params *)&(ci->data);
2007 
2008 	mtx_lock(&cfiscsi_softc.lock);
2009 	TAILQ_FOREACH(cs, &cfiscsi_softc.sessions, cs_next) {
2010 		if (cs->cs_id == cirp->connection_id)
2011 			break;
2012 	}
2013 	if (cs == NULL) {
2014 		mtx_unlock(&cfiscsi_softc.lock);
2015 		snprintf(ci->error_str, sizeof(ci->error_str),
2016 		    "connection not found");
2017 		ci->status = CTL_ISCSI_ERROR;
2018 		return;
2019 	}
2020 	mtx_unlock(&cfiscsi_softc.lock);
2021 
2022 #if 0
2023 	if (is->is_login_phase == false)
2024 		return (EBUSY);
2025 #endif
2026 
2027 	CFISCSI_SESSION_LOCK(cs);
2028 	while (cs->cs_login_pdu == NULL && cs->cs_terminating == false) {
2029 		error = cv_wait_sig(&cs->cs_login_cv, &cs->cs_lock);
2030 		if (error != 0) {
2031 			CFISCSI_SESSION_UNLOCK(cs);
2032 			snprintf(ci->error_str, sizeof(ci->error_str),
2033 			    "interrupted by signal");
2034 			ci->status = CTL_ISCSI_ERROR;
2035 			return;
2036 		}
2037 	}
2038 
2039 	if (cs->cs_terminating) {
2040 		CFISCSI_SESSION_UNLOCK(cs);
2041 		snprintf(ci->error_str, sizeof(ci->error_str),
2042 		    "connection terminating");
2043 		ci->status = CTL_ISCSI_ERROR;
2044 		return;
2045 	}
2046 	ip = cs->cs_login_pdu;
2047 	cs->cs_login_pdu = NULL;
2048 	CFISCSI_SESSION_UNLOCK(cs);
2049 
2050 	if (ip->ip_data_len > cirp->data_segment_len) {
2051 		icl_pdu_free(ip);
2052 		snprintf(ci->error_str, sizeof(ci->error_str),
2053 		    "data segment too big");
2054 		ci->status = CTL_ISCSI_ERROR;
2055 		return;
2056 	}
2057 
2058 	copyout(ip->ip_bhs, cirp->bhs, sizeof(*ip->ip_bhs));
2059 	if (ip->ip_data_len > 0) {
2060 		data = malloc(ip->ip_data_len, M_CFISCSI, M_WAITOK);
2061 		icl_pdu_get_data(ip, 0, data, ip->ip_data_len);
2062 		copyout(data, cirp->data_segment, ip->ip_data_len);
2063 		free(data, M_CFISCSI);
2064 	}
2065 
2066 	icl_pdu_free(ip);
2067 	ci->status = CTL_ISCSI_OK;
2068 }
2069 
2070 #endif /* !ICL_KERNEL_PROXY */
2071 
2072 static void
2073 cfiscsi_ioctl_port_create(struct ctl_req *req)
2074 {
2075 	struct cfiscsi_target *ct;
2076 	struct ctl_port *port;
2077 	const char *target, *alias, *tags;
2078 	struct scsi_vpd_id_descriptor *desc;
2079 	ctl_options_t opts;
2080 	int retval, len, idlen;
2081 	uint16_t tag;
2082 
2083 	ctl_init_opts(&opts, req->num_args, req->kern_args);
2084 	target = ctl_get_opt(&opts, "cfiscsi_target");
2085 	alias = ctl_get_opt(&opts, "cfiscsi_target_alias");
2086 	tags = ctl_get_opt(&opts, "cfiscsi_portal_group_tag");
2087 	if (target == NULL || tags == NULL) {
2088 		req->status = CTL_LUN_ERROR;
2089 		snprintf(req->error_str, sizeof(req->error_str),
2090 		    "Missing required argument");
2091 		ctl_free_opts(&opts);
2092 		return;
2093 	}
2094 	tag = strtol(tags, (char **)NULL, 10);
2095 	ct = cfiscsi_target_find_or_create(&cfiscsi_softc, target, alias, tag);
2096 	if (ct == NULL) {
2097 		req->status = CTL_LUN_ERROR;
2098 		snprintf(req->error_str, sizeof(req->error_str),
2099 		    "failed to create target \"%s\"", target);
2100 		ctl_free_opts(&opts);
2101 		return;
2102 	}
2103 	if (ct->ct_state == CFISCSI_TARGET_STATE_ACTIVE) {
2104 		req->status = CTL_LUN_ERROR;
2105 		snprintf(req->error_str, sizeof(req->error_str),
2106 		    "target \"%s\" for portal group tag %u already exists",
2107 		    target, tag);
2108 		cfiscsi_target_release(ct);
2109 		ctl_free_opts(&opts);
2110 		return;
2111 	}
2112 	port = &ct->ct_port;
2113 	// WAT
2114 	if (ct->ct_state == CFISCSI_TARGET_STATE_DYING)
2115 		goto done;
2116 
2117 	port->frontend = &cfiscsi_frontend;
2118 	port->port_type = CTL_PORT_ISCSI;
2119 	/* XXX KDM what should the real number be here? */
2120 	port->num_requested_ctl_io = 4096;
2121 	port->port_name = "iscsi";
2122 	port->physical_port = tag;
2123 	port->virtual_port = ct->ct_target_id;
2124 	port->port_online = cfiscsi_online;
2125 	port->port_offline = cfiscsi_offline;
2126 	port->port_info = cfiscsi_info;
2127 	port->onoff_arg = ct;
2128 	port->fe_datamove = cfiscsi_datamove;
2129 	port->fe_done = cfiscsi_done;
2130 	port->targ_port = -1;
2131 
2132 	port->options = opts;
2133 	STAILQ_INIT(&opts);
2134 
2135 	/* Generate Port ID. */
2136 	idlen = strlen(target) + strlen(",t,0x0001") + 1;
2137 	idlen = roundup2(idlen, 4);
2138 	len = sizeof(struct scsi_vpd_device_id) + idlen;
2139 	port->port_devid = malloc(sizeof(struct ctl_devid) + len,
2140 	    M_CTL, M_WAITOK | M_ZERO);
2141 	port->port_devid->len = len;
2142 	desc = (struct scsi_vpd_id_descriptor *)port->port_devid->data;
2143 	desc->proto_codeset = (SCSI_PROTO_ISCSI << 4) | SVPD_ID_CODESET_UTF8;
2144 	desc->id_type = SVPD_ID_PIV | SVPD_ID_ASSOC_PORT |
2145 	    SVPD_ID_TYPE_SCSI_NAME;
2146 	desc->length = idlen;
2147 	snprintf(desc->identifier, idlen, "%s,t,0x%4.4x", target, tag);
2148 
2149 	/* Generate Target ID. */
2150 	idlen = strlen(target) + 1;
2151 	idlen = roundup2(idlen, 4);
2152 	len = sizeof(struct scsi_vpd_device_id) + idlen;
2153 	port->target_devid = malloc(sizeof(struct ctl_devid) + len,
2154 	    M_CTL, M_WAITOK | M_ZERO);
2155 	port->target_devid->len = len;
2156 	desc = (struct scsi_vpd_id_descriptor *)port->target_devid->data;
2157 	desc->proto_codeset = (SCSI_PROTO_ISCSI << 4) | SVPD_ID_CODESET_UTF8;
2158 	desc->id_type = SVPD_ID_PIV | SVPD_ID_ASSOC_TARGET |
2159 	    SVPD_ID_TYPE_SCSI_NAME;
2160 	desc->length = idlen;
2161 	strlcpy(desc->identifier, target, idlen);
2162 
2163 	retval = ctl_port_register(port);
2164 	if (retval != 0) {
2165 		ctl_free_opts(&port->options);
2166 		cfiscsi_target_release(ct);
2167 		free(port->port_devid, M_CFISCSI);
2168 		free(port->target_devid, M_CFISCSI);
2169 		req->status = CTL_LUN_ERROR;
2170 		snprintf(req->error_str, sizeof(req->error_str),
2171 		    "ctl_port_register() failed with error %d", retval);
2172 		return;
2173 	}
2174 done:
2175 	ct->ct_state = CFISCSI_TARGET_STATE_ACTIVE;
2176 	req->status = CTL_LUN_OK;
2177 	memcpy(req->kern_args[0].kvalue, &port->targ_port,
2178 	    sizeof(port->targ_port)); //XXX
2179 }
2180 
2181 static void
2182 cfiscsi_ioctl_port_remove(struct ctl_req *req)
2183 {
2184 	struct cfiscsi_target *ct;
2185 	const char *target, *tags;
2186 	ctl_options_t opts;
2187 	uint16_t tag;
2188 
2189 	ctl_init_opts(&opts, req->num_args, req->kern_args);
2190 	target = ctl_get_opt(&opts, "cfiscsi_target");
2191 	tags = ctl_get_opt(&opts, "cfiscsi_portal_group_tag");
2192 	if (target == NULL || tags == NULL) {
2193 		ctl_free_opts(&opts);
2194 		req->status = CTL_LUN_ERROR;
2195 		snprintf(req->error_str, sizeof(req->error_str),
2196 		    "Missing required argument");
2197 		return;
2198 	}
2199 	tag = strtol(tags, (char **)NULL, 10);
2200 	ct = cfiscsi_target_find(&cfiscsi_softc, target, tag);
2201 	if (ct == NULL) {
2202 		ctl_free_opts(&opts);
2203 		req->status = CTL_LUN_ERROR;
2204 		snprintf(req->error_str, sizeof(req->error_str),
2205 		    "can't find target \"%s\"", target);
2206 		return;
2207 	}
2208 	if (ct->ct_state != CFISCSI_TARGET_STATE_ACTIVE) {
2209 		ctl_free_opts(&opts);
2210 		req->status = CTL_LUN_ERROR;
2211 		snprintf(req->error_str, sizeof(req->error_str),
2212 		    "target \"%s\" is already dying", target);
2213 		return;
2214 	}
2215 	ctl_free_opts(&opts);
2216 
2217 	ct->ct_state = CFISCSI_TARGET_STATE_DYING;
2218 	ctl_port_offline(&ct->ct_port);
2219 	cfiscsi_target_release(ct);
2220 	cfiscsi_target_release(ct);
2221 	req->status = CTL_LUN_OK;
2222 }
2223 
2224 static int
2225 cfiscsi_ioctl(struct cdev *dev,
2226     u_long cmd, caddr_t addr, int flag, struct thread *td)
2227 {
2228 	struct ctl_iscsi *ci;
2229 	struct ctl_req *req;
2230 
2231 	if (cmd == CTL_PORT_REQ) {
2232 		req = (struct ctl_req *)addr;
2233 		switch (req->reqtype) {
2234 		case CTL_REQ_CREATE:
2235 			cfiscsi_ioctl_port_create(req);
2236 			break;
2237 		case CTL_REQ_REMOVE:
2238 			cfiscsi_ioctl_port_remove(req);
2239 			break;
2240 		default:
2241 			req->status = CTL_LUN_ERROR;
2242 			snprintf(req->error_str, sizeof(req->error_str),
2243 			    "Unsupported request type %d", req->reqtype);
2244 		}
2245 		return (0);
2246 	}
2247 
2248 	if (cmd != CTL_ISCSI)
2249 		return (ENOTTY);
2250 
2251 	ci = (struct ctl_iscsi *)addr;
2252 	switch (ci->type) {
2253 	case CTL_ISCSI_HANDOFF:
2254 		cfiscsi_ioctl_handoff(ci);
2255 		break;
2256 	case CTL_ISCSI_LIST:
2257 		cfiscsi_ioctl_list(ci);
2258 		break;
2259 	case CTL_ISCSI_LOGOUT:
2260 		cfiscsi_ioctl_logout(ci);
2261 		break;
2262 	case CTL_ISCSI_TERMINATE:
2263 		cfiscsi_ioctl_terminate(ci);
2264 		break;
2265 	case CTL_ISCSI_LIMITS:
2266 		cfiscsi_ioctl_limits(ci);
2267 		break;
2268 #ifdef ICL_KERNEL_PROXY
2269 	case CTL_ISCSI_LISTEN:
2270 		cfiscsi_ioctl_listen(ci);
2271 		break;
2272 	case CTL_ISCSI_ACCEPT:
2273 		cfiscsi_ioctl_accept(ci);
2274 		break;
2275 	case CTL_ISCSI_SEND:
2276 		cfiscsi_ioctl_send(ci);
2277 		break;
2278 	case CTL_ISCSI_RECEIVE:
2279 		cfiscsi_ioctl_receive(ci);
2280 		break;
2281 #else
2282 	case CTL_ISCSI_LISTEN:
2283 	case CTL_ISCSI_ACCEPT:
2284 	case CTL_ISCSI_SEND:
2285 	case CTL_ISCSI_RECEIVE:
2286 		ci->status = CTL_ISCSI_ERROR;
2287 		snprintf(ci->error_str, sizeof(ci->error_str),
2288 		    "%s: CTL compiled without ICL_KERNEL_PROXY",
2289 		    __func__);
2290 		break;
2291 #endif /* !ICL_KERNEL_PROXY */
2292 	default:
2293 		ci->status = CTL_ISCSI_ERROR;
2294 		snprintf(ci->error_str, sizeof(ci->error_str),
2295 		    "%s: invalid iSCSI request type %d", __func__, ci->type);
2296 		break;
2297 	}
2298 
2299 	return (0);
2300 }
2301 
2302 static void
2303 cfiscsi_target_hold(struct cfiscsi_target *ct)
2304 {
2305 
2306 	refcount_acquire(&ct->ct_refcount);
2307 }
2308 
2309 static void
2310 cfiscsi_target_release(struct cfiscsi_target *ct)
2311 {
2312 	struct cfiscsi_softc *softc;
2313 
2314 	softc = ct->ct_softc;
2315 	mtx_lock(&softc->lock);
2316 	if (refcount_release(&ct->ct_refcount)) {
2317 		TAILQ_REMOVE(&softc->targets, ct, ct_next);
2318 		mtx_unlock(&softc->lock);
2319 		if (ct->ct_state != CFISCSI_TARGET_STATE_INVALID) {
2320 			ct->ct_state = CFISCSI_TARGET_STATE_INVALID;
2321 			if (ctl_port_deregister(&ct->ct_port) != 0)
2322 				printf("%s: ctl_port_deregister() failed\n",
2323 				    __func__);
2324 		}
2325 		free(ct, M_CFISCSI);
2326 
2327 		return;
2328 	}
2329 	mtx_unlock(&softc->lock);
2330 }
2331 
2332 static struct cfiscsi_target *
2333 cfiscsi_target_find(struct cfiscsi_softc *softc, const char *name, uint16_t tag)
2334 {
2335 	struct cfiscsi_target *ct;
2336 
2337 	mtx_lock(&softc->lock);
2338 	TAILQ_FOREACH(ct, &softc->targets, ct_next) {
2339 		if (ct->ct_tag != tag ||
2340 		    strcmp(name, ct->ct_name) != 0 ||
2341 		    ct->ct_state != CFISCSI_TARGET_STATE_ACTIVE)
2342 			continue;
2343 		cfiscsi_target_hold(ct);
2344 		mtx_unlock(&softc->lock);
2345 		return (ct);
2346 	}
2347 	mtx_unlock(&softc->lock);
2348 
2349 	return (NULL);
2350 }
2351 
2352 static struct cfiscsi_target *
2353 cfiscsi_target_find_or_create(struct cfiscsi_softc *softc, const char *name,
2354     const char *alias, uint16_t tag)
2355 {
2356 	struct cfiscsi_target *ct, *newct;
2357 
2358 	if (name[0] == '\0' || strlen(name) >= CTL_ISCSI_NAME_LEN)
2359 		return (NULL);
2360 
2361 	newct = malloc(sizeof(*newct), M_CFISCSI, M_WAITOK | M_ZERO);
2362 
2363 	mtx_lock(&softc->lock);
2364 	TAILQ_FOREACH(ct, &softc->targets, ct_next) {
2365 		if (ct->ct_tag != tag ||
2366 		    strcmp(name, ct->ct_name) != 0 ||
2367 		    ct->ct_state == CFISCSI_TARGET_STATE_INVALID)
2368 			continue;
2369 		cfiscsi_target_hold(ct);
2370 		mtx_unlock(&softc->lock);
2371 		free(newct, M_CFISCSI);
2372 		return (ct);
2373 	}
2374 
2375 	strlcpy(newct->ct_name, name, sizeof(newct->ct_name));
2376 	if (alias != NULL)
2377 		strlcpy(newct->ct_alias, alias, sizeof(newct->ct_alias));
2378 	newct->ct_tag = tag;
2379 	refcount_init(&newct->ct_refcount, 1);
2380 	newct->ct_softc = softc;
2381 	if (TAILQ_EMPTY(&softc->targets))
2382 		softc->last_target_id = 0;
2383 	newct->ct_target_id = ++softc->last_target_id;
2384 	TAILQ_INSERT_TAIL(&softc->targets, newct, ct_next);
2385 	mtx_unlock(&softc->lock);
2386 
2387 	return (newct);
2388 }
2389 
2390 static void
2391 cfiscsi_datamove_in(union ctl_io *io)
2392 {
2393 	struct cfiscsi_session *cs;
2394 	struct icl_pdu *request, *response;
2395 	const struct iscsi_bhs_scsi_command *bhssc;
2396 	struct iscsi_bhs_data_in *bhsdi;
2397 	struct ctl_sg_entry ctl_sg_entry, *ctl_sglist;
2398 	size_t len, expected_len, sg_len, buffer_offset;
2399 	const char *sg_addr;
2400 	int ctl_sg_count, error, i;
2401 
2402 	request = io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr;
2403 	cs = PDU_SESSION(request);
2404 
2405 	bhssc = (const struct iscsi_bhs_scsi_command *)request->ip_bhs;
2406 	KASSERT((bhssc->bhssc_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
2407 	    ISCSI_BHS_OPCODE_SCSI_COMMAND,
2408 	    ("bhssc->bhssc_opcode != ISCSI_BHS_OPCODE_SCSI_COMMAND"));
2409 
2410 	if (io->scsiio.kern_sg_entries > 0) {
2411 		ctl_sglist = (struct ctl_sg_entry *)io->scsiio.kern_data_ptr;
2412 		ctl_sg_count = io->scsiio.kern_sg_entries;
2413 	} else {
2414 		ctl_sglist = &ctl_sg_entry;
2415 		ctl_sglist->addr = io->scsiio.kern_data_ptr;
2416 		ctl_sglist->len = io->scsiio.kern_data_len;
2417 		ctl_sg_count = 1;
2418 	}
2419 
2420 	/*
2421 	 * This is the total amount of data to be transferred within the current
2422 	 * SCSI command.  We need to record it so that we can properly report
2423 	 * underflow/underflow.
2424 	 */
2425 	PDU_TOTAL_TRANSFER_LEN(request) = io->scsiio.kern_total_len;
2426 
2427 	/*
2428 	 * This is the offset within the current SCSI command; for the first
2429 	 * call to cfiscsi_datamove() it will be 0, and for subsequent ones
2430 	 * it will be the sum of lengths of previous ones.
2431 	 */
2432 	buffer_offset = io->scsiio.kern_rel_offset;
2433 
2434 	/*
2435 	 * This is the transfer length expected by the initiator.  In theory,
2436 	 * it could be different from the correct amount of data from the SCSI
2437 	 * point of view, even if that doesn't make any sense.
2438 	 */
2439 	expected_len = ntohl(bhssc->bhssc_expected_data_transfer_length);
2440 #if 0
2441 	if (expected_len != io->scsiio.kern_total_len) {
2442 		CFISCSI_SESSION_DEBUG(cs, "expected transfer length %zd, "
2443 		    "actual length %zd", expected_len,
2444 		    (size_t)io->scsiio.kern_total_len);
2445 	}
2446 #endif
2447 
2448 	if (buffer_offset >= expected_len) {
2449 #if 0
2450 		CFISCSI_SESSION_DEBUG(cs, "buffer_offset = %zd, "
2451 		    "already sent the expected len", buffer_offset);
2452 #endif
2453 		io->scsiio.be_move_done(io);
2454 		return;
2455 	}
2456 
2457 	i = 0;
2458 	sg_addr = NULL;
2459 	sg_len = 0;
2460 	response = NULL;
2461 	bhsdi = NULL;
2462 	for (;;) {
2463 		if (response == NULL) {
2464 			response = cfiscsi_pdu_new_response(request, M_NOWAIT);
2465 			if (response == NULL) {
2466 				CFISCSI_SESSION_WARN(cs, "failed to "
2467 				    "allocate memory; dropping connection");
2468 				ctl_set_busy(&io->scsiio);
2469 				io->scsiio.be_move_done(io);
2470 				cfiscsi_session_terminate(cs);
2471 				return;
2472 			}
2473 			bhsdi = (struct iscsi_bhs_data_in *)response->ip_bhs;
2474 			bhsdi->bhsdi_opcode = ISCSI_BHS_OPCODE_SCSI_DATA_IN;
2475 			bhsdi->bhsdi_initiator_task_tag =
2476 			    bhssc->bhssc_initiator_task_tag;
2477 			bhsdi->bhsdi_target_transfer_tag = 0xffffffff;
2478 			bhsdi->bhsdi_datasn = htonl(PDU_EXPDATASN(request));
2479 			PDU_EXPDATASN(request)++;
2480 			bhsdi->bhsdi_buffer_offset = htonl(buffer_offset);
2481 		}
2482 
2483 		KASSERT(i < ctl_sg_count, ("i >= ctl_sg_count"));
2484 		if (sg_len == 0) {
2485 			sg_addr = ctl_sglist[i].addr;
2486 			sg_len = ctl_sglist[i].len;
2487 			KASSERT(sg_len > 0, ("sg_len <= 0"));
2488 		}
2489 
2490 		len = sg_len;
2491 
2492 		/*
2493 		 * Truncate to maximum data segment length.
2494 		 */
2495 		KASSERT(response->ip_data_len < cs->cs_max_send_data_segment_length,
2496 		    ("ip_data_len %zd >= max_send_data_segment_length %d",
2497 		    response->ip_data_len, cs->cs_max_send_data_segment_length));
2498 		if (response->ip_data_len + len >
2499 		    cs->cs_max_send_data_segment_length) {
2500 			len = cs->cs_max_send_data_segment_length -
2501 			    response->ip_data_len;
2502 			KASSERT(len <= sg_len, ("len %zd > sg_len %zd",
2503 			    len, sg_len));
2504 		}
2505 
2506 		/*
2507 		 * Truncate to expected data transfer length.
2508 		 */
2509 		KASSERT(buffer_offset + response->ip_data_len < expected_len,
2510 		    ("buffer_offset %zd + ip_data_len %zd >= expected_len %zd",
2511 		    buffer_offset, response->ip_data_len, expected_len));
2512 		if (buffer_offset + response->ip_data_len + len > expected_len) {
2513 			CFISCSI_SESSION_DEBUG(cs, "truncating from %zd "
2514 			    "to expected data transfer length %zd",
2515 			    buffer_offset + response->ip_data_len + len, expected_len);
2516 			len = expected_len - (buffer_offset + response->ip_data_len);
2517 			KASSERT(len <= sg_len, ("len %zd > sg_len %zd",
2518 			    len, sg_len));
2519 		}
2520 
2521 		error = icl_pdu_append_data(response, sg_addr, len, M_NOWAIT);
2522 		if (error != 0) {
2523 			CFISCSI_SESSION_WARN(cs, "failed to "
2524 			    "allocate memory; dropping connection");
2525 			icl_pdu_free(response);
2526 			ctl_set_busy(&io->scsiio);
2527 			io->scsiio.be_move_done(io);
2528 			cfiscsi_session_terminate(cs);
2529 			return;
2530 		}
2531 		sg_addr += len;
2532 		sg_len -= len;
2533 		io->scsiio.kern_data_resid -= len;
2534 
2535 		KASSERT(buffer_offset + response->ip_data_len <= expected_len,
2536 		    ("buffer_offset %zd + ip_data_len %zd > expected_len %zd",
2537 		    buffer_offset, response->ip_data_len, expected_len));
2538 		if (buffer_offset + response->ip_data_len == expected_len) {
2539 			/*
2540 			 * Already have the amount of data the initiator wanted.
2541 			 */
2542 			break;
2543 		}
2544 
2545 		if (sg_len == 0) {
2546 			/*
2547 			 * End of scatter-gather segment;
2548 			 * proceed to the next one...
2549 			 */
2550 			if (i == ctl_sg_count - 1) {
2551 				/*
2552 				 * ... unless this was the last one.
2553 				 */
2554 				break;
2555 			}
2556 			i++;
2557 		}
2558 
2559 		if (response->ip_data_len == cs->cs_max_send_data_segment_length) {
2560 			/*
2561 			 * Can't stuff more data into the current PDU;
2562 			 * queue it.  Note that's not enough to check
2563 			 * for kern_data_resid == 0 instead; there
2564 			 * may be several Data-In PDUs for the final
2565 			 * call to cfiscsi_datamove(), and we want
2566 			 * to set the F flag only on the last of them.
2567 			 */
2568 			buffer_offset += response->ip_data_len;
2569 			if (buffer_offset == io->scsiio.kern_total_len ||
2570 			    buffer_offset == expected_len) {
2571 				buffer_offset -= response->ip_data_len;
2572 				break;
2573 			}
2574 			cfiscsi_pdu_queue(response);
2575 			response = NULL;
2576 			bhsdi = NULL;
2577 		}
2578 	}
2579 	if (response != NULL) {
2580 		buffer_offset += response->ip_data_len;
2581 		if (buffer_offset == io->scsiio.kern_total_len ||
2582 		    buffer_offset == expected_len) {
2583 			bhsdi->bhsdi_flags |= BHSDI_FLAGS_F;
2584 			if (io->io_hdr.status == CTL_SUCCESS) {
2585 				bhsdi->bhsdi_flags |= BHSDI_FLAGS_S;
2586 				if (PDU_TOTAL_TRANSFER_LEN(request) <
2587 				    ntohl(bhssc->bhssc_expected_data_transfer_length)) {
2588 					bhsdi->bhsdi_flags |= BHSSR_FLAGS_RESIDUAL_UNDERFLOW;
2589 					bhsdi->bhsdi_residual_count =
2590 					    htonl(ntohl(bhssc->bhssc_expected_data_transfer_length) -
2591 					    PDU_TOTAL_TRANSFER_LEN(request));
2592 				} else if (PDU_TOTAL_TRANSFER_LEN(request) >
2593 				    ntohl(bhssc->bhssc_expected_data_transfer_length)) {
2594 					bhsdi->bhsdi_flags |= BHSSR_FLAGS_RESIDUAL_OVERFLOW;
2595 					bhsdi->bhsdi_residual_count =
2596 					    htonl(PDU_TOTAL_TRANSFER_LEN(request) -
2597 					    ntohl(bhssc->bhssc_expected_data_transfer_length));
2598 				}
2599 				bhsdi->bhsdi_status = io->scsiio.scsi_status;
2600 				io->io_hdr.flags |= CTL_FLAG_STATUS_SENT;
2601 			}
2602 		}
2603 		KASSERT(response->ip_data_len > 0, ("sending empty Data-In"));
2604 		cfiscsi_pdu_queue(response);
2605 	}
2606 
2607 	io->scsiio.be_move_done(io);
2608 }
2609 
2610 static void
2611 cfiscsi_datamove_out(union ctl_io *io)
2612 {
2613 	struct cfiscsi_session *cs;
2614 	struct icl_pdu *request, *response;
2615 	const struct iscsi_bhs_scsi_command *bhssc;
2616 	struct iscsi_bhs_r2t *bhsr2t;
2617 	struct cfiscsi_data_wait *cdw;
2618 	struct ctl_sg_entry ctl_sg_entry, *ctl_sglist;
2619 	uint32_t expected_len, datamove_len, r2t_off, r2t_len;
2620 	uint32_t target_transfer_tag;
2621 	bool done;
2622 
2623 	request = io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr;
2624 	cs = PDU_SESSION(request);
2625 
2626 	bhssc = (const struct iscsi_bhs_scsi_command *)request->ip_bhs;
2627 	KASSERT((bhssc->bhssc_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
2628 	    ISCSI_BHS_OPCODE_SCSI_COMMAND,
2629 	    ("bhssc->bhssc_opcode != ISCSI_BHS_OPCODE_SCSI_COMMAND"));
2630 
2631 	/*
2632 	 * We need to record it so that we can properly report
2633 	 * underflow/underflow.
2634 	 */
2635 	PDU_TOTAL_TRANSFER_LEN(request) = io->scsiio.kern_total_len;
2636 
2637 	/*
2638 	 * Complete write underflow.  Not a single byte to read.  Return.
2639 	 */
2640 	expected_len = ntohl(bhssc->bhssc_expected_data_transfer_length);
2641 	if (io->scsiio.kern_rel_offset >= expected_len) {
2642 		io->scsiio.be_move_done(io);
2643 		return;
2644 	}
2645 	datamove_len = MIN(io->scsiio.kern_data_len,
2646 	    expected_len - io->scsiio.kern_rel_offset);
2647 
2648 	target_transfer_tag =
2649 	    atomic_fetchadd_32(&cs->cs_target_transfer_tag, 1);
2650 	cdw = cfiscsi_data_wait_new(cs, io, bhssc->bhssc_initiator_task_tag,
2651 	    &target_transfer_tag);
2652 	if (cdw == NULL) {
2653 		CFISCSI_SESSION_WARN(cs, "failed to "
2654 		    "allocate memory; dropping connection");
2655 		ctl_set_busy(&io->scsiio);
2656 		io->scsiio.be_move_done(io);
2657 		cfiscsi_session_terminate(cs);
2658 		return;
2659 	}
2660 #if 0
2661 	CFISCSI_SESSION_DEBUG(cs, "expecting Data-Out with initiator "
2662 	    "task tag 0x%x, target transfer tag 0x%x",
2663 	    bhssc->bhssc_initiator_task_tag, target_transfer_tag);
2664 #endif
2665 
2666 	cdw->cdw_ctl_io = io;
2667 	cdw->cdw_target_transfer_tag = target_transfer_tag;
2668 	cdw->cdw_initiator_task_tag = bhssc->bhssc_initiator_task_tag;
2669 	cdw->cdw_r2t_end = datamove_len;
2670 	cdw->cdw_datasn = 0;
2671 
2672 	/* Set initial data pointer for the CDW respecting ext_data_filled. */
2673 	if (io->scsiio.kern_sg_entries > 0) {
2674 		ctl_sglist = (struct ctl_sg_entry *)io->scsiio.kern_data_ptr;
2675 	} else {
2676 		ctl_sglist = &ctl_sg_entry;
2677 		ctl_sglist->addr = io->scsiio.kern_data_ptr;
2678 		ctl_sglist->len = datamove_len;
2679 	}
2680 	cdw->cdw_sg_index = 0;
2681 	cdw->cdw_sg_addr = ctl_sglist[cdw->cdw_sg_index].addr;
2682 	cdw->cdw_sg_len = ctl_sglist[cdw->cdw_sg_index].len;
2683 	r2t_off = io->scsiio.ext_data_filled;
2684 	while (r2t_off > 0) {
2685 		if (r2t_off >= cdw->cdw_sg_len) {
2686 			r2t_off -= cdw->cdw_sg_len;
2687 			cdw->cdw_sg_index++;
2688 			cdw->cdw_sg_addr = ctl_sglist[cdw->cdw_sg_index].addr;
2689 			cdw->cdw_sg_len = ctl_sglist[cdw->cdw_sg_index].len;
2690 			continue;
2691 		}
2692 		cdw->cdw_sg_addr += r2t_off;
2693 		cdw->cdw_sg_len -= r2t_off;
2694 		r2t_off = 0;
2695 	}
2696 
2697 	if (cs->cs_immediate_data &&
2698 	    io->scsiio.kern_rel_offset + io->scsiio.ext_data_filled <
2699 	    icl_pdu_data_segment_length(request)) {
2700 		done = cfiscsi_handle_data_segment(request, cdw);
2701 		if (done) {
2702 			cfiscsi_data_wait_free(cs, cdw);
2703 			io->scsiio.be_move_done(io);
2704 			return;
2705 		}
2706 	}
2707 
2708 	r2t_off = io->scsiio.kern_rel_offset + io->scsiio.ext_data_filled;
2709 	r2t_len = MIN(datamove_len - io->scsiio.ext_data_filled,
2710 	    cs->cs_max_burst_length);
2711 	cdw->cdw_r2t_end = io->scsiio.ext_data_filled + r2t_len;
2712 
2713 	CFISCSI_SESSION_LOCK(cs);
2714 	TAILQ_INSERT_TAIL(&cs->cs_waiting_for_data_out, cdw, cdw_next);
2715 	CFISCSI_SESSION_UNLOCK(cs);
2716 
2717 	/*
2718 	 * XXX: We should limit the number of outstanding R2T PDUs
2719 	 * 	per task to MaxOutstandingR2T.
2720 	 */
2721 	response = cfiscsi_pdu_new_response(request, M_NOWAIT);
2722 	if (response == NULL) {
2723 		CFISCSI_SESSION_WARN(cs, "failed to "
2724 		    "allocate memory; dropping connection");
2725 		ctl_set_busy(&io->scsiio);
2726 		io->scsiio.be_move_done(io);
2727 		cfiscsi_session_terminate(cs);
2728 		return;
2729 	}
2730 	io->io_hdr.flags |= CTL_FLAG_DMA_INPROG;
2731 	bhsr2t = (struct iscsi_bhs_r2t *)response->ip_bhs;
2732 	bhsr2t->bhsr2t_opcode = ISCSI_BHS_OPCODE_R2T;
2733 	bhsr2t->bhsr2t_flags = 0x80;
2734 	bhsr2t->bhsr2t_lun = bhssc->bhssc_lun;
2735 	bhsr2t->bhsr2t_initiator_task_tag = bhssc->bhssc_initiator_task_tag;
2736 	bhsr2t->bhsr2t_target_transfer_tag = target_transfer_tag;
2737 	/*
2738 	 * XXX: Here we assume that cfiscsi_datamove() won't ever
2739 	 *	be running concurrently on several CPUs for a given
2740 	 *	command.
2741 	 */
2742 	bhsr2t->bhsr2t_r2tsn = htonl(PDU_R2TSN(request));
2743 	PDU_R2TSN(request)++;
2744 	/*
2745 	 * This is the offset within the current SCSI command;
2746 	 * i.e. for the first call of datamove(), it will be 0,
2747 	 * and for subsequent ones it will be the sum of lengths
2748 	 * of previous ones.
2749 	 *
2750 	 * The ext_data_filled is to account for unsolicited
2751 	 * (immediate) data that might have already arrived.
2752 	 */
2753 	bhsr2t->bhsr2t_buffer_offset = htonl(r2t_off);
2754 	/*
2755 	 * This is the total length (sum of S/G lengths) this call
2756 	 * to cfiscsi_datamove() is supposed to handle, limited by
2757 	 * MaxBurstLength.
2758 	 */
2759 	bhsr2t->bhsr2t_desired_data_transfer_length = htonl(r2t_len);
2760 	cfiscsi_pdu_queue(response);
2761 }
2762 
2763 static void
2764 cfiscsi_datamove(union ctl_io *io)
2765 {
2766 
2767 	if ((io->io_hdr.flags & CTL_FLAG_DATA_MASK) == CTL_FLAG_DATA_IN)
2768 		cfiscsi_datamove_in(io);
2769 	else {
2770 		/* We hadn't received anything during this datamove yet. */
2771 		io->scsiio.ext_data_filled = 0;
2772 		cfiscsi_datamove_out(io);
2773 	}
2774 }
2775 
2776 static void
2777 cfiscsi_scsi_command_done(union ctl_io *io)
2778 {
2779 	struct icl_pdu *request, *response;
2780 	struct iscsi_bhs_scsi_command *bhssc;
2781 	struct iscsi_bhs_scsi_response *bhssr;
2782 #ifdef DIAGNOSTIC
2783 	struct cfiscsi_data_wait *cdw;
2784 #endif
2785 	struct cfiscsi_session *cs;
2786 	uint16_t sense_length;
2787 
2788 	request = io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr;
2789 	cs = PDU_SESSION(request);
2790 	bhssc = (struct iscsi_bhs_scsi_command *)request->ip_bhs;
2791 	KASSERT((bhssc->bhssc_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
2792 	    ISCSI_BHS_OPCODE_SCSI_COMMAND,
2793 	    ("replying to wrong opcode 0x%x", bhssc->bhssc_opcode));
2794 
2795 	//CFISCSI_SESSION_DEBUG(cs, "initiator task tag 0x%x",
2796 	//    bhssc->bhssc_initiator_task_tag);
2797 
2798 #ifdef DIAGNOSTIC
2799 	CFISCSI_SESSION_LOCK(cs);
2800 	TAILQ_FOREACH(cdw, &cs->cs_waiting_for_data_out, cdw_next)
2801 		KASSERT(bhssc->bhssc_initiator_task_tag !=
2802 		    cdw->cdw_initiator_task_tag, ("dangling cdw"));
2803 	CFISCSI_SESSION_UNLOCK(cs);
2804 #endif
2805 
2806 	/*
2807 	 * Do not return status for aborted commands.
2808 	 * There are exceptions, but none supported by CTL yet.
2809 	 */
2810 	if (((io->io_hdr.flags & CTL_FLAG_ABORT) &&
2811 	     (io->io_hdr.flags & CTL_FLAG_ABORT_STATUS) == 0) ||
2812 	    (io->io_hdr.flags & CTL_FLAG_STATUS_SENT)) {
2813 		ctl_free_io(io);
2814 		icl_pdu_free(request);
2815 		return;
2816 	}
2817 
2818 	response = cfiscsi_pdu_new_response(request, M_WAITOK);
2819 	bhssr = (struct iscsi_bhs_scsi_response *)response->ip_bhs;
2820 	bhssr->bhssr_opcode = ISCSI_BHS_OPCODE_SCSI_RESPONSE;
2821 	bhssr->bhssr_flags = 0x80;
2822 	/*
2823 	 * XXX: We don't deal with bidirectional under/overflows;
2824 	 *	does anything actually support those?
2825 	 */
2826 	if (PDU_TOTAL_TRANSFER_LEN(request) <
2827 	    ntohl(bhssc->bhssc_expected_data_transfer_length)) {
2828 		bhssr->bhssr_flags |= BHSSR_FLAGS_RESIDUAL_UNDERFLOW;
2829 		bhssr->bhssr_residual_count =
2830 		    htonl(ntohl(bhssc->bhssc_expected_data_transfer_length) -
2831 		    PDU_TOTAL_TRANSFER_LEN(request));
2832 		//CFISCSI_SESSION_DEBUG(cs, "underflow; residual count %d",
2833 		//    ntohl(bhssr->bhssr_residual_count));
2834 	} else if (PDU_TOTAL_TRANSFER_LEN(request) >
2835 	    ntohl(bhssc->bhssc_expected_data_transfer_length)) {
2836 		bhssr->bhssr_flags |= BHSSR_FLAGS_RESIDUAL_OVERFLOW;
2837 		bhssr->bhssr_residual_count =
2838 		    htonl(PDU_TOTAL_TRANSFER_LEN(request) -
2839 		    ntohl(bhssc->bhssc_expected_data_transfer_length));
2840 		//CFISCSI_SESSION_DEBUG(cs, "overflow; residual count %d",
2841 		//    ntohl(bhssr->bhssr_residual_count));
2842 	}
2843 	bhssr->bhssr_response = BHSSR_RESPONSE_COMMAND_COMPLETED;
2844 	bhssr->bhssr_status = io->scsiio.scsi_status;
2845 	bhssr->bhssr_initiator_task_tag = bhssc->bhssc_initiator_task_tag;
2846 	bhssr->bhssr_expdatasn = htonl(PDU_EXPDATASN(request));
2847 
2848 	if (io->scsiio.sense_len > 0) {
2849 #if 0
2850 		CFISCSI_SESSION_DEBUG(cs, "returning %d bytes of sense data",
2851 		    io->scsiio.sense_len);
2852 #endif
2853 		sense_length = htons(io->scsiio.sense_len);
2854 		icl_pdu_append_data(response,
2855 		    &sense_length, sizeof(sense_length), M_WAITOK);
2856 		icl_pdu_append_data(response,
2857 		    &io->scsiio.sense_data, io->scsiio.sense_len, M_WAITOK);
2858 	}
2859 
2860 	ctl_free_io(io);
2861 	icl_pdu_free(request);
2862 	cfiscsi_pdu_queue(response);
2863 }
2864 
2865 static void
2866 cfiscsi_task_management_done(union ctl_io *io)
2867 {
2868 	struct icl_pdu *request, *response;
2869 	struct iscsi_bhs_task_management_request *bhstmr;
2870 	struct iscsi_bhs_task_management_response *bhstmr2;
2871 	struct cfiscsi_data_wait *cdw, *tmpcdw;
2872 	struct cfiscsi_session *cs, *tcs;
2873 	struct cfiscsi_softc *softc;
2874 	int cold_reset = 0;
2875 
2876 	request = io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr;
2877 	cs = PDU_SESSION(request);
2878 	bhstmr = (struct iscsi_bhs_task_management_request *)request->ip_bhs;
2879 	KASSERT((bhstmr->bhstmr_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
2880 	    ISCSI_BHS_OPCODE_TASK_REQUEST,
2881 	    ("replying to wrong opcode 0x%x", bhstmr->bhstmr_opcode));
2882 
2883 #if 0
2884 	CFISCSI_SESSION_DEBUG(cs, "initiator task tag 0x%x; referenced task tag 0x%x",
2885 	    bhstmr->bhstmr_initiator_task_tag,
2886 	    bhstmr->bhstmr_referenced_task_tag);
2887 #endif
2888 
2889 	if ((bhstmr->bhstmr_function & ~0x80) ==
2890 	    BHSTMR_FUNCTION_ABORT_TASK) {
2891 		/*
2892 		 * Make sure we no longer wait for Data-Out for this command.
2893 		 */
2894 		CFISCSI_SESSION_LOCK(cs);
2895 		TAILQ_FOREACH_SAFE(cdw,
2896 		    &cs->cs_waiting_for_data_out, cdw_next, tmpcdw) {
2897 			if (bhstmr->bhstmr_referenced_task_tag !=
2898 			    cdw->cdw_initiator_task_tag)
2899 				continue;
2900 
2901 #if 0
2902 			CFISCSI_SESSION_DEBUG(cs, "removing csw for initiator task "
2903 			    "tag 0x%x", bhstmr->bhstmr_initiator_task_tag);
2904 #endif
2905 			TAILQ_REMOVE(&cs->cs_waiting_for_data_out,
2906 			    cdw, cdw_next);
2907 			io->io_hdr.flags &= ~CTL_FLAG_DMA_INPROG;
2908 			cdw->cdw_ctl_io->scsiio.io_hdr.port_status = 43;
2909 			cdw->cdw_ctl_io->scsiio.be_move_done(cdw->cdw_ctl_io);
2910 			cfiscsi_data_wait_free(cs, cdw);
2911 		}
2912 		CFISCSI_SESSION_UNLOCK(cs);
2913 	}
2914 	if ((bhstmr->bhstmr_function & ~0x80) ==
2915 	    BHSTMR_FUNCTION_TARGET_COLD_RESET &&
2916 	    io->io_hdr.status == CTL_SUCCESS)
2917 		cold_reset = 1;
2918 
2919 	response = cfiscsi_pdu_new_response(request, M_WAITOK);
2920 	bhstmr2 = (struct iscsi_bhs_task_management_response *)
2921 	    response->ip_bhs;
2922 	bhstmr2->bhstmr_opcode = ISCSI_BHS_OPCODE_TASK_RESPONSE;
2923 	bhstmr2->bhstmr_flags = 0x80;
2924 	switch (io->taskio.task_status) {
2925 	case CTL_TASK_FUNCTION_COMPLETE:
2926 		bhstmr2->bhstmr_response = BHSTMR_RESPONSE_FUNCTION_COMPLETE;
2927 		break;
2928 	case CTL_TASK_FUNCTION_SUCCEEDED:
2929 		bhstmr2->bhstmr_response = BHSTMR_RESPONSE_FUNCTION_SUCCEEDED;
2930 		break;
2931 	case CTL_TASK_LUN_DOES_NOT_EXIST:
2932 		bhstmr2->bhstmr_response = BHSTMR_RESPONSE_LUN_DOES_NOT_EXIST;
2933 		break;
2934 	case CTL_TASK_FUNCTION_NOT_SUPPORTED:
2935 	default:
2936 		bhstmr2->bhstmr_response = BHSTMR_RESPONSE_FUNCTION_NOT_SUPPORTED;
2937 		break;
2938 	}
2939 	memcpy(bhstmr2->bhstmr_additional_reponse_information,
2940 	    io->taskio.task_resp, sizeof(io->taskio.task_resp));
2941 	bhstmr2->bhstmr_initiator_task_tag = bhstmr->bhstmr_initiator_task_tag;
2942 
2943 	ctl_free_io(io);
2944 	icl_pdu_free(request);
2945 	cfiscsi_pdu_queue(response);
2946 
2947 	if (cold_reset) {
2948 		softc = cs->cs_target->ct_softc;
2949 		mtx_lock(&softc->lock);
2950 		TAILQ_FOREACH(tcs, &softc->sessions, cs_next) {
2951 			if (tcs->cs_target == cs->cs_target)
2952 				cfiscsi_session_terminate(tcs);
2953 		}
2954 		mtx_unlock(&softc->lock);
2955 	}
2956 }
2957 
2958 static void
2959 cfiscsi_done(union ctl_io *io)
2960 {
2961 	struct icl_pdu *request;
2962 	struct cfiscsi_session *cs;
2963 
2964 	KASSERT(((io->io_hdr.status & CTL_STATUS_MASK) != CTL_STATUS_NONE),
2965 		("invalid CTL status %#x", io->io_hdr.status));
2966 
2967 	if (io->io_hdr.io_type == CTL_IO_TASK &&
2968 	    io->taskio.task_action == CTL_TASK_I_T_NEXUS_RESET) {
2969 		/*
2970 		 * Implicit task termination has just completed; nothing to do.
2971 		 */
2972 		cs = io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr;
2973 		cs->cs_tasks_aborted = true;
2974 		refcount_release(&cs->cs_outstanding_ctl_pdus);
2975 		wakeup(__DEVOLATILE(void *, &cs->cs_outstanding_ctl_pdus));
2976 		ctl_free_io(io);
2977 		return;
2978 	}
2979 
2980 	request = io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr;
2981 	cs = PDU_SESSION(request);
2982 
2983 	switch (request->ip_bhs->bhs_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) {
2984 	case ISCSI_BHS_OPCODE_SCSI_COMMAND:
2985 		cfiscsi_scsi_command_done(io);
2986 		break;
2987 	case ISCSI_BHS_OPCODE_TASK_REQUEST:
2988 		cfiscsi_task_management_done(io);
2989 		break;
2990 	default:
2991 		panic("cfiscsi_done called with wrong opcode 0x%x",
2992 		    request->ip_bhs->bhs_opcode);
2993 	}
2994 
2995 	refcount_release(&cs->cs_outstanding_ctl_pdus);
2996 }
2997