xref: /freebsd/sys/cam/ctl/ctl_frontend_iscsi.c (revision 19261079b74319502c6ffa1249920079f0f69a72)
1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3  *
4  * Copyright (c) 2012 The FreeBSD Foundation
5  *
6  * This software was developed by Edward Tomasz Napierala under sponsorship
7  * from the FreeBSD Foundation.
8  *
9  * Redistribution and use in source and binary forms, with or without
10  * modification, are permitted provided that the following conditions
11  * are met:
12  * 1. Redistributions of source code must retain the above copyright
13  *    notice, this list of conditions and the following disclaimer.
14  * 2. Redistributions in binary form must reproduce the above copyright
15  *    notice, this list of conditions and the following disclaimer in the
16  *    documentation and/or other materials provided with the distribution.
17  *
18  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
19  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
22  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28  * SUCH DAMAGE.
29  *
30  * $FreeBSD$
31  */
32 
33 /*
34  * CTL frontend for the iSCSI protocol.
35  */
36 
37 #include <sys/cdefs.h>
38 __FBSDID("$FreeBSD$");
39 
40 #include <sys/param.h>
41 #include <sys/capsicum.h>
42 #include <sys/condvar.h>
43 #include <sys/endian.h>
44 #include <sys/file.h>
45 #include <sys/kernel.h>
46 #include <sys/kthread.h>
47 #include <sys/lock.h>
48 #include <sys/malloc.h>
49 #include <sys/module.h>
50 #include <sys/mutex.h>
51 #include <sys/queue.h>
52 #include <sys/sbuf.h>
53 #include <sys/socket.h>
54 #include <sys/sysctl.h>
55 #include <sys/systm.h>
56 #include <sys/uio.h>
57 #include <sys/unistd.h>
58 #include <sys/nv.h>
59 #include <sys/dnv.h>
60 #include <vm/uma.h>
61 
62 #include <cam/scsi/scsi_all.h>
63 #include <cam/scsi/scsi_da.h>
64 #include <cam/ctl/ctl_io.h>
65 #include <cam/ctl/ctl.h>
66 #include <cam/ctl/ctl_backend.h>
67 #include <cam/ctl/ctl_error.h>
68 #include <cam/ctl/ctl_frontend.h>
69 #include <cam/ctl/ctl_debug.h>
70 #include <cam/ctl/ctl_ha.h>
71 #include <cam/ctl/ctl_ioctl.h>
72 #include <cam/ctl/ctl_private.h>
73 
74 #include <dev/iscsi/icl.h>
75 #include <dev/iscsi/icl_wrappers.h>
76 #include <dev/iscsi/iscsi_proto.h>
77 #include <cam/ctl/ctl_frontend_iscsi.h>
78 
79 #ifdef ICL_KERNEL_PROXY
80 #include <sys/socketvar.h>
81 #endif
82 
83 #ifdef ICL_KERNEL_PROXY
84 FEATURE(cfiscsi_kernel_proxy, "iSCSI target built with ICL_KERNEL_PROXY");
85 #endif
86 
87 static MALLOC_DEFINE(M_CFISCSI, "cfiscsi", "Memory used for CTL iSCSI frontend");
88 static uma_zone_t cfiscsi_data_wait_zone;
89 
90 SYSCTL_NODE(_kern_cam_ctl, OID_AUTO, iscsi, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
91     "CAM Target Layer iSCSI Frontend");
92 static int debug = 1;
93 SYSCTL_INT(_kern_cam_ctl_iscsi, OID_AUTO, debug, CTLFLAG_RWTUN,
94     &debug, 1, "Enable debug messages");
95 static int ping_timeout = 5;
96 SYSCTL_INT(_kern_cam_ctl_iscsi, OID_AUTO, ping_timeout, CTLFLAG_RWTUN,
97     &ping_timeout, 5, "Interval between ping (NOP-Out) requests, in seconds");
98 static int login_timeout = 60;
99 SYSCTL_INT(_kern_cam_ctl_iscsi, OID_AUTO, login_timeout, CTLFLAG_RWTUN,
100     &login_timeout, 60, "Time to wait for ctld(8) to finish Login Phase, in seconds");
101 static int maxtags = 256;
102 SYSCTL_INT(_kern_cam_ctl_iscsi, OID_AUTO, maxtags, CTLFLAG_RWTUN,
103     &maxtags, 0, "Max number of requests queued by initiator");
104 
105 #define	CFISCSI_DEBUG(X, ...)						\
106 	do {								\
107 		if (debug > 1) {					\
108 			printf("%s: " X "\n",				\
109 			    __func__, ## __VA_ARGS__);			\
110 		}							\
111 	} while (0)
112 
113 #define	CFISCSI_WARN(X, ...)						\
114 	do {								\
115 		if (debug > 0) {					\
116 			printf("WARNING: %s: " X "\n",			\
117 			    __func__, ## __VA_ARGS__);			\
118 		}							\
119 	} while (0)
120 
121 #define	CFISCSI_SESSION_DEBUG(S, X, ...)				\
122 	do {								\
123 		if (debug > 1) {					\
124 			printf("%s: %s (%s): " X "\n",			\
125 			    __func__, S->cs_initiator_addr,		\
126 			    S->cs_initiator_name, ## __VA_ARGS__);	\
127 		}							\
128 	} while (0)
129 
130 #define	CFISCSI_SESSION_WARN(S, X, ...)					\
131 	do  {								\
132 		if (debug > 0) {					\
133 			printf("WARNING: %s (%s): " X "\n",		\
134 			    S->cs_initiator_addr,			\
135 			    S->cs_initiator_name, ## __VA_ARGS__);	\
136 		}							\
137 	} while (0)
138 
139 #define CFISCSI_SESSION_LOCK(X)		mtx_lock(&X->cs_lock)
140 #define CFISCSI_SESSION_UNLOCK(X)	mtx_unlock(&X->cs_lock)
141 #define CFISCSI_SESSION_LOCK_ASSERT(X)	mtx_assert(&X->cs_lock, MA_OWNED)
142 
143 #define	CONN_SESSION(X)			((struct cfiscsi_session *)(X)->ic_prv0)
144 #define	PDU_SESSION(X)			CONN_SESSION((X)->ip_conn)
145 
146 struct cfiscsi_priv {
147 	void		*request;
148 	uint32_t	 expdatasn;
149 	uint32_t	 r2tsn;
150 };
151 #define	PRIV(io)	\
152     ((struct cfiscsi_priv *)&(io)->io_hdr.ctl_private[CTL_PRIV_FRONTEND])
153 #define	PRIV_REQUEST(io)		PRIV(io)->request
154 #define	PRIV_EXPDATASN(io)		PRIV(io)->expdatasn
155 #define	PRIV_R2TSN(io)			PRIV(io)->r2tsn
156 
157 static int	cfiscsi_init(void);
158 static int	cfiscsi_shutdown(void);
159 static void	cfiscsi_online(void *arg);
160 static void	cfiscsi_offline(void *arg);
161 static int	cfiscsi_info(void *arg, struct sbuf *sb);
162 static int	cfiscsi_ioctl(struct cdev *dev,
163 		    u_long cmd, caddr_t addr, int flag, struct thread *td);
164 static void	cfiscsi_datamove(union ctl_io *io);
165 static void	cfiscsi_datamove_in(union ctl_io *io);
166 static void	cfiscsi_datamove_out(union ctl_io *io);
167 static void	cfiscsi_done(union ctl_io *io);
168 static bool	cfiscsi_pdu_update_cmdsn(const struct icl_pdu *request);
169 static void	cfiscsi_pdu_handle_nop_out(struct icl_pdu *request);
170 static void	cfiscsi_pdu_handle_scsi_command(struct icl_pdu *request);
171 static void	cfiscsi_pdu_handle_task_request(struct icl_pdu *request);
172 static void	cfiscsi_pdu_handle_data_out(struct icl_pdu *request);
173 static void	cfiscsi_pdu_handle_logout_request(struct icl_pdu *request);
174 static void	cfiscsi_session_terminate(struct cfiscsi_session *cs);
175 static struct cfiscsi_data_wait	*cfiscsi_data_wait_new(
176 		    struct cfiscsi_session *cs, union ctl_io *io,
177 		    uint32_t initiator_task_tag,
178 		    uint32_t *target_transfer_tagp);
179 static void	cfiscsi_data_wait_free(struct cfiscsi_session *cs,
180 		    struct cfiscsi_data_wait *cdw);
181 static struct cfiscsi_target	*cfiscsi_target_find(struct cfiscsi_softc
182 		    *softc, const char *name, uint16_t tag);
183 static struct cfiscsi_target	*cfiscsi_target_find_or_create(
184     struct cfiscsi_softc *softc, const char *name, const char *alias,
185     uint16_t tag);
186 static void	cfiscsi_target_release(struct cfiscsi_target *ct);
187 static void	cfiscsi_session_delete(struct cfiscsi_session *cs);
188 
189 static struct cfiscsi_softc cfiscsi_softc;
190 
191 static struct ctl_frontend cfiscsi_frontend =
192 {
193 	.name = "iscsi",
194 	.init = cfiscsi_init,
195 	.ioctl = cfiscsi_ioctl,
196 	.shutdown = cfiscsi_shutdown,
197 };
198 CTL_FRONTEND_DECLARE(cfiscsi, cfiscsi_frontend);
199 MODULE_DEPEND(cfiscsi, icl, 1, 1, 1);
200 
201 static struct icl_pdu *
202 cfiscsi_pdu_new_response(struct icl_pdu *request, int flags)
203 {
204 
205 	return (icl_pdu_new(request->ip_conn, flags));
206 }
207 
208 static bool
209 cfiscsi_pdu_update_cmdsn(const struct icl_pdu *request)
210 {
211 	const struct iscsi_bhs_scsi_command *bhssc;
212 	struct cfiscsi_session *cs;
213 	uint32_t cmdsn, curcmdsn;
214 
215 	cs = PDU_SESSION(request);
216 
217 	/*
218 	 * Every incoming PDU - not just NOP-Out - resets the ping timer.
219 	 * The purpose of the timeout is to reset the connection when it stalls;
220 	 * we don't want this to happen when NOP-In or NOP-Out ends up delayed
221 	 * in some queue.
222 	 */
223 	cs->cs_timeout = 0;
224 
225 	/*
226 	 * Immediate commands carry cmdsn, but it is neither incremented nor
227 	 * verified.
228 	 */
229 	if (request->ip_bhs->bhs_opcode & ISCSI_BHS_OPCODE_IMMEDIATE)
230 		return (false);
231 
232 	/*
233 	 * Data-Out PDUs don't contain CmdSN.
234 	 */
235 	if (request->ip_bhs->bhs_opcode == ISCSI_BHS_OPCODE_SCSI_DATA_OUT)
236 		return (false);
237 
238 	/*
239 	 * We're only using fields common for all the request
240 	 * (initiator -> target) PDUs.
241 	 */
242 	bhssc = (const struct iscsi_bhs_scsi_command *)request->ip_bhs;
243 	curcmdsn = cmdsn = ntohl(bhssc->bhssc_cmdsn);
244 
245 	/*
246 	 * Increment session cmdsn and exit if we received the expected value.
247 	 */
248 	do {
249 		if (atomic_fcmpset_32(&cs->cs_cmdsn, &curcmdsn, cmdsn + 1))
250 			return (false);
251 	} while (curcmdsn == cmdsn);
252 
253 	/*
254 	 * The target MUST silently ignore any non-immediate command outside
255 	 * of this range.
256 	 */
257 	if (ISCSI_SNLT(cmdsn, curcmdsn) ||
258 	    ISCSI_SNGT(cmdsn, curcmdsn - 1 + maxtags)) {
259 		CFISCSI_SESSION_WARN(cs, "received PDU with CmdSN %u, "
260 		    "while expected %u", cmdsn, curcmdsn);
261 		return (true);
262 	}
263 
264 	/*
265 	 * We don't support multiple connections now, so any discontinuity in
266 	 * CmdSN means lost PDUs.  Since we don't support PDU retransmission --
267 	 * terminate the connection.
268 	 */
269 	CFISCSI_SESSION_WARN(cs, "received PDU with CmdSN %u, "
270 	    "while expected %u; dropping connection",
271 	    cmdsn, curcmdsn);
272 	cfiscsi_session_terminate(cs);
273 	return (true);
274 }
275 
276 static void
277 cfiscsi_pdu_handle(struct icl_pdu *request)
278 {
279 	struct cfiscsi_session *cs;
280 	bool ignore;
281 
282 	cs = PDU_SESSION(request);
283 
284 	ignore = cfiscsi_pdu_update_cmdsn(request);
285 	if (ignore) {
286 		icl_pdu_free(request);
287 		return;
288 	}
289 
290 	/*
291 	 * Handle the PDU; this includes e.g. receiving the remaining
292 	 * part of PDU and submitting the SCSI command to CTL
293 	 * or queueing a reply.  The handling routine is responsible
294 	 * for freeing the PDU when it's no longer needed.
295 	 */
296 	switch (request->ip_bhs->bhs_opcode &
297 	    ~ISCSI_BHS_OPCODE_IMMEDIATE) {
298 	case ISCSI_BHS_OPCODE_NOP_OUT:
299 		cfiscsi_pdu_handle_nop_out(request);
300 		break;
301 	case ISCSI_BHS_OPCODE_SCSI_COMMAND:
302 		cfiscsi_pdu_handle_scsi_command(request);
303 		break;
304 	case ISCSI_BHS_OPCODE_TASK_REQUEST:
305 		cfiscsi_pdu_handle_task_request(request);
306 		break;
307 	case ISCSI_BHS_OPCODE_SCSI_DATA_OUT:
308 		cfiscsi_pdu_handle_data_out(request);
309 		break;
310 	case ISCSI_BHS_OPCODE_LOGOUT_REQUEST:
311 		cfiscsi_pdu_handle_logout_request(request);
312 		break;
313 	default:
314 		CFISCSI_SESSION_WARN(cs, "received PDU with unsupported "
315 		    "opcode 0x%x; dropping connection",
316 		    request->ip_bhs->bhs_opcode);
317 		icl_pdu_free(request);
318 		cfiscsi_session_terminate(cs);
319 	}
320 
321 }
322 
323 static void
324 cfiscsi_receive_callback(struct icl_pdu *request)
325 {
326 #ifdef ICL_KERNEL_PROXY
327 	struct cfiscsi_session *cs;
328 
329 	cs = PDU_SESSION(request);
330 	if (cs->cs_waiting_for_ctld || cs->cs_login_phase) {
331 		if (cs->cs_login_pdu == NULL)
332 			cs->cs_login_pdu = request;
333 		else
334 			icl_pdu_free(request);
335 		cv_signal(&cs->cs_login_cv);
336 		return;
337 	}
338 #endif
339 
340 	cfiscsi_pdu_handle(request);
341 }
342 
343 static void
344 cfiscsi_error_callback(struct icl_conn *ic)
345 {
346 	struct cfiscsi_session *cs;
347 
348 	cs = CONN_SESSION(ic);
349 
350 	CFISCSI_SESSION_WARN(cs, "connection error; dropping connection");
351 	cfiscsi_session_terminate(cs);
352 }
353 
354 static int
355 cfiscsi_pdu_prepare(struct icl_pdu *response)
356 {
357 	struct cfiscsi_session *cs;
358 	struct iscsi_bhs_scsi_response *bhssr;
359 	bool advance_statsn = true;
360 	uint32_t cmdsn;
361 
362 	cs = PDU_SESSION(response);
363 
364 	CFISCSI_SESSION_LOCK_ASSERT(cs);
365 
366 	/*
367 	 * We're only using fields common for all the response
368 	 * (target -> initiator) PDUs.
369 	 */
370 	bhssr = (struct iscsi_bhs_scsi_response *)response->ip_bhs;
371 
372 	/*
373 	 * 10.8.3: "The StatSN for this connection is not advanced
374 	 * after this PDU is sent."
375 	 */
376 	if (bhssr->bhssr_opcode == ISCSI_BHS_OPCODE_R2T)
377 		advance_statsn = false;
378 
379 	/*
380 	 * 10.19.2: "However, when the Initiator Task Tag is set to 0xffffffff,
381 	 * StatSN for the connection is not advanced after this PDU is sent."
382 	 */
383 	if (bhssr->bhssr_opcode == ISCSI_BHS_OPCODE_NOP_IN &&
384 	    bhssr->bhssr_initiator_task_tag == 0xffffffff)
385 		advance_statsn = false;
386 
387 	/*
388 	 * See the comment below - StatSN is not meaningful and must
389 	 * not be advanced.
390 	 */
391 	if (bhssr->bhssr_opcode == ISCSI_BHS_OPCODE_SCSI_DATA_IN &&
392 	    (bhssr->bhssr_flags & BHSDI_FLAGS_S) == 0)
393 		advance_statsn = false;
394 
395 	/*
396 	 * 10.7.3: "The fields StatSN, Status, and Residual Count
397 	 * only have meaningful content if the S bit is set to 1."
398 	 */
399 	if (bhssr->bhssr_opcode != ISCSI_BHS_OPCODE_SCSI_DATA_IN ||
400 	    (bhssr->bhssr_flags & BHSDI_FLAGS_S))
401 		bhssr->bhssr_statsn = htonl(cs->cs_statsn);
402 	cmdsn = cs->cs_cmdsn;
403 	bhssr->bhssr_expcmdsn = htonl(cmdsn);
404 	bhssr->bhssr_maxcmdsn = htonl(cmdsn - 1 +
405 	    imax(0, maxtags - cs->cs_outstanding_ctl_pdus));
406 
407 	if (advance_statsn)
408 		cs->cs_statsn++;
409 
410 	return (0);
411 }
412 
413 static void
414 cfiscsi_pdu_queue(struct icl_pdu *response)
415 {
416 	struct cfiscsi_session *cs;
417 
418 	cs = PDU_SESSION(response);
419 
420 	CFISCSI_SESSION_LOCK(cs);
421 	cfiscsi_pdu_prepare(response);
422 	icl_pdu_queue(response);
423 	CFISCSI_SESSION_UNLOCK(cs);
424 }
425 
426  static void
427 cfiscsi_pdu_queue_cb(struct icl_pdu *response, icl_pdu_cb cb)
428 {
429 	struct cfiscsi_session *cs = PDU_SESSION(response);
430 
431 	CFISCSI_SESSION_LOCK(cs);
432 	cfiscsi_pdu_prepare(response);
433 	icl_pdu_queue_cb(response, cb);
434 	CFISCSI_SESSION_UNLOCK(cs);
435 }
436 
437 static void
438 cfiscsi_pdu_handle_nop_out(struct icl_pdu *request)
439 {
440 	struct cfiscsi_session *cs;
441 	struct iscsi_bhs_nop_out *bhsno;
442 	struct iscsi_bhs_nop_in *bhsni;
443 	struct icl_pdu *response;
444 	void *data = NULL;
445 	size_t datasize;
446 	int error;
447 
448 	cs = PDU_SESSION(request);
449 	bhsno = (struct iscsi_bhs_nop_out *)request->ip_bhs;
450 
451 	if (bhsno->bhsno_initiator_task_tag == 0xffffffff) {
452 		/*
453 		 * Nothing to do, iscsi_pdu_update_statsn() already
454 		 * zeroed the timeout.
455 		 */
456 		icl_pdu_free(request);
457 		return;
458 	}
459 
460 	datasize = icl_pdu_data_segment_length(request);
461 	if (datasize > 0) {
462 		data = malloc(datasize, M_CFISCSI, M_NOWAIT | M_ZERO);
463 		if (data == NULL) {
464 			CFISCSI_SESSION_WARN(cs, "failed to allocate memory; "
465 			    "dropping connection");
466 			icl_pdu_free(request);
467 			cfiscsi_session_terminate(cs);
468 			return;
469 		}
470 		icl_pdu_get_data(request, 0, data, datasize);
471 	}
472 
473 	response = cfiscsi_pdu_new_response(request, M_NOWAIT);
474 	if (response == NULL) {
475 		CFISCSI_SESSION_WARN(cs, "failed to allocate memory; "
476 		    "droppping connection");
477 		free(data, M_CFISCSI);
478 		icl_pdu_free(request);
479 		cfiscsi_session_terminate(cs);
480 		return;
481 	}
482 	bhsni = (struct iscsi_bhs_nop_in *)response->ip_bhs;
483 	bhsni->bhsni_opcode = ISCSI_BHS_OPCODE_NOP_IN;
484 	bhsni->bhsni_flags = 0x80;
485 	bhsni->bhsni_initiator_task_tag = bhsno->bhsno_initiator_task_tag;
486 	bhsni->bhsni_target_transfer_tag = 0xffffffff;
487 	if (datasize > 0) {
488 		error = icl_pdu_append_data(response, data, datasize, M_NOWAIT);
489 		if (error != 0) {
490 			CFISCSI_SESSION_WARN(cs, "failed to allocate memory; "
491 			    "dropping connection");
492 			free(data, M_CFISCSI);
493 			icl_pdu_free(request);
494 			icl_pdu_free(response);
495 			cfiscsi_session_terminate(cs);
496 			return;
497 		}
498 		free(data, M_CFISCSI);
499 	}
500 
501 	icl_pdu_free(request);
502 	cfiscsi_pdu_queue(response);
503 }
504 
505 static void
506 cfiscsi_pdu_handle_scsi_command(struct icl_pdu *request)
507 {
508 	struct iscsi_bhs_scsi_command *bhssc;
509 	struct cfiscsi_session *cs;
510 	union ctl_io *io;
511 	int error;
512 
513 	cs = PDU_SESSION(request);
514 	bhssc = (struct iscsi_bhs_scsi_command *)request->ip_bhs;
515 	//CFISCSI_SESSION_DEBUG(cs, "initiator task tag 0x%x",
516 	//    bhssc->bhssc_initiator_task_tag);
517 
518 	if (request->ip_data_len > 0 && cs->cs_immediate_data == false) {
519 		CFISCSI_SESSION_WARN(cs, "unsolicited data with "
520 		    "ImmediateData=No; dropping connection");
521 		icl_pdu_free(request);
522 		cfiscsi_session_terminate(cs);
523 		return;
524 	}
525 	io = ctl_alloc_io(cs->cs_target->ct_port.ctl_pool_ref);
526 	ctl_zero_io(io);
527 	PRIV_REQUEST(io) = request;
528 	io->io_hdr.io_type = CTL_IO_SCSI;
529 	io->io_hdr.nexus.initid = cs->cs_ctl_initid;
530 	io->io_hdr.nexus.targ_port = cs->cs_target->ct_port.targ_port;
531 	io->io_hdr.nexus.targ_lun = ctl_decode_lun(be64toh(bhssc->bhssc_lun));
532 	io->scsiio.priority = (bhssc->bhssc_pri & BHSSC_PRI_MASK) >>
533 	    BHSSC_PRI_SHIFT;
534 	io->scsiio.tag_num = bhssc->bhssc_initiator_task_tag;
535 	switch ((bhssc->bhssc_flags & BHSSC_FLAGS_ATTR)) {
536 	case BHSSC_FLAGS_ATTR_UNTAGGED:
537 		io->scsiio.tag_type = CTL_TAG_UNTAGGED;
538 		break;
539 	case BHSSC_FLAGS_ATTR_SIMPLE:
540 		io->scsiio.tag_type = CTL_TAG_SIMPLE;
541 		break;
542 	case BHSSC_FLAGS_ATTR_ORDERED:
543         	io->scsiio.tag_type = CTL_TAG_ORDERED;
544 		break;
545 	case BHSSC_FLAGS_ATTR_HOQ:
546         	io->scsiio.tag_type = CTL_TAG_HEAD_OF_QUEUE;
547 		break;
548 	case BHSSC_FLAGS_ATTR_ACA:
549 		io->scsiio.tag_type = CTL_TAG_ACA;
550 		break;
551 	default:
552 		io->scsiio.tag_type = CTL_TAG_UNTAGGED;
553 		CFISCSI_SESSION_WARN(cs, "unhandled tag type %d",
554 		    bhssc->bhssc_flags & BHSSC_FLAGS_ATTR);
555 		break;
556 	}
557 	io->scsiio.cdb_len = sizeof(bhssc->bhssc_cdb); /* Which is 16. */
558 	memcpy(io->scsiio.cdb, bhssc->bhssc_cdb, sizeof(bhssc->bhssc_cdb));
559 	refcount_acquire(&cs->cs_outstanding_ctl_pdus);
560 	error = ctl_run(io);
561 	if (error != CTL_RETVAL_COMPLETE) {
562 		CFISCSI_SESSION_WARN(cs, "ctl_run() failed; error %d; "
563 		    "dropping connection", error);
564 		ctl_free_io(io);
565 		refcount_release(&cs->cs_outstanding_ctl_pdus);
566 		icl_pdu_free(request);
567 		cfiscsi_session_terminate(cs);
568 	}
569 }
570 
571 static void
572 cfiscsi_pdu_handle_task_request(struct icl_pdu *request)
573 {
574 	struct iscsi_bhs_task_management_request *bhstmr;
575 	struct iscsi_bhs_task_management_response *bhstmr2;
576 	struct icl_pdu *response;
577 	struct cfiscsi_session *cs;
578 	union ctl_io *io;
579 	int error;
580 
581 	cs = PDU_SESSION(request);
582 	bhstmr = (struct iscsi_bhs_task_management_request *)request->ip_bhs;
583 	io = ctl_alloc_io(cs->cs_target->ct_port.ctl_pool_ref);
584 	ctl_zero_io(io);
585 	PRIV_REQUEST(io) = request;
586 	io->io_hdr.io_type = CTL_IO_TASK;
587 	io->io_hdr.nexus.initid = cs->cs_ctl_initid;
588 	io->io_hdr.nexus.targ_port = cs->cs_target->ct_port.targ_port;
589 	io->io_hdr.nexus.targ_lun = ctl_decode_lun(be64toh(bhstmr->bhstmr_lun));
590 	io->taskio.tag_type = CTL_TAG_SIMPLE; /* XXX */
591 
592 	switch (bhstmr->bhstmr_function & ~0x80) {
593 	case BHSTMR_FUNCTION_ABORT_TASK:
594 #if 0
595 		CFISCSI_SESSION_DEBUG(cs, "BHSTMR_FUNCTION_ABORT_TASK");
596 #endif
597 		io->taskio.task_action = CTL_TASK_ABORT_TASK;
598 		io->taskio.tag_num = bhstmr->bhstmr_referenced_task_tag;
599 		break;
600 	case BHSTMR_FUNCTION_ABORT_TASK_SET:
601 #if 0
602 		CFISCSI_SESSION_DEBUG(cs, "BHSTMR_FUNCTION_ABORT_TASK_SET");
603 #endif
604 		io->taskio.task_action = CTL_TASK_ABORT_TASK_SET;
605 		break;
606 	case BHSTMR_FUNCTION_CLEAR_TASK_SET:
607 #if 0
608 		CFISCSI_SESSION_DEBUG(cs, "BHSTMR_FUNCTION_CLEAR_TASK_SET");
609 #endif
610 		io->taskio.task_action = CTL_TASK_CLEAR_TASK_SET;
611 		break;
612 	case BHSTMR_FUNCTION_LOGICAL_UNIT_RESET:
613 #if 0
614 		CFISCSI_SESSION_DEBUG(cs, "BHSTMR_FUNCTION_LOGICAL_UNIT_RESET");
615 #endif
616 		io->taskio.task_action = CTL_TASK_LUN_RESET;
617 		break;
618 	case BHSTMR_FUNCTION_TARGET_WARM_RESET:
619 #if 0
620 		CFISCSI_SESSION_DEBUG(cs, "BHSTMR_FUNCTION_TARGET_WARM_RESET");
621 #endif
622 		io->taskio.task_action = CTL_TASK_TARGET_RESET;
623 		break;
624 	case BHSTMR_FUNCTION_TARGET_COLD_RESET:
625 #if 0
626 		CFISCSI_SESSION_DEBUG(cs, "BHSTMR_FUNCTION_TARGET_COLD_RESET");
627 #endif
628 		io->taskio.task_action = CTL_TASK_TARGET_RESET;
629 		break;
630 	case BHSTMR_FUNCTION_QUERY_TASK:
631 #if 0
632 		CFISCSI_SESSION_DEBUG(cs, "BHSTMR_FUNCTION_QUERY_TASK");
633 #endif
634 		io->taskio.task_action = CTL_TASK_QUERY_TASK;
635 		io->taskio.tag_num = bhstmr->bhstmr_referenced_task_tag;
636 		break;
637 	case BHSTMR_FUNCTION_QUERY_TASK_SET:
638 #if 0
639 		CFISCSI_SESSION_DEBUG(cs, "BHSTMR_FUNCTION_QUERY_TASK_SET");
640 #endif
641 		io->taskio.task_action = CTL_TASK_QUERY_TASK_SET;
642 		break;
643 	case BHSTMR_FUNCTION_I_T_NEXUS_RESET:
644 #if 0
645 		CFISCSI_SESSION_DEBUG(cs, "BHSTMR_FUNCTION_I_T_NEXUS_RESET");
646 #endif
647 		io->taskio.task_action = CTL_TASK_I_T_NEXUS_RESET;
648 		break;
649 	case BHSTMR_FUNCTION_QUERY_ASYNC_EVENT:
650 #if 0
651 		CFISCSI_SESSION_DEBUG(cs, "BHSTMR_FUNCTION_QUERY_ASYNC_EVENT");
652 #endif
653 		io->taskio.task_action = CTL_TASK_QUERY_ASYNC_EVENT;
654 		break;
655 	default:
656 		CFISCSI_SESSION_DEBUG(cs, "unsupported function 0x%x",
657 		    bhstmr->bhstmr_function & ~0x80);
658 		ctl_free_io(io);
659 
660 		response = cfiscsi_pdu_new_response(request, M_NOWAIT);
661 		if (response == NULL) {
662 			CFISCSI_SESSION_WARN(cs, "failed to allocate memory; "
663 			    "dropping connection");
664 			icl_pdu_free(request);
665 			cfiscsi_session_terminate(cs);
666 			return;
667 		}
668 		bhstmr2 = (struct iscsi_bhs_task_management_response *)
669 		    response->ip_bhs;
670 		bhstmr2->bhstmr_opcode = ISCSI_BHS_OPCODE_TASK_RESPONSE;
671 		bhstmr2->bhstmr_flags = 0x80;
672 		bhstmr2->bhstmr_response =
673 		    BHSTMR_RESPONSE_FUNCTION_NOT_SUPPORTED;
674 		bhstmr2->bhstmr_initiator_task_tag =
675 		    bhstmr->bhstmr_initiator_task_tag;
676 		icl_pdu_free(request);
677 		cfiscsi_pdu_queue(response);
678 		return;
679 	}
680 
681 	refcount_acquire(&cs->cs_outstanding_ctl_pdus);
682 	error = ctl_run(io);
683 	if (error != CTL_RETVAL_COMPLETE) {
684 		CFISCSI_SESSION_WARN(cs, "ctl_run() failed; error %d; "
685 		    "dropping connection", error);
686 		ctl_free_io(io);
687 		refcount_release(&cs->cs_outstanding_ctl_pdus);
688 		icl_pdu_free(request);
689 		cfiscsi_session_terminate(cs);
690 	}
691 }
692 
693 static bool
694 cfiscsi_handle_data_segment(struct icl_pdu *request, struct cfiscsi_data_wait *cdw)
695 {
696 	struct iscsi_bhs_data_out *bhsdo;
697 	struct cfiscsi_session *cs;
698 	struct ctl_sg_entry ctl_sg_entry, *ctl_sglist;
699 	size_t copy_len, len, off, buffer_offset;
700 	int ctl_sg_count;
701 	union ctl_io *io;
702 
703 	cs = PDU_SESSION(request);
704 
705 	KASSERT((request->ip_bhs->bhs_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
706 	    ISCSI_BHS_OPCODE_SCSI_DATA_OUT ||
707 	    (request->ip_bhs->bhs_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
708 	    ISCSI_BHS_OPCODE_SCSI_COMMAND,
709 	    ("bad opcode 0x%x", request->ip_bhs->bhs_opcode));
710 
711 	/*
712 	 * We're only using fields common for Data-Out and SCSI Command PDUs.
713 	 */
714 	bhsdo = (struct iscsi_bhs_data_out *)request->ip_bhs;
715 
716 	io = cdw->cdw_ctl_io;
717 	KASSERT((io->io_hdr.flags & CTL_FLAG_DATA_MASK) != CTL_FLAG_DATA_IN,
718 	    ("CTL_FLAG_DATA_IN"));
719 
720 #if 0
721 	CFISCSI_SESSION_DEBUG(cs, "received %zd bytes out of %d",
722 	    request->ip_data_len, io->scsiio.kern_total_len);
723 #endif
724 
725 	if (io->scsiio.kern_sg_entries > 0) {
726 		ctl_sglist = (struct ctl_sg_entry *)io->scsiio.kern_data_ptr;
727 		ctl_sg_count = io->scsiio.kern_sg_entries;
728 	} else {
729 		ctl_sglist = &ctl_sg_entry;
730 		ctl_sglist->addr = io->scsiio.kern_data_ptr;
731 		ctl_sglist->len = io->scsiio.kern_data_len;
732 		ctl_sg_count = 1;
733 	}
734 
735 	if ((request->ip_bhs->bhs_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
736 	    ISCSI_BHS_OPCODE_SCSI_DATA_OUT)
737 		buffer_offset = ntohl(bhsdo->bhsdo_buffer_offset);
738 	else
739 		buffer_offset = 0;
740 	len = icl_pdu_data_segment_length(request);
741 
742 	/*
743 	 * Make sure the offset, as sent by the initiator, matches the offset
744 	 * we're supposed to be at in the scatter-gather list.
745 	 */
746 	if (buffer_offset >
747 	    io->scsiio.kern_rel_offset + io->scsiio.ext_data_filled ||
748 	    buffer_offset + len <=
749 	    io->scsiio.kern_rel_offset + io->scsiio.ext_data_filled) {
750 		CFISCSI_SESSION_WARN(cs, "received bad buffer offset %zd, "
751 		    "expected %zd; dropping connection", buffer_offset,
752 		    (size_t)io->scsiio.kern_rel_offset +
753 		    (size_t)io->scsiio.ext_data_filled);
754 		ctl_set_data_phase_error(&io->scsiio);
755 		cfiscsi_session_terminate(cs);
756 		return (true);
757 	}
758 
759 	/*
760 	 * This is the offset within the PDU data segment, as opposed
761 	 * to buffer_offset, which is the offset within the task (SCSI
762 	 * command).
763 	 */
764 	off = io->scsiio.kern_rel_offset + io->scsiio.ext_data_filled -
765 	    buffer_offset;
766 
767 	/*
768 	 * Iterate over the scatter/gather segments, filling them with data
769 	 * from the PDU data segment.  Note that this can get called multiple
770 	 * times for one SCSI command; the cdw structure holds state for the
771 	 * scatter/gather list.
772 	 */
773 	for (;;) {
774 		KASSERT(cdw->cdw_sg_index < ctl_sg_count,
775 		    ("cdw->cdw_sg_index >= ctl_sg_count"));
776 		if (cdw->cdw_sg_len == 0) {
777 			cdw->cdw_sg_addr = ctl_sglist[cdw->cdw_sg_index].addr;
778 			cdw->cdw_sg_len = ctl_sglist[cdw->cdw_sg_index].len;
779 		}
780 		KASSERT(off <= len, ("len > off"));
781 		copy_len = len - off;
782 		if (copy_len > cdw->cdw_sg_len)
783 			copy_len = cdw->cdw_sg_len;
784 
785 		icl_pdu_get_data(request, off, cdw->cdw_sg_addr, copy_len);
786 		cdw->cdw_sg_addr += copy_len;
787 		cdw->cdw_sg_len -= copy_len;
788 		off += copy_len;
789 		io->scsiio.ext_data_filled += copy_len;
790 		io->scsiio.kern_data_resid -= copy_len;
791 
792 		if (cdw->cdw_sg_len == 0) {
793 			/*
794 			 * End of current segment.
795 			 */
796 			if (cdw->cdw_sg_index == ctl_sg_count - 1) {
797 				/*
798 				 * Last segment in scatter/gather list.
799 				 */
800 				break;
801 			}
802 			cdw->cdw_sg_index++;
803 		}
804 
805 		if (off == len) {
806 			/*
807 			 * End of PDU payload.
808 			 */
809 			break;
810 		}
811 	}
812 
813 	if (len > off) {
814 		/*
815 		 * In case of unsolicited data, it's possible that the buffer
816 		 * provided by CTL is smaller than negotiated FirstBurstLength.
817 		 * Just ignore the superfluous data; will ask for them with R2T
818 		 * on next call to cfiscsi_datamove().
819 		 *
820 		 * This obviously can only happen with SCSI Command PDU.
821 		 */
822 		if ((request->ip_bhs->bhs_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
823 		    ISCSI_BHS_OPCODE_SCSI_COMMAND)
824 			return (true);
825 
826 		CFISCSI_SESSION_WARN(cs, "received too much data: got %zd bytes, "
827 		    "expected %zd; dropping connection",
828 		    icl_pdu_data_segment_length(request), off);
829 		ctl_set_data_phase_error(&io->scsiio);
830 		cfiscsi_session_terminate(cs);
831 		return (true);
832 	}
833 
834 	if (io->scsiio.ext_data_filled == cdw->cdw_r2t_end &&
835 	    (bhsdo->bhsdo_flags & BHSDO_FLAGS_F) == 0) {
836 		CFISCSI_SESSION_WARN(cs, "got the final packet without "
837 		    "the F flag; flags = 0x%x; dropping connection",
838 		    bhsdo->bhsdo_flags);
839 		ctl_set_data_phase_error(&io->scsiio);
840 		cfiscsi_session_terminate(cs);
841 		return (true);
842 	}
843 
844 	if (io->scsiio.ext_data_filled != cdw->cdw_r2t_end &&
845 	    (bhsdo->bhsdo_flags & BHSDO_FLAGS_F) != 0) {
846 		if ((request->ip_bhs->bhs_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
847 		    ISCSI_BHS_OPCODE_SCSI_DATA_OUT) {
848 			CFISCSI_SESSION_WARN(cs, "got the final packet, but the "
849 			    "transmitted size was %zd bytes instead of %d; "
850 			    "dropping connection",
851 			    (size_t)io->scsiio.ext_data_filled,
852 			    cdw->cdw_r2t_end);
853 			ctl_set_data_phase_error(&io->scsiio);
854 			cfiscsi_session_terminate(cs);
855 			return (true);
856 		} else {
857 			/*
858 			 * For SCSI Command PDU, this just means we need to
859 			 * solicit more data by sending R2T.
860 			 */
861 			return (false);
862 		}
863 	}
864 
865 	if (io->scsiio.ext_data_filled == cdw->cdw_r2t_end) {
866 #if 0
867 		CFISCSI_SESSION_DEBUG(cs, "no longer expecting Data-Out with target "
868 		    "transfer tag 0x%x", cdw->cdw_target_transfer_tag);
869 #endif
870 
871 		return (true);
872 	}
873 
874 	return (false);
875 }
876 
877 static void
878 cfiscsi_pdu_handle_data_out(struct icl_pdu *request)
879 {
880 	struct iscsi_bhs_data_out *bhsdo;
881 	struct cfiscsi_session *cs;
882 	struct cfiscsi_data_wait *cdw = NULL;
883 	union ctl_io *io;
884 	bool done;
885 
886 	cs = PDU_SESSION(request);
887 	bhsdo = (struct iscsi_bhs_data_out *)request->ip_bhs;
888 
889 	CFISCSI_SESSION_LOCK(cs);
890 	TAILQ_FOREACH(cdw, &cs->cs_waiting_for_data_out, cdw_next) {
891 #if 0
892 		CFISCSI_SESSION_DEBUG(cs, "have ttt 0x%x, itt 0x%x; looking for "
893 		    "ttt 0x%x, itt 0x%x",
894 		    bhsdo->bhsdo_target_transfer_tag,
895 		    bhsdo->bhsdo_initiator_task_tag,
896 		    cdw->cdw_target_transfer_tag, cdw->cdw_initiator_task_tag));
897 #endif
898 		if (bhsdo->bhsdo_target_transfer_tag ==
899 		    cdw->cdw_target_transfer_tag)
900 			break;
901 	}
902 	CFISCSI_SESSION_UNLOCK(cs);
903 	if (cdw == NULL) {
904 		CFISCSI_SESSION_WARN(cs, "data transfer tag 0x%x, initiator task tag "
905 		    "0x%x, not found; dropping connection",
906 		    bhsdo->bhsdo_target_transfer_tag, bhsdo->bhsdo_initiator_task_tag);
907 		icl_pdu_free(request);
908 		cfiscsi_session_terminate(cs);
909 		return;
910 	}
911 
912 	if (cdw->cdw_datasn != ntohl(bhsdo->bhsdo_datasn)) {
913 		CFISCSI_SESSION_WARN(cs, "received Data-Out PDU with "
914 		    "DataSN %u, while expected %u; dropping connection",
915 		    ntohl(bhsdo->bhsdo_datasn), cdw->cdw_datasn);
916 		icl_pdu_free(request);
917 		cfiscsi_session_terminate(cs);
918 		return;
919 	}
920 	cdw->cdw_datasn += request->ip_additional_pdus + 1;
921 
922 	io = cdw->cdw_ctl_io;
923 	KASSERT((io->io_hdr.flags & CTL_FLAG_DATA_MASK) != CTL_FLAG_DATA_IN,
924 	    ("CTL_FLAG_DATA_IN"));
925 
926 	done = cfiscsi_handle_data_segment(request, cdw);
927 	if (done) {
928 		CFISCSI_SESSION_LOCK(cs);
929 		TAILQ_REMOVE(&cs->cs_waiting_for_data_out, cdw, cdw_next);
930 		CFISCSI_SESSION_UNLOCK(cs);
931 		done = (io->scsiio.ext_data_filled != cdw->cdw_r2t_end ||
932 		    io->scsiio.ext_data_filled == io->scsiio.kern_data_len);
933 		cfiscsi_data_wait_free(cs, cdw);
934 		io->io_hdr.flags &= ~CTL_FLAG_DMA_INPROG;
935 		if (done)
936 			ctl_datamove_done(io, false);
937 		else
938 			cfiscsi_datamove_out(io);
939 	}
940 
941 	icl_pdu_free(request);
942 }
943 
944 static void
945 cfiscsi_pdu_handle_logout_request(struct icl_pdu *request)
946 {
947 	struct iscsi_bhs_logout_request *bhslr;
948 	struct iscsi_bhs_logout_response *bhslr2;
949 	struct icl_pdu *response;
950 	struct cfiscsi_session *cs;
951 
952 	cs = PDU_SESSION(request);
953 	bhslr = (struct iscsi_bhs_logout_request *)request->ip_bhs;
954 	switch (bhslr->bhslr_reason & 0x7f) {
955 	case BHSLR_REASON_CLOSE_SESSION:
956 	case BHSLR_REASON_CLOSE_CONNECTION:
957 		response = cfiscsi_pdu_new_response(request, M_NOWAIT);
958 		if (response == NULL) {
959 			CFISCSI_SESSION_DEBUG(cs, "failed to allocate memory");
960 			icl_pdu_free(request);
961 			cfiscsi_session_terminate(cs);
962 			return;
963 		}
964 		bhslr2 = (struct iscsi_bhs_logout_response *)response->ip_bhs;
965 		bhslr2->bhslr_opcode = ISCSI_BHS_OPCODE_LOGOUT_RESPONSE;
966 		bhslr2->bhslr_flags = 0x80;
967 		bhslr2->bhslr_response = BHSLR_RESPONSE_CLOSED_SUCCESSFULLY;
968 		bhslr2->bhslr_initiator_task_tag =
969 		    bhslr->bhslr_initiator_task_tag;
970 		icl_pdu_free(request);
971 		cfiscsi_pdu_queue(response);
972 		cfiscsi_session_terminate(cs);
973 		break;
974 	case BHSLR_REASON_REMOVE_FOR_RECOVERY:
975 		response = cfiscsi_pdu_new_response(request, M_NOWAIT);
976 		if (response == NULL) {
977 			CFISCSI_SESSION_WARN(cs,
978 			    "failed to allocate memory; dropping connection");
979 			icl_pdu_free(request);
980 			cfiscsi_session_terminate(cs);
981 			return;
982 		}
983 		bhslr2 = (struct iscsi_bhs_logout_response *)response->ip_bhs;
984 		bhslr2->bhslr_opcode = ISCSI_BHS_OPCODE_LOGOUT_RESPONSE;
985 		bhslr2->bhslr_flags = 0x80;
986 		bhslr2->bhslr_response = BHSLR_RESPONSE_RECOVERY_NOT_SUPPORTED;
987 		bhslr2->bhslr_initiator_task_tag =
988 		    bhslr->bhslr_initiator_task_tag;
989 		icl_pdu_free(request);
990 		cfiscsi_pdu_queue(response);
991 		break;
992 	default:
993 		CFISCSI_SESSION_WARN(cs, "invalid reason 0%x; dropping connection",
994 		    bhslr->bhslr_reason);
995 		icl_pdu_free(request);
996 		cfiscsi_session_terminate(cs);
997 		break;
998 	}
999 }
1000 
1001 static void
1002 cfiscsi_callout(void *context)
1003 {
1004 	struct icl_pdu *cp;
1005 	struct iscsi_bhs_nop_in *bhsni;
1006 	struct cfiscsi_session *cs;
1007 
1008 	cs = context;
1009 
1010 	if (cs->cs_terminating)
1011 		return;
1012 
1013 	callout_schedule(&cs->cs_callout, 1 * hz);
1014 
1015 	atomic_add_int(&cs->cs_timeout, 1);
1016 
1017 #ifdef ICL_KERNEL_PROXY
1018 	if (cs->cs_waiting_for_ctld || cs->cs_login_phase) {
1019 		if (login_timeout > 0 && cs->cs_timeout > login_timeout) {
1020 			CFISCSI_SESSION_WARN(cs, "login timed out after "
1021 			    "%d seconds; dropping connection", cs->cs_timeout);
1022 			cfiscsi_session_terminate(cs);
1023 		}
1024 		return;
1025 	}
1026 #endif
1027 
1028 	if (ping_timeout <= 0) {
1029 		/*
1030 		 * Pings are disabled.  Don't send NOP-In in this case;
1031 		 * user might have disabled pings to work around problems
1032 		 * with certain initiators that can't properly handle
1033 		 * NOP-In, such as iPXE.  Reset the timeout, to avoid
1034 		 * triggering reconnection, should the user decide to
1035 		 * reenable them.
1036 		 */
1037 		cs->cs_timeout = 0;
1038 		return;
1039 	}
1040 
1041 	if (cs->cs_timeout >= ping_timeout) {
1042 		CFISCSI_SESSION_WARN(cs, "no ping reply (NOP-Out) after %d seconds; "
1043 		    "dropping connection",  ping_timeout);
1044 		cfiscsi_session_terminate(cs);
1045 		return;
1046 	}
1047 
1048 	/*
1049 	 * If the ping was reset less than one second ago - which means
1050 	 * that we've received some PDU during the last second - assume
1051 	 * the traffic flows correctly and don't bother sending a NOP-Out.
1052 	 *
1053 	 * (It's 2 - one for one second, and one for incrementing is_timeout
1054 	 * earlier in this routine.)
1055 	 */
1056 	if (cs->cs_timeout < 2)
1057 		return;
1058 
1059 	cp = icl_pdu_new(cs->cs_conn, M_NOWAIT);
1060 	if (cp == NULL) {
1061 		CFISCSI_SESSION_WARN(cs, "failed to allocate memory");
1062 		return;
1063 	}
1064 	bhsni = (struct iscsi_bhs_nop_in *)cp->ip_bhs;
1065 	bhsni->bhsni_opcode = ISCSI_BHS_OPCODE_NOP_IN;
1066 	bhsni->bhsni_flags = 0x80;
1067 	bhsni->bhsni_initiator_task_tag = 0xffffffff;
1068 
1069 	cfiscsi_pdu_queue(cp);
1070 }
1071 
1072 static struct cfiscsi_data_wait *
1073 cfiscsi_data_wait_new(struct cfiscsi_session *cs, union ctl_io *io,
1074     uint32_t initiator_task_tag, uint32_t *target_transfer_tagp)
1075 {
1076 	struct cfiscsi_data_wait *cdw;
1077 	int error;
1078 
1079 	cdw = uma_zalloc(cfiscsi_data_wait_zone, M_NOWAIT | M_ZERO);
1080 	if (cdw == NULL) {
1081 		CFISCSI_SESSION_WARN(cs,
1082 		    "failed to allocate %zd bytes", sizeof(*cdw));
1083 		return (NULL);
1084 	}
1085 
1086 	error = icl_conn_transfer_setup(cs->cs_conn, io, target_transfer_tagp,
1087 	    &cdw->cdw_icl_prv);
1088 	if (error != 0) {
1089 		CFISCSI_SESSION_WARN(cs,
1090 		    "icl_conn_transfer_setup() failed with error %d", error);
1091 		uma_zfree(cfiscsi_data_wait_zone, cdw);
1092 		return (NULL);
1093 	}
1094 
1095 	cdw->cdw_ctl_io = io;
1096 	cdw->cdw_target_transfer_tag = *target_transfer_tagp;
1097 	cdw->cdw_initiator_task_tag = initiator_task_tag;
1098 
1099 	return (cdw);
1100 }
1101 
1102 static void
1103 cfiscsi_data_wait_free(struct cfiscsi_session *cs,
1104     struct cfiscsi_data_wait *cdw)
1105 {
1106 
1107 	icl_conn_transfer_done(cs->cs_conn, cdw->cdw_icl_prv);
1108 	uma_zfree(cfiscsi_data_wait_zone, cdw);
1109 }
1110 
1111 static void
1112 cfiscsi_session_terminate_tasks(struct cfiscsi_session *cs)
1113 {
1114 	struct cfiscsi_data_wait *cdw;
1115 	union ctl_io *io, *cdw_io;
1116 	int error, last, wait;
1117 
1118 	if (cs->cs_target == NULL)
1119 		return;		/* No target yet, so nothing to do. */
1120 	io = ctl_alloc_io(cs->cs_target->ct_port.ctl_pool_ref);
1121 	ctl_zero_io(io);
1122 	PRIV_REQUEST(io) = cs;
1123 	io->io_hdr.io_type = CTL_IO_TASK;
1124 	io->io_hdr.nexus.initid = cs->cs_ctl_initid;
1125 	io->io_hdr.nexus.targ_port = cs->cs_target->ct_port.targ_port;
1126 	io->io_hdr.nexus.targ_lun = 0;
1127 	io->taskio.tag_type = CTL_TAG_SIMPLE; /* XXX */
1128 	io->taskio.task_action = CTL_TASK_I_T_NEXUS_RESET;
1129 	wait = cs->cs_outstanding_ctl_pdus;
1130 	refcount_acquire(&cs->cs_outstanding_ctl_pdus);
1131 	error = ctl_run(io);
1132 	if (error != CTL_RETVAL_COMPLETE) {
1133 		CFISCSI_SESSION_WARN(cs, "ctl_run() failed; error %d", error);
1134 		refcount_release(&cs->cs_outstanding_ctl_pdus);
1135 		ctl_free_io(io);
1136 	}
1137 
1138 	CFISCSI_SESSION_LOCK(cs);
1139 	while ((cdw = TAILQ_FIRST(&cs->cs_waiting_for_data_out)) != NULL) {
1140 		TAILQ_REMOVE(&cs->cs_waiting_for_data_out, cdw, cdw_next);
1141 		CFISCSI_SESSION_UNLOCK(cs);
1142 		/*
1143 		 * Set nonzero port status; this prevents backends from
1144 		 * assuming that the data transfer actually succeeded
1145 		 * and writing uninitialized data to disk.
1146 		 */
1147 		cdw_io = cdw->cdw_ctl_io;
1148 		cdw_io->io_hdr.flags &= ~CTL_FLAG_DMA_INPROG;
1149 		cdw_io->scsiio.io_hdr.port_status = 42;
1150 		cfiscsi_data_wait_free(cs, cdw);
1151 		ctl_datamove_done(cdw_io, false);
1152 		CFISCSI_SESSION_LOCK(cs);
1153 	}
1154 	CFISCSI_SESSION_UNLOCK(cs);
1155 
1156 	/*
1157 	 * Wait for CTL to terminate all the tasks.
1158 	 */
1159 	if (wait > 0)
1160 		CFISCSI_SESSION_WARN(cs,
1161 		    "waiting for CTL to terminate %d tasks", wait);
1162 	for (;;) {
1163 		refcount_acquire(&cs->cs_outstanding_ctl_pdus);
1164 		last = refcount_release(&cs->cs_outstanding_ctl_pdus);
1165 		if (last != 0)
1166 			break;
1167 		tsleep(__DEVOLATILE(void *, &cs->cs_outstanding_ctl_pdus),
1168 		    0, "cfiscsi_terminate", hz / 100);
1169 	}
1170 	if (wait > 0)
1171 		CFISCSI_SESSION_WARN(cs, "tasks terminated");
1172 }
1173 
1174 static void
1175 cfiscsi_maintenance_thread(void *arg)
1176 {
1177 	struct cfiscsi_session *cs;
1178 
1179 	cs = arg;
1180 
1181 	for (;;) {
1182 		CFISCSI_SESSION_LOCK(cs);
1183 		if (cs->cs_terminating == false || cs->cs_handoff_in_progress)
1184 			cv_wait(&cs->cs_maintenance_cv, &cs->cs_lock);
1185 		CFISCSI_SESSION_UNLOCK(cs);
1186 
1187 		if (cs->cs_terminating && cs->cs_handoff_in_progress == false) {
1188 			/*
1189 			 * We used to wait up to 30 seconds to deliver queued
1190 			 * PDUs to the initiator.  We also tried hard to deliver
1191 			 * SCSI Responses for the aborted PDUs.  We don't do
1192 			 * that anymore.  We might need to revisit that.
1193 			 */
1194 			callout_drain(&cs->cs_callout);
1195 			icl_conn_close(cs->cs_conn);
1196 
1197 			/*
1198 			 * At this point ICL receive thread is no longer
1199 			 * running; no new tasks can be queued.
1200 			 */
1201 			cfiscsi_session_terminate_tasks(cs);
1202 			cfiscsi_session_delete(cs);
1203 			kthread_exit();
1204 			return;
1205 		}
1206 		CFISCSI_SESSION_DEBUG(cs, "nothing to do");
1207 	}
1208 }
1209 
1210 static void
1211 cfiscsi_session_terminate(struct cfiscsi_session *cs)
1212 {
1213 
1214 	cs->cs_terminating = true;
1215 	cv_signal(&cs->cs_maintenance_cv);
1216 #ifdef ICL_KERNEL_PROXY
1217 	cv_signal(&cs->cs_login_cv);
1218 #endif
1219 }
1220 
1221 static int
1222 cfiscsi_session_register_initiator(struct cfiscsi_session *cs)
1223 {
1224 	struct cfiscsi_target *ct;
1225 	char *name;
1226 	int i;
1227 
1228 	KASSERT(cs->cs_ctl_initid == -1, ("already registered"));
1229 
1230 	ct = cs->cs_target;
1231 	name = strdup(cs->cs_initiator_id, M_CTL);
1232 	i = ctl_add_initiator(&ct->ct_port, -1, 0, name);
1233 	if (i < 0) {
1234 		CFISCSI_SESSION_WARN(cs, "ctl_add_initiator failed with error %d",
1235 		    i);
1236 		cs->cs_ctl_initid = -1;
1237 		return (1);
1238 	}
1239 	cs->cs_ctl_initid = i;
1240 #if 0
1241 	CFISCSI_SESSION_DEBUG(cs, "added initiator id %d", i);
1242 #endif
1243 
1244 	return (0);
1245 }
1246 
1247 static void
1248 cfiscsi_session_unregister_initiator(struct cfiscsi_session *cs)
1249 {
1250 	int error;
1251 
1252 	if (cs->cs_ctl_initid == -1)
1253 		return;
1254 
1255 	error = ctl_remove_initiator(&cs->cs_target->ct_port, cs->cs_ctl_initid);
1256 	if (error != 0) {
1257 		CFISCSI_SESSION_WARN(cs, "ctl_remove_initiator failed with error %d",
1258 		    error);
1259 	}
1260 	cs->cs_ctl_initid = -1;
1261 }
1262 
1263 static struct cfiscsi_session *
1264 cfiscsi_session_new(struct cfiscsi_softc *softc, const char *offload)
1265 {
1266 	struct cfiscsi_session *cs;
1267 	int error;
1268 
1269 	cs = malloc(sizeof(*cs), M_CFISCSI, M_NOWAIT | M_ZERO);
1270 	if (cs == NULL) {
1271 		CFISCSI_WARN("malloc failed");
1272 		return (NULL);
1273 	}
1274 	cs->cs_ctl_initid = -1;
1275 
1276 	refcount_init(&cs->cs_outstanding_ctl_pdus, 0);
1277 	TAILQ_INIT(&cs->cs_waiting_for_data_out);
1278 	mtx_init(&cs->cs_lock, "cfiscsi_lock", NULL, MTX_DEF);
1279 	cv_init(&cs->cs_maintenance_cv, "cfiscsi_mt");
1280 #ifdef ICL_KERNEL_PROXY
1281 	cv_init(&cs->cs_login_cv, "cfiscsi_login");
1282 #endif
1283 
1284 	/*
1285 	 * The purpose of this is to avoid racing with session shutdown.
1286 	 * Otherwise we could have the maintenance thread call icl_conn_close()
1287 	 * before we call icl_conn_handoff().
1288 	 */
1289 	cs->cs_handoff_in_progress = true;
1290 
1291 	cs->cs_conn = icl_new_conn(offload, false, "cfiscsi", &cs->cs_lock);
1292 	if (cs->cs_conn == NULL) {
1293 		free(cs, M_CFISCSI);
1294 		return (NULL);
1295 	}
1296 	cs->cs_conn->ic_receive = cfiscsi_receive_callback;
1297 	cs->cs_conn->ic_error = cfiscsi_error_callback;
1298 	cs->cs_conn->ic_prv0 = cs;
1299 
1300 	error = kthread_add(cfiscsi_maintenance_thread, cs, NULL, NULL, 0, 0, "cfiscsimt");
1301 	if (error != 0) {
1302 		CFISCSI_SESSION_WARN(cs, "kthread_add(9) failed with error %d", error);
1303 		free(cs, M_CFISCSI);
1304 		return (NULL);
1305 	}
1306 
1307 	mtx_lock(&softc->lock);
1308 	cs->cs_id = ++softc->last_session_id;
1309 	TAILQ_INSERT_TAIL(&softc->sessions, cs, cs_next);
1310 	mtx_unlock(&softc->lock);
1311 
1312 	/*
1313 	 * Start pinging the initiator.
1314 	 */
1315 	callout_init(&cs->cs_callout, 1);
1316 	callout_reset(&cs->cs_callout, 1 * hz, cfiscsi_callout, cs);
1317 
1318 	return (cs);
1319 }
1320 
1321 static void
1322 cfiscsi_session_delete(struct cfiscsi_session *cs)
1323 {
1324 	struct cfiscsi_softc *softc;
1325 
1326 	softc = &cfiscsi_softc;
1327 
1328 	KASSERT(cs->cs_outstanding_ctl_pdus == 0,
1329 	    ("destroying session with outstanding CTL pdus"));
1330 	KASSERT(TAILQ_EMPTY(&cs->cs_waiting_for_data_out),
1331 	    ("destroying session with non-empty queue"));
1332 
1333 	mtx_lock(&softc->lock);
1334 	TAILQ_REMOVE(&softc->sessions, cs, cs_next);
1335 	mtx_unlock(&softc->lock);
1336 
1337 	cfiscsi_session_unregister_initiator(cs);
1338 	if (cs->cs_target != NULL)
1339 		cfiscsi_target_release(cs->cs_target);
1340 	icl_conn_close(cs->cs_conn);
1341 	icl_conn_free(cs->cs_conn);
1342 	free(cs, M_CFISCSI);
1343 	cv_signal(&softc->sessions_cv);
1344 }
1345 
1346 static int
1347 cfiscsi_init(void)
1348 {
1349 	struct cfiscsi_softc *softc;
1350 
1351 	softc = &cfiscsi_softc;
1352 	bzero(softc, sizeof(*softc));
1353 	mtx_init(&softc->lock, "cfiscsi", NULL, MTX_DEF);
1354 
1355 	cv_init(&softc->sessions_cv, "cfiscsi_sessions");
1356 #ifdef ICL_KERNEL_PROXY
1357 	cv_init(&softc->accept_cv, "cfiscsi_accept");
1358 #endif
1359 	TAILQ_INIT(&softc->sessions);
1360 	TAILQ_INIT(&softc->targets);
1361 
1362 	cfiscsi_data_wait_zone = uma_zcreate("cfiscsi_data_wait",
1363 	    sizeof(struct cfiscsi_data_wait), NULL, NULL, NULL, NULL,
1364 	    UMA_ALIGN_PTR, 0);
1365 
1366 	return (0);
1367 }
1368 
1369 static int
1370 cfiscsi_shutdown(void)
1371 {
1372 	struct cfiscsi_softc *softc = &cfiscsi_softc;
1373 
1374 	if (!TAILQ_EMPTY(&softc->sessions) || !TAILQ_EMPTY(&softc->targets))
1375 		return (EBUSY);
1376 
1377 	uma_zdestroy(cfiscsi_data_wait_zone);
1378 #ifdef ICL_KERNEL_PROXY
1379 	cv_destroy(&softc->accept_cv);
1380 #endif
1381 	cv_destroy(&softc->sessions_cv);
1382 	mtx_destroy(&softc->lock);
1383 	return (0);
1384 }
1385 
1386 #ifdef ICL_KERNEL_PROXY
1387 static void
1388 cfiscsi_accept(struct socket *so, struct sockaddr *sa, int portal_id)
1389 {
1390 	struct cfiscsi_session *cs;
1391 
1392 	cs = cfiscsi_session_new(&cfiscsi_softc, NULL);
1393 	if (cs == NULL) {
1394 		CFISCSI_WARN("failed to create session");
1395 		return;
1396 	}
1397 
1398 	icl_conn_handoff_sock(cs->cs_conn, so);
1399 	cs->cs_initiator_sa = sa;
1400 	cs->cs_portal_id = portal_id;
1401 	cs->cs_handoff_in_progress = false;
1402 	cs->cs_waiting_for_ctld = true;
1403 	cv_signal(&cfiscsi_softc.accept_cv);
1404 
1405 	CFISCSI_SESSION_LOCK(cs);
1406 	/*
1407 	 * Wake up the maintenance thread if we got scheduled for termination
1408 	 * somewhere between cfiscsi_session_new() and icl_conn_handoff_sock().
1409 	 */
1410 	if (cs->cs_terminating)
1411 		cfiscsi_session_terminate(cs);
1412 	CFISCSI_SESSION_UNLOCK(cs);
1413 }
1414 #endif
1415 
1416 static void
1417 cfiscsi_online(void *arg)
1418 {
1419 	struct cfiscsi_softc *softc;
1420 	struct cfiscsi_target *ct;
1421 	int online;
1422 
1423 	ct = (struct cfiscsi_target *)arg;
1424 	softc = ct->ct_softc;
1425 
1426 	mtx_lock(&softc->lock);
1427 	if (ct->ct_online) {
1428 		mtx_unlock(&softc->lock);
1429 		return;
1430 	}
1431 	ct->ct_online = 1;
1432 	online = softc->online++;
1433 	mtx_unlock(&softc->lock);
1434 	if (online > 0)
1435 		return;
1436 
1437 #ifdef ICL_KERNEL_PROXY
1438 	if (softc->listener != NULL)
1439 		icl_listen_free(softc->listener);
1440 	softc->listener = icl_listen_new(cfiscsi_accept);
1441 #endif
1442 }
1443 
1444 static void
1445 cfiscsi_offline(void *arg)
1446 {
1447 	struct cfiscsi_softc *softc;
1448 	struct cfiscsi_target *ct;
1449 	struct cfiscsi_session *cs;
1450 	int error, online;
1451 
1452 	ct = (struct cfiscsi_target *)arg;
1453 	softc = ct->ct_softc;
1454 
1455 	mtx_lock(&softc->lock);
1456 	if (!ct->ct_online) {
1457 		mtx_unlock(&softc->lock);
1458 		return;
1459 	}
1460 	ct->ct_online = 0;
1461 	online = --softc->online;
1462 
1463 	do {
1464 		TAILQ_FOREACH(cs, &softc->sessions, cs_next) {
1465 			if (cs->cs_target == ct)
1466 				cfiscsi_session_terminate(cs);
1467 		}
1468 		TAILQ_FOREACH(cs, &softc->sessions, cs_next) {
1469 			if (cs->cs_target == ct)
1470 				break;
1471 		}
1472 		if (cs != NULL) {
1473 			error = cv_wait_sig(&softc->sessions_cv, &softc->lock);
1474 			if (error != 0) {
1475 				CFISCSI_SESSION_DEBUG(cs,
1476 				    "cv_wait failed with error %d\n", error);
1477 				break;
1478 			}
1479 		}
1480 	} while (cs != NULL && ct->ct_online == 0);
1481 	mtx_unlock(&softc->lock);
1482 	if (online > 0)
1483 		return;
1484 
1485 #ifdef ICL_KERNEL_PROXY
1486 	icl_listen_free(softc->listener);
1487 	softc->listener = NULL;
1488 #endif
1489 }
1490 
1491 static int
1492 cfiscsi_info(void *arg, struct sbuf *sb)
1493 {
1494 	struct cfiscsi_target *ct = (struct cfiscsi_target *)arg;
1495 	int retval;
1496 
1497 	retval = sbuf_printf(sb, "\t<cfiscsi_state>%d</cfiscsi_state>\n",
1498 	    ct->ct_state);
1499 	return (retval);
1500 }
1501 
1502 static void
1503 cfiscsi_ioctl_handoff(struct ctl_iscsi *ci)
1504 {
1505 	struct cfiscsi_softc *softc;
1506 	struct cfiscsi_session *cs, *cs2;
1507 	struct cfiscsi_target *ct;
1508 	struct ctl_iscsi_handoff_params *cihp;
1509 	int error;
1510 
1511 	cihp = (struct ctl_iscsi_handoff_params *)&(ci->data);
1512 	softc = &cfiscsi_softc;
1513 
1514 	CFISCSI_DEBUG("new connection from %s (%s) to %s",
1515 	    cihp->initiator_name, cihp->initiator_addr,
1516 	    cihp->target_name);
1517 
1518 	ct = cfiscsi_target_find(softc, cihp->target_name,
1519 	    cihp->portal_group_tag);
1520 	if (ct == NULL) {
1521 		ci->status = CTL_ISCSI_ERROR;
1522 		snprintf(ci->error_str, sizeof(ci->error_str),
1523 		    "%s: target not found", __func__);
1524 		return;
1525 	}
1526 
1527 #ifdef ICL_KERNEL_PROXY
1528 	if (cihp->socket > 0 && cihp->connection_id > 0) {
1529 		snprintf(ci->error_str, sizeof(ci->error_str),
1530 		    "both socket and connection_id set");
1531 		ci->status = CTL_ISCSI_ERROR;
1532 		cfiscsi_target_release(ct);
1533 		return;
1534 	}
1535 	if (cihp->socket == 0) {
1536 		mtx_lock(&cfiscsi_softc.lock);
1537 		TAILQ_FOREACH(cs, &cfiscsi_softc.sessions, cs_next) {
1538 			if (cs->cs_id == cihp->connection_id)
1539 				break;
1540 		}
1541 		if (cs == NULL) {
1542 			mtx_unlock(&cfiscsi_softc.lock);
1543 			snprintf(ci->error_str, sizeof(ci->error_str),
1544 			    "connection not found");
1545 			ci->status = CTL_ISCSI_ERROR;
1546 			cfiscsi_target_release(ct);
1547 			return;
1548 		}
1549 		mtx_unlock(&cfiscsi_softc.lock);
1550 	} else {
1551 #endif
1552 		cs = cfiscsi_session_new(softc, cihp->offload);
1553 		if (cs == NULL) {
1554 			ci->status = CTL_ISCSI_ERROR;
1555 			snprintf(ci->error_str, sizeof(ci->error_str),
1556 			    "%s: cfiscsi_session_new failed", __func__);
1557 			cfiscsi_target_release(ct);
1558 			return;
1559 		}
1560 #ifdef ICL_KERNEL_PROXY
1561 	}
1562 #endif
1563 
1564 	/*
1565 	 * First PDU of Full Feature phase has the same CmdSN as the last
1566 	 * PDU from the Login Phase received from the initiator.  Thus,
1567 	 * the -1 below.
1568 	 */
1569 	cs->cs_cmdsn = cihp->cmdsn;
1570 	cs->cs_statsn = cihp->statsn;
1571 	cs->cs_conn->ic_max_recv_data_segment_length =
1572 	    cihp->max_recv_data_segment_length;
1573 	cs->cs_conn->ic_max_send_data_segment_length =
1574 	    cihp->max_send_data_segment_length;
1575 	cs->cs_max_burst_length = cihp->max_burst_length;
1576 	cs->cs_first_burst_length = cihp->first_burst_length;
1577 	cs->cs_immediate_data = !!cihp->immediate_data;
1578 	if (cihp->header_digest == CTL_ISCSI_DIGEST_CRC32C)
1579 		cs->cs_conn->ic_header_crc32c = true;
1580 	if (cihp->data_digest == CTL_ISCSI_DIGEST_CRC32C)
1581 		cs->cs_conn->ic_data_crc32c = true;
1582 
1583 	strlcpy(cs->cs_initiator_name,
1584 	    cihp->initiator_name, sizeof(cs->cs_initiator_name));
1585 	strlcpy(cs->cs_initiator_addr,
1586 	    cihp->initiator_addr, sizeof(cs->cs_initiator_addr));
1587 	strlcpy(cs->cs_initiator_alias,
1588 	    cihp->initiator_alias, sizeof(cs->cs_initiator_alias));
1589 	memcpy(cs->cs_initiator_isid,
1590 	    cihp->initiator_isid, sizeof(cs->cs_initiator_isid));
1591 	snprintf(cs->cs_initiator_id, sizeof(cs->cs_initiator_id),
1592 	    "%s,i,0x%02x%02x%02x%02x%02x%02x", cs->cs_initiator_name,
1593 	    cihp->initiator_isid[0], cihp->initiator_isid[1],
1594 	    cihp->initiator_isid[2], cihp->initiator_isid[3],
1595 	    cihp->initiator_isid[4], cihp->initiator_isid[5]);
1596 
1597 	mtx_lock(&softc->lock);
1598 	if (ct->ct_online == 0) {
1599 		mtx_unlock(&softc->lock);
1600 		CFISCSI_SESSION_LOCK(cs);
1601 		cs->cs_handoff_in_progress = false;
1602 		cfiscsi_session_terminate(cs);
1603 		CFISCSI_SESSION_UNLOCK(cs);
1604 		cfiscsi_target_release(ct);
1605 		ci->status = CTL_ISCSI_ERROR;
1606 		snprintf(ci->error_str, sizeof(ci->error_str),
1607 		    "%s: port offline", __func__);
1608 		return;
1609 	}
1610 	cs->cs_target = ct;
1611 	mtx_unlock(&softc->lock);
1612 
1613 restart:
1614 	if (!cs->cs_terminating) {
1615 		mtx_lock(&softc->lock);
1616 		TAILQ_FOREACH(cs2, &softc->sessions, cs_next) {
1617 			if (cs2 != cs && cs2->cs_tasks_aborted == false &&
1618 			    cs->cs_target == cs2->cs_target &&
1619 			    strcmp(cs->cs_initiator_id, cs2->cs_initiator_id) == 0) {
1620 				if (strcmp(cs->cs_initiator_addr,
1621 				    cs2->cs_initiator_addr) != 0) {
1622 					CFISCSI_SESSION_WARN(cs2,
1623 					    "session reinstatement from "
1624 					    "different address %s",
1625 					    cs->cs_initiator_addr);
1626 				} else {
1627 					CFISCSI_SESSION_DEBUG(cs2,
1628 					    "session reinstatement");
1629 				}
1630 				cfiscsi_session_terminate(cs2);
1631 				mtx_unlock(&softc->lock);
1632 				pause("cfiscsi_reinstate", 1);
1633 				goto restart;
1634 			}
1635 		}
1636 		mtx_unlock(&softc->lock);
1637 	}
1638 
1639 	/*
1640 	 * Register initiator with CTL.
1641 	 */
1642 	cfiscsi_session_register_initiator(cs);
1643 
1644 #ifdef ICL_KERNEL_PROXY
1645 	if (cihp->socket > 0) {
1646 #endif
1647 		error = icl_conn_handoff(cs->cs_conn, cihp->socket);
1648 		if (error != 0) {
1649 			CFISCSI_SESSION_LOCK(cs);
1650 			cs->cs_handoff_in_progress = false;
1651 			cfiscsi_session_terminate(cs);
1652 			CFISCSI_SESSION_UNLOCK(cs);
1653 			ci->status = CTL_ISCSI_ERROR;
1654 			snprintf(ci->error_str, sizeof(ci->error_str),
1655 			    "%s: icl_conn_handoff failed with error %d",
1656 			    __func__, error);
1657 			return;
1658 		}
1659 #ifdef ICL_KERNEL_PROXY
1660 	}
1661 #endif
1662 
1663 #ifdef ICL_KERNEL_PROXY
1664 	cs->cs_login_phase = false;
1665 
1666 	/*
1667 	 * First PDU of the Full Feature phase has likely already arrived.
1668 	 * We have to pick it up and execute properly.
1669 	 */
1670 	if (cs->cs_login_pdu != NULL) {
1671 		CFISCSI_SESSION_DEBUG(cs, "picking up first PDU");
1672 		cfiscsi_pdu_handle(cs->cs_login_pdu);
1673 		cs->cs_login_pdu = NULL;
1674 	}
1675 #endif
1676 
1677 	CFISCSI_SESSION_LOCK(cs);
1678 	cs->cs_handoff_in_progress = false;
1679 
1680 	/*
1681 	 * Wake up the maintenance thread if we got scheduled for termination.
1682 	 */
1683 	if (cs->cs_terminating)
1684 		cfiscsi_session_terminate(cs);
1685 	CFISCSI_SESSION_UNLOCK(cs);
1686 
1687 	ci->status = CTL_ISCSI_OK;
1688 }
1689 
1690 static void
1691 cfiscsi_ioctl_list(struct ctl_iscsi *ci)
1692 {
1693 	struct ctl_iscsi_list_params *cilp;
1694 	struct cfiscsi_session *cs;
1695 	struct cfiscsi_softc *softc;
1696 	struct sbuf *sb;
1697 	int error;
1698 
1699 	cilp = (struct ctl_iscsi_list_params *)&(ci->data);
1700 	softc = &cfiscsi_softc;
1701 
1702 	sb = sbuf_new(NULL, NULL, cilp->alloc_len, SBUF_FIXEDLEN);
1703 	if (sb == NULL) {
1704 		ci->status = CTL_ISCSI_ERROR;
1705 		snprintf(ci->error_str, sizeof(ci->error_str),
1706 		    "Unable to allocate %d bytes for iSCSI session list",
1707 		    cilp->alloc_len);
1708 		return;
1709 	}
1710 
1711 	sbuf_printf(sb, "<ctlislist>\n");
1712 	mtx_lock(&softc->lock);
1713 	TAILQ_FOREACH(cs, &softc->sessions, cs_next) {
1714 		if (cs->cs_target == NULL)
1715 			continue;
1716 		error = sbuf_printf(sb, "<connection id=\"%d\">"
1717 		    "<initiator>%s</initiator>"
1718 		    "<initiator_addr>%s</initiator_addr>"
1719 		    "<initiator_alias>%s</initiator_alias>"
1720 		    "<target>%s</target>"
1721 		    "<target_alias>%s</target_alias>"
1722 		    "<target_portal_group_tag>%u</target_portal_group_tag>"
1723 		    "<header_digest>%s</header_digest>"
1724 		    "<data_digest>%s</data_digest>"
1725 		    "<max_recv_data_segment_length>%d</max_recv_data_segment_length>"
1726 		    "<max_send_data_segment_length>%d</max_send_data_segment_length>"
1727 		    "<max_burst_length>%d</max_burst_length>"
1728 		    "<first_burst_length>%d</first_burst_length>"
1729 		    "<immediate_data>%d</immediate_data>"
1730 		    "<iser>%d</iser>"
1731 		    "<offload>%s</offload>"
1732 		    "</connection>\n",
1733 		    cs->cs_id,
1734 		    cs->cs_initiator_name, cs->cs_initiator_addr, cs->cs_initiator_alias,
1735 		    cs->cs_target->ct_name, cs->cs_target->ct_alias,
1736 		    cs->cs_target->ct_tag,
1737 		    cs->cs_conn->ic_header_crc32c ? "CRC32C" : "None",
1738 		    cs->cs_conn->ic_data_crc32c ? "CRC32C" : "None",
1739 		    cs->cs_conn->ic_max_recv_data_segment_length,
1740 		    cs->cs_conn->ic_max_send_data_segment_length,
1741 		    cs->cs_max_burst_length,
1742 		    cs->cs_first_burst_length,
1743 		    cs->cs_immediate_data,
1744 		    cs->cs_conn->ic_iser,
1745 		    cs->cs_conn->ic_offload);
1746 		if (error != 0)
1747 			break;
1748 	}
1749 	mtx_unlock(&softc->lock);
1750 	error = sbuf_printf(sb, "</ctlislist>\n");
1751 	if (error != 0) {
1752 		sbuf_delete(sb);
1753 		ci->status = CTL_ISCSI_LIST_NEED_MORE_SPACE;
1754 		snprintf(ci->error_str, sizeof(ci->error_str),
1755 		    "Out of space, %d bytes is too small", cilp->alloc_len);
1756 		return;
1757 	}
1758 	sbuf_finish(sb);
1759 
1760 	error = copyout(sbuf_data(sb), cilp->conn_xml, sbuf_len(sb) + 1);
1761 	if (error != 0) {
1762 		sbuf_delete(sb);
1763 		snprintf(ci->error_str, sizeof(ci->error_str),
1764 		    "copyout failed with error %d", error);
1765 		ci->status = CTL_ISCSI_ERROR;
1766 		return;
1767 	}
1768 	cilp->fill_len = sbuf_len(sb) + 1;
1769 	ci->status = CTL_ISCSI_OK;
1770 	sbuf_delete(sb);
1771 }
1772 
1773 static void
1774 cfiscsi_ioctl_logout(struct ctl_iscsi *ci)
1775 {
1776 	struct icl_pdu *response;
1777 	struct iscsi_bhs_asynchronous_message *bhsam;
1778 	struct ctl_iscsi_logout_params *cilp;
1779 	struct cfiscsi_session *cs;
1780 	struct cfiscsi_softc *softc;
1781 	int found = 0;
1782 
1783 	cilp = (struct ctl_iscsi_logout_params *)&(ci->data);
1784 	softc = &cfiscsi_softc;
1785 
1786 	mtx_lock(&softc->lock);
1787 	TAILQ_FOREACH(cs, &softc->sessions, cs_next) {
1788 		if (cilp->all == 0 && cs->cs_id != cilp->connection_id &&
1789 		    strcmp(cs->cs_initiator_name, cilp->initiator_name) != 0 &&
1790 		    strcmp(cs->cs_initiator_addr, cilp->initiator_addr) != 0)
1791 			continue;
1792 
1793 		response = icl_pdu_new(cs->cs_conn, M_NOWAIT);
1794 		if (response == NULL) {
1795 			ci->status = CTL_ISCSI_ERROR;
1796 			snprintf(ci->error_str, sizeof(ci->error_str),
1797 			    "Unable to allocate memory");
1798 			mtx_unlock(&softc->lock);
1799 			return;
1800 		}
1801 		bhsam =
1802 		    (struct iscsi_bhs_asynchronous_message *)response->ip_bhs;
1803 		bhsam->bhsam_opcode = ISCSI_BHS_OPCODE_ASYNC_MESSAGE;
1804 		bhsam->bhsam_flags = 0x80;
1805 		bhsam->bhsam_async_event = BHSAM_EVENT_TARGET_REQUESTS_LOGOUT;
1806 		bhsam->bhsam_parameter3 = htons(10);
1807 		cfiscsi_pdu_queue(response);
1808 		found++;
1809 	}
1810 	mtx_unlock(&softc->lock);
1811 
1812 	if (found == 0) {
1813 		ci->status = CTL_ISCSI_SESSION_NOT_FOUND;
1814 		snprintf(ci->error_str, sizeof(ci->error_str),
1815 		    "No matching connections found");
1816 		return;
1817 	}
1818 
1819 	ci->status = CTL_ISCSI_OK;
1820 }
1821 
1822 static void
1823 cfiscsi_ioctl_terminate(struct ctl_iscsi *ci)
1824 {
1825 	struct icl_pdu *response;
1826 	struct iscsi_bhs_asynchronous_message *bhsam;
1827 	struct ctl_iscsi_terminate_params *citp;
1828 	struct cfiscsi_session *cs;
1829 	struct cfiscsi_softc *softc;
1830 	int found = 0;
1831 
1832 	citp = (struct ctl_iscsi_terminate_params *)&(ci->data);
1833 	softc = &cfiscsi_softc;
1834 
1835 	mtx_lock(&softc->lock);
1836 	TAILQ_FOREACH(cs, &softc->sessions, cs_next) {
1837 		if (citp->all == 0 && cs->cs_id != citp->connection_id &&
1838 		    strcmp(cs->cs_initiator_name, citp->initiator_name) != 0 &&
1839 		    strcmp(cs->cs_initiator_addr, citp->initiator_addr) != 0)
1840 			continue;
1841 
1842 		response = icl_pdu_new(cs->cs_conn, M_NOWAIT);
1843 		if (response == NULL) {
1844 			/*
1845 			 * Oh well.  Just terminate the connection.
1846 			 */
1847 		} else {
1848 			bhsam = (struct iscsi_bhs_asynchronous_message *)
1849 			    response->ip_bhs;
1850 			bhsam->bhsam_opcode = ISCSI_BHS_OPCODE_ASYNC_MESSAGE;
1851 			bhsam->bhsam_flags = 0x80;
1852 			bhsam->bhsam_0xffffffff = 0xffffffff;
1853 			bhsam->bhsam_async_event =
1854 			    BHSAM_EVENT_TARGET_TERMINATES_SESSION;
1855 			cfiscsi_pdu_queue(response);
1856 		}
1857 		cfiscsi_session_terminate(cs);
1858 		found++;
1859 	}
1860 	mtx_unlock(&softc->lock);
1861 
1862 	if (found == 0) {
1863 		ci->status = CTL_ISCSI_SESSION_NOT_FOUND;
1864 		snprintf(ci->error_str, sizeof(ci->error_str),
1865 		    "No matching connections found");
1866 		return;
1867 	}
1868 
1869 	ci->status = CTL_ISCSI_OK;
1870 }
1871 
1872 static void
1873 cfiscsi_ioctl_limits(struct ctl_iscsi *ci)
1874 {
1875 	struct ctl_iscsi_limits_params *cilp;
1876 	struct icl_drv_limits idl;
1877 	int error;
1878 
1879 	cilp = (struct ctl_iscsi_limits_params *)&(ci->data);
1880 
1881 	error = icl_limits(cilp->offload, false, &idl);
1882 	if (error != 0) {
1883 		ci->status = CTL_ISCSI_ERROR;
1884 		snprintf(ci->error_str, sizeof(ci->error_str),
1885 			"%s: icl_limits failed with error %d",
1886 			__func__, error);
1887 		return;
1888 	}
1889 
1890 	cilp->max_recv_data_segment_length =
1891 	    idl.idl_max_recv_data_segment_length;
1892 	cilp->max_send_data_segment_length =
1893 	    idl.idl_max_send_data_segment_length;
1894 	cilp->max_burst_length = idl.idl_max_burst_length;
1895 	cilp->first_burst_length = idl.idl_first_burst_length;
1896 
1897 	ci->status = CTL_ISCSI_OK;
1898 }
1899 
1900 #ifdef ICL_KERNEL_PROXY
1901 static void
1902 cfiscsi_ioctl_listen(struct ctl_iscsi *ci)
1903 {
1904 	struct ctl_iscsi_listen_params *cilp;
1905 	struct sockaddr *sa;
1906 	int error;
1907 
1908 	cilp = (struct ctl_iscsi_listen_params *)&(ci->data);
1909 
1910 	if (cfiscsi_softc.listener == NULL) {
1911 		CFISCSI_DEBUG("no listener");
1912 		snprintf(ci->error_str, sizeof(ci->error_str), "no listener");
1913 		ci->status = CTL_ISCSI_ERROR;
1914 		return;
1915 	}
1916 
1917 	error = getsockaddr(&sa, (void *)cilp->addr, cilp->addrlen);
1918 	if (error != 0) {
1919 		CFISCSI_DEBUG("getsockaddr, error %d", error);
1920 		snprintf(ci->error_str, sizeof(ci->error_str), "getsockaddr failed");
1921 		ci->status = CTL_ISCSI_ERROR;
1922 		return;
1923 	}
1924 
1925 	error = icl_listen_add(cfiscsi_softc.listener, cilp->iser, cilp->domain,
1926 	    cilp->socktype, cilp->protocol, sa, cilp->portal_id);
1927 	if (error != 0) {
1928 		free(sa, M_SONAME);
1929 		CFISCSI_DEBUG("icl_listen_add, error %d", error);
1930 		snprintf(ci->error_str, sizeof(ci->error_str),
1931 		    "icl_listen_add failed, error %d", error);
1932 		ci->status = CTL_ISCSI_ERROR;
1933 		return;
1934 	}
1935 
1936 	ci->status = CTL_ISCSI_OK;
1937 }
1938 
1939 static void
1940 cfiscsi_ioctl_accept(struct ctl_iscsi *ci)
1941 {
1942 	struct ctl_iscsi_accept_params *ciap;
1943 	struct cfiscsi_session *cs;
1944 	int error;
1945 
1946 	ciap = (struct ctl_iscsi_accept_params *)&(ci->data);
1947 
1948 	mtx_lock(&cfiscsi_softc.lock);
1949 	for (;;) {
1950 		TAILQ_FOREACH(cs, &cfiscsi_softc.sessions, cs_next) {
1951 			if (cs->cs_waiting_for_ctld)
1952 				break;
1953 		}
1954 		if (cs != NULL)
1955 			break;
1956 		error = cv_wait_sig(&cfiscsi_softc.accept_cv, &cfiscsi_softc.lock);
1957 		if (error != 0) {
1958 			mtx_unlock(&cfiscsi_softc.lock);
1959 			snprintf(ci->error_str, sizeof(ci->error_str), "interrupted");
1960 			ci->status = CTL_ISCSI_ERROR;
1961 			return;
1962 		}
1963 	}
1964 	mtx_unlock(&cfiscsi_softc.lock);
1965 
1966 	cs->cs_waiting_for_ctld = false;
1967 	cs->cs_login_phase = true;
1968 
1969 	ciap->connection_id = cs->cs_id;
1970 	ciap->portal_id = cs->cs_portal_id;
1971 	ciap->initiator_addrlen = cs->cs_initiator_sa->sa_len;
1972 	error = copyout(cs->cs_initiator_sa, ciap->initiator_addr,
1973 	    cs->cs_initiator_sa->sa_len);
1974 	if (error != 0) {
1975 		snprintf(ci->error_str, sizeof(ci->error_str),
1976 		    "copyout failed with error %d", error);
1977 		ci->status = CTL_ISCSI_ERROR;
1978 		return;
1979 	}
1980 
1981 	ci->status = CTL_ISCSI_OK;
1982 }
1983 
1984 static void
1985 cfiscsi_ioctl_send(struct ctl_iscsi *ci)
1986 {
1987 	struct ctl_iscsi_send_params *cisp;
1988 	struct cfiscsi_session *cs;
1989 	struct icl_pdu *ip;
1990 	size_t datalen;
1991 	void *data;
1992 	int error;
1993 
1994 	cisp = (struct ctl_iscsi_send_params *)&(ci->data);
1995 
1996 	mtx_lock(&cfiscsi_softc.lock);
1997 	TAILQ_FOREACH(cs, &cfiscsi_softc.sessions, cs_next) {
1998 		if (cs->cs_id == cisp->connection_id)
1999 			break;
2000 	}
2001 	if (cs == NULL) {
2002 		mtx_unlock(&cfiscsi_softc.lock);
2003 		snprintf(ci->error_str, sizeof(ci->error_str), "connection not found");
2004 		ci->status = CTL_ISCSI_ERROR;
2005 		return;
2006 	}
2007 	mtx_unlock(&cfiscsi_softc.lock);
2008 
2009 #if 0
2010 	if (cs->cs_login_phase == false)
2011 		return (EBUSY);
2012 #endif
2013 
2014 	if (cs->cs_terminating) {
2015 		snprintf(ci->error_str, sizeof(ci->error_str), "connection is terminating");
2016 		ci->status = CTL_ISCSI_ERROR;
2017 		return;
2018 	}
2019 
2020 	datalen = cisp->data_segment_len;
2021 	/*
2022 	 * XXX
2023 	 */
2024 	//if (datalen > CFISCSI_MAX_DATA_SEGMENT_LENGTH) {
2025 	if (datalen > 65535) {
2026 		snprintf(ci->error_str, sizeof(ci->error_str), "data segment too big");
2027 		ci->status = CTL_ISCSI_ERROR;
2028 		return;
2029 	}
2030 	if (datalen > 0) {
2031 		data = malloc(datalen, M_CFISCSI, M_WAITOK);
2032 		error = copyin(cisp->data_segment, data, datalen);
2033 		if (error != 0) {
2034 			free(data, M_CFISCSI);
2035 			snprintf(ci->error_str, sizeof(ci->error_str), "copyin error %d", error);
2036 			ci->status = CTL_ISCSI_ERROR;
2037 			return;
2038 		}
2039 	}
2040 
2041 	ip = icl_pdu_new(cs->cs_conn, M_WAITOK);
2042 	memcpy(ip->ip_bhs, cisp->bhs, sizeof(*ip->ip_bhs));
2043 	if (datalen > 0) {
2044 		icl_pdu_append_data(ip, data, datalen, M_WAITOK);
2045 		free(data, M_CFISCSI);
2046 	}
2047 	CFISCSI_SESSION_LOCK(cs);
2048 	icl_pdu_queue(ip);
2049 	CFISCSI_SESSION_UNLOCK(cs);
2050 	ci->status = CTL_ISCSI_OK;
2051 }
2052 
2053 static void
2054 cfiscsi_ioctl_receive(struct ctl_iscsi *ci)
2055 {
2056 	struct ctl_iscsi_receive_params *cirp;
2057 	struct cfiscsi_session *cs;
2058 	struct icl_pdu *ip;
2059 	void *data;
2060 	int error;
2061 
2062 	cirp = (struct ctl_iscsi_receive_params *)&(ci->data);
2063 
2064 	mtx_lock(&cfiscsi_softc.lock);
2065 	TAILQ_FOREACH(cs, &cfiscsi_softc.sessions, cs_next) {
2066 		if (cs->cs_id == cirp->connection_id)
2067 			break;
2068 	}
2069 	if (cs == NULL) {
2070 		mtx_unlock(&cfiscsi_softc.lock);
2071 		snprintf(ci->error_str, sizeof(ci->error_str),
2072 		    "connection not found");
2073 		ci->status = CTL_ISCSI_ERROR;
2074 		return;
2075 	}
2076 	mtx_unlock(&cfiscsi_softc.lock);
2077 
2078 #if 0
2079 	if (is->is_login_phase == false)
2080 		return (EBUSY);
2081 #endif
2082 
2083 	CFISCSI_SESSION_LOCK(cs);
2084 	while (cs->cs_login_pdu == NULL && cs->cs_terminating == false) {
2085 		error = cv_wait_sig(&cs->cs_login_cv, &cs->cs_lock);
2086 		if (error != 0) {
2087 			CFISCSI_SESSION_UNLOCK(cs);
2088 			snprintf(ci->error_str, sizeof(ci->error_str),
2089 			    "interrupted by signal");
2090 			ci->status = CTL_ISCSI_ERROR;
2091 			return;
2092 		}
2093 	}
2094 
2095 	if (cs->cs_terminating) {
2096 		CFISCSI_SESSION_UNLOCK(cs);
2097 		snprintf(ci->error_str, sizeof(ci->error_str),
2098 		    "connection terminating");
2099 		ci->status = CTL_ISCSI_ERROR;
2100 		return;
2101 	}
2102 	ip = cs->cs_login_pdu;
2103 	cs->cs_login_pdu = NULL;
2104 	CFISCSI_SESSION_UNLOCK(cs);
2105 
2106 	if (ip->ip_data_len > cirp->data_segment_len) {
2107 		icl_pdu_free(ip);
2108 		snprintf(ci->error_str, sizeof(ci->error_str),
2109 		    "data segment too big");
2110 		ci->status = CTL_ISCSI_ERROR;
2111 		return;
2112 	}
2113 
2114 	copyout(ip->ip_bhs, cirp->bhs, sizeof(*ip->ip_bhs));
2115 	if (ip->ip_data_len > 0) {
2116 		data = malloc(ip->ip_data_len, M_CFISCSI, M_WAITOK);
2117 		icl_pdu_get_data(ip, 0, data, ip->ip_data_len);
2118 		copyout(data, cirp->data_segment, ip->ip_data_len);
2119 		free(data, M_CFISCSI);
2120 	}
2121 
2122 	icl_pdu_free(ip);
2123 	ci->status = CTL_ISCSI_OK;
2124 }
2125 
2126 #endif /* !ICL_KERNEL_PROXY */
2127 
2128 static void
2129 cfiscsi_ioctl_port_create(struct ctl_req *req)
2130 {
2131 	struct cfiscsi_target *ct;
2132 	struct ctl_port *port;
2133 	const char *target, *alias, *val;
2134 	struct scsi_vpd_id_descriptor *desc;
2135 	int retval, len, idlen;
2136 	uint16_t tag;
2137 
2138 	target = dnvlist_get_string(req->args_nvl, "cfiscsi_target", NULL);
2139 	alias = dnvlist_get_string(req->args_nvl, "cfiscsi_target_alias", NULL);
2140 	val = dnvlist_get_string(req->args_nvl, "cfiscsi_portal_group_tag",
2141 	    NULL);
2142 
2143 	if (target == NULL || val == NULL) {
2144 		req->status = CTL_LUN_ERROR;
2145 		snprintf(req->error_str, sizeof(req->error_str),
2146 		    "Missing required argument");
2147 		return;
2148 	}
2149 
2150 	tag = strtoul(val, NULL, 0);
2151 	ct = cfiscsi_target_find_or_create(&cfiscsi_softc, target, alias, tag);
2152 	if (ct == NULL) {
2153 		req->status = CTL_LUN_ERROR;
2154 		snprintf(req->error_str, sizeof(req->error_str),
2155 		    "failed to create target \"%s\"", target);
2156 		return;
2157 	}
2158 	if (ct->ct_state == CFISCSI_TARGET_STATE_ACTIVE) {
2159 		req->status = CTL_LUN_ERROR;
2160 		snprintf(req->error_str, sizeof(req->error_str),
2161 		    "target \"%s\" for portal group tag %u already exists",
2162 		    target, tag);
2163 		cfiscsi_target_release(ct);
2164 		return;
2165 	}
2166 	port = &ct->ct_port;
2167 	// WAT
2168 	if (ct->ct_state == CFISCSI_TARGET_STATE_DYING)
2169 		goto done;
2170 
2171 	port->frontend = &cfiscsi_frontend;
2172 	port->port_type = CTL_PORT_ISCSI;
2173 	/* XXX KDM what should the real number be here? */
2174 	port->num_requested_ctl_io = 4096;
2175 	port->port_name = "iscsi";
2176 	port->physical_port = (int)tag;
2177 	port->virtual_port = ct->ct_target_id;
2178 	port->port_online = cfiscsi_online;
2179 	port->port_offline = cfiscsi_offline;
2180 	port->port_info = cfiscsi_info;
2181 	port->onoff_arg = ct;
2182 	port->fe_datamove = cfiscsi_datamove;
2183 	port->fe_done = cfiscsi_done;
2184 	port->targ_port = -1;
2185 	port->options = nvlist_clone(req->args_nvl);
2186 
2187 	/* Generate Port ID. */
2188 	idlen = strlen(target) + strlen(",t,0x0001") + 1;
2189 	idlen = roundup2(idlen, 4);
2190 	len = sizeof(struct scsi_vpd_device_id) + idlen;
2191 	port->port_devid = malloc(sizeof(struct ctl_devid) + len,
2192 	    M_CTL, M_WAITOK | M_ZERO);
2193 	port->port_devid->len = len;
2194 	desc = (struct scsi_vpd_id_descriptor *)port->port_devid->data;
2195 	desc->proto_codeset = (SCSI_PROTO_ISCSI << 4) | SVPD_ID_CODESET_UTF8;
2196 	desc->id_type = SVPD_ID_PIV | SVPD_ID_ASSOC_PORT |
2197 	    SVPD_ID_TYPE_SCSI_NAME;
2198 	desc->length = idlen;
2199 	snprintf(desc->identifier, idlen, "%s,t,0x%4.4x", target, tag);
2200 
2201 	/* Generate Target ID. */
2202 	idlen = strlen(target) + 1;
2203 	idlen = roundup2(idlen, 4);
2204 	len = sizeof(struct scsi_vpd_device_id) + idlen;
2205 	port->target_devid = malloc(sizeof(struct ctl_devid) + len,
2206 	    M_CTL, M_WAITOK | M_ZERO);
2207 	port->target_devid->len = len;
2208 	desc = (struct scsi_vpd_id_descriptor *)port->target_devid->data;
2209 	desc->proto_codeset = (SCSI_PROTO_ISCSI << 4) | SVPD_ID_CODESET_UTF8;
2210 	desc->id_type = SVPD_ID_PIV | SVPD_ID_ASSOC_TARGET |
2211 	    SVPD_ID_TYPE_SCSI_NAME;
2212 	desc->length = idlen;
2213 	strlcpy(desc->identifier, target, idlen);
2214 
2215 	retval = ctl_port_register(port);
2216 	if (retval != 0) {
2217 		free(port->port_devid, M_CFISCSI);
2218 		free(port->target_devid, M_CFISCSI);
2219 		cfiscsi_target_release(ct);
2220 		req->status = CTL_LUN_ERROR;
2221 		snprintf(req->error_str, sizeof(req->error_str),
2222 		    "ctl_port_register() failed with error %d", retval);
2223 		return;
2224 	}
2225 done:
2226 	ct->ct_state = CFISCSI_TARGET_STATE_ACTIVE;
2227 	req->status = CTL_LUN_OK;
2228 	req->result_nvl = nvlist_create(0);
2229 	nvlist_add_number(req->result_nvl, "port_id", port->targ_port);
2230 }
2231 
2232 static void
2233 cfiscsi_ioctl_port_remove(struct ctl_req *req)
2234 {
2235 	struct cfiscsi_target *ct;
2236 	const char *target, *val;
2237 	uint16_t tag;
2238 
2239 	target = dnvlist_get_string(req->args_nvl, "cfiscsi_target", NULL);
2240 	val = dnvlist_get_string(req->args_nvl, "cfiscsi_portal_group_tag",
2241 	    NULL);
2242 
2243 	if (target == NULL || val == NULL) {
2244 		req->status = CTL_LUN_ERROR;
2245 		snprintf(req->error_str, sizeof(req->error_str),
2246 		    "Missing required argument");
2247 		return;
2248 	}
2249 
2250 	tag = strtoul(val, NULL, 0);
2251 	ct = cfiscsi_target_find(&cfiscsi_softc, target, tag);
2252 	if (ct == NULL) {
2253 		req->status = CTL_LUN_ERROR;
2254 		snprintf(req->error_str, sizeof(req->error_str),
2255 		    "can't find target \"%s\"", target);
2256 		return;
2257 	}
2258 
2259 	ct->ct_state = CFISCSI_TARGET_STATE_DYING;
2260 	ctl_port_offline(&ct->ct_port);
2261 	cfiscsi_target_release(ct);
2262 	cfiscsi_target_release(ct);
2263 	req->status = CTL_LUN_OK;
2264 }
2265 
2266 static int
2267 cfiscsi_ioctl(struct cdev *dev,
2268     u_long cmd, caddr_t addr, int flag, struct thread *td)
2269 {
2270 	struct ctl_iscsi *ci;
2271 	struct ctl_req *req;
2272 
2273 	if (cmd == CTL_PORT_REQ) {
2274 		req = (struct ctl_req *)addr;
2275 		switch (req->reqtype) {
2276 		case CTL_REQ_CREATE:
2277 			cfiscsi_ioctl_port_create(req);
2278 			break;
2279 		case CTL_REQ_REMOVE:
2280 			cfiscsi_ioctl_port_remove(req);
2281 			break;
2282 		default:
2283 			req->status = CTL_LUN_ERROR;
2284 			snprintf(req->error_str, sizeof(req->error_str),
2285 			    "Unsupported request type %d", req->reqtype);
2286 		}
2287 		return (0);
2288 	}
2289 
2290 	if (cmd != CTL_ISCSI)
2291 		return (ENOTTY);
2292 
2293 	ci = (struct ctl_iscsi *)addr;
2294 	switch (ci->type) {
2295 	case CTL_ISCSI_HANDOFF:
2296 		cfiscsi_ioctl_handoff(ci);
2297 		break;
2298 	case CTL_ISCSI_LIST:
2299 		cfiscsi_ioctl_list(ci);
2300 		break;
2301 	case CTL_ISCSI_LOGOUT:
2302 		cfiscsi_ioctl_logout(ci);
2303 		break;
2304 	case CTL_ISCSI_TERMINATE:
2305 		cfiscsi_ioctl_terminate(ci);
2306 		break;
2307 	case CTL_ISCSI_LIMITS:
2308 		cfiscsi_ioctl_limits(ci);
2309 		break;
2310 #ifdef ICL_KERNEL_PROXY
2311 	case CTL_ISCSI_LISTEN:
2312 		cfiscsi_ioctl_listen(ci);
2313 		break;
2314 	case CTL_ISCSI_ACCEPT:
2315 		cfiscsi_ioctl_accept(ci);
2316 		break;
2317 	case CTL_ISCSI_SEND:
2318 		cfiscsi_ioctl_send(ci);
2319 		break;
2320 	case CTL_ISCSI_RECEIVE:
2321 		cfiscsi_ioctl_receive(ci);
2322 		break;
2323 #else
2324 	case CTL_ISCSI_LISTEN:
2325 	case CTL_ISCSI_ACCEPT:
2326 	case CTL_ISCSI_SEND:
2327 	case CTL_ISCSI_RECEIVE:
2328 		ci->status = CTL_ISCSI_ERROR;
2329 		snprintf(ci->error_str, sizeof(ci->error_str),
2330 		    "%s: CTL compiled without ICL_KERNEL_PROXY",
2331 		    __func__);
2332 		break;
2333 #endif /* !ICL_KERNEL_PROXY */
2334 	default:
2335 		ci->status = CTL_ISCSI_ERROR;
2336 		snprintf(ci->error_str, sizeof(ci->error_str),
2337 		    "%s: invalid iSCSI request type %d", __func__, ci->type);
2338 		break;
2339 	}
2340 
2341 	return (0);
2342 }
2343 
2344 static void
2345 cfiscsi_target_hold(struct cfiscsi_target *ct)
2346 {
2347 
2348 	refcount_acquire(&ct->ct_refcount);
2349 }
2350 
2351 static void
2352 cfiscsi_target_release(struct cfiscsi_target *ct)
2353 {
2354 	struct cfiscsi_softc *softc;
2355 
2356 	softc = ct->ct_softc;
2357 	mtx_lock(&softc->lock);
2358 	if (refcount_release(&ct->ct_refcount)) {
2359 		TAILQ_REMOVE(&softc->targets, ct, ct_next);
2360 		mtx_unlock(&softc->lock);
2361 		if (ct->ct_state != CFISCSI_TARGET_STATE_INVALID) {
2362 			ct->ct_state = CFISCSI_TARGET_STATE_INVALID;
2363 			if (ctl_port_deregister(&ct->ct_port) != 0)
2364 				printf("%s: ctl_port_deregister() failed\n",
2365 				    __func__);
2366 		}
2367 		free(ct, M_CFISCSI);
2368 
2369 		return;
2370 	}
2371 	mtx_unlock(&softc->lock);
2372 }
2373 
2374 static struct cfiscsi_target *
2375 cfiscsi_target_find(struct cfiscsi_softc *softc, const char *name, uint16_t tag)
2376 {
2377 	struct cfiscsi_target *ct;
2378 
2379 	mtx_lock(&softc->lock);
2380 	TAILQ_FOREACH(ct, &softc->targets, ct_next) {
2381 		if (ct->ct_tag != tag ||
2382 		    strcmp(name, ct->ct_name) != 0 ||
2383 		    ct->ct_state != CFISCSI_TARGET_STATE_ACTIVE)
2384 			continue;
2385 		cfiscsi_target_hold(ct);
2386 		mtx_unlock(&softc->lock);
2387 		return (ct);
2388 	}
2389 	mtx_unlock(&softc->lock);
2390 
2391 	return (NULL);
2392 }
2393 
2394 static struct cfiscsi_target *
2395 cfiscsi_target_find_or_create(struct cfiscsi_softc *softc, const char *name,
2396     const char *alias, uint16_t tag)
2397 {
2398 	struct cfiscsi_target *ct, *newct;
2399 
2400 	if (name[0] == '\0' || strlen(name) >= CTL_ISCSI_NAME_LEN)
2401 		return (NULL);
2402 
2403 	newct = malloc(sizeof(*newct), M_CFISCSI, M_WAITOK | M_ZERO);
2404 
2405 	mtx_lock(&softc->lock);
2406 	TAILQ_FOREACH(ct, &softc->targets, ct_next) {
2407 		if (ct->ct_tag != tag ||
2408 		    strcmp(name, ct->ct_name) != 0 ||
2409 		    ct->ct_state == CFISCSI_TARGET_STATE_INVALID)
2410 			continue;
2411 		cfiscsi_target_hold(ct);
2412 		mtx_unlock(&softc->lock);
2413 		free(newct, M_CFISCSI);
2414 		return (ct);
2415 	}
2416 
2417 	strlcpy(newct->ct_name, name, sizeof(newct->ct_name));
2418 	if (alias != NULL)
2419 		strlcpy(newct->ct_alias, alias, sizeof(newct->ct_alias));
2420 	newct->ct_tag = tag;
2421 	refcount_init(&newct->ct_refcount, 1);
2422 	newct->ct_softc = softc;
2423 	if (TAILQ_EMPTY(&softc->targets))
2424 		softc->last_target_id = 0;
2425 	newct->ct_target_id = ++softc->last_target_id;
2426 	TAILQ_INSERT_TAIL(&softc->targets, newct, ct_next);
2427 	mtx_unlock(&softc->lock);
2428 
2429 	return (newct);
2430 }
2431 
2432 static void
2433 cfiscsi_pdu_done(struct icl_pdu *ip, int error)
2434 {
2435 
2436 	if (error != 0)
2437 		; // XXX: Do something on error?
2438 	((ctl_ref)ip->ip_prv0)(ip->ip_prv1, -1);
2439 }
2440 
2441 static void
2442 cfiscsi_datamove_in(union ctl_io *io)
2443 {
2444 	struct cfiscsi_session *cs;
2445 	struct icl_pdu *request, *response;
2446 	const struct iscsi_bhs_scsi_command *bhssc;
2447 	struct iscsi_bhs_data_in *bhsdi;
2448 	struct ctl_sg_entry ctl_sg_entry, *ctl_sglist;
2449 	size_t len, expected_len, sg_len, buffer_offset;
2450 	size_t max_send_data_segment_length;
2451 	const char *sg_addr;
2452 	icl_pdu_cb cb;
2453 	int ctl_sg_count, error, i;
2454 
2455 	request = PRIV_REQUEST(io);
2456 	cs = PDU_SESSION(request);
2457 
2458 	bhssc = (const struct iscsi_bhs_scsi_command *)request->ip_bhs;
2459 	KASSERT((bhssc->bhssc_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
2460 	    ISCSI_BHS_OPCODE_SCSI_COMMAND,
2461 	    ("bhssc->bhssc_opcode != ISCSI_BHS_OPCODE_SCSI_COMMAND"));
2462 
2463 	if (io->scsiio.kern_sg_entries > 0) {
2464 		ctl_sglist = (struct ctl_sg_entry *)io->scsiio.kern_data_ptr;
2465 		ctl_sg_count = io->scsiio.kern_sg_entries;
2466 	} else {
2467 		ctl_sglist = &ctl_sg_entry;
2468 		ctl_sglist->addr = io->scsiio.kern_data_ptr;
2469 		ctl_sglist->len = io->scsiio.kern_data_len;
2470 		ctl_sg_count = 1;
2471 	}
2472 
2473 	/*
2474 	 * This is the offset within the current SCSI command; for the first
2475 	 * call to cfiscsi_datamove() it will be 0, and for subsequent ones
2476 	 * it will be the sum of lengths of previous ones.
2477 	 */
2478 	buffer_offset = io->scsiio.kern_rel_offset;
2479 
2480 	/*
2481 	 * This is the transfer length expected by the initiator.  It can be
2482 	 * different from the amount of data from the SCSI point of view.
2483 	 */
2484 	expected_len = ntohl(bhssc->bhssc_expected_data_transfer_length);
2485 
2486 	/*
2487 	 * If the transfer is outside of expected length -- we are done.
2488 	 */
2489 	if (buffer_offset >= expected_len) {
2490 #if 0
2491 		CFISCSI_SESSION_DEBUG(cs, "buffer_offset = %zd, "
2492 		    "already sent the expected len", buffer_offset);
2493 #endif
2494 		ctl_datamove_done(io, true);
2495 		return;
2496 	}
2497 
2498 	if (io->scsiio.kern_data_ref != NULL)
2499 		cb = cfiscsi_pdu_done;
2500 	else
2501 		cb = NULL;
2502 
2503 	i = 0;
2504 	sg_addr = NULL;
2505 	sg_len = 0;
2506 	response = NULL;
2507 	bhsdi = NULL;
2508 	if (cs->cs_conn->ic_hw_isomax != 0)
2509 		max_send_data_segment_length = cs->cs_conn->ic_hw_isomax;
2510 	else
2511 		max_send_data_segment_length =
2512 		    cs->cs_conn->ic_max_send_data_segment_length;
2513 	for (;;) {
2514 		if (response == NULL) {
2515 			response = cfiscsi_pdu_new_response(request, M_NOWAIT);
2516 			if (response == NULL) {
2517 				CFISCSI_SESSION_WARN(cs, "failed to "
2518 				    "allocate memory; dropping connection");
2519 				ctl_set_busy(&io->scsiio);
2520 				ctl_datamove_done(io, true);
2521 				cfiscsi_session_terminate(cs);
2522 				return;
2523 			}
2524 			bhsdi = (struct iscsi_bhs_data_in *)response->ip_bhs;
2525 			bhsdi->bhsdi_opcode = ISCSI_BHS_OPCODE_SCSI_DATA_IN;
2526 			bhsdi->bhsdi_initiator_task_tag =
2527 			    bhssc->bhssc_initiator_task_tag;
2528 			bhsdi->bhsdi_target_transfer_tag = 0xffffffff;
2529 			bhsdi->bhsdi_datasn = htonl(PRIV_EXPDATASN(io));
2530 			bhsdi->bhsdi_buffer_offset = htonl(buffer_offset);
2531 		}
2532 
2533 		KASSERT(i < ctl_sg_count, ("i >= ctl_sg_count"));
2534 		if (sg_len == 0) {
2535 			sg_addr = ctl_sglist[i].addr;
2536 			sg_len = ctl_sglist[i].len;
2537 			KASSERT(sg_len > 0, ("sg_len <= 0"));
2538 		}
2539 
2540 		len = sg_len;
2541 
2542 		/*
2543 		 * Truncate to maximum data segment length.
2544 		 */
2545 		KASSERT(response->ip_data_len < max_send_data_segment_length,
2546 		    ("ip_data_len %zd >= max_send_data_segment_length %zd",
2547 		    response->ip_data_len, max_send_data_segment_length));
2548 		if (response->ip_data_len + len > max_send_data_segment_length) {
2549 			len = max_send_data_segment_length - response->ip_data_len;
2550 			KASSERT(len <= sg_len, ("len %zd > sg_len %zd",
2551 			    len, sg_len));
2552 		}
2553 
2554 		/*
2555 		 * Truncate to expected data transfer length.
2556 		 */
2557 		KASSERT(buffer_offset + response->ip_data_len < expected_len,
2558 		    ("buffer_offset %zd + ip_data_len %zd >= expected_len %zd",
2559 		    buffer_offset, response->ip_data_len, expected_len));
2560 		if (buffer_offset + response->ip_data_len + len > expected_len) {
2561 			CFISCSI_SESSION_DEBUG(cs, "truncating from %zd "
2562 			    "to expected data transfer length %zd",
2563 			    buffer_offset + response->ip_data_len + len, expected_len);
2564 			len = expected_len - (buffer_offset + response->ip_data_len);
2565 			KASSERT(len <= sg_len, ("len %zd > sg_len %zd",
2566 			    len, sg_len));
2567 		}
2568 
2569 		error = icl_pdu_append_data(response, sg_addr, len,
2570 		    M_NOWAIT | (cb ? ICL_NOCOPY : 0));
2571 		if (error != 0) {
2572 			CFISCSI_SESSION_WARN(cs, "failed to "
2573 			    "allocate memory; dropping connection");
2574 			icl_pdu_free(response);
2575 			ctl_set_busy(&io->scsiio);
2576 			ctl_datamove_done(io, true);
2577 			cfiscsi_session_terminate(cs);
2578 			return;
2579 		}
2580 		sg_addr += len;
2581 		sg_len -= len;
2582 		io->scsiio.kern_data_resid -= len;
2583 
2584 		KASSERT(buffer_offset + response->ip_data_len <= expected_len,
2585 		    ("buffer_offset %zd + ip_data_len %zd > expected_len %zd",
2586 		    buffer_offset, response->ip_data_len, expected_len));
2587 		if (buffer_offset + response->ip_data_len == expected_len) {
2588 			/*
2589 			 * Already have the amount of data the initiator wanted.
2590 			 */
2591 			break;
2592 		}
2593 
2594 		if (sg_len == 0) {
2595 			/*
2596 			 * End of scatter-gather segment;
2597 			 * proceed to the next one...
2598 			 */
2599 			if (i == ctl_sg_count - 1) {
2600 				/*
2601 				 * ... unless this was the last one.
2602 				 */
2603 				break;
2604 			}
2605 			i++;
2606 		}
2607 
2608 		if (response->ip_data_len == max_send_data_segment_length) {
2609 			/*
2610 			 * Can't stuff more data into the current PDU;
2611 			 * queue it.  Note that's not enough to check
2612 			 * for kern_data_resid == 0 instead; there
2613 			 * may be several Data-In PDUs for the final
2614 			 * call to cfiscsi_datamove(), and we want
2615 			 * to set the F flag only on the last of them.
2616 			 */
2617 			buffer_offset += response->ip_data_len;
2618 			if (buffer_offset == io->scsiio.kern_total_len ||
2619 			    buffer_offset == expected_len) {
2620 				buffer_offset -= response->ip_data_len;
2621 				break;
2622 			}
2623 			PRIV_EXPDATASN(io) += howmany(response->ip_data_len,
2624 			    cs->cs_conn->ic_max_send_data_segment_length);
2625 			if (cb != NULL) {
2626 				response->ip_prv0 = io->scsiio.kern_data_ref;
2627 				response->ip_prv1 = io->scsiio.kern_data_arg;
2628 				io->scsiio.kern_data_ref(io->scsiio.kern_data_arg, 1);
2629 			}
2630 			cfiscsi_pdu_queue_cb(response, cb);
2631 			response = NULL;
2632 			bhsdi = NULL;
2633 		}
2634 	}
2635 	if (response != NULL) {
2636 		buffer_offset += response->ip_data_len;
2637 		if (buffer_offset == io->scsiio.kern_total_len ||
2638 		    buffer_offset == expected_len) {
2639 			bhsdi->bhsdi_flags |= BHSDI_FLAGS_F;
2640 			if (io->io_hdr.status == CTL_SUCCESS) {
2641 				bhsdi->bhsdi_flags |= BHSDI_FLAGS_S;
2642 				if (io->scsiio.kern_total_len <
2643 				    ntohl(bhssc->bhssc_expected_data_transfer_length)) {
2644 					bhsdi->bhsdi_flags |= BHSSR_FLAGS_RESIDUAL_UNDERFLOW;
2645 					bhsdi->bhsdi_residual_count =
2646 					    htonl(ntohl(bhssc->bhssc_expected_data_transfer_length) -
2647 					    io->scsiio.kern_total_len);
2648 				} else if (io->scsiio.kern_total_len >
2649 				    ntohl(bhssc->bhssc_expected_data_transfer_length)) {
2650 					bhsdi->bhsdi_flags |= BHSSR_FLAGS_RESIDUAL_OVERFLOW;
2651 					bhsdi->bhsdi_residual_count =
2652 					    htonl(io->scsiio.kern_total_len -
2653 					    ntohl(bhssc->bhssc_expected_data_transfer_length));
2654 				}
2655 				bhsdi->bhsdi_status = io->scsiio.scsi_status;
2656 				io->io_hdr.flags |= CTL_FLAG_STATUS_SENT;
2657 			}
2658 		}
2659 		KASSERT(response->ip_data_len > 0, ("sending empty Data-In"));
2660 		PRIV_EXPDATASN(io) += howmany(response->ip_data_len,
2661 		    cs->cs_conn->ic_max_send_data_segment_length);
2662 		if (cb != NULL) {
2663 			response->ip_prv0 = io->scsiio.kern_data_ref;
2664 			response->ip_prv1 = io->scsiio.kern_data_arg;
2665 			io->scsiio.kern_data_ref(io->scsiio.kern_data_arg, 1);
2666 		}
2667 		cfiscsi_pdu_queue_cb(response, cb);
2668 	}
2669 
2670 	ctl_datamove_done(io, true);
2671 }
2672 
2673 static void
2674 cfiscsi_datamove_out(union ctl_io *io)
2675 {
2676 	struct cfiscsi_session *cs;
2677 	struct icl_pdu *request, *response;
2678 	const struct iscsi_bhs_scsi_command *bhssc;
2679 	struct iscsi_bhs_r2t *bhsr2t;
2680 	struct cfiscsi_data_wait *cdw;
2681 	struct ctl_sg_entry ctl_sg_entry, *ctl_sglist;
2682 	uint32_t expected_len, datamove_len, r2t_off, r2t_len;
2683 	uint32_t target_transfer_tag;
2684 	bool done;
2685 
2686 	request = PRIV_REQUEST(io);
2687 	cs = PDU_SESSION(request);
2688 
2689 	bhssc = (const struct iscsi_bhs_scsi_command *)request->ip_bhs;
2690 	KASSERT((bhssc->bhssc_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
2691 	    ISCSI_BHS_OPCODE_SCSI_COMMAND,
2692 	    ("bhssc->bhssc_opcode != ISCSI_BHS_OPCODE_SCSI_COMMAND"));
2693 
2694 	/*
2695 	 * Complete write underflow.  Not a single byte to read.  Return.
2696 	 */
2697 	expected_len = ntohl(bhssc->bhssc_expected_data_transfer_length);
2698 	if (io->scsiio.kern_rel_offset >= expected_len) {
2699 		ctl_datamove_done(io, true);
2700 		return;
2701 	}
2702 
2703 	datamove_len = MIN(io->scsiio.kern_data_len,
2704 	    expected_len - io->scsiio.kern_rel_offset);
2705 
2706 	target_transfer_tag =
2707 	    atomic_fetchadd_32(&cs->cs_target_transfer_tag, 1);
2708 	if (target_transfer_tag == 0xffffffff) {
2709 		target_transfer_tag =
2710 		    atomic_fetchadd_32(&cs->cs_target_transfer_tag, 1);
2711 	}
2712 	cdw = cfiscsi_data_wait_new(cs, io, bhssc->bhssc_initiator_task_tag,
2713 	    &target_transfer_tag);
2714 	if (cdw == NULL) {
2715 		CFISCSI_SESSION_WARN(cs, "failed to "
2716 		    "allocate memory; dropping connection");
2717 		ctl_set_busy(&io->scsiio);
2718 		ctl_datamove_done(io, true);
2719 		cfiscsi_session_terminate(cs);
2720 		return;
2721 	}
2722 #if 0
2723 	CFISCSI_SESSION_DEBUG(cs, "expecting Data-Out with initiator "
2724 	    "task tag 0x%x, target transfer tag 0x%x",
2725 	    bhssc->bhssc_initiator_task_tag, target_transfer_tag);
2726 #endif
2727 
2728 	cdw->cdw_ctl_io = io;
2729 	cdw->cdw_target_transfer_tag = target_transfer_tag;
2730 	cdw->cdw_initiator_task_tag = bhssc->bhssc_initiator_task_tag;
2731 	cdw->cdw_r2t_end = datamove_len;
2732 	cdw->cdw_datasn = 0;
2733 
2734 	/* Set initial data pointer for the CDW respecting ext_data_filled. */
2735 	if (io->scsiio.kern_sg_entries > 0) {
2736 		ctl_sglist = (struct ctl_sg_entry *)io->scsiio.kern_data_ptr;
2737 	} else {
2738 		ctl_sglist = &ctl_sg_entry;
2739 		ctl_sglist->addr = io->scsiio.kern_data_ptr;
2740 		ctl_sglist->len = datamove_len;
2741 	}
2742 	cdw->cdw_sg_index = 0;
2743 	cdw->cdw_sg_addr = ctl_sglist[cdw->cdw_sg_index].addr;
2744 	cdw->cdw_sg_len = ctl_sglist[cdw->cdw_sg_index].len;
2745 	r2t_off = io->scsiio.ext_data_filled;
2746 	while (r2t_off > 0) {
2747 		if (r2t_off >= cdw->cdw_sg_len) {
2748 			r2t_off -= cdw->cdw_sg_len;
2749 			cdw->cdw_sg_index++;
2750 			cdw->cdw_sg_addr = ctl_sglist[cdw->cdw_sg_index].addr;
2751 			cdw->cdw_sg_len = ctl_sglist[cdw->cdw_sg_index].len;
2752 			continue;
2753 		}
2754 		cdw->cdw_sg_addr += r2t_off;
2755 		cdw->cdw_sg_len -= r2t_off;
2756 		r2t_off = 0;
2757 	}
2758 
2759 	if (cs->cs_immediate_data &&
2760 	    io->scsiio.kern_rel_offset + io->scsiio.ext_data_filled <
2761 	    icl_pdu_data_segment_length(request)) {
2762 		done = cfiscsi_handle_data_segment(request, cdw);
2763 		if (done) {
2764 			cfiscsi_data_wait_free(cs, cdw);
2765 			ctl_datamove_done(io, true);
2766 			return;
2767 		}
2768 	}
2769 
2770 	r2t_off = io->scsiio.kern_rel_offset + io->scsiio.ext_data_filled;
2771 	r2t_len = MIN(datamove_len - io->scsiio.ext_data_filled,
2772 	    cs->cs_max_burst_length);
2773 	cdw->cdw_r2t_end = io->scsiio.ext_data_filled + r2t_len;
2774 
2775 	CFISCSI_SESSION_LOCK(cs);
2776 	TAILQ_INSERT_TAIL(&cs->cs_waiting_for_data_out, cdw, cdw_next);
2777 	CFISCSI_SESSION_UNLOCK(cs);
2778 
2779 	/*
2780 	 * XXX: We should limit the number of outstanding R2T PDUs
2781 	 * 	per task to MaxOutstandingR2T.
2782 	 */
2783 	response = cfiscsi_pdu_new_response(request, M_NOWAIT);
2784 	if (response == NULL) {
2785 		CFISCSI_SESSION_WARN(cs, "failed to "
2786 		    "allocate memory; dropping connection");
2787 		ctl_set_busy(&io->scsiio);
2788 		ctl_datamove_done(io, true);
2789 		cfiscsi_session_terminate(cs);
2790 		return;
2791 	}
2792 	io->io_hdr.flags |= CTL_FLAG_DMA_INPROG;
2793 	bhsr2t = (struct iscsi_bhs_r2t *)response->ip_bhs;
2794 	bhsr2t->bhsr2t_opcode = ISCSI_BHS_OPCODE_R2T;
2795 	bhsr2t->bhsr2t_flags = 0x80;
2796 	bhsr2t->bhsr2t_lun = bhssc->bhssc_lun;
2797 	bhsr2t->bhsr2t_initiator_task_tag = bhssc->bhssc_initiator_task_tag;
2798 	bhsr2t->bhsr2t_target_transfer_tag = target_transfer_tag;
2799 	/*
2800 	 * XXX: Here we assume that cfiscsi_datamove() won't ever
2801 	 *	be running concurrently on several CPUs for a given
2802 	 *	command.
2803 	 */
2804 	bhsr2t->bhsr2t_r2tsn = htonl(PRIV_R2TSN(io)++);
2805 	/*
2806 	 * This is the offset within the current SCSI command;
2807 	 * i.e. for the first call of datamove(), it will be 0,
2808 	 * and for subsequent ones it will be the sum of lengths
2809 	 * of previous ones.
2810 	 *
2811 	 * The ext_data_filled is to account for unsolicited
2812 	 * (immediate) data that might have already arrived.
2813 	 */
2814 	bhsr2t->bhsr2t_buffer_offset = htonl(r2t_off);
2815 	/*
2816 	 * This is the total length (sum of S/G lengths) this call
2817 	 * to cfiscsi_datamove() is supposed to handle, limited by
2818 	 * MaxBurstLength.
2819 	 */
2820 	bhsr2t->bhsr2t_desired_data_transfer_length = htonl(r2t_len);
2821 	cfiscsi_pdu_queue(response);
2822 }
2823 
2824 static void
2825 cfiscsi_datamove(union ctl_io *io)
2826 {
2827 
2828 	if ((io->io_hdr.flags & CTL_FLAG_DATA_MASK) == CTL_FLAG_DATA_IN)
2829 		cfiscsi_datamove_in(io);
2830 	else {
2831 		/* We hadn't received anything during this datamove yet. */
2832 		io->scsiio.ext_data_filled = 0;
2833 		cfiscsi_datamove_out(io);
2834 	}
2835 }
2836 
2837 static void
2838 cfiscsi_scsi_command_done(union ctl_io *io)
2839 {
2840 	struct icl_pdu *request, *response;
2841 	struct iscsi_bhs_scsi_command *bhssc;
2842 	struct iscsi_bhs_scsi_response *bhssr;
2843 #ifdef DIAGNOSTIC
2844 	struct cfiscsi_data_wait *cdw;
2845 #endif
2846 	struct cfiscsi_session *cs;
2847 	uint16_t sense_length;
2848 
2849 	request = PRIV_REQUEST(io);
2850 	cs = PDU_SESSION(request);
2851 	bhssc = (struct iscsi_bhs_scsi_command *)request->ip_bhs;
2852 	KASSERT((bhssc->bhssc_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
2853 	    ISCSI_BHS_OPCODE_SCSI_COMMAND,
2854 	    ("replying to wrong opcode 0x%x", bhssc->bhssc_opcode));
2855 
2856 	//CFISCSI_SESSION_DEBUG(cs, "initiator task tag 0x%x",
2857 	//    bhssc->bhssc_initiator_task_tag);
2858 
2859 #ifdef DIAGNOSTIC
2860 	CFISCSI_SESSION_LOCK(cs);
2861 	TAILQ_FOREACH(cdw, &cs->cs_waiting_for_data_out, cdw_next)
2862 		KASSERT(bhssc->bhssc_initiator_task_tag !=
2863 		    cdw->cdw_initiator_task_tag, ("dangling cdw"));
2864 	CFISCSI_SESSION_UNLOCK(cs);
2865 #endif
2866 
2867 	/*
2868 	 * Do not return status for aborted commands.
2869 	 * There are exceptions, but none supported by CTL yet.
2870 	 */
2871 	if (((io->io_hdr.flags & CTL_FLAG_ABORT) &&
2872 	     (io->io_hdr.flags & CTL_FLAG_ABORT_STATUS) == 0) ||
2873 	    (io->io_hdr.flags & CTL_FLAG_STATUS_SENT)) {
2874 		ctl_free_io(io);
2875 		icl_pdu_free(request);
2876 		return;
2877 	}
2878 
2879 	response = cfiscsi_pdu_new_response(request, M_WAITOK);
2880 	bhssr = (struct iscsi_bhs_scsi_response *)response->ip_bhs;
2881 	bhssr->bhssr_opcode = ISCSI_BHS_OPCODE_SCSI_RESPONSE;
2882 	bhssr->bhssr_flags = 0x80;
2883 	/*
2884 	 * XXX: We don't deal with bidirectional under/overflows;
2885 	 *	does anything actually support those?
2886 	 */
2887 	if (io->scsiio.kern_total_len <
2888 	    ntohl(bhssc->bhssc_expected_data_transfer_length)) {
2889 		bhssr->bhssr_flags |= BHSSR_FLAGS_RESIDUAL_UNDERFLOW;
2890 		bhssr->bhssr_residual_count =
2891 		    htonl(ntohl(bhssc->bhssc_expected_data_transfer_length) -
2892 		    io->scsiio.kern_total_len);
2893 		//CFISCSI_SESSION_DEBUG(cs, "underflow; residual count %d",
2894 		//    ntohl(bhssr->bhssr_residual_count));
2895 	} else if (io->scsiio.kern_total_len >
2896 	    ntohl(bhssc->bhssc_expected_data_transfer_length)) {
2897 		bhssr->bhssr_flags |= BHSSR_FLAGS_RESIDUAL_OVERFLOW;
2898 		bhssr->bhssr_residual_count = htonl(io->scsiio.kern_total_len -
2899 		    ntohl(bhssc->bhssc_expected_data_transfer_length));
2900 		//CFISCSI_SESSION_DEBUG(cs, "overflow; residual count %d",
2901 		//    ntohl(bhssr->bhssr_residual_count));
2902 	}
2903 	bhssr->bhssr_response = BHSSR_RESPONSE_COMMAND_COMPLETED;
2904 	bhssr->bhssr_status = io->scsiio.scsi_status;
2905 	bhssr->bhssr_initiator_task_tag = bhssc->bhssc_initiator_task_tag;
2906 	bhssr->bhssr_expdatasn = htonl(PRIV_EXPDATASN(io));
2907 
2908 	if (io->scsiio.sense_len > 0) {
2909 #if 0
2910 		CFISCSI_SESSION_DEBUG(cs, "returning %d bytes of sense data",
2911 		    io->scsiio.sense_len);
2912 #endif
2913 		sense_length = htons(io->scsiio.sense_len);
2914 		icl_pdu_append_data(response,
2915 		    &sense_length, sizeof(sense_length), M_WAITOK);
2916 		icl_pdu_append_data(response,
2917 		    &io->scsiio.sense_data, io->scsiio.sense_len, M_WAITOK);
2918 	}
2919 
2920 	ctl_free_io(io);
2921 	icl_pdu_free(request);
2922 	cfiscsi_pdu_queue(response);
2923 }
2924 
2925 static void
2926 cfiscsi_task_management_done(union ctl_io *io)
2927 {
2928 	struct icl_pdu *request, *response;
2929 	struct iscsi_bhs_task_management_request *bhstmr;
2930 	struct iscsi_bhs_task_management_response *bhstmr2;
2931 	struct cfiscsi_data_wait *cdw, *tmpcdw;
2932 	struct cfiscsi_session *cs, *tcs;
2933 	struct cfiscsi_softc *softc;
2934 	union ctl_io *cdw_io;
2935 	int cold_reset = 0;
2936 
2937 	request = PRIV_REQUEST(io);
2938 	cs = PDU_SESSION(request);
2939 	bhstmr = (struct iscsi_bhs_task_management_request *)request->ip_bhs;
2940 	KASSERT((bhstmr->bhstmr_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
2941 	    ISCSI_BHS_OPCODE_TASK_REQUEST,
2942 	    ("replying to wrong opcode 0x%x", bhstmr->bhstmr_opcode));
2943 
2944 #if 0
2945 	CFISCSI_SESSION_DEBUG(cs, "initiator task tag 0x%x; referenced task tag 0x%x",
2946 	    bhstmr->bhstmr_initiator_task_tag,
2947 	    bhstmr->bhstmr_referenced_task_tag);
2948 #endif
2949 
2950 	if ((bhstmr->bhstmr_function & ~0x80) ==
2951 	    BHSTMR_FUNCTION_ABORT_TASK) {
2952 		/*
2953 		 * Make sure we no longer wait for Data-Out for this command.
2954 		 */
2955 		CFISCSI_SESSION_LOCK(cs);
2956 		TAILQ_FOREACH_SAFE(cdw,
2957 		    &cs->cs_waiting_for_data_out, cdw_next, tmpcdw) {
2958 			if (bhstmr->bhstmr_referenced_task_tag !=
2959 			    cdw->cdw_initiator_task_tag)
2960 				continue;
2961 
2962 #if 0
2963 			CFISCSI_SESSION_DEBUG(cs, "removing csw for initiator task "
2964 			    "tag 0x%x", bhstmr->bhstmr_initiator_task_tag);
2965 #endif
2966 			TAILQ_REMOVE(&cs->cs_waiting_for_data_out,
2967 			    cdw, cdw_next);
2968 			cdw_io = cdw->cdw_ctl_io;
2969 			cdw_io->io_hdr.flags &= ~CTL_FLAG_DMA_INPROG;
2970 			cdw_io->scsiio.io_hdr.port_status = 43;
2971 			cfiscsi_data_wait_free(cs, cdw);
2972 			ctl_datamove_done(cdw_io, false);
2973 		}
2974 		CFISCSI_SESSION_UNLOCK(cs);
2975 	}
2976 	if ((bhstmr->bhstmr_function & ~0x80) ==
2977 	    BHSTMR_FUNCTION_TARGET_COLD_RESET &&
2978 	    io->io_hdr.status == CTL_SUCCESS)
2979 		cold_reset = 1;
2980 
2981 	response = cfiscsi_pdu_new_response(request, M_WAITOK);
2982 	bhstmr2 = (struct iscsi_bhs_task_management_response *)
2983 	    response->ip_bhs;
2984 	bhstmr2->bhstmr_opcode = ISCSI_BHS_OPCODE_TASK_RESPONSE;
2985 	bhstmr2->bhstmr_flags = 0x80;
2986 	switch (io->taskio.task_status) {
2987 	case CTL_TASK_FUNCTION_COMPLETE:
2988 		bhstmr2->bhstmr_response = BHSTMR_RESPONSE_FUNCTION_COMPLETE;
2989 		break;
2990 	case CTL_TASK_FUNCTION_SUCCEEDED:
2991 		bhstmr2->bhstmr_response = BHSTMR_RESPONSE_FUNCTION_SUCCEEDED;
2992 		break;
2993 	case CTL_TASK_LUN_DOES_NOT_EXIST:
2994 		bhstmr2->bhstmr_response = BHSTMR_RESPONSE_LUN_DOES_NOT_EXIST;
2995 		break;
2996 	case CTL_TASK_FUNCTION_NOT_SUPPORTED:
2997 	default:
2998 		bhstmr2->bhstmr_response = BHSTMR_RESPONSE_FUNCTION_NOT_SUPPORTED;
2999 		break;
3000 	}
3001 	memcpy(bhstmr2->bhstmr_additional_reponse_information,
3002 	    io->taskio.task_resp, sizeof(io->taskio.task_resp));
3003 	bhstmr2->bhstmr_initiator_task_tag = bhstmr->bhstmr_initiator_task_tag;
3004 
3005 	ctl_free_io(io);
3006 	icl_pdu_free(request);
3007 	cfiscsi_pdu_queue(response);
3008 
3009 	if (cold_reset) {
3010 		softc = cs->cs_target->ct_softc;
3011 		mtx_lock(&softc->lock);
3012 		TAILQ_FOREACH(tcs, &softc->sessions, cs_next) {
3013 			if (tcs->cs_target == cs->cs_target)
3014 				cfiscsi_session_terminate(tcs);
3015 		}
3016 		mtx_unlock(&softc->lock);
3017 	}
3018 }
3019 
3020 static void
3021 cfiscsi_done(union ctl_io *io)
3022 {
3023 	struct icl_pdu *request;
3024 	struct cfiscsi_session *cs;
3025 
3026 	KASSERT(((io->io_hdr.status & CTL_STATUS_MASK) != CTL_STATUS_NONE),
3027 		("invalid CTL status %#x", io->io_hdr.status));
3028 
3029 	if (io->io_hdr.io_type == CTL_IO_TASK &&
3030 	    io->taskio.task_action == CTL_TASK_I_T_NEXUS_RESET) {
3031 		/*
3032 		 * Implicit task termination has just completed; nothing to do.
3033 		 */
3034 		cs = PRIV_REQUEST(io);
3035 		cs->cs_tasks_aborted = true;
3036 		refcount_release(&cs->cs_outstanding_ctl_pdus);
3037 		wakeup(__DEVOLATILE(void *, &cs->cs_outstanding_ctl_pdus));
3038 		ctl_free_io(io);
3039 		return;
3040 	}
3041 
3042 	request = PRIV_REQUEST(io);
3043 	cs = PDU_SESSION(request);
3044 
3045 	switch (request->ip_bhs->bhs_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) {
3046 	case ISCSI_BHS_OPCODE_SCSI_COMMAND:
3047 		cfiscsi_scsi_command_done(io);
3048 		break;
3049 	case ISCSI_BHS_OPCODE_TASK_REQUEST:
3050 		cfiscsi_task_management_done(io);
3051 		break;
3052 	default:
3053 		panic("cfiscsi_done called with wrong opcode 0x%x",
3054 		    request->ip_bhs->bhs_opcode);
3055 	}
3056 
3057 	refcount_release(&cs->cs_outstanding_ctl_pdus);
3058 }
3059