xref: /freebsd/sys/cam/ctl/ctl_frontend_iscsi.c (revision f61e92ca5a23450bc28169bbdd71d7674df98c19)
1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3  *
4  * Copyright (c) 2012 The FreeBSD Foundation
5  *
6  * This software was developed by Edward Tomasz Napierala under sponsorship
7  * from the FreeBSD Foundation.
8  *
9  * Redistribution and use in source and binary forms, with or without
10  * modification, are permitted provided that the following conditions
11  * are met:
12  * 1. Redistributions of source code must retain the above copyright
13  *    notice, this list of conditions and the following disclaimer.
14  * 2. Redistributions in binary form must reproduce the above copyright
15  *    notice, this list of conditions and the following disclaimer in the
16  *    documentation and/or other materials provided with the distribution.
17  *
18  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
19  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
22  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28  * SUCH DAMAGE.
29  *
30  * $FreeBSD$
31  */
32 
33 /*
34  * CTL frontend for the iSCSI protocol.
35  */
36 
37 #include <sys/cdefs.h>
38 __FBSDID("$FreeBSD$");
39 
40 #include <sys/param.h>
41 #include <sys/capsicum.h>
42 #include <sys/condvar.h>
43 #include <sys/endian.h>
44 #include <sys/file.h>
45 #include <sys/kernel.h>
46 #include <sys/kthread.h>
47 #include <sys/lock.h>
48 #include <sys/malloc.h>
49 #include <sys/module.h>
50 #include <sys/mutex.h>
51 #include <sys/queue.h>
52 #include <sys/sbuf.h>
53 #include <sys/socket.h>
54 #include <sys/sysctl.h>
55 #include <sys/systm.h>
56 #include <sys/uio.h>
57 #include <sys/unistd.h>
58 #include <sys/nv.h>
59 #include <sys/dnv.h>
60 #include <vm/uma.h>
61 
62 #include <cam/scsi/scsi_all.h>
63 #include <cam/scsi/scsi_da.h>
64 #include <cam/ctl/ctl_io.h>
65 #include <cam/ctl/ctl.h>
66 #include <cam/ctl/ctl_backend.h>
67 #include <cam/ctl/ctl_error.h>
68 #include <cam/ctl/ctl_frontend.h>
69 #include <cam/ctl/ctl_debug.h>
70 #include <cam/ctl/ctl_ha.h>
71 #include <cam/ctl/ctl_ioctl.h>
72 #include <cam/ctl/ctl_private.h>
73 
74 #include <dev/iscsi/icl.h>
75 #include <dev/iscsi/icl_wrappers.h>
76 #include <dev/iscsi/iscsi_proto.h>
77 #include <cam/ctl/ctl_frontend_iscsi.h>
78 
79 #ifdef ICL_KERNEL_PROXY
80 #include <sys/socketvar.h>
81 #endif
82 
83 #ifdef ICL_KERNEL_PROXY
84 FEATURE(cfiscsi_kernel_proxy, "iSCSI target built with ICL_KERNEL_PROXY");
85 #endif
86 
87 static MALLOC_DEFINE(M_CFISCSI, "cfiscsi", "Memory used for CTL iSCSI frontend");
88 static uma_zone_t cfiscsi_data_wait_zone;
89 
90 SYSCTL_NODE(_kern_cam_ctl, OID_AUTO, iscsi, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
91     "CAM Target Layer iSCSI Frontend");
92 static int debug = 1;
93 SYSCTL_INT(_kern_cam_ctl_iscsi, OID_AUTO, debug, CTLFLAG_RWTUN,
94     &debug, 1, "Enable debug messages");
95 static int ping_timeout = 5;
96 SYSCTL_INT(_kern_cam_ctl_iscsi, OID_AUTO, ping_timeout, CTLFLAG_RWTUN,
97     &ping_timeout, 5, "Interval between ping (NOP-Out) requests, in seconds");
98 static int login_timeout = 60;
99 SYSCTL_INT(_kern_cam_ctl_iscsi, OID_AUTO, login_timeout, CTLFLAG_RWTUN,
100     &login_timeout, 60, "Time to wait for ctld(8) to finish Login Phase, in seconds");
101 static int maxtags = 256;
102 SYSCTL_INT(_kern_cam_ctl_iscsi, OID_AUTO, maxtags, CTLFLAG_RWTUN,
103     &maxtags, 0, "Max number of requests queued by initiator");
104 
105 #define	CFISCSI_DEBUG(X, ...)						\
106 	do {								\
107 		if (debug > 1) {					\
108 			printf("%s: " X "\n",				\
109 			    __func__, ## __VA_ARGS__);			\
110 		}							\
111 	} while (0)
112 
113 #define	CFISCSI_WARN(X, ...)						\
114 	do {								\
115 		if (debug > 0) {					\
116 			printf("WARNING: %s: " X "\n",			\
117 			    __func__, ## __VA_ARGS__);			\
118 		}							\
119 	} while (0)
120 
121 #define	CFISCSI_SESSION_DEBUG(S, X, ...)				\
122 	do {								\
123 		if (debug > 1) {					\
124 			printf("%s: %s (%s): " X "\n",			\
125 			    __func__, S->cs_initiator_addr,		\
126 			    S->cs_initiator_name, ## __VA_ARGS__);	\
127 		}							\
128 	} while (0)
129 
130 #define	CFISCSI_SESSION_WARN(S, X, ...)					\
131 	do  {								\
132 		if (debug > 0) {					\
133 			printf("WARNING: %s (%s): " X "\n",		\
134 			    S->cs_initiator_addr,			\
135 			    S->cs_initiator_name, ## __VA_ARGS__);	\
136 		}							\
137 	} while (0)
138 
139 #define CFISCSI_SESSION_LOCK(X)		mtx_lock(&X->cs_lock)
140 #define CFISCSI_SESSION_UNLOCK(X)	mtx_unlock(&X->cs_lock)
141 #define CFISCSI_SESSION_LOCK_ASSERT(X)	mtx_assert(&X->cs_lock, MA_OWNED)
142 
143 #define	CONN_SESSION(X)			((struct cfiscsi_session *)(X)->ic_prv0)
144 #define	PDU_SESSION(X)			CONN_SESSION((X)->ip_conn)
145 
146 struct cfiscsi_priv {
147 	void		*request;
148 	uint32_t	 expdatasn;
149 	uint32_t	 r2tsn;
150 };
151 #define	PRIV(io)	\
152     ((struct cfiscsi_priv *)&(io)->io_hdr.ctl_private[CTL_PRIV_FRONTEND])
153 #define	PRIV_REQUEST(io)		PRIV(io)->request
154 #define	PRIV_EXPDATASN(io)		PRIV(io)->expdatasn
155 #define	PRIV_R2TSN(io)			PRIV(io)->r2tsn
156 
157 static int	cfiscsi_init(void);
158 static int	cfiscsi_shutdown(void);
159 static void	cfiscsi_online(void *arg);
160 static void	cfiscsi_offline(void *arg);
161 static int	cfiscsi_info(void *arg, struct sbuf *sb);
162 static int	cfiscsi_ioctl(struct cdev *dev,
163 		    u_long cmd, caddr_t addr, int flag, struct thread *td);
164 static void	cfiscsi_datamove(union ctl_io *io);
165 static void	cfiscsi_datamove_in(union ctl_io *io);
166 static void	cfiscsi_datamove_out(union ctl_io *io);
167 static void	cfiscsi_done(union ctl_io *io);
168 static bool	cfiscsi_pdu_update_cmdsn(const struct icl_pdu *request);
169 static void	cfiscsi_pdu_handle_nop_out(struct icl_pdu *request);
170 static void	cfiscsi_pdu_handle_scsi_command(struct icl_pdu *request);
171 static void	cfiscsi_pdu_handle_task_request(struct icl_pdu *request);
172 static void	cfiscsi_pdu_handle_data_out(struct icl_pdu *request);
173 static void	cfiscsi_pdu_handle_logout_request(struct icl_pdu *request);
174 static void	cfiscsi_session_terminate(struct cfiscsi_session *cs);
175 static struct cfiscsi_data_wait	*cfiscsi_data_wait_new(
176 		    struct cfiscsi_session *cs, union ctl_io *io,
177 		    uint32_t initiator_task_tag,
178 		    uint32_t *target_transfer_tagp);
179 static void	cfiscsi_data_wait_free(struct cfiscsi_session *cs,
180 		    struct cfiscsi_data_wait *cdw);
181 static struct cfiscsi_target	*cfiscsi_target_find(struct cfiscsi_softc
182 		    *softc, const char *name, uint16_t tag);
183 static struct cfiscsi_target	*cfiscsi_target_find_or_create(
184     struct cfiscsi_softc *softc, const char *name, const char *alias,
185     uint16_t tag);
186 static void	cfiscsi_target_release(struct cfiscsi_target *ct);
187 static void	cfiscsi_session_delete(struct cfiscsi_session *cs);
188 
189 static struct cfiscsi_softc cfiscsi_softc;
190 
191 static struct ctl_frontend cfiscsi_frontend =
192 {
193 	.name = "iscsi",
194 	.init = cfiscsi_init,
195 	.ioctl = cfiscsi_ioctl,
196 	.shutdown = cfiscsi_shutdown,
197 };
198 CTL_FRONTEND_DECLARE(cfiscsi, cfiscsi_frontend);
199 MODULE_DEPEND(cfiscsi, icl, 1, 1, 1);
200 
201 static struct icl_pdu *
202 cfiscsi_pdu_new_response(struct icl_pdu *request, int flags)
203 {
204 
205 	return (icl_pdu_new(request->ip_conn, flags));
206 }
207 
208 static bool
209 cfiscsi_pdu_update_cmdsn(const struct icl_pdu *request)
210 {
211 	const struct iscsi_bhs_scsi_command *bhssc;
212 	struct cfiscsi_session *cs;
213 	uint32_t cmdsn, curcmdsn;
214 
215 	cs = PDU_SESSION(request);
216 
217 	/*
218 	 * Every incoming PDU - not just NOP-Out - resets the ping timer.
219 	 * The purpose of the timeout is to reset the connection when it stalls;
220 	 * we don't want this to happen when NOP-In or NOP-Out ends up delayed
221 	 * in some queue.
222 	 */
223 	cs->cs_timeout = 0;
224 
225 	/*
226 	 * Immediate commands carry cmdsn, but it is neither incremented nor
227 	 * verified.
228 	 */
229 	if (request->ip_bhs->bhs_opcode & ISCSI_BHS_OPCODE_IMMEDIATE)
230 		return (false);
231 
232 	/*
233 	 * Data-Out PDUs don't contain CmdSN.
234 	 */
235 	if (request->ip_bhs->bhs_opcode == ISCSI_BHS_OPCODE_SCSI_DATA_OUT)
236 		return (false);
237 
238 	/*
239 	 * We're only using fields common for all the request
240 	 * (initiator -> target) PDUs.
241 	 */
242 	bhssc = (const struct iscsi_bhs_scsi_command *)request->ip_bhs;
243 	curcmdsn = cmdsn = ntohl(bhssc->bhssc_cmdsn);
244 
245 	/*
246 	 * Increment session cmdsn and exit if we received the expected value.
247 	 */
248 	do {
249 		if (atomic_fcmpset_32(&cs->cs_cmdsn, &curcmdsn, cmdsn + 1))
250 			return (false);
251 	} while (curcmdsn == cmdsn);
252 
253 	/*
254 	 * The target MUST silently ignore any non-immediate command outside
255 	 * of this range.
256 	 */
257 	if (ISCSI_SNLT(cmdsn, curcmdsn) ||
258 	    ISCSI_SNGT(cmdsn, curcmdsn - 1 + maxtags)) {
259 		CFISCSI_SESSION_WARN(cs, "received PDU with CmdSN %u, "
260 		    "while expected %u", cmdsn, curcmdsn);
261 		return (true);
262 	}
263 
264 	/*
265 	 * We don't support multiple connections now, so any discontinuity in
266 	 * CmdSN means lost PDUs.  Since we don't support PDU retransmission --
267 	 * terminate the connection.
268 	 */
269 	CFISCSI_SESSION_WARN(cs, "received PDU with CmdSN %u, "
270 	    "while expected %u; dropping connection",
271 	    cmdsn, curcmdsn);
272 	cfiscsi_session_terminate(cs);
273 	return (true);
274 }
275 
276 static void
277 cfiscsi_pdu_handle(struct icl_pdu *request)
278 {
279 	struct cfiscsi_session *cs;
280 	bool ignore;
281 
282 	cs = PDU_SESSION(request);
283 
284 	ignore = cfiscsi_pdu_update_cmdsn(request);
285 	if (ignore) {
286 		icl_pdu_free(request);
287 		return;
288 	}
289 
290 	/*
291 	 * Handle the PDU; this includes e.g. receiving the remaining
292 	 * part of PDU and submitting the SCSI command to CTL
293 	 * or queueing a reply.  The handling routine is responsible
294 	 * for freeing the PDU when it's no longer needed.
295 	 */
296 	switch (request->ip_bhs->bhs_opcode &
297 	    ~ISCSI_BHS_OPCODE_IMMEDIATE) {
298 	case ISCSI_BHS_OPCODE_NOP_OUT:
299 		cfiscsi_pdu_handle_nop_out(request);
300 		break;
301 	case ISCSI_BHS_OPCODE_SCSI_COMMAND:
302 		cfiscsi_pdu_handle_scsi_command(request);
303 		break;
304 	case ISCSI_BHS_OPCODE_TASK_REQUEST:
305 		cfiscsi_pdu_handle_task_request(request);
306 		break;
307 	case ISCSI_BHS_OPCODE_SCSI_DATA_OUT:
308 		cfiscsi_pdu_handle_data_out(request);
309 		break;
310 	case ISCSI_BHS_OPCODE_LOGOUT_REQUEST:
311 		cfiscsi_pdu_handle_logout_request(request);
312 		break;
313 	default:
314 		CFISCSI_SESSION_WARN(cs, "received PDU with unsupported "
315 		    "opcode 0x%x; dropping connection",
316 		    request->ip_bhs->bhs_opcode);
317 		icl_pdu_free(request);
318 		cfiscsi_session_terminate(cs);
319 	}
320 
321 }
322 
323 static void
324 cfiscsi_receive_callback(struct icl_pdu *request)
325 {
326 #ifdef ICL_KERNEL_PROXY
327 	struct cfiscsi_session *cs;
328 
329 	cs = PDU_SESSION(request);
330 	if (cs->cs_waiting_for_ctld || cs->cs_login_phase) {
331 		if (cs->cs_login_pdu == NULL)
332 			cs->cs_login_pdu = request;
333 		else
334 			icl_pdu_free(request);
335 		cv_signal(&cs->cs_login_cv);
336 		return;
337 	}
338 #endif
339 
340 	cfiscsi_pdu_handle(request);
341 }
342 
343 static void
344 cfiscsi_error_callback(struct icl_conn *ic)
345 {
346 	struct cfiscsi_session *cs;
347 
348 	cs = CONN_SESSION(ic);
349 
350 	CFISCSI_SESSION_WARN(cs, "connection error; dropping connection");
351 	cfiscsi_session_terminate(cs);
352 }
353 
354 static int
355 cfiscsi_pdu_prepare(struct icl_pdu *response)
356 {
357 	struct cfiscsi_session *cs;
358 	struct iscsi_bhs_scsi_response *bhssr;
359 	bool advance_statsn = true;
360 	uint32_t cmdsn;
361 
362 	cs = PDU_SESSION(response);
363 
364 	CFISCSI_SESSION_LOCK_ASSERT(cs);
365 
366 	/*
367 	 * We're only using fields common for all the response
368 	 * (target -> initiator) PDUs.
369 	 */
370 	bhssr = (struct iscsi_bhs_scsi_response *)response->ip_bhs;
371 
372 	/*
373 	 * 10.8.3: "The StatSN for this connection is not advanced
374 	 * after this PDU is sent."
375 	 */
376 	if (bhssr->bhssr_opcode == ISCSI_BHS_OPCODE_R2T)
377 		advance_statsn = false;
378 
379 	/*
380 	 * 10.19.2: "However, when the Initiator Task Tag is set to 0xffffffff,
381 	 * StatSN for the connection is not advanced after this PDU is sent."
382 	 */
383 	if (bhssr->bhssr_opcode == ISCSI_BHS_OPCODE_NOP_IN &&
384 	    bhssr->bhssr_initiator_task_tag == 0xffffffff)
385 		advance_statsn = false;
386 
387 	/*
388 	 * See the comment below - StatSN is not meaningful and must
389 	 * not be advanced.
390 	 */
391 	if (bhssr->bhssr_opcode == ISCSI_BHS_OPCODE_SCSI_DATA_IN &&
392 	    (bhssr->bhssr_flags & BHSDI_FLAGS_S) == 0)
393 		advance_statsn = false;
394 
395 	/*
396 	 * 10.7.3: "The fields StatSN, Status, and Residual Count
397 	 * only have meaningful content if the S bit is set to 1."
398 	 */
399 	if (bhssr->bhssr_opcode != ISCSI_BHS_OPCODE_SCSI_DATA_IN ||
400 	    (bhssr->bhssr_flags & BHSDI_FLAGS_S))
401 		bhssr->bhssr_statsn = htonl(cs->cs_statsn);
402 	cmdsn = cs->cs_cmdsn;
403 	bhssr->bhssr_expcmdsn = htonl(cmdsn);
404 	bhssr->bhssr_maxcmdsn = htonl(cmdsn - 1 +
405 	    imax(0, maxtags - cs->cs_outstanding_ctl_pdus));
406 
407 	if (advance_statsn)
408 		cs->cs_statsn++;
409 
410 	return (0);
411 }
412 
413 static void
414 cfiscsi_pdu_queue(struct icl_pdu *response)
415 {
416 	struct cfiscsi_session *cs;
417 
418 	cs = PDU_SESSION(response);
419 
420 	CFISCSI_SESSION_LOCK(cs);
421 	cfiscsi_pdu_prepare(response);
422 	icl_pdu_queue(response);
423 	CFISCSI_SESSION_UNLOCK(cs);
424 }
425 
426  static void
427 cfiscsi_pdu_queue_cb(struct icl_pdu *response, icl_pdu_cb cb)
428 {
429 	struct cfiscsi_session *cs = PDU_SESSION(response);
430 
431 	CFISCSI_SESSION_LOCK(cs);
432 	cfiscsi_pdu_prepare(response);
433 	icl_pdu_queue_cb(response, cb);
434 	CFISCSI_SESSION_UNLOCK(cs);
435 }
436 
437 static void
438 cfiscsi_pdu_handle_nop_out(struct icl_pdu *request)
439 {
440 	struct cfiscsi_session *cs;
441 	struct iscsi_bhs_nop_out *bhsno;
442 	struct iscsi_bhs_nop_in *bhsni;
443 	struct icl_pdu *response;
444 	void *data = NULL;
445 	size_t datasize;
446 	int error;
447 
448 	cs = PDU_SESSION(request);
449 	bhsno = (struct iscsi_bhs_nop_out *)request->ip_bhs;
450 
451 	if (bhsno->bhsno_initiator_task_tag == 0xffffffff) {
452 		/*
453 		 * Nothing to do, iscsi_pdu_update_statsn() already
454 		 * zeroed the timeout.
455 		 */
456 		icl_pdu_free(request);
457 		return;
458 	}
459 
460 	datasize = icl_pdu_data_segment_length(request);
461 	if (datasize > 0) {
462 		data = malloc(datasize, M_CFISCSI, M_NOWAIT | M_ZERO);
463 		if (data == NULL) {
464 			CFISCSI_SESSION_WARN(cs, "failed to allocate memory; "
465 			    "dropping connection");
466 			icl_pdu_free(request);
467 			cfiscsi_session_terminate(cs);
468 			return;
469 		}
470 		icl_pdu_get_data(request, 0, data, datasize);
471 	}
472 
473 	response = cfiscsi_pdu_new_response(request, M_NOWAIT);
474 	if (response == NULL) {
475 		CFISCSI_SESSION_WARN(cs, "failed to allocate memory; "
476 		    "droppping connection");
477 		free(data, M_CFISCSI);
478 		icl_pdu_free(request);
479 		cfiscsi_session_terminate(cs);
480 		return;
481 	}
482 	bhsni = (struct iscsi_bhs_nop_in *)response->ip_bhs;
483 	bhsni->bhsni_opcode = ISCSI_BHS_OPCODE_NOP_IN;
484 	bhsni->bhsni_flags = 0x80;
485 	bhsni->bhsni_initiator_task_tag = bhsno->bhsno_initiator_task_tag;
486 	bhsni->bhsni_target_transfer_tag = 0xffffffff;
487 	if (datasize > 0) {
488 		error = icl_pdu_append_data(response, data, datasize, M_NOWAIT);
489 		if (error != 0) {
490 			CFISCSI_SESSION_WARN(cs, "failed to allocate memory; "
491 			    "dropping connection");
492 			free(data, M_CFISCSI);
493 			icl_pdu_free(request);
494 			icl_pdu_free(response);
495 			cfiscsi_session_terminate(cs);
496 			return;
497 		}
498 		free(data, M_CFISCSI);
499 	}
500 
501 	icl_pdu_free(request);
502 	cfiscsi_pdu_queue(response);
503 }
504 
505 static void
506 cfiscsi_pdu_handle_scsi_command(struct icl_pdu *request)
507 {
508 	struct iscsi_bhs_scsi_command *bhssc;
509 	struct cfiscsi_session *cs;
510 	union ctl_io *io;
511 	int error;
512 
513 	cs = PDU_SESSION(request);
514 	bhssc = (struct iscsi_bhs_scsi_command *)request->ip_bhs;
515 	//CFISCSI_SESSION_DEBUG(cs, "initiator task tag 0x%x",
516 	//    bhssc->bhssc_initiator_task_tag);
517 
518 	if (request->ip_data_len > 0 && cs->cs_immediate_data == false) {
519 		CFISCSI_SESSION_WARN(cs, "unsolicited data with "
520 		    "ImmediateData=No; dropping connection");
521 		icl_pdu_free(request);
522 		cfiscsi_session_terminate(cs);
523 		return;
524 	}
525 	io = ctl_alloc_io(cs->cs_target->ct_port.ctl_pool_ref);
526 	ctl_zero_io(io);
527 	PRIV_REQUEST(io) = request;
528 	io->io_hdr.io_type = CTL_IO_SCSI;
529 	io->io_hdr.nexus.initid = cs->cs_ctl_initid;
530 	io->io_hdr.nexus.targ_port = cs->cs_target->ct_port.targ_port;
531 	io->io_hdr.nexus.targ_lun = ctl_decode_lun(be64toh(bhssc->bhssc_lun));
532 	io->scsiio.priority = (bhssc->bhssc_pri & BHSSC_PRI_MASK) >>
533 	    BHSSC_PRI_SHIFT;
534 	io->scsiio.tag_num = bhssc->bhssc_initiator_task_tag;
535 	switch ((bhssc->bhssc_flags & BHSSC_FLAGS_ATTR)) {
536 	case BHSSC_FLAGS_ATTR_UNTAGGED:
537 		io->scsiio.tag_type = CTL_TAG_UNTAGGED;
538 		break;
539 	case BHSSC_FLAGS_ATTR_SIMPLE:
540 		io->scsiio.tag_type = CTL_TAG_SIMPLE;
541 		break;
542 	case BHSSC_FLAGS_ATTR_ORDERED:
543         	io->scsiio.tag_type = CTL_TAG_ORDERED;
544 		break;
545 	case BHSSC_FLAGS_ATTR_HOQ:
546         	io->scsiio.tag_type = CTL_TAG_HEAD_OF_QUEUE;
547 		break;
548 	case BHSSC_FLAGS_ATTR_ACA:
549 		io->scsiio.tag_type = CTL_TAG_ACA;
550 		break;
551 	default:
552 		io->scsiio.tag_type = CTL_TAG_UNTAGGED;
553 		CFISCSI_SESSION_WARN(cs, "unhandled tag type %d",
554 		    bhssc->bhssc_flags & BHSSC_FLAGS_ATTR);
555 		break;
556 	}
557 	io->scsiio.cdb_len = sizeof(bhssc->bhssc_cdb); /* Which is 16. */
558 	memcpy(io->scsiio.cdb, bhssc->bhssc_cdb, sizeof(bhssc->bhssc_cdb));
559 	refcount_acquire(&cs->cs_outstanding_ctl_pdus);
560 	error = ctl_queue(io);
561 	if (error != CTL_RETVAL_COMPLETE) {
562 		CFISCSI_SESSION_WARN(cs, "ctl_queue() failed; error %d; "
563 		    "dropping connection", error);
564 		ctl_free_io(io);
565 		refcount_release(&cs->cs_outstanding_ctl_pdus);
566 		icl_pdu_free(request);
567 		cfiscsi_session_terminate(cs);
568 	}
569 }
570 
571 static void
572 cfiscsi_pdu_handle_task_request(struct icl_pdu *request)
573 {
574 	struct iscsi_bhs_task_management_request *bhstmr;
575 	struct iscsi_bhs_task_management_response *bhstmr2;
576 	struct icl_pdu *response;
577 	struct cfiscsi_session *cs;
578 	union ctl_io *io;
579 	int error;
580 
581 	cs = PDU_SESSION(request);
582 	bhstmr = (struct iscsi_bhs_task_management_request *)request->ip_bhs;
583 	io = ctl_alloc_io(cs->cs_target->ct_port.ctl_pool_ref);
584 	ctl_zero_io(io);
585 	PRIV_REQUEST(io) = request;
586 	io->io_hdr.io_type = CTL_IO_TASK;
587 	io->io_hdr.nexus.initid = cs->cs_ctl_initid;
588 	io->io_hdr.nexus.targ_port = cs->cs_target->ct_port.targ_port;
589 	io->io_hdr.nexus.targ_lun = ctl_decode_lun(be64toh(bhstmr->bhstmr_lun));
590 	io->taskio.tag_type = CTL_TAG_SIMPLE; /* XXX */
591 
592 	switch (bhstmr->bhstmr_function & ~0x80) {
593 	case BHSTMR_FUNCTION_ABORT_TASK:
594 #if 0
595 		CFISCSI_SESSION_DEBUG(cs, "BHSTMR_FUNCTION_ABORT_TASK");
596 #endif
597 		io->taskio.task_action = CTL_TASK_ABORT_TASK;
598 		io->taskio.tag_num = bhstmr->bhstmr_referenced_task_tag;
599 		break;
600 	case BHSTMR_FUNCTION_ABORT_TASK_SET:
601 #if 0
602 		CFISCSI_SESSION_DEBUG(cs, "BHSTMR_FUNCTION_ABORT_TASK_SET");
603 #endif
604 		io->taskio.task_action = CTL_TASK_ABORT_TASK_SET;
605 		break;
606 	case BHSTMR_FUNCTION_CLEAR_TASK_SET:
607 #if 0
608 		CFISCSI_SESSION_DEBUG(cs, "BHSTMR_FUNCTION_CLEAR_TASK_SET");
609 #endif
610 		io->taskio.task_action = CTL_TASK_CLEAR_TASK_SET;
611 		break;
612 	case BHSTMR_FUNCTION_LOGICAL_UNIT_RESET:
613 #if 0
614 		CFISCSI_SESSION_DEBUG(cs, "BHSTMR_FUNCTION_LOGICAL_UNIT_RESET");
615 #endif
616 		io->taskio.task_action = CTL_TASK_LUN_RESET;
617 		break;
618 	case BHSTMR_FUNCTION_TARGET_WARM_RESET:
619 #if 0
620 		CFISCSI_SESSION_DEBUG(cs, "BHSTMR_FUNCTION_TARGET_WARM_RESET");
621 #endif
622 		io->taskio.task_action = CTL_TASK_TARGET_RESET;
623 		break;
624 	case BHSTMR_FUNCTION_TARGET_COLD_RESET:
625 #if 0
626 		CFISCSI_SESSION_DEBUG(cs, "BHSTMR_FUNCTION_TARGET_COLD_RESET");
627 #endif
628 		io->taskio.task_action = CTL_TASK_TARGET_RESET;
629 		break;
630 	case BHSTMR_FUNCTION_QUERY_TASK:
631 #if 0
632 		CFISCSI_SESSION_DEBUG(cs, "BHSTMR_FUNCTION_QUERY_TASK");
633 #endif
634 		io->taskio.task_action = CTL_TASK_QUERY_TASK;
635 		io->taskio.tag_num = bhstmr->bhstmr_referenced_task_tag;
636 		break;
637 	case BHSTMR_FUNCTION_QUERY_TASK_SET:
638 #if 0
639 		CFISCSI_SESSION_DEBUG(cs, "BHSTMR_FUNCTION_QUERY_TASK_SET");
640 #endif
641 		io->taskio.task_action = CTL_TASK_QUERY_TASK_SET;
642 		break;
643 	case BHSTMR_FUNCTION_I_T_NEXUS_RESET:
644 #if 0
645 		CFISCSI_SESSION_DEBUG(cs, "BHSTMR_FUNCTION_I_T_NEXUS_RESET");
646 #endif
647 		io->taskio.task_action = CTL_TASK_I_T_NEXUS_RESET;
648 		break;
649 	case BHSTMR_FUNCTION_QUERY_ASYNC_EVENT:
650 #if 0
651 		CFISCSI_SESSION_DEBUG(cs, "BHSTMR_FUNCTION_QUERY_ASYNC_EVENT");
652 #endif
653 		io->taskio.task_action = CTL_TASK_QUERY_ASYNC_EVENT;
654 		break;
655 	default:
656 		CFISCSI_SESSION_DEBUG(cs, "unsupported function 0x%x",
657 		    bhstmr->bhstmr_function & ~0x80);
658 		ctl_free_io(io);
659 
660 		response = cfiscsi_pdu_new_response(request, M_NOWAIT);
661 		if (response == NULL) {
662 			CFISCSI_SESSION_WARN(cs, "failed to allocate memory; "
663 			    "dropping connection");
664 			icl_pdu_free(request);
665 			cfiscsi_session_terminate(cs);
666 			return;
667 		}
668 		bhstmr2 = (struct iscsi_bhs_task_management_response *)
669 		    response->ip_bhs;
670 		bhstmr2->bhstmr_opcode = ISCSI_BHS_OPCODE_TASK_RESPONSE;
671 		bhstmr2->bhstmr_flags = 0x80;
672 		bhstmr2->bhstmr_response =
673 		    BHSTMR_RESPONSE_FUNCTION_NOT_SUPPORTED;
674 		bhstmr2->bhstmr_initiator_task_tag =
675 		    bhstmr->bhstmr_initiator_task_tag;
676 		icl_pdu_free(request);
677 		cfiscsi_pdu_queue(response);
678 		return;
679 	}
680 
681 	refcount_acquire(&cs->cs_outstanding_ctl_pdus);
682 	error = ctl_queue(io);
683 	if (error != CTL_RETVAL_COMPLETE) {
684 		CFISCSI_SESSION_WARN(cs, "ctl_queue() failed; error %d; "
685 		    "dropping connection", error);
686 		ctl_free_io(io);
687 		refcount_release(&cs->cs_outstanding_ctl_pdus);
688 		icl_pdu_free(request);
689 		cfiscsi_session_terminate(cs);
690 	}
691 }
692 
693 static bool
694 cfiscsi_handle_data_segment(struct icl_pdu *request, struct cfiscsi_data_wait *cdw)
695 {
696 	struct iscsi_bhs_data_out *bhsdo;
697 	struct cfiscsi_session *cs;
698 	struct ctl_sg_entry ctl_sg_entry, *ctl_sglist;
699 	size_t copy_len, len, off, buffer_offset;
700 	int ctl_sg_count;
701 	union ctl_io *io;
702 
703 	cs = PDU_SESSION(request);
704 
705 	KASSERT((request->ip_bhs->bhs_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
706 	    ISCSI_BHS_OPCODE_SCSI_DATA_OUT ||
707 	    (request->ip_bhs->bhs_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
708 	    ISCSI_BHS_OPCODE_SCSI_COMMAND,
709 	    ("bad opcode 0x%x", request->ip_bhs->bhs_opcode));
710 
711 	/*
712 	 * We're only using fields common for Data-Out and SCSI Command PDUs.
713 	 */
714 	bhsdo = (struct iscsi_bhs_data_out *)request->ip_bhs;
715 
716 	io = cdw->cdw_ctl_io;
717 	KASSERT((io->io_hdr.flags & CTL_FLAG_DATA_MASK) != CTL_FLAG_DATA_IN,
718 	    ("CTL_FLAG_DATA_IN"));
719 
720 #if 0
721 	CFISCSI_SESSION_DEBUG(cs, "received %zd bytes out of %d",
722 	    request->ip_data_len, io->scsiio.kern_total_len);
723 #endif
724 
725 	if (io->scsiio.kern_sg_entries > 0) {
726 		ctl_sglist = (struct ctl_sg_entry *)io->scsiio.kern_data_ptr;
727 		ctl_sg_count = io->scsiio.kern_sg_entries;
728 	} else {
729 		ctl_sglist = &ctl_sg_entry;
730 		ctl_sglist->addr = io->scsiio.kern_data_ptr;
731 		ctl_sglist->len = io->scsiio.kern_data_len;
732 		ctl_sg_count = 1;
733 	}
734 
735 	if ((request->ip_bhs->bhs_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
736 	    ISCSI_BHS_OPCODE_SCSI_DATA_OUT)
737 		buffer_offset = ntohl(bhsdo->bhsdo_buffer_offset);
738 	else
739 		buffer_offset = 0;
740 	len = icl_pdu_data_segment_length(request);
741 
742 	/*
743 	 * Make sure the offset, as sent by the initiator, matches the offset
744 	 * we're supposed to be at in the scatter-gather list.
745 	 */
746 	if (buffer_offset >
747 	    io->scsiio.kern_rel_offset + io->scsiio.ext_data_filled ||
748 	    buffer_offset + len <=
749 	    io->scsiio.kern_rel_offset + io->scsiio.ext_data_filled) {
750 		CFISCSI_SESSION_WARN(cs, "received bad buffer offset %zd, "
751 		    "expected %zd; dropping connection", buffer_offset,
752 		    (size_t)io->scsiio.kern_rel_offset +
753 		    (size_t)io->scsiio.ext_data_filled);
754 		ctl_set_data_phase_error(&io->scsiio);
755 		cfiscsi_session_terminate(cs);
756 		return (true);
757 	}
758 
759 	/*
760 	 * This is the offset within the PDU data segment, as opposed
761 	 * to buffer_offset, which is the offset within the task (SCSI
762 	 * command).
763 	 */
764 	off = io->scsiio.kern_rel_offset + io->scsiio.ext_data_filled -
765 	    buffer_offset;
766 
767 	/*
768 	 * Iterate over the scatter/gather segments, filling them with data
769 	 * from the PDU data segment.  Note that this can get called multiple
770 	 * times for one SCSI command; the cdw structure holds state for the
771 	 * scatter/gather list.
772 	 */
773 	for (;;) {
774 		KASSERT(cdw->cdw_sg_index < ctl_sg_count,
775 		    ("cdw->cdw_sg_index >= ctl_sg_count"));
776 		if (cdw->cdw_sg_len == 0) {
777 			cdw->cdw_sg_addr = ctl_sglist[cdw->cdw_sg_index].addr;
778 			cdw->cdw_sg_len = ctl_sglist[cdw->cdw_sg_index].len;
779 		}
780 		KASSERT(off <= len, ("len > off"));
781 		copy_len = len - off;
782 		if (copy_len > cdw->cdw_sg_len)
783 			copy_len = cdw->cdw_sg_len;
784 
785 		icl_pdu_get_data(request, off, cdw->cdw_sg_addr, copy_len);
786 		cdw->cdw_sg_addr += copy_len;
787 		cdw->cdw_sg_len -= copy_len;
788 		off += copy_len;
789 		io->scsiio.ext_data_filled += copy_len;
790 		io->scsiio.kern_data_resid -= copy_len;
791 
792 		if (cdw->cdw_sg_len == 0) {
793 			/*
794 			 * End of current segment.
795 			 */
796 			if (cdw->cdw_sg_index == ctl_sg_count - 1) {
797 				/*
798 				 * Last segment in scatter/gather list.
799 				 */
800 				break;
801 			}
802 			cdw->cdw_sg_index++;
803 		}
804 
805 		if (off == len) {
806 			/*
807 			 * End of PDU payload.
808 			 */
809 			break;
810 		}
811 	}
812 
813 	if (len > off) {
814 		/*
815 		 * In case of unsolicited data, it's possible that the buffer
816 		 * provided by CTL is smaller than negotiated FirstBurstLength.
817 		 * Just ignore the superfluous data; will ask for them with R2T
818 		 * on next call to cfiscsi_datamove().
819 		 *
820 		 * This obviously can only happen with SCSI Command PDU.
821 		 */
822 		if ((request->ip_bhs->bhs_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
823 		    ISCSI_BHS_OPCODE_SCSI_COMMAND)
824 			return (true);
825 
826 		CFISCSI_SESSION_WARN(cs, "received too much data: got %zd bytes, "
827 		    "expected %zd; dropping connection",
828 		    icl_pdu_data_segment_length(request), off);
829 		ctl_set_data_phase_error(&io->scsiio);
830 		cfiscsi_session_terminate(cs);
831 		return (true);
832 	}
833 
834 	if (io->scsiio.ext_data_filled == cdw->cdw_r2t_end &&
835 	    (bhsdo->bhsdo_flags & BHSDO_FLAGS_F) == 0) {
836 		CFISCSI_SESSION_WARN(cs, "got the final packet without "
837 		    "the F flag; flags = 0x%x; dropping connection",
838 		    bhsdo->bhsdo_flags);
839 		ctl_set_data_phase_error(&io->scsiio);
840 		cfiscsi_session_terminate(cs);
841 		return (true);
842 	}
843 
844 	if (io->scsiio.ext_data_filled != cdw->cdw_r2t_end &&
845 	    (bhsdo->bhsdo_flags & BHSDO_FLAGS_F) != 0) {
846 		if ((request->ip_bhs->bhs_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
847 		    ISCSI_BHS_OPCODE_SCSI_DATA_OUT) {
848 			CFISCSI_SESSION_WARN(cs, "got the final packet, but the "
849 			    "transmitted size was %zd bytes instead of %d; "
850 			    "dropping connection",
851 			    (size_t)io->scsiio.ext_data_filled,
852 			    cdw->cdw_r2t_end);
853 			ctl_set_data_phase_error(&io->scsiio);
854 			cfiscsi_session_terminate(cs);
855 			return (true);
856 		} else {
857 			/*
858 			 * For SCSI Command PDU, this just means we need to
859 			 * solicit more data by sending R2T.
860 			 */
861 			return (false);
862 		}
863 	}
864 
865 	if (io->scsiio.ext_data_filled == cdw->cdw_r2t_end) {
866 #if 0
867 		CFISCSI_SESSION_DEBUG(cs, "no longer expecting Data-Out with target "
868 		    "transfer tag 0x%x", cdw->cdw_target_transfer_tag);
869 #endif
870 
871 		return (true);
872 	}
873 
874 	return (false);
875 }
876 
877 static void
878 cfiscsi_pdu_handle_data_out(struct icl_pdu *request)
879 {
880 	struct iscsi_bhs_data_out *bhsdo;
881 	struct cfiscsi_session *cs;
882 	struct cfiscsi_data_wait *cdw = NULL;
883 	union ctl_io *io;
884 	bool done;
885 
886 	cs = PDU_SESSION(request);
887 	bhsdo = (struct iscsi_bhs_data_out *)request->ip_bhs;
888 
889 	CFISCSI_SESSION_LOCK(cs);
890 	TAILQ_FOREACH(cdw, &cs->cs_waiting_for_data_out, cdw_next) {
891 #if 0
892 		CFISCSI_SESSION_DEBUG(cs, "have ttt 0x%x, itt 0x%x; looking for "
893 		    "ttt 0x%x, itt 0x%x",
894 		    bhsdo->bhsdo_target_transfer_tag,
895 		    bhsdo->bhsdo_initiator_task_tag,
896 		    cdw->cdw_target_transfer_tag, cdw->cdw_initiator_task_tag));
897 #endif
898 		if (bhsdo->bhsdo_target_transfer_tag ==
899 		    cdw->cdw_target_transfer_tag)
900 			break;
901 	}
902 	CFISCSI_SESSION_UNLOCK(cs);
903 	if (cdw == NULL) {
904 		CFISCSI_SESSION_WARN(cs, "data transfer tag 0x%x, initiator task tag "
905 		    "0x%x, not found; dropping connection",
906 		    bhsdo->bhsdo_target_transfer_tag, bhsdo->bhsdo_initiator_task_tag);
907 		icl_pdu_free(request);
908 		cfiscsi_session_terminate(cs);
909 		return;
910 	}
911 
912 	if (cdw->cdw_datasn != ntohl(bhsdo->bhsdo_datasn)) {
913 		CFISCSI_SESSION_WARN(cs, "received Data-Out PDU with "
914 		    "DataSN %u, while expected %u; dropping connection",
915 		    ntohl(bhsdo->bhsdo_datasn), cdw->cdw_datasn);
916 		icl_pdu_free(request);
917 		cfiscsi_session_terminate(cs);
918 		return;
919 	}
920 	cdw->cdw_datasn++;
921 
922 	io = cdw->cdw_ctl_io;
923 	KASSERT((io->io_hdr.flags & CTL_FLAG_DATA_MASK) != CTL_FLAG_DATA_IN,
924 	    ("CTL_FLAG_DATA_IN"));
925 
926 	done = cfiscsi_handle_data_segment(request, cdw);
927 	if (done) {
928 		CFISCSI_SESSION_LOCK(cs);
929 		TAILQ_REMOVE(&cs->cs_waiting_for_data_out, cdw, cdw_next);
930 		CFISCSI_SESSION_UNLOCK(cs);
931 		done = (io->scsiio.ext_data_filled != cdw->cdw_r2t_end ||
932 		    io->scsiio.ext_data_filled == io->scsiio.kern_data_len);
933 		cfiscsi_data_wait_free(cs, cdw);
934 		io->io_hdr.flags &= ~CTL_FLAG_DMA_INPROG;
935 		if (done)
936 			io->scsiio.be_move_done(io);
937 		else
938 			cfiscsi_datamove_out(io);
939 	}
940 
941 	icl_pdu_free(request);
942 }
943 
944 static void
945 cfiscsi_pdu_handle_logout_request(struct icl_pdu *request)
946 {
947 	struct iscsi_bhs_logout_request *bhslr;
948 	struct iscsi_bhs_logout_response *bhslr2;
949 	struct icl_pdu *response;
950 	struct cfiscsi_session *cs;
951 
952 	cs = PDU_SESSION(request);
953 	bhslr = (struct iscsi_bhs_logout_request *)request->ip_bhs;
954 	switch (bhslr->bhslr_reason & 0x7f) {
955 	case BHSLR_REASON_CLOSE_SESSION:
956 	case BHSLR_REASON_CLOSE_CONNECTION:
957 		response = cfiscsi_pdu_new_response(request, M_NOWAIT);
958 		if (response == NULL) {
959 			CFISCSI_SESSION_DEBUG(cs, "failed to allocate memory");
960 			icl_pdu_free(request);
961 			cfiscsi_session_terminate(cs);
962 			return;
963 		}
964 		bhslr2 = (struct iscsi_bhs_logout_response *)response->ip_bhs;
965 		bhslr2->bhslr_opcode = ISCSI_BHS_OPCODE_LOGOUT_RESPONSE;
966 		bhslr2->bhslr_flags = 0x80;
967 		bhslr2->bhslr_response = BHSLR_RESPONSE_CLOSED_SUCCESSFULLY;
968 		bhslr2->bhslr_initiator_task_tag =
969 		    bhslr->bhslr_initiator_task_tag;
970 		icl_pdu_free(request);
971 		cfiscsi_pdu_queue(response);
972 		cfiscsi_session_terminate(cs);
973 		break;
974 	case BHSLR_REASON_REMOVE_FOR_RECOVERY:
975 		response = cfiscsi_pdu_new_response(request, M_NOWAIT);
976 		if (response == NULL) {
977 			CFISCSI_SESSION_WARN(cs,
978 			    "failed to allocate memory; dropping connection");
979 			icl_pdu_free(request);
980 			cfiscsi_session_terminate(cs);
981 			return;
982 		}
983 		bhslr2 = (struct iscsi_bhs_logout_response *)response->ip_bhs;
984 		bhslr2->bhslr_opcode = ISCSI_BHS_OPCODE_LOGOUT_RESPONSE;
985 		bhslr2->bhslr_flags = 0x80;
986 		bhslr2->bhslr_response = BHSLR_RESPONSE_RECOVERY_NOT_SUPPORTED;
987 		bhslr2->bhslr_initiator_task_tag =
988 		    bhslr->bhslr_initiator_task_tag;
989 		icl_pdu_free(request);
990 		cfiscsi_pdu_queue(response);
991 		break;
992 	default:
993 		CFISCSI_SESSION_WARN(cs, "invalid reason 0%x; dropping connection",
994 		    bhslr->bhslr_reason);
995 		icl_pdu_free(request);
996 		cfiscsi_session_terminate(cs);
997 		break;
998 	}
999 }
1000 
1001 static void
1002 cfiscsi_callout(void *context)
1003 {
1004 	struct icl_pdu *cp;
1005 	struct iscsi_bhs_nop_in *bhsni;
1006 	struct cfiscsi_session *cs;
1007 
1008 	cs = context;
1009 
1010 	if (cs->cs_terminating)
1011 		return;
1012 
1013 	callout_schedule(&cs->cs_callout, 1 * hz);
1014 
1015 	atomic_add_int(&cs->cs_timeout, 1);
1016 
1017 #ifdef ICL_KERNEL_PROXY
1018 	if (cs->cs_waiting_for_ctld || cs->cs_login_phase) {
1019 		if (login_timeout > 0 && cs->cs_timeout > login_timeout) {
1020 			CFISCSI_SESSION_WARN(cs, "login timed out after "
1021 			    "%d seconds; dropping connection", cs->cs_timeout);
1022 			cfiscsi_session_terminate(cs);
1023 		}
1024 		return;
1025 	}
1026 #endif
1027 
1028 	if (ping_timeout <= 0) {
1029 		/*
1030 		 * Pings are disabled.  Don't send NOP-In in this case;
1031 		 * user might have disabled pings to work around problems
1032 		 * with certain initiators that can't properly handle
1033 		 * NOP-In, such as iPXE.  Reset the timeout, to avoid
1034 		 * triggering reconnection, should the user decide to
1035 		 * reenable them.
1036 		 */
1037 		cs->cs_timeout = 0;
1038 		return;
1039 	}
1040 
1041 	if (cs->cs_timeout >= ping_timeout) {
1042 		CFISCSI_SESSION_WARN(cs, "no ping reply (NOP-Out) after %d seconds; "
1043 		    "dropping connection",  ping_timeout);
1044 		cfiscsi_session_terminate(cs);
1045 		return;
1046 	}
1047 
1048 	/*
1049 	 * If the ping was reset less than one second ago - which means
1050 	 * that we've received some PDU during the last second - assume
1051 	 * the traffic flows correctly and don't bother sending a NOP-Out.
1052 	 *
1053 	 * (It's 2 - one for one second, and one for incrementing is_timeout
1054 	 * earlier in this routine.)
1055 	 */
1056 	if (cs->cs_timeout < 2)
1057 		return;
1058 
1059 	cp = icl_pdu_new(cs->cs_conn, M_NOWAIT);
1060 	if (cp == NULL) {
1061 		CFISCSI_SESSION_WARN(cs, "failed to allocate memory");
1062 		return;
1063 	}
1064 	bhsni = (struct iscsi_bhs_nop_in *)cp->ip_bhs;
1065 	bhsni->bhsni_opcode = ISCSI_BHS_OPCODE_NOP_IN;
1066 	bhsni->bhsni_flags = 0x80;
1067 	bhsni->bhsni_initiator_task_tag = 0xffffffff;
1068 
1069 	cfiscsi_pdu_queue(cp);
1070 }
1071 
1072 static struct cfiscsi_data_wait *
1073 cfiscsi_data_wait_new(struct cfiscsi_session *cs, union ctl_io *io,
1074     uint32_t initiator_task_tag, uint32_t *target_transfer_tagp)
1075 {
1076 	struct cfiscsi_data_wait *cdw;
1077 	int error;
1078 
1079 	cdw = uma_zalloc(cfiscsi_data_wait_zone, M_NOWAIT | M_ZERO);
1080 	if (cdw == NULL) {
1081 		CFISCSI_SESSION_WARN(cs,
1082 		    "failed to allocate %zd bytes", sizeof(*cdw));
1083 		return (NULL);
1084 	}
1085 
1086 	error = icl_conn_transfer_setup(cs->cs_conn, io, target_transfer_tagp,
1087 	    &cdw->cdw_icl_prv);
1088 	if (error != 0) {
1089 		CFISCSI_SESSION_WARN(cs,
1090 		    "icl_conn_transfer_setup() failed with error %d", error);
1091 		uma_zfree(cfiscsi_data_wait_zone, cdw);
1092 		return (NULL);
1093 	}
1094 
1095 	cdw->cdw_ctl_io = io;
1096 	cdw->cdw_target_transfer_tag = *target_transfer_tagp;
1097 	cdw->cdw_initiator_task_tag = initiator_task_tag;
1098 
1099 	return (cdw);
1100 }
1101 
1102 static void
1103 cfiscsi_data_wait_free(struct cfiscsi_session *cs,
1104     struct cfiscsi_data_wait *cdw)
1105 {
1106 
1107 	icl_conn_transfer_done(cs->cs_conn, cdw->cdw_icl_prv);
1108 	uma_zfree(cfiscsi_data_wait_zone, cdw);
1109 }
1110 
1111 static void
1112 cfiscsi_session_terminate_tasks(struct cfiscsi_session *cs)
1113 {
1114 	struct cfiscsi_data_wait *cdw;
1115 	union ctl_io *io;
1116 	int error, last, wait;
1117 
1118 	if (cs->cs_target == NULL)
1119 		return;		/* No target yet, so nothing to do. */
1120 	io = ctl_alloc_io(cs->cs_target->ct_port.ctl_pool_ref);
1121 	ctl_zero_io(io);
1122 	PRIV_REQUEST(io) = cs;
1123 	io->io_hdr.io_type = CTL_IO_TASK;
1124 	io->io_hdr.nexus.initid = cs->cs_ctl_initid;
1125 	io->io_hdr.nexus.targ_port = cs->cs_target->ct_port.targ_port;
1126 	io->io_hdr.nexus.targ_lun = 0;
1127 	io->taskio.tag_type = CTL_TAG_SIMPLE; /* XXX */
1128 	io->taskio.task_action = CTL_TASK_I_T_NEXUS_RESET;
1129 	wait = cs->cs_outstanding_ctl_pdus;
1130 	refcount_acquire(&cs->cs_outstanding_ctl_pdus);
1131 	error = ctl_queue(io);
1132 	if (error != CTL_RETVAL_COMPLETE) {
1133 		CFISCSI_SESSION_WARN(cs, "ctl_queue() failed; error %d", error);
1134 		refcount_release(&cs->cs_outstanding_ctl_pdus);
1135 		ctl_free_io(io);
1136 	}
1137 
1138 	CFISCSI_SESSION_LOCK(cs);
1139 	while ((cdw = TAILQ_FIRST(&cs->cs_waiting_for_data_out)) != NULL) {
1140 		TAILQ_REMOVE(&cs->cs_waiting_for_data_out, cdw, cdw_next);
1141 		CFISCSI_SESSION_UNLOCK(cs);
1142 		/*
1143 		 * Set nonzero port status; this prevents backends from
1144 		 * assuming that the data transfer actually succeeded
1145 		 * and writing uninitialized data to disk.
1146 		 */
1147 		cdw->cdw_ctl_io->io_hdr.flags &= ~CTL_FLAG_DMA_INPROG;
1148 		cdw->cdw_ctl_io->scsiio.io_hdr.port_status = 42;
1149 		cdw->cdw_ctl_io->scsiio.be_move_done(cdw->cdw_ctl_io);
1150 		cfiscsi_data_wait_free(cs, cdw);
1151 		CFISCSI_SESSION_LOCK(cs);
1152 	}
1153 	CFISCSI_SESSION_UNLOCK(cs);
1154 
1155 	/*
1156 	 * Wait for CTL to terminate all the tasks.
1157 	 */
1158 	if (wait > 0)
1159 		CFISCSI_SESSION_WARN(cs,
1160 		    "waiting for CTL to terminate %d tasks", wait);
1161 	for (;;) {
1162 		refcount_acquire(&cs->cs_outstanding_ctl_pdus);
1163 		last = refcount_release(&cs->cs_outstanding_ctl_pdus);
1164 		if (last != 0)
1165 			break;
1166 		tsleep(__DEVOLATILE(void *, &cs->cs_outstanding_ctl_pdus),
1167 		    0, "cfiscsi_terminate", hz / 100);
1168 	}
1169 	if (wait > 0)
1170 		CFISCSI_SESSION_WARN(cs, "tasks terminated");
1171 }
1172 
1173 static void
1174 cfiscsi_maintenance_thread(void *arg)
1175 {
1176 	struct cfiscsi_session *cs;
1177 
1178 	cs = arg;
1179 
1180 	for (;;) {
1181 		CFISCSI_SESSION_LOCK(cs);
1182 		if (cs->cs_terminating == false || cs->cs_handoff_in_progress)
1183 			cv_wait(&cs->cs_maintenance_cv, &cs->cs_lock);
1184 		CFISCSI_SESSION_UNLOCK(cs);
1185 
1186 		if (cs->cs_terminating && cs->cs_handoff_in_progress == false) {
1187 			/*
1188 			 * We used to wait up to 30 seconds to deliver queued
1189 			 * PDUs to the initiator.  We also tried hard to deliver
1190 			 * SCSI Responses for the aborted PDUs.  We don't do
1191 			 * that anymore.  We might need to revisit that.
1192 			 */
1193 			callout_drain(&cs->cs_callout);
1194 			icl_conn_close(cs->cs_conn);
1195 
1196 			/*
1197 			 * At this point ICL receive thread is no longer
1198 			 * running; no new tasks can be queued.
1199 			 */
1200 			cfiscsi_session_terminate_tasks(cs);
1201 			cfiscsi_session_delete(cs);
1202 			kthread_exit();
1203 			return;
1204 		}
1205 		CFISCSI_SESSION_DEBUG(cs, "nothing to do");
1206 	}
1207 }
1208 
1209 static void
1210 cfiscsi_session_terminate(struct cfiscsi_session *cs)
1211 {
1212 
1213 	cs->cs_terminating = true;
1214 	cv_signal(&cs->cs_maintenance_cv);
1215 #ifdef ICL_KERNEL_PROXY
1216 	cv_signal(&cs->cs_login_cv);
1217 #endif
1218 }
1219 
1220 static int
1221 cfiscsi_session_register_initiator(struct cfiscsi_session *cs)
1222 {
1223 	struct cfiscsi_target *ct;
1224 	char *name;
1225 	int i;
1226 
1227 	KASSERT(cs->cs_ctl_initid == -1, ("already registered"));
1228 
1229 	ct = cs->cs_target;
1230 	name = strdup(cs->cs_initiator_id, M_CTL);
1231 	i = ctl_add_initiator(&ct->ct_port, -1, 0, name);
1232 	if (i < 0) {
1233 		CFISCSI_SESSION_WARN(cs, "ctl_add_initiator failed with error %d",
1234 		    i);
1235 		cs->cs_ctl_initid = -1;
1236 		return (1);
1237 	}
1238 	cs->cs_ctl_initid = i;
1239 #if 0
1240 	CFISCSI_SESSION_DEBUG(cs, "added initiator id %d", i);
1241 #endif
1242 
1243 	return (0);
1244 }
1245 
1246 static void
1247 cfiscsi_session_unregister_initiator(struct cfiscsi_session *cs)
1248 {
1249 	int error;
1250 
1251 	if (cs->cs_ctl_initid == -1)
1252 		return;
1253 
1254 	error = ctl_remove_initiator(&cs->cs_target->ct_port, cs->cs_ctl_initid);
1255 	if (error != 0) {
1256 		CFISCSI_SESSION_WARN(cs, "ctl_remove_initiator failed with error %d",
1257 		    error);
1258 	}
1259 	cs->cs_ctl_initid = -1;
1260 }
1261 
1262 static struct cfiscsi_session *
1263 cfiscsi_session_new(struct cfiscsi_softc *softc, const char *offload)
1264 {
1265 	struct cfiscsi_session *cs;
1266 	int error;
1267 
1268 	cs = malloc(sizeof(*cs), M_CFISCSI, M_NOWAIT | M_ZERO);
1269 	if (cs == NULL) {
1270 		CFISCSI_WARN("malloc failed");
1271 		return (NULL);
1272 	}
1273 	cs->cs_ctl_initid = -1;
1274 
1275 	refcount_init(&cs->cs_outstanding_ctl_pdus, 0);
1276 	TAILQ_INIT(&cs->cs_waiting_for_data_out);
1277 	mtx_init(&cs->cs_lock, "cfiscsi_lock", NULL, MTX_DEF);
1278 	cv_init(&cs->cs_maintenance_cv, "cfiscsi_mt");
1279 #ifdef ICL_KERNEL_PROXY
1280 	cv_init(&cs->cs_login_cv, "cfiscsi_login");
1281 #endif
1282 
1283 	/*
1284 	 * The purpose of this is to avoid racing with session shutdown.
1285 	 * Otherwise we could have the maintenance thread call icl_conn_close()
1286 	 * before we call icl_conn_handoff().
1287 	 */
1288 	cs->cs_handoff_in_progress = true;
1289 
1290 	cs->cs_conn = icl_new_conn(offload, false, "cfiscsi", &cs->cs_lock);
1291 	if (cs->cs_conn == NULL) {
1292 		free(cs, M_CFISCSI);
1293 		return (NULL);
1294 	}
1295 	cs->cs_conn->ic_receive = cfiscsi_receive_callback;
1296 	cs->cs_conn->ic_error = cfiscsi_error_callback;
1297 	cs->cs_conn->ic_prv0 = cs;
1298 
1299 	error = kthread_add(cfiscsi_maintenance_thread, cs, NULL, NULL, 0, 0, "cfiscsimt");
1300 	if (error != 0) {
1301 		CFISCSI_SESSION_WARN(cs, "kthread_add(9) failed with error %d", error);
1302 		free(cs, M_CFISCSI);
1303 		return (NULL);
1304 	}
1305 
1306 	mtx_lock(&softc->lock);
1307 	cs->cs_id = ++softc->last_session_id;
1308 	TAILQ_INSERT_TAIL(&softc->sessions, cs, cs_next);
1309 	mtx_unlock(&softc->lock);
1310 
1311 	/*
1312 	 * Start pinging the initiator.
1313 	 */
1314 	callout_init(&cs->cs_callout, 1);
1315 	callout_reset(&cs->cs_callout, 1 * hz, cfiscsi_callout, cs);
1316 
1317 	return (cs);
1318 }
1319 
1320 static void
1321 cfiscsi_session_delete(struct cfiscsi_session *cs)
1322 {
1323 	struct cfiscsi_softc *softc;
1324 
1325 	softc = &cfiscsi_softc;
1326 
1327 	KASSERT(cs->cs_outstanding_ctl_pdus == 0,
1328 	    ("destroying session with outstanding CTL pdus"));
1329 	KASSERT(TAILQ_EMPTY(&cs->cs_waiting_for_data_out),
1330 	    ("destroying session with non-empty queue"));
1331 
1332 	mtx_lock(&softc->lock);
1333 	TAILQ_REMOVE(&softc->sessions, cs, cs_next);
1334 	mtx_unlock(&softc->lock);
1335 
1336 	cfiscsi_session_unregister_initiator(cs);
1337 	if (cs->cs_target != NULL)
1338 		cfiscsi_target_release(cs->cs_target);
1339 	icl_conn_close(cs->cs_conn);
1340 	icl_conn_free(cs->cs_conn);
1341 	free(cs, M_CFISCSI);
1342 	cv_signal(&softc->sessions_cv);
1343 }
1344 
1345 static int
1346 cfiscsi_init(void)
1347 {
1348 	struct cfiscsi_softc *softc;
1349 
1350 	softc = &cfiscsi_softc;
1351 	bzero(softc, sizeof(*softc));
1352 	mtx_init(&softc->lock, "cfiscsi", NULL, MTX_DEF);
1353 
1354 	cv_init(&softc->sessions_cv, "cfiscsi_sessions");
1355 #ifdef ICL_KERNEL_PROXY
1356 	cv_init(&softc->accept_cv, "cfiscsi_accept");
1357 #endif
1358 	TAILQ_INIT(&softc->sessions);
1359 	TAILQ_INIT(&softc->targets);
1360 
1361 	cfiscsi_data_wait_zone = uma_zcreate("cfiscsi_data_wait",
1362 	    sizeof(struct cfiscsi_data_wait), NULL, NULL, NULL, NULL,
1363 	    UMA_ALIGN_PTR, 0);
1364 
1365 	return (0);
1366 }
1367 
1368 static int
1369 cfiscsi_shutdown(void)
1370 {
1371 	struct cfiscsi_softc *softc = &cfiscsi_softc;
1372 
1373 	if (!TAILQ_EMPTY(&softc->sessions) || !TAILQ_EMPTY(&softc->targets))
1374 		return (EBUSY);
1375 
1376 	uma_zdestroy(cfiscsi_data_wait_zone);
1377 #ifdef ICL_KERNEL_PROXY
1378 	cv_destroy(&softc->accept_cv);
1379 #endif
1380 	cv_destroy(&softc->sessions_cv);
1381 	mtx_destroy(&softc->lock);
1382 	return (0);
1383 }
1384 
1385 #ifdef ICL_KERNEL_PROXY
1386 static void
1387 cfiscsi_accept(struct socket *so, struct sockaddr *sa, int portal_id)
1388 {
1389 	struct cfiscsi_session *cs;
1390 
1391 	cs = cfiscsi_session_new(&cfiscsi_softc, NULL);
1392 	if (cs == NULL) {
1393 		CFISCSI_WARN("failed to create session");
1394 		return;
1395 	}
1396 
1397 	icl_conn_handoff_sock(cs->cs_conn, so);
1398 	cs->cs_initiator_sa = sa;
1399 	cs->cs_portal_id = portal_id;
1400 	cs->cs_handoff_in_progress = false;
1401 	cs->cs_waiting_for_ctld = true;
1402 	cv_signal(&cfiscsi_softc.accept_cv);
1403 
1404 	CFISCSI_SESSION_LOCK(cs);
1405 	/*
1406 	 * Wake up the maintenance thread if we got scheduled for termination
1407 	 * somewhere between cfiscsi_session_new() and icl_conn_handoff_sock().
1408 	 */
1409 	if (cs->cs_terminating)
1410 		cfiscsi_session_terminate(cs);
1411 	CFISCSI_SESSION_UNLOCK(cs);
1412 }
1413 #endif
1414 
1415 static void
1416 cfiscsi_online(void *arg)
1417 {
1418 	struct cfiscsi_softc *softc;
1419 	struct cfiscsi_target *ct;
1420 	int online;
1421 
1422 	ct = (struct cfiscsi_target *)arg;
1423 	softc = ct->ct_softc;
1424 
1425 	mtx_lock(&softc->lock);
1426 	if (ct->ct_online) {
1427 		mtx_unlock(&softc->lock);
1428 		return;
1429 	}
1430 	ct->ct_online = 1;
1431 	online = softc->online++;
1432 	mtx_unlock(&softc->lock);
1433 	if (online > 0)
1434 		return;
1435 
1436 #ifdef ICL_KERNEL_PROXY
1437 	if (softc->listener != NULL)
1438 		icl_listen_free(softc->listener);
1439 	softc->listener = icl_listen_new(cfiscsi_accept);
1440 #endif
1441 }
1442 
1443 static void
1444 cfiscsi_offline(void *arg)
1445 {
1446 	struct cfiscsi_softc *softc;
1447 	struct cfiscsi_target *ct;
1448 	struct cfiscsi_session *cs;
1449 	int error, online;
1450 
1451 	ct = (struct cfiscsi_target *)arg;
1452 	softc = ct->ct_softc;
1453 
1454 	mtx_lock(&softc->lock);
1455 	if (!ct->ct_online) {
1456 		mtx_unlock(&softc->lock);
1457 		return;
1458 	}
1459 	ct->ct_online = 0;
1460 	online = --softc->online;
1461 
1462 	do {
1463 		TAILQ_FOREACH(cs, &softc->sessions, cs_next) {
1464 			if (cs->cs_target == ct)
1465 				cfiscsi_session_terminate(cs);
1466 		}
1467 		TAILQ_FOREACH(cs, &softc->sessions, cs_next) {
1468 			if (cs->cs_target == ct)
1469 				break;
1470 		}
1471 		if (cs != NULL) {
1472 			error = cv_wait_sig(&softc->sessions_cv, &softc->lock);
1473 			if (error != 0) {
1474 				CFISCSI_SESSION_DEBUG(cs,
1475 				    "cv_wait failed with error %d\n", error);
1476 				break;
1477 			}
1478 		}
1479 	} while (cs != NULL && ct->ct_online == 0);
1480 	mtx_unlock(&softc->lock);
1481 	if (online > 0)
1482 		return;
1483 
1484 #ifdef ICL_KERNEL_PROXY
1485 	icl_listen_free(softc->listener);
1486 	softc->listener = NULL;
1487 #endif
1488 }
1489 
1490 static int
1491 cfiscsi_info(void *arg, struct sbuf *sb)
1492 {
1493 	struct cfiscsi_target *ct = (struct cfiscsi_target *)arg;
1494 	int retval;
1495 
1496 	retval = sbuf_printf(sb, "\t<cfiscsi_state>%d</cfiscsi_state>\n",
1497 	    ct->ct_state);
1498 	return (retval);
1499 }
1500 
1501 static void
1502 cfiscsi_ioctl_handoff(struct ctl_iscsi *ci)
1503 {
1504 	struct cfiscsi_softc *softc;
1505 	struct cfiscsi_session *cs, *cs2;
1506 	struct cfiscsi_target *ct;
1507 	struct ctl_iscsi_handoff_params *cihp;
1508 	int error;
1509 
1510 	cihp = (struct ctl_iscsi_handoff_params *)&(ci->data);
1511 	softc = &cfiscsi_softc;
1512 
1513 	CFISCSI_DEBUG("new connection from %s (%s) to %s",
1514 	    cihp->initiator_name, cihp->initiator_addr,
1515 	    cihp->target_name);
1516 
1517 	ct = cfiscsi_target_find(softc, cihp->target_name,
1518 	    cihp->portal_group_tag);
1519 	if (ct == NULL) {
1520 		ci->status = CTL_ISCSI_ERROR;
1521 		snprintf(ci->error_str, sizeof(ci->error_str),
1522 		    "%s: target not found", __func__);
1523 		return;
1524 	}
1525 
1526 #ifdef ICL_KERNEL_PROXY
1527 	if (cihp->socket > 0 && cihp->connection_id > 0) {
1528 		snprintf(ci->error_str, sizeof(ci->error_str),
1529 		    "both socket and connection_id set");
1530 		ci->status = CTL_ISCSI_ERROR;
1531 		cfiscsi_target_release(ct);
1532 		return;
1533 	}
1534 	if (cihp->socket == 0) {
1535 		mtx_lock(&cfiscsi_softc.lock);
1536 		TAILQ_FOREACH(cs, &cfiscsi_softc.sessions, cs_next) {
1537 			if (cs->cs_id == cihp->connection_id)
1538 				break;
1539 		}
1540 		if (cs == NULL) {
1541 			mtx_unlock(&cfiscsi_softc.lock);
1542 			snprintf(ci->error_str, sizeof(ci->error_str),
1543 			    "connection not found");
1544 			ci->status = CTL_ISCSI_ERROR;
1545 			cfiscsi_target_release(ct);
1546 			return;
1547 		}
1548 		mtx_unlock(&cfiscsi_softc.lock);
1549 	} else {
1550 #endif
1551 		cs = cfiscsi_session_new(softc, cihp->offload);
1552 		if (cs == NULL) {
1553 			ci->status = CTL_ISCSI_ERROR;
1554 			snprintf(ci->error_str, sizeof(ci->error_str),
1555 			    "%s: cfiscsi_session_new failed", __func__);
1556 			cfiscsi_target_release(ct);
1557 			return;
1558 		}
1559 #ifdef ICL_KERNEL_PROXY
1560 	}
1561 #endif
1562 
1563 	/*
1564 	 * First PDU of Full Feature phase has the same CmdSN as the last
1565 	 * PDU from the Login Phase received from the initiator.  Thus,
1566 	 * the -1 below.
1567 	 */
1568 	cs->cs_cmdsn = cihp->cmdsn;
1569 	cs->cs_statsn = cihp->statsn;
1570 	cs->cs_max_recv_data_segment_length = cihp->max_recv_data_segment_length;
1571 	cs->cs_max_send_data_segment_length = cihp->max_send_data_segment_length;
1572 	cs->cs_max_burst_length = cihp->max_burst_length;
1573 	cs->cs_first_burst_length = cihp->first_burst_length;
1574 	cs->cs_immediate_data = !!cihp->immediate_data;
1575 	if (cihp->header_digest == CTL_ISCSI_DIGEST_CRC32C)
1576 		cs->cs_conn->ic_header_crc32c = true;
1577 	if (cihp->data_digest == CTL_ISCSI_DIGEST_CRC32C)
1578 		cs->cs_conn->ic_data_crc32c = true;
1579 
1580 	strlcpy(cs->cs_initiator_name,
1581 	    cihp->initiator_name, sizeof(cs->cs_initiator_name));
1582 	strlcpy(cs->cs_initiator_addr,
1583 	    cihp->initiator_addr, sizeof(cs->cs_initiator_addr));
1584 	strlcpy(cs->cs_initiator_alias,
1585 	    cihp->initiator_alias, sizeof(cs->cs_initiator_alias));
1586 	memcpy(cs->cs_initiator_isid,
1587 	    cihp->initiator_isid, sizeof(cs->cs_initiator_isid));
1588 	snprintf(cs->cs_initiator_id, sizeof(cs->cs_initiator_id),
1589 	    "%s,i,0x%02x%02x%02x%02x%02x%02x", cs->cs_initiator_name,
1590 	    cihp->initiator_isid[0], cihp->initiator_isid[1],
1591 	    cihp->initiator_isid[2], cihp->initiator_isid[3],
1592 	    cihp->initiator_isid[4], cihp->initiator_isid[5]);
1593 
1594 	mtx_lock(&softc->lock);
1595 	if (ct->ct_online == 0) {
1596 		mtx_unlock(&softc->lock);
1597 		CFISCSI_SESSION_LOCK(cs);
1598 		cs->cs_handoff_in_progress = false;
1599 		cfiscsi_session_terminate(cs);
1600 		CFISCSI_SESSION_UNLOCK(cs);
1601 		cfiscsi_target_release(ct);
1602 		ci->status = CTL_ISCSI_ERROR;
1603 		snprintf(ci->error_str, sizeof(ci->error_str),
1604 		    "%s: port offline", __func__);
1605 		return;
1606 	}
1607 	cs->cs_target = ct;
1608 	mtx_unlock(&softc->lock);
1609 
1610 restart:
1611 	if (!cs->cs_terminating) {
1612 		mtx_lock(&softc->lock);
1613 		TAILQ_FOREACH(cs2, &softc->sessions, cs_next) {
1614 			if (cs2 != cs && cs2->cs_tasks_aborted == false &&
1615 			    cs->cs_target == cs2->cs_target &&
1616 			    strcmp(cs->cs_initiator_id, cs2->cs_initiator_id) == 0) {
1617 				if (strcmp(cs->cs_initiator_addr,
1618 				    cs2->cs_initiator_addr) != 0) {
1619 					CFISCSI_SESSION_WARN(cs2,
1620 					    "session reinstatement from "
1621 					    "different address %s",
1622 					    cs->cs_initiator_addr);
1623 				} else {
1624 					CFISCSI_SESSION_DEBUG(cs2,
1625 					    "session reinstatement");
1626 				}
1627 				cfiscsi_session_terminate(cs2);
1628 				mtx_unlock(&softc->lock);
1629 				pause("cfiscsi_reinstate", 1);
1630 				goto restart;
1631 			}
1632 		}
1633 		mtx_unlock(&softc->lock);
1634 	}
1635 
1636 	/*
1637 	 * Register initiator with CTL.
1638 	 */
1639 	cfiscsi_session_register_initiator(cs);
1640 
1641 #ifdef ICL_KERNEL_PROXY
1642 	if (cihp->socket > 0) {
1643 #endif
1644 		error = icl_conn_handoff(cs->cs_conn, cihp->socket);
1645 		if (error != 0) {
1646 			CFISCSI_SESSION_LOCK(cs);
1647 			cs->cs_handoff_in_progress = false;
1648 			cfiscsi_session_terminate(cs);
1649 			CFISCSI_SESSION_UNLOCK(cs);
1650 			ci->status = CTL_ISCSI_ERROR;
1651 			snprintf(ci->error_str, sizeof(ci->error_str),
1652 			    "%s: icl_conn_handoff failed with error %d",
1653 			    __func__, error);
1654 			return;
1655 		}
1656 #ifdef ICL_KERNEL_PROXY
1657 	}
1658 #endif
1659 
1660 #ifdef ICL_KERNEL_PROXY
1661 	cs->cs_login_phase = false;
1662 
1663 	/*
1664 	 * First PDU of the Full Feature phase has likely already arrived.
1665 	 * We have to pick it up and execute properly.
1666 	 */
1667 	if (cs->cs_login_pdu != NULL) {
1668 		CFISCSI_SESSION_DEBUG(cs, "picking up first PDU");
1669 		cfiscsi_pdu_handle(cs->cs_login_pdu);
1670 		cs->cs_login_pdu = NULL;
1671 	}
1672 #endif
1673 
1674 	CFISCSI_SESSION_LOCK(cs);
1675 	cs->cs_handoff_in_progress = false;
1676 
1677 	/*
1678 	 * Wake up the maintenance thread if we got scheduled for termination.
1679 	 */
1680 	if (cs->cs_terminating)
1681 		cfiscsi_session_terminate(cs);
1682 	CFISCSI_SESSION_UNLOCK(cs);
1683 
1684 	ci->status = CTL_ISCSI_OK;
1685 }
1686 
1687 static void
1688 cfiscsi_ioctl_list(struct ctl_iscsi *ci)
1689 {
1690 	struct ctl_iscsi_list_params *cilp;
1691 	struct cfiscsi_session *cs;
1692 	struct cfiscsi_softc *softc;
1693 	struct sbuf *sb;
1694 	int error;
1695 
1696 	cilp = (struct ctl_iscsi_list_params *)&(ci->data);
1697 	softc = &cfiscsi_softc;
1698 
1699 	sb = sbuf_new(NULL, NULL, cilp->alloc_len, SBUF_FIXEDLEN);
1700 	if (sb == NULL) {
1701 		ci->status = CTL_ISCSI_ERROR;
1702 		snprintf(ci->error_str, sizeof(ci->error_str),
1703 		    "Unable to allocate %d bytes for iSCSI session list",
1704 		    cilp->alloc_len);
1705 		return;
1706 	}
1707 
1708 	sbuf_printf(sb, "<ctlislist>\n");
1709 	mtx_lock(&softc->lock);
1710 	TAILQ_FOREACH(cs, &softc->sessions, cs_next) {
1711 		if (cs->cs_target == NULL)
1712 			continue;
1713 		error = sbuf_printf(sb, "<connection id=\"%d\">"
1714 		    "<initiator>%s</initiator>"
1715 		    "<initiator_addr>%s</initiator_addr>"
1716 		    "<initiator_alias>%s</initiator_alias>"
1717 		    "<target>%s</target>"
1718 		    "<target_alias>%s</target_alias>"
1719 		    "<target_portal_group_tag>%u</target_portal_group_tag>"
1720 		    "<header_digest>%s</header_digest>"
1721 		    "<data_digest>%s</data_digest>"
1722 		    "<max_recv_data_segment_length>%d</max_recv_data_segment_length>"
1723 		    "<max_send_data_segment_length>%d</max_send_data_segment_length>"
1724 		    "<max_burst_length>%d</max_burst_length>"
1725 		    "<first_burst_length>%d</first_burst_length>"
1726 		    "<immediate_data>%d</immediate_data>"
1727 		    "<iser>%d</iser>"
1728 		    "<offload>%s</offload>"
1729 		    "</connection>\n",
1730 		    cs->cs_id,
1731 		    cs->cs_initiator_name, cs->cs_initiator_addr, cs->cs_initiator_alias,
1732 		    cs->cs_target->ct_name, cs->cs_target->ct_alias,
1733 		    cs->cs_target->ct_tag,
1734 		    cs->cs_conn->ic_header_crc32c ? "CRC32C" : "None",
1735 		    cs->cs_conn->ic_data_crc32c ? "CRC32C" : "None",
1736 		    cs->cs_max_recv_data_segment_length,
1737 		    cs->cs_max_send_data_segment_length,
1738 		    cs->cs_max_burst_length,
1739 		    cs->cs_first_burst_length,
1740 		    cs->cs_immediate_data,
1741 		    cs->cs_conn->ic_iser,
1742 		    cs->cs_conn->ic_offload);
1743 		if (error != 0)
1744 			break;
1745 	}
1746 	mtx_unlock(&softc->lock);
1747 	error = sbuf_printf(sb, "</ctlislist>\n");
1748 	if (error != 0) {
1749 		sbuf_delete(sb);
1750 		ci->status = CTL_ISCSI_LIST_NEED_MORE_SPACE;
1751 		snprintf(ci->error_str, sizeof(ci->error_str),
1752 		    "Out of space, %d bytes is too small", cilp->alloc_len);
1753 		return;
1754 	}
1755 	sbuf_finish(sb);
1756 
1757 	error = copyout(sbuf_data(sb), cilp->conn_xml, sbuf_len(sb) + 1);
1758 	if (error != 0) {
1759 		sbuf_delete(sb);
1760 		snprintf(ci->error_str, sizeof(ci->error_str),
1761 		    "copyout failed with error %d", error);
1762 		ci->status = CTL_ISCSI_ERROR;
1763 		return;
1764 	}
1765 	cilp->fill_len = sbuf_len(sb) + 1;
1766 	ci->status = CTL_ISCSI_OK;
1767 	sbuf_delete(sb);
1768 }
1769 
1770 static void
1771 cfiscsi_ioctl_logout(struct ctl_iscsi *ci)
1772 {
1773 	struct icl_pdu *response;
1774 	struct iscsi_bhs_asynchronous_message *bhsam;
1775 	struct ctl_iscsi_logout_params *cilp;
1776 	struct cfiscsi_session *cs;
1777 	struct cfiscsi_softc *softc;
1778 	int found = 0;
1779 
1780 	cilp = (struct ctl_iscsi_logout_params *)&(ci->data);
1781 	softc = &cfiscsi_softc;
1782 
1783 	mtx_lock(&softc->lock);
1784 	TAILQ_FOREACH(cs, &softc->sessions, cs_next) {
1785 		if (cilp->all == 0 && cs->cs_id != cilp->connection_id &&
1786 		    strcmp(cs->cs_initiator_name, cilp->initiator_name) != 0 &&
1787 		    strcmp(cs->cs_initiator_addr, cilp->initiator_addr) != 0)
1788 			continue;
1789 
1790 		response = icl_pdu_new(cs->cs_conn, M_NOWAIT);
1791 		if (response == NULL) {
1792 			ci->status = CTL_ISCSI_ERROR;
1793 			snprintf(ci->error_str, sizeof(ci->error_str),
1794 			    "Unable to allocate memory");
1795 			mtx_unlock(&softc->lock);
1796 			return;
1797 		}
1798 		bhsam =
1799 		    (struct iscsi_bhs_asynchronous_message *)response->ip_bhs;
1800 		bhsam->bhsam_opcode = ISCSI_BHS_OPCODE_ASYNC_MESSAGE;
1801 		bhsam->bhsam_flags = 0x80;
1802 		bhsam->bhsam_async_event = BHSAM_EVENT_TARGET_REQUESTS_LOGOUT;
1803 		bhsam->bhsam_parameter3 = htons(10);
1804 		cfiscsi_pdu_queue(response);
1805 		found++;
1806 	}
1807 	mtx_unlock(&softc->lock);
1808 
1809 	if (found == 0) {
1810 		ci->status = CTL_ISCSI_SESSION_NOT_FOUND;
1811 		snprintf(ci->error_str, sizeof(ci->error_str),
1812 		    "No matching connections found");
1813 		return;
1814 	}
1815 
1816 	ci->status = CTL_ISCSI_OK;
1817 }
1818 
1819 static void
1820 cfiscsi_ioctl_terminate(struct ctl_iscsi *ci)
1821 {
1822 	struct icl_pdu *response;
1823 	struct iscsi_bhs_asynchronous_message *bhsam;
1824 	struct ctl_iscsi_terminate_params *citp;
1825 	struct cfiscsi_session *cs;
1826 	struct cfiscsi_softc *softc;
1827 	int found = 0;
1828 
1829 	citp = (struct ctl_iscsi_terminate_params *)&(ci->data);
1830 	softc = &cfiscsi_softc;
1831 
1832 	mtx_lock(&softc->lock);
1833 	TAILQ_FOREACH(cs, &softc->sessions, cs_next) {
1834 		if (citp->all == 0 && cs->cs_id != citp->connection_id &&
1835 		    strcmp(cs->cs_initiator_name, citp->initiator_name) != 0 &&
1836 		    strcmp(cs->cs_initiator_addr, citp->initiator_addr) != 0)
1837 			continue;
1838 
1839 		response = icl_pdu_new(cs->cs_conn, M_NOWAIT);
1840 		if (response == NULL) {
1841 			/*
1842 			 * Oh well.  Just terminate the connection.
1843 			 */
1844 		} else {
1845 			bhsam = (struct iscsi_bhs_asynchronous_message *)
1846 			    response->ip_bhs;
1847 			bhsam->bhsam_opcode = ISCSI_BHS_OPCODE_ASYNC_MESSAGE;
1848 			bhsam->bhsam_flags = 0x80;
1849 			bhsam->bhsam_0xffffffff = 0xffffffff;
1850 			bhsam->bhsam_async_event =
1851 			    BHSAM_EVENT_TARGET_TERMINATES_SESSION;
1852 			cfiscsi_pdu_queue(response);
1853 		}
1854 		cfiscsi_session_terminate(cs);
1855 		found++;
1856 	}
1857 	mtx_unlock(&softc->lock);
1858 
1859 	if (found == 0) {
1860 		ci->status = CTL_ISCSI_SESSION_NOT_FOUND;
1861 		snprintf(ci->error_str, sizeof(ci->error_str),
1862 		    "No matching connections found");
1863 		return;
1864 	}
1865 
1866 	ci->status = CTL_ISCSI_OK;
1867 }
1868 
1869 static void
1870 cfiscsi_ioctl_limits(struct ctl_iscsi *ci)
1871 {
1872 	struct ctl_iscsi_limits_params *cilp;
1873 	struct icl_drv_limits idl;
1874 	int error;
1875 
1876 	cilp = (struct ctl_iscsi_limits_params *)&(ci->data);
1877 
1878 	error = icl_limits(cilp->offload, false, &idl);
1879 	if (error != 0) {
1880 		ci->status = CTL_ISCSI_ERROR;
1881 		snprintf(ci->error_str, sizeof(ci->error_str),
1882 			"%s: icl_limits failed with error %d",
1883 			__func__, error);
1884 		return;
1885 	}
1886 
1887 	cilp->max_recv_data_segment_length =
1888 	    idl.idl_max_recv_data_segment_length;
1889 	cilp->max_send_data_segment_length =
1890 	    idl.idl_max_send_data_segment_length;
1891 	cilp->max_burst_length = idl.idl_max_burst_length;
1892 	cilp->first_burst_length = idl.idl_first_burst_length;
1893 
1894 	ci->status = CTL_ISCSI_OK;
1895 }
1896 
1897 #ifdef ICL_KERNEL_PROXY
1898 static void
1899 cfiscsi_ioctl_listen(struct ctl_iscsi *ci)
1900 {
1901 	struct ctl_iscsi_listen_params *cilp;
1902 	struct sockaddr *sa;
1903 	int error;
1904 
1905 	cilp = (struct ctl_iscsi_listen_params *)&(ci->data);
1906 
1907 	if (cfiscsi_softc.listener == NULL) {
1908 		CFISCSI_DEBUG("no listener");
1909 		snprintf(ci->error_str, sizeof(ci->error_str), "no listener");
1910 		ci->status = CTL_ISCSI_ERROR;
1911 		return;
1912 	}
1913 
1914 	error = getsockaddr(&sa, (void *)cilp->addr, cilp->addrlen);
1915 	if (error != 0) {
1916 		CFISCSI_DEBUG("getsockaddr, error %d", error);
1917 		snprintf(ci->error_str, sizeof(ci->error_str), "getsockaddr failed");
1918 		ci->status = CTL_ISCSI_ERROR;
1919 		return;
1920 	}
1921 
1922 	error = icl_listen_add(cfiscsi_softc.listener, cilp->iser, cilp->domain,
1923 	    cilp->socktype, cilp->protocol, sa, cilp->portal_id);
1924 	if (error != 0) {
1925 		free(sa, M_SONAME);
1926 		CFISCSI_DEBUG("icl_listen_add, error %d", error);
1927 		snprintf(ci->error_str, sizeof(ci->error_str),
1928 		    "icl_listen_add failed, error %d", error);
1929 		ci->status = CTL_ISCSI_ERROR;
1930 		return;
1931 	}
1932 
1933 	ci->status = CTL_ISCSI_OK;
1934 }
1935 
1936 static void
1937 cfiscsi_ioctl_accept(struct ctl_iscsi *ci)
1938 {
1939 	struct ctl_iscsi_accept_params *ciap;
1940 	struct cfiscsi_session *cs;
1941 	int error;
1942 
1943 	ciap = (struct ctl_iscsi_accept_params *)&(ci->data);
1944 
1945 	mtx_lock(&cfiscsi_softc.lock);
1946 	for (;;) {
1947 		TAILQ_FOREACH(cs, &cfiscsi_softc.sessions, cs_next) {
1948 			if (cs->cs_waiting_for_ctld)
1949 				break;
1950 		}
1951 		if (cs != NULL)
1952 			break;
1953 		error = cv_wait_sig(&cfiscsi_softc.accept_cv, &cfiscsi_softc.lock);
1954 		if (error != 0) {
1955 			mtx_unlock(&cfiscsi_softc.lock);
1956 			snprintf(ci->error_str, sizeof(ci->error_str), "interrupted");
1957 			ci->status = CTL_ISCSI_ERROR;
1958 			return;
1959 		}
1960 	}
1961 	mtx_unlock(&cfiscsi_softc.lock);
1962 
1963 	cs->cs_waiting_for_ctld = false;
1964 	cs->cs_login_phase = true;
1965 
1966 	ciap->connection_id = cs->cs_id;
1967 	ciap->portal_id = cs->cs_portal_id;
1968 	ciap->initiator_addrlen = cs->cs_initiator_sa->sa_len;
1969 	error = copyout(cs->cs_initiator_sa, ciap->initiator_addr,
1970 	    cs->cs_initiator_sa->sa_len);
1971 	if (error != 0) {
1972 		snprintf(ci->error_str, sizeof(ci->error_str),
1973 		    "copyout failed with error %d", error);
1974 		ci->status = CTL_ISCSI_ERROR;
1975 		return;
1976 	}
1977 
1978 	ci->status = CTL_ISCSI_OK;
1979 }
1980 
1981 static void
1982 cfiscsi_ioctl_send(struct ctl_iscsi *ci)
1983 {
1984 	struct ctl_iscsi_send_params *cisp;
1985 	struct cfiscsi_session *cs;
1986 	struct icl_pdu *ip;
1987 	size_t datalen;
1988 	void *data;
1989 	int error;
1990 
1991 	cisp = (struct ctl_iscsi_send_params *)&(ci->data);
1992 
1993 	mtx_lock(&cfiscsi_softc.lock);
1994 	TAILQ_FOREACH(cs, &cfiscsi_softc.sessions, cs_next) {
1995 		if (cs->cs_id == cisp->connection_id)
1996 			break;
1997 	}
1998 	if (cs == NULL) {
1999 		mtx_unlock(&cfiscsi_softc.lock);
2000 		snprintf(ci->error_str, sizeof(ci->error_str), "connection not found");
2001 		ci->status = CTL_ISCSI_ERROR;
2002 		return;
2003 	}
2004 	mtx_unlock(&cfiscsi_softc.lock);
2005 
2006 #if 0
2007 	if (cs->cs_login_phase == false)
2008 		return (EBUSY);
2009 #endif
2010 
2011 	if (cs->cs_terminating) {
2012 		snprintf(ci->error_str, sizeof(ci->error_str), "connection is terminating");
2013 		ci->status = CTL_ISCSI_ERROR;
2014 		return;
2015 	}
2016 
2017 	datalen = cisp->data_segment_len;
2018 	/*
2019 	 * XXX
2020 	 */
2021 	//if (datalen > CFISCSI_MAX_DATA_SEGMENT_LENGTH) {
2022 	if (datalen > 65535) {
2023 		snprintf(ci->error_str, sizeof(ci->error_str), "data segment too big");
2024 		ci->status = CTL_ISCSI_ERROR;
2025 		return;
2026 	}
2027 	if (datalen > 0) {
2028 		data = malloc(datalen, M_CFISCSI, M_WAITOK);
2029 		error = copyin(cisp->data_segment, data, datalen);
2030 		if (error != 0) {
2031 			free(data, M_CFISCSI);
2032 			snprintf(ci->error_str, sizeof(ci->error_str), "copyin error %d", error);
2033 			ci->status = CTL_ISCSI_ERROR;
2034 			return;
2035 		}
2036 	}
2037 
2038 	ip = icl_pdu_new(cs->cs_conn, M_WAITOK);
2039 	memcpy(ip->ip_bhs, cisp->bhs, sizeof(*ip->ip_bhs));
2040 	if (datalen > 0) {
2041 		icl_pdu_append_data(ip, data, datalen, M_WAITOK);
2042 		free(data, M_CFISCSI);
2043 	}
2044 	CFISCSI_SESSION_LOCK(cs);
2045 	icl_pdu_queue(ip);
2046 	CFISCSI_SESSION_UNLOCK(cs);
2047 	ci->status = CTL_ISCSI_OK;
2048 }
2049 
2050 static void
2051 cfiscsi_ioctl_receive(struct ctl_iscsi *ci)
2052 {
2053 	struct ctl_iscsi_receive_params *cirp;
2054 	struct cfiscsi_session *cs;
2055 	struct icl_pdu *ip;
2056 	void *data;
2057 	int error;
2058 
2059 	cirp = (struct ctl_iscsi_receive_params *)&(ci->data);
2060 
2061 	mtx_lock(&cfiscsi_softc.lock);
2062 	TAILQ_FOREACH(cs, &cfiscsi_softc.sessions, cs_next) {
2063 		if (cs->cs_id == cirp->connection_id)
2064 			break;
2065 	}
2066 	if (cs == NULL) {
2067 		mtx_unlock(&cfiscsi_softc.lock);
2068 		snprintf(ci->error_str, sizeof(ci->error_str),
2069 		    "connection not found");
2070 		ci->status = CTL_ISCSI_ERROR;
2071 		return;
2072 	}
2073 	mtx_unlock(&cfiscsi_softc.lock);
2074 
2075 #if 0
2076 	if (is->is_login_phase == false)
2077 		return (EBUSY);
2078 #endif
2079 
2080 	CFISCSI_SESSION_LOCK(cs);
2081 	while (cs->cs_login_pdu == NULL && cs->cs_terminating == false) {
2082 		error = cv_wait_sig(&cs->cs_login_cv, &cs->cs_lock);
2083 		if (error != 0) {
2084 			CFISCSI_SESSION_UNLOCK(cs);
2085 			snprintf(ci->error_str, sizeof(ci->error_str),
2086 			    "interrupted by signal");
2087 			ci->status = CTL_ISCSI_ERROR;
2088 			return;
2089 		}
2090 	}
2091 
2092 	if (cs->cs_terminating) {
2093 		CFISCSI_SESSION_UNLOCK(cs);
2094 		snprintf(ci->error_str, sizeof(ci->error_str),
2095 		    "connection terminating");
2096 		ci->status = CTL_ISCSI_ERROR;
2097 		return;
2098 	}
2099 	ip = cs->cs_login_pdu;
2100 	cs->cs_login_pdu = NULL;
2101 	CFISCSI_SESSION_UNLOCK(cs);
2102 
2103 	if (ip->ip_data_len > cirp->data_segment_len) {
2104 		icl_pdu_free(ip);
2105 		snprintf(ci->error_str, sizeof(ci->error_str),
2106 		    "data segment too big");
2107 		ci->status = CTL_ISCSI_ERROR;
2108 		return;
2109 	}
2110 
2111 	copyout(ip->ip_bhs, cirp->bhs, sizeof(*ip->ip_bhs));
2112 	if (ip->ip_data_len > 0) {
2113 		data = malloc(ip->ip_data_len, M_CFISCSI, M_WAITOK);
2114 		icl_pdu_get_data(ip, 0, data, ip->ip_data_len);
2115 		copyout(data, cirp->data_segment, ip->ip_data_len);
2116 		free(data, M_CFISCSI);
2117 	}
2118 
2119 	icl_pdu_free(ip);
2120 	ci->status = CTL_ISCSI_OK;
2121 }
2122 
2123 #endif /* !ICL_KERNEL_PROXY */
2124 
2125 static void
2126 cfiscsi_ioctl_port_create(struct ctl_req *req)
2127 {
2128 	struct cfiscsi_target *ct;
2129 	struct ctl_port *port;
2130 	const char *target, *alias, *val;
2131 	struct scsi_vpd_id_descriptor *desc;
2132 	int retval, len, idlen;
2133 	uint16_t tag;
2134 
2135 	target = dnvlist_get_string(req->args_nvl, "cfiscsi_target", NULL);
2136 	alias = dnvlist_get_string(req->args_nvl, "cfiscsi_target_alias", NULL);
2137 	val = dnvlist_get_string(req->args_nvl, "cfiscsi_portal_group_tag",
2138 	    NULL);
2139 
2140 	if (target == NULL || val == NULL) {
2141 		req->status = CTL_LUN_ERROR;
2142 		snprintf(req->error_str, sizeof(req->error_str),
2143 		    "Missing required argument");
2144 		return;
2145 	}
2146 
2147 	tag = strtoul(val, NULL, 0);
2148 	ct = cfiscsi_target_find_or_create(&cfiscsi_softc, target, alias, tag);
2149 	if (ct == NULL) {
2150 		req->status = CTL_LUN_ERROR;
2151 		snprintf(req->error_str, sizeof(req->error_str),
2152 		    "failed to create target \"%s\"", target);
2153 		return;
2154 	}
2155 	if (ct->ct_state == CFISCSI_TARGET_STATE_ACTIVE) {
2156 		req->status = CTL_LUN_ERROR;
2157 		snprintf(req->error_str, sizeof(req->error_str),
2158 		    "target \"%s\" for portal group tag %u already exists",
2159 		    target, tag);
2160 		cfiscsi_target_release(ct);
2161 		return;
2162 	}
2163 	port = &ct->ct_port;
2164 	// WAT
2165 	if (ct->ct_state == CFISCSI_TARGET_STATE_DYING)
2166 		goto done;
2167 
2168 	port->frontend = &cfiscsi_frontend;
2169 	port->port_type = CTL_PORT_ISCSI;
2170 	/* XXX KDM what should the real number be here? */
2171 	port->num_requested_ctl_io = 4096;
2172 	port->port_name = "iscsi";
2173 	port->physical_port = (int)tag;
2174 	port->virtual_port = ct->ct_target_id;
2175 	port->port_online = cfiscsi_online;
2176 	port->port_offline = cfiscsi_offline;
2177 	port->port_info = cfiscsi_info;
2178 	port->onoff_arg = ct;
2179 	port->fe_datamove = cfiscsi_datamove;
2180 	port->fe_done = cfiscsi_done;
2181 	port->targ_port = -1;
2182 	port->options = nvlist_clone(req->args_nvl);
2183 
2184 	/* Generate Port ID. */
2185 	idlen = strlen(target) + strlen(",t,0x0001") + 1;
2186 	idlen = roundup2(idlen, 4);
2187 	len = sizeof(struct scsi_vpd_device_id) + idlen;
2188 	port->port_devid = malloc(sizeof(struct ctl_devid) + len,
2189 	    M_CTL, M_WAITOK | M_ZERO);
2190 	port->port_devid->len = len;
2191 	desc = (struct scsi_vpd_id_descriptor *)port->port_devid->data;
2192 	desc->proto_codeset = (SCSI_PROTO_ISCSI << 4) | SVPD_ID_CODESET_UTF8;
2193 	desc->id_type = SVPD_ID_PIV | SVPD_ID_ASSOC_PORT |
2194 	    SVPD_ID_TYPE_SCSI_NAME;
2195 	desc->length = idlen;
2196 	snprintf(desc->identifier, idlen, "%s,t,0x%4.4x", target, tag);
2197 
2198 	/* Generate Target ID. */
2199 	idlen = strlen(target) + 1;
2200 	idlen = roundup2(idlen, 4);
2201 	len = sizeof(struct scsi_vpd_device_id) + idlen;
2202 	port->target_devid = malloc(sizeof(struct ctl_devid) + len,
2203 	    M_CTL, M_WAITOK | M_ZERO);
2204 	port->target_devid->len = len;
2205 	desc = (struct scsi_vpd_id_descriptor *)port->target_devid->data;
2206 	desc->proto_codeset = (SCSI_PROTO_ISCSI << 4) | SVPD_ID_CODESET_UTF8;
2207 	desc->id_type = SVPD_ID_PIV | SVPD_ID_ASSOC_TARGET |
2208 	    SVPD_ID_TYPE_SCSI_NAME;
2209 	desc->length = idlen;
2210 	strlcpy(desc->identifier, target, idlen);
2211 
2212 	retval = ctl_port_register(port);
2213 	if (retval != 0) {
2214 		free(port->port_devid, M_CFISCSI);
2215 		free(port->target_devid, M_CFISCSI);
2216 		cfiscsi_target_release(ct);
2217 		req->status = CTL_LUN_ERROR;
2218 		snprintf(req->error_str, sizeof(req->error_str),
2219 		    "ctl_port_register() failed with error %d", retval);
2220 		return;
2221 	}
2222 done:
2223 	ct->ct_state = CFISCSI_TARGET_STATE_ACTIVE;
2224 	req->status = CTL_LUN_OK;
2225 	req->result_nvl = nvlist_create(0);
2226 	nvlist_add_number(req->result_nvl, "port_id", port->targ_port);
2227 }
2228 
2229 static void
2230 cfiscsi_ioctl_port_remove(struct ctl_req *req)
2231 {
2232 	struct cfiscsi_target *ct;
2233 	const char *target, *val;
2234 	uint16_t tag;
2235 
2236 	target = dnvlist_get_string(req->args_nvl, "cfiscsi_target", NULL);
2237 	val = dnvlist_get_string(req->args_nvl, "cfiscsi_portal_group_tag",
2238 	    NULL);
2239 
2240 	if (target == NULL || val == NULL) {
2241 		req->status = CTL_LUN_ERROR;
2242 		snprintf(req->error_str, sizeof(req->error_str),
2243 		    "Missing required argument");
2244 		return;
2245 	}
2246 
2247 	tag = strtoul(val, NULL, 0);
2248 	ct = cfiscsi_target_find(&cfiscsi_softc, target, tag);
2249 	if (ct == NULL) {
2250 		req->status = CTL_LUN_ERROR;
2251 		snprintf(req->error_str, sizeof(req->error_str),
2252 		    "can't find target \"%s\"", target);
2253 		return;
2254 	}
2255 
2256 	ct->ct_state = CFISCSI_TARGET_STATE_DYING;
2257 	ctl_port_offline(&ct->ct_port);
2258 	cfiscsi_target_release(ct);
2259 	cfiscsi_target_release(ct);
2260 	req->status = CTL_LUN_OK;
2261 }
2262 
2263 static int
2264 cfiscsi_ioctl(struct cdev *dev,
2265     u_long cmd, caddr_t addr, int flag, struct thread *td)
2266 {
2267 	struct ctl_iscsi *ci;
2268 	struct ctl_req *req;
2269 
2270 	if (cmd == CTL_PORT_REQ) {
2271 		req = (struct ctl_req *)addr;
2272 		switch (req->reqtype) {
2273 		case CTL_REQ_CREATE:
2274 			cfiscsi_ioctl_port_create(req);
2275 			break;
2276 		case CTL_REQ_REMOVE:
2277 			cfiscsi_ioctl_port_remove(req);
2278 			break;
2279 		default:
2280 			req->status = CTL_LUN_ERROR;
2281 			snprintf(req->error_str, sizeof(req->error_str),
2282 			    "Unsupported request type %d", req->reqtype);
2283 		}
2284 		return (0);
2285 	}
2286 
2287 	if (cmd != CTL_ISCSI)
2288 		return (ENOTTY);
2289 
2290 	ci = (struct ctl_iscsi *)addr;
2291 	switch (ci->type) {
2292 	case CTL_ISCSI_HANDOFF:
2293 		cfiscsi_ioctl_handoff(ci);
2294 		break;
2295 	case CTL_ISCSI_LIST:
2296 		cfiscsi_ioctl_list(ci);
2297 		break;
2298 	case CTL_ISCSI_LOGOUT:
2299 		cfiscsi_ioctl_logout(ci);
2300 		break;
2301 	case CTL_ISCSI_TERMINATE:
2302 		cfiscsi_ioctl_terminate(ci);
2303 		break;
2304 	case CTL_ISCSI_LIMITS:
2305 		cfiscsi_ioctl_limits(ci);
2306 		break;
2307 #ifdef ICL_KERNEL_PROXY
2308 	case CTL_ISCSI_LISTEN:
2309 		cfiscsi_ioctl_listen(ci);
2310 		break;
2311 	case CTL_ISCSI_ACCEPT:
2312 		cfiscsi_ioctl_accept(ci);
2313 		break;
2314 	case CTL_ISCSI_SEND:
2315 		cfiscsi_ioctl_send(ci);
2316 		break;
2317 	case CTL_ISCSI_RECEIVE:
2318 		cfiscsi_ioctl_receive(ci);
2319 		break;
2320 #else
2321 	case CTL_ISCSI_LISTEN:
2322 	case CTL_ISCSI_ACCEPT:
2323 	case CTL_ISCSI_SEND:
2324 	case CTL_ISCSI_RECEIVE:
2325 		ci->status = CTL_ISCSI_ERROR;
2326 		snprintf(ci->error_str, sizeof(ci->error_str),
2327 		    "%s: CTL compiled without ICL_KERNEL_PROXY",
2328 		    __func__);
2329 		break;
2330 #endif /* !ICL_KERNEL_PROXY */
2331 	default:
2332 		ci->status = CTL_ISCSI_ERROR;
2333 		snprintf(ci->error_str, sizeof(ci->error_str),
2334 		    "%s: invalid iSCSI request type %d", __func__, ci->type);
2335 		break;
2336 	}
2337 
2338 	return (0);
2339 }
2340 
2341 static void
2342 cfiscsi_target_hold(struct cfiscsi_target *ct)
2343 {
2344 
2345 	refcount_acquire(&ct->ct_refcount);
2346 }
2347 
2348 static void
2349 cfiscsi_target_release(struct cfiscsi_target *ct)
2350 {
2351 	struct cfiscsi_softc *softc;
2352 
2353 	softc = ct->ct_softc;
2354 	mtx_lock(&softc->lock);
2355 	if (refcount_release(&ct->ct_refcount)) {
2356 		TAILQ_REMOVE(&softc->targets, ct, ct_next);
2357 		mtx_unlock(&softc->lock);
2358 		if (ct->ct_state != CFISCSI_TARGET_STATE_INVALID) {
2359 			ct->ct_state = CFISCSI_TARGET_STATE_INVALID;
2360 			if (ctl_port_deregister(&ct->ct_port) != 0)
2361 				printf("%s: ctl_port_deregister() failed\n",
2362 				    __func__);
2363 		}
2364 		free(ct, M_CFISCSI);
2365 
2366 		return;
2367 	}
2368 	mtx_unlock(&softc->lock);
2369 }
2370 
2371 static struct cfiscsi_target *
2372 cfiscsi_target_find(struct cfiscsi_softc *softc, const char *name, uint16_t tag)
2373 {
2374 	struct cfiscsi_target *ct;
2375 
2376 	mtx_lock(&softc->lock);
2377 	TAILQ_FOREACH(ct, &softc->targets, ct_next) {
2378 		if (ct->ct_tag != tag ||
2379 		    strcmp(name, ct->ct_name) != 0 ||
2380 		    ct->ct_state != CFISCSI_TARGET_STATE_ACTIVE)
2381 			continue;
2382 		cfiscsi_target_hold(ct);
2383 		mtx_unlock(&softc->lock);
2384 		return (ct);
2385 	}
2386 	mtx_unlock(&softc->lock);
2387 
2388 	return (NULL);
2389 }
2390 
2391 static struct cfiscsi_target *
2392 cfiscsi_target_find_or_create(struct cfiscsi_softc *softc, const char *name,
2393     const char *alias, uint16_t tag)
2394 {
2395 	struct cfiscsi_target *ct, *newct;
2396 
2397 	if (name[0] == '\0' || strlen(name) >= CTL_ISCSI_NAME_LEN)
2398 		return (NULL);
2399 
2400 	newct = malloc(sizeof(*newct), M_CFISCSI, M_WAITOK | M_ZERO);
2401 
2402 	mtx_lock(&softc->lock);
2403 	TAILQ_FOREACH(ct, &softc->targets, ct_next) {
2404 		if (ct->ct_tag != tag ||
2405 		    strcmp(name, ct->ct_name) != 0 ||
2406 		    ct->ct_state == CFISCSI_TARGET_STATE_INVALID)
2407 			continue;
2408 		cfiscsi_target_hold(ct);
2409 		mtx_unlock(&softc->lock);
2410 		free(newct, M_CFISCSI);
2411 		return (ct);
2412 	}
2413 
2414 	strlcpy(newct->ct_name, name, sizeof(newct->ct_name));
2415 	if (alias != NULL)
2416 		strlcpy(newct->ct_alias, alias, sizeof(newct->ct_alias));
2417 	newct->ct_tag = tag;
2418 	refcount_init(&newct->ct_refcount, 1);
2419 	newct->ct_softc = softc;
2420 	if (TAILQ_EMPTY(&softc->targets))
2421 		softc->last_target_id = 0;
2422 	newct->ct_target_id = ++softc->last_target_id;
2423 	TAILQ_INSERT_TAIL(&softc->targets, newct, ct_next);
2424 	mtx_unlock(&softc->lock);
2425 
2426 	return (newct);
2427 }
2428 
2429 static void
2430 cfiscsi_pdu_done(struct icl_pdu *ip, int error)
2431 {
2432 
2433 	if (error != 0)
2434 		; // XXX: Do something on error?
2435 	((ctl_ref)ip->ip_prv0)(ip->ip_prv1, -1);
2436 }
2437 
2438 static void
2439 cfiscsi_datamove_in(union ctl_io *io)
2440 {
2441 	struct cfiscsi_session *cs;
2442 	struct icl_pdu *request, *response;
2443 	const struct iscsi_bhs_scsi_command *bhssc;
2444 	struct iscsi_bhs_data_in *bhsdi;
2445 	struct ctl_sg_entry ctl_sg_entry, *ctl_sglist;
2446 	size_t len, expected_len, sg_len, buffer_offset;
2447 	const char *sg_addr;
2448 	icl_pdu_cb cb;
2449 	int ctl_sg_count, error, i;
2450 
2451 	request = PRIV_REQUEST(io);
2452 	cs = PDU_SESSION(request);
2453 
2454 	bhssc = (const struct iscsi_bhs_scsi_command *)request->ip_bhs;
2455 	KASSERT((bhssc->bhssc_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
2456 	    ISCSI_BHS_OPCODE_SCSI_COMMAND,
2457 	    ("bhssc->bhssc_opcode != ISCSI_BHS_OPCODE_SCSI_COMMAND"));
2458 
2459 	if (io->scsiio.kern_sg_entries > 0) {
2460 		ctl_sglist = (struct ctl_sg_entry *)io->scsiio.kern_data_ptr;
2461 		ctl_sg_count = io->scsiio.kern_sg_entries;
2462 	} else {
2463 		ctl_sglist = &ctl_sg_entry;
2464 		ctl_sglist->addr = io->scsiio.kern_data_ptr;
2465 		ctl_sglist->len = io->scsiio.kern_data_len;
2466 		ctl_sg_count = 1;
2467 	}
2468 
2469 	/*
2470 	 * This is the offset within the current SCSI command; for the first
2471 	 * call to cfiscsi_datamove() it will be 0, and for subsequent ones
2472 	 * it will be the sum of lengths of previous ones.
2473 	 */
2474 	buffer_offset = io->scsiio.kern_rel_offset;
2475 
2476 	/*
2477 	 * This is the transfer length expected by the initiator.  It can be
2478 	 * different from the amount of data from the SCSI point of view.
2479 	 */
2480 	expected_len = ntohl(bhssc->bhssc_expected_data_transfer_length);
2481 
2482 	/*
2483 	 * If the transfer is outside of expected length -- we are done.
2484 	 */
2485 	if (buffer_offset >= expected_len) {
2486 #if 0
2487 		CFISCSI_SESSION_DEBUG(cs, "buffer_offset = %zd, "
2488 		    "already sent the expected len", buffer_offset);
2489 #endif
2490 		io->scsiio.be_move_done(io);
2491 		return;
2492 	}
2493 
2494 	if (io->scsiio.kern_data_ref != NULL)
2495 		cb = cfiscsi_pdu_done;
2496 	else
2497 		cb = NULL;
2498 
2499 	i = 0;
2500 	sg_addr = NULL;
2501 	sg_len = 0;
2502 	response = NULL;
2503 	bhsdi = NULL;
2504 	for (;;) {
2505 		if (response == NULL) {
2506 			response = cfiscsi_pdu_new_response(request, M_NOWAIT);
2507 			if (response == NULL) {
2508 				CFISCSI_SESSION_WARN(cs, "failed to "
2509 				    "allocate memory; dropping connection");
2510 				ctl_set_busy(&io->scsiio);
2511 				io->scsiio.be_move_done(io);
2512 				cfiscsi_session_terminate(cs);
2513 				return;
2514 			}
2515 			bhsdi = (struct iscsi_bhs_data_in *)response->ip_bhs;
2516 			bhsdi->bhsdi_opcode = ISCSI_BHS_OPCODE_SCSI_DATA_IN;
2517 			bhsdi->bhsdi_initiator_task_tag =
2518 			    bhssc->bhssc_initiator_task_tag;
2519 			bhsdi->bhsdi_target_transfer_tag = 0xffffffff;
2520 			bhsdi->bhsdi_datasn = htonl(PRIV_EXPDATASN(io)++);
2521 			bhsdi->bhsdi_buffer_offset = htonl(buffer_offset);
2522 		}
2523 
2524 		KASSERT(i < ctl_sg_count, ("i >= ctl_sg_count"));
2525 		if (sg_len == 0) {
2526 			sg_addr = ctl_sglist[i].addr;
2527 			sg_len = ctl_sglist[i].len;
2528 			KASSERT(sg_len > 0, ("sg_len <= 0"));
2529 		}
2530 
2531 		len = sg_len;
2532 
2533 		/*
2534 		 * Truncate to maximum data segment length.
2535 		 */
2536 		KASSERT(response->ip_data_len < cs->cs_max_send_data_segment_length,
2537 		    ("ip_data_len %zd >= max_send_data_segment_length %d",
2538 		    response->ip_data_len, cs->cs_max_send_data_segment_length));
2539 		if (response->ip_data_len + len >
2540 		    cs->cs_max_send_data_segment_length) {
2541 			len = cs->cs_max_send_data_segment_length -
2542 			    response->ip_data_len;
2543 			KASSERT(len <= sg_len, ("len %zd > sg_len %zd",
2544 			    len, sg_len));
2545 		}
2546 
2547 		/*
2548 		 * Truncate to expected data transfer length.
2549 		 */
2550 		KASSERT(buffer_offset + response->ip_data_len < expected_len,
2551 		    ("buffer_offset %zd + ip_data_len %zd >= expected_len %zd",
2552 		    buffer_offset, response->ip_data_len, expected_len));
2553 		if (buffer_offset + response->ip_data_len + len > expected_len) {
2554 			CFISCSI_SESSION_DEBUG(cs, "truncating from %zd "
2555 			    "to expected data transfer length %zd",
2556 			    buffer_offset + response->ip_data_len + len, expected_len);
2557 			len = expected_len - (buffer_offset + response->ip_data_len);
2558 			KASSERT(len <= sg_len, ("len %zd > sg_len %zd",
2559 			    len, sg_len));
2560 		}
2561 
2562 		error = icl_pdu_append_data(response, sg_addr, len,
2563 		    M_NOWAIT | (cb ? ICL_NOCOPY : 0));
2564 		if (error != 0) {
2565 			CFISCSI_SESSION_WARN(cs, "failed to "
2566 			    "allocate memory; dropping connection");
2567 			icl_pdu_free(response);
2568 			ctl_set_busy(&io->scsiio);
2569 			io->scsiio.be_move_done(io);
2570 			cfiscsi_session_terminate(cs);
2571 			return;
2572 		}
2573 		sg_addr += len;
2574 		sg_len -= len;
2575 		io->scsiio.kern_data_resid -= len;
2576 
2577 		KASSERT(buffer_offset + response->ip_data_len <= expected_len,
2578 		    ("buffer_offset %zd + ip_data_len %zd > expected_len %zd",
2579 		    buffer_offset, response->ip_data_len, expected_len));
2580 		if (buffer_offset + response->ip_data_len == expected_len) {
2581 			/*
2582 			 * Already have the amount of data the initiator wanted.
2583 			 */
2584 			break;
2585 		}
2586 
2587 		if (sg_len == 0) {
2588 			/*
2589 			 * End of scatter-gather segment;
2590 			 * proceed to the next one...
2591 			 */
2592 			if (i == ctl_sg_count - 1) {
2593 				/*
2594 				 * ... unless this was the last one.
2595 				 */
2596 				break;
2597 			}
2598 			i++;
2599 		}
2600 
2601 		if (response->ip_data_len == cs->cs_max_send_data_segment_length) {
2602 			/*
2603 			 * Can't stuff more data into the current PDU;
2604 			 * queue it.  Note that's not enough to check
2605 			 * for kern_data_resid == 0 instead; there
2606 			 * may be several Data-In PDUs for the final
2607 			 * call to cfiscsi_datamove(), and we want
2608 			 * to set the F flag only on the last of them.
2609 			 */
2610 			buffer_offset += response->ip_data_len;
2611 			if (buffer_offset == io->scsiio.kern_total_len ||
2612 			    buffer_offset == expected_len) {
2613 				buffer_offset -= response->ip_data_len;
2614 				break;
2615 			}
2616 			if (cb != NULL) {
2617 				response->ip_prv0 = io->scsiio.kern_data_ref;
2618 				response->ip_prv1 = io->scsiio.kern_data_arg;
2619 				io->scsiio.kern_data_ref(io->scsiio.kern_data_arg, 1);
2620 			}
2621 			cfiscsi_pdu_queue_cb(response, cb);
2622 			response = NULL;
2623 			bhsdi = NULL;
2624 		}
2625 	}
2626 	if (response != NULL) {
2627 		buffer_offset += response->ip_data_len;
2628 		if (buffer_offset == io->scsiio.kern_total_len ||
2629 		    buffer_offset == expected_len) {
2630 			bhsdi->bhsdi_flags |= BHSDI_FLAGS_F;
2631 			if (io->io_hdr.status == CTL_SUCCESS) {
2632 				bhsdi->bhsdi_flags |= BHSDI_FLAGS_S;
2633 				if (io->scsiio.kern_total_len <
2634 				    ntohl(bhssc->bhssc_expected_data_transfer_length)) {
2635 					bhsdi->bhsdi_flags |= BHSSR_FLAGS_RESIDUAL_UNDERFLOW;
2636 					bhsdi->bhsdi_residual_count =
2637 					    htonl(ntohl(bhssc->bhssc_expected_data_transfer_length) -
2638 					    io->scsiio.kern_total_len);
2639 				} else if (io->scsiio.kern_total_len >
2640 				    ntohl(bhssc->bhssc_expected_data_transfer_length)) {
2641 					bhsdi->bhsdi_flags |= BHSSR_FLAGS_RESIDUAL_OVERFLOW;
2642 					bhsdi->bhsdi_residual_count =
2643 					    htonl(io->scsiio.kern_total_len -
2644 					    ntohl(bhssc->bhssc_expected_data_transfer_length));
2645 				}
2646 				bhsdi->bhsdi_status = io->scsiio.scsi_status;
2647 				io->io_hdr.flags |= CTL_FLAG_STATUS_SENT;
2648 			}
2649 		}
2650 		KASSERT(response->ip_data_len > 0, ("sending empty Data-In"));
2651 		if (cb != NULL) {
2652 			response->ip_prv0 = io->scsiio.kern_data_ref;
2653 			response->ip_prv1 = io->scsiio.kern_data_arg;
2654 			io->scsiio.kern_data_ref(io->scsiio.kern_data_arg, 1);
2655 		}
2656 		cfiscsi_pdu_queue_cb(response, cb);
2657 	}
2658 
2659 	io->scsiio.be_move_done(io);
2660 }
2661 
2662 static void
2663 cfiscsi_datamove_out(union ctl_io *io)
2664 {
2665 	struct cfiscsi_session *cs;
2666 	struct icl_pdu *request, *response;
2667 	const struct iscsi_bhs_scsi_command *bhssc;
2668 	struct iscsi_bhs_r2t *bhsr2t;
2669 	struct cfiscsi_data_wait *cdw;
2670 	struct ctl_sg_entry ctl_sg_entry, *ctl_sglist;
2671 	uint32_t expected_len, datamove_len, r2t_off, r2t_len;
2672 	uint32_t target_transfer_tag;
2673 	bool done;
2674 
2675 	request = PRIV_REQUEST(io);
2676 	cs = PDU_SESSION(request);
2677 
2678 	bhssc = (const struct iscsi_bhs_scsi_command *)request->ip_bhs;
2679 	KASSERT((bhssc->bhssc_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
2680 	    ISCSI_BHS_OPCODE_SCSI_COMMAND,
2681 	    ("bhssc->bhssc_opcode != ISCSI_BHS_OPCODE_SCSI_COMMAND"));
2682 
2683 	/*
2684 	 * Complete write underflow.  Not a single byte to read.  Return.
2685 	 */
2686 	expected_len = ntohl(bhssc->bhssc_expected_data_transfer_length);
2687 	if (io->scsiio.kern_rel_offset >= expected_len) {
2688 		io->scsiio.be_move_done(io);
2689 		return;
2690 	}
2691 	datamove_len = MIN(io->scsiio.kern_data_len,
2692 	    expected_len - io->scsiio.kern_rel_offset);
2693 
2694 	target_transfer_tag =
2695 	    atomic_fetchadd_32(&cs->cs_target_transfer_tag, 1);
2696 	if (target_transfer_tag == 0xffffffff) {
2697 		target_transfer_tag =
2698 		    atomic_fetchadd_32(&cs->cs_target_transfer_tag, 1);
2699 	}
2700 	cdw = cfiscsi_data_wait_new(cs, io, bhssc->bhssc_initiator_task_tag,
2701 	    &target_transfer_tag);
2702 	if (cdw == NULL) {
2703 		CFISCSI_SESSION_WARN(cs, "failed to "
2704 		    "allocate memory; dropping connection");
2705 		ctl_set_busy(&io->scsiio);
2706 		io->scsiio.be_move_done(io);
2707 		cfiscsi_session_terminate(cs);
2708 		return;
2709 	}
2710 #if 0
2711 	CFISCSI_SESSION_DEBUG(cs, "expecting Data-Out with initiator "
2712 	    "task tag 0x%x, target transfer tag 0x%x",
2713 	    bhssc->bhssc_initiator_task_tag, target_transfer_tag);
2714 #endif
2715 
2716 	cdw->cdw_ctl_io = io;
2717 	cdw->cdw_target_transfer_tag = target_transfer_tag;
2718 	cdw->cdw_initiator_task_tag = bhssc->bhssc_initiator_task_tag;
2719 	cdw->cdw_r2t_end = datamove_len;
2720 	cdw->cdw_datasn = 0;
2721 
2722 	/* Set initial data pointer for the CDW respecting ext_data_filled. */
2723 	if (io->scsiio.kern_sg_entries > 0) {
2724 		ctl_sglist = (struct ctl_sg_entry *)io->scsiio.kern_data_ptr;
2725 	} else {
2726 		ctl_sglist = &ctl_sg_entry;
2727 		ctl_sglist->addr = io->scsiio.kern_data_ptr;
2728 		ctl_sglist->len = datamove_len;
2729 	}
2730 	cdw->cdw_sg_index = 0;
2731 	cdw->cdw_sg_addr = ctl_sglist[cdw->cdw_sg_index].addr;
2732 	cdw->cdw_sg_len = ctl_sglist[cdw->cdw_sg_index].len;
2733 	r2t_off = io->scsiio.ext_data_filled;
2734 	while (r2t_off > 0) {
2735 		if (r2t_off >= cdw->cdw_sg_len) {
2736 			r2t_off -= cdw->cdw_sg_len;
2737 			cdw->cdw_sg_index++;
2738 			cdw->cdw_sg_addr = ctl_sglist[cdw->cdw_sg_index].addr;
2739 			cdw->cdw_sg_len = ctl_sglist[cdw->cdw_sg_index].len;
2740 			continue;
2741 		}
2742 		cdw->cdw_sg_addr += r2t_off;
2743 		cdw->cdw_sg_len -= r2t_off;
2744 		r2t_off = 0;
2745 	}
2746 
2747 	if (cs->cs_immediate_data &&
2748 	    io->scsiio.kern_rel_offset + io->scsiio.ext_data_filled <
2749 	    icl_pdu_data_segment_length(request)) {
2750 		done = cfiscsi_handle_data_segment(request, cdw);
2751 		if (done) {
2752 			cfiscsi_data_wait_free(cs, cdw);
2753 			io->scsiio.be_move_done(io);
2754 			return;
2755 		}
2756 	}
2757 
2758 	r2t_off = io->scsiio.kern_rel_offset + io->scsiio.ext_data_filled;
2759 	r2t_len = MIN(datamove_len - io->scsiio.ext_data_filled,
2760 	    cs->cs_max_burst_length);
2761 	cdw->cdw_r2t_end = io->scsiio.ext_data_filled + r2t_len;
2762 
2763 	CFISCSI_SESSION_LOCK(cs);
2764 	TAILQ_INSERT_TAIL(&cs->cs_waiting_for_data_out, cdw, cdw_next);
2765 	CFISCSI_SESSION_UNLOCK(cs);
2766 
2767 	/*
2768 	 * XXX: We should limit the number of outstanding R2T PDUs
2769 	 * 	per task to MaxOutstandingR2T.
2770 	 */
2771 	response = cfiscsi_pdu_new_response(request, M_NOWAIT);
2772 	if (response == NULL) {
2773 		CFISCSI_SESSION_WARN(cs, "failed to "
2774 		    "allocate memory; dropping connection");
2775 		ctl_set_busy(&io->scsiio);
2776 		io->scsiio.be_move_done(io);
2777 		cfiscsi_session_terminate(cs);
2778 		return;
2779 	}
2780 	io->io_hdr.flags |= CTL_FLAG_DMA_INPROG;
2781 	bhsr2t = (struct iscsi_bhs_r2t *)response->ip_bhs;
2782 	bhsr2t->bhsr2t_opcode = ISCSI_BHS_OPCODE_R2T;
2783 	bhsr2t->bhsr2t_flags = 0x80;
2784 	bhsr2t->bhsr2t_lun = bhssc->bhssc_lun;
2785 	bhsr2t->bhsr2t_initiator_task_tag = bhssc->bhssc_initiator_task_tag;
2786 	bhsr2t->bhsr2t_target_transfer_tag = target_transfer_tag;
2787 	/*
2788 	 * XXX: Here we assume that cfiscsi_datamove() won't ever
2789 	 *	be running concurrently on several CPUs for a given
2790 	 *	command.
2791 	 */
2792 	bhsr2t->bhsr2t_r2tsn = htonl(PRIV_R2TSN(io)++);
2793 	/*
2794 	 * This is the offset within the current SCSI command;
2795 	 * i.e. for the first call of datamove(), it will be 0,
2796 	 * and for subsequent ones it will be the sum of lengths
2797 	 * of previous ones.
2798 	 *
2799 	 * The ext_data_filled is to account for unsolicited
2800 	 * (immediate) data that might have already arrived.
2801 	 */
2802 	bhsr2t->bhsr2t_buffer_offset = htonl(r2t_off);
2803 	/*
2804 	 * This is the total length (sum of S/G lengths) this call
2805 	 * to cfiscsi_datamove() is supposed to handle, limited by
2806 	 * MaxBurstLength.
2807 	 */
2808 	bhsr2t->bhsr2t_desired_data_transfer_length = htonl(r2t_len);
2809 	cfiscsi_pdu_queue(response);
2810 }
2811 
2812 static void
2813 cfiscsi_datamove(union ctl_io *io)
2814 {
2815 
2816 	if ((io->io_hdr.flags & CTL_FLAG_DATA_MASK) == CTL_FLAG_DATA_IN)
2817 		cfiscsi_datamove_in(io);
2818 	else {
2819 		/* We hadn't received anything during this datamove yet. */
2820 		io->scsiio.ext_data_filled = 0;
2821 		cfiscsi_datamove_out(io);
2822 	}
2823 }
2824 
2825 static void
2826 cfiscsi_scsi_command_done(union ctl_io *io)
2827 {
2828 	struct icl_pdu *request, *response;
2829 	struct iscsi_bhs_scsi_command *bhssc;
2830 	struct iscsi_bhs_scsi_response *bhssr;
2831 #ifdef DIAGNOSTIC
2832 	struct cfiscsi_data_wait *cdw;
2833 #endif
2834 	struct cfiscsi_session *cs;
2835 	uint16_t sense_length;
2836 
2837 	request = PRIV_REQUEST(io);
2838 	cs = PDU_SESSION(request);
2839 	bhssc = (struct iscsi_bhs_scsi_command *)request->ip_bhs;
2840 	KASSERT((bhssc->bhssc_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
2841 	    ISCSI_BHS_OPCODE_SCSI_COMMAND,
2842 	    ("replying to wrong opcode 0x%x", bhssc->bhssc_opcode));
2843 
2844 	//CFISCSI_SESSION_DEBUG(cs, "initiator task tag 0x%x",
2845 	//    bhssc->bhssc_initiator_task_tag);
2846 
2847 #ifdef DIAGNOSTIC
2848 	CFISCSI_SESSION_LOCK(cs);
2849 	TAILQ_FOREACH(cdw, &cs->cs_waiting_for_data_out, cdw_next)
2850 		KASSERT(bhssc->bhssc_initiator_task_tag !=
2851 		    cdw->cdw_initiator_task_tag, ("dangling cdw"));
2852 	CFISCSI_SESSION_UNLOCK(cs);
2853 #endif
2854 
2855 	/*
2856 	 * Do not return status for aborted commands.
2857 	 * There are exceptions, but none supported by CTL yet.
2858 	 */
2859 	if (((io->io_hdr.flags & CTL_FLAG_ABORT) &&
2860 	     (io->io_hdr.flags & CTL_FLAG_ABORT_STATUS) == 0) ||
2861 	    (io->io_hdr.flags & CTL_FLAG_STATUS_SENT)) {
2862 		ctl_free_io(io);
2863 		icl_pdu_free(request);
2864 		return;
2865 	}
2866 
2867 	response = cfiscsi_pdu_new_response(request, M_WAITOK);
2868 	bhssr = (struct iscsi_bhs_scsi_response *)response->ip_bhs;
2869 	bhssr->bhssr_opcode = ISCSI_BHS_OPCODE_SCSI_RESPONSE;
2870 	bhssr->bhssr_flags = 0x80;
2871 	/*
2872 	 * XXX: We don't deal with bidirectional under/overflows;
2873 	 *	does anything actually support those?
2874 	 */
2875 	if (io->scsiio.kern_total_len <
2876 	    ntohl(bhssc->bhssc_expected_data_transfer_length)) {
2877 		bhssr->bhssr_flags |= BHSSR_FLAGS_RESIDUAL_UNDERFLOW;
2878 		bhssr->bhssr_residual_count =
2879 		    htonl(ntohl(bhssc->bhssc_expected_data_transfer_length) -
2880 		    io->scsiio.kern_total_len);
2881 		//CFISCSI_SESSION_DEBUG(cs, "underflow; residual count %d",
2882 		//    ntohl(bhssr->bhssr_residual_count));
2883 	} else if (io->scsiio.kern_total_len >
2884 	    ntohl(bhssc->bhssc_expected_data_transfer_length)) {
2885 		bhssr->bhssr_flags |= BHSSR_FLAGS_RESIDUAL_OVERFLOW;
2886 		bhssr->bhssr_residual_count = htonl(io->scsiio.kern_total_len -
2887 		    ntohl(bhssc->bhssc_expected_data_transfer_length));
2888 		//CFISCSI_SESSION_DEBUG(cs, "overflow; residual count %d",
2889 		//    ntohl(bhssr->bhssr_residual_count));
2890 	}
2891 	bhssr->bhssr_response = BHSSR_RESPONSE_COMMAND_COMPLETED;
2892 	bhssr->bhssr_status = io->scsiio.scsi_status;
2893 	bhssr->bhssr_initiator_task_tag = bhssc->bhssc_initiator_task_tag;
2894 	bhssr->bhssr_expdatasn = htonl(PRIV_EXPDATASN(io));
2895 
2896 	if (io->scsiio.sense_len > 0) {
2897 #if 0
2898 		CFISCSI_SESSION_DEBUG(cs, "returning %d bytes of sense data",
2899 		    io->scsiio.sense_len);
2900 #endif
2901 		sense_length = htons(io->scsiio.sense_len);
2902 		icl_pdu_append_data(response,
2903 		    &sense_length, sizeof(sense_length), M_WAITOK);
2904 		icl_pdu_append_data(response,
2905 		    &io->scsiio.sense_data, io->scsiio.sense_len, M_WAITOK);
2906 	}
2907 
2908 	ctl_free_io(io);
2909 	icl_pdu_free(request);
2910 	cfiscsi_pdu_queue(response);
2911 }
2912 
2913 static void
2914 cfiscsi_task_management_done(union ctl_io *io)
2915 {
2916 	struct icl_pdu *request, *response;
2917 	struct iscsi_bhs_task_management_request *bhstmr;
2918 	struct iscsi_bhs_task_management_response *bhstmr2;
2919 	struct cfiscsi_data_wait *cdw, *tmpcdw;
2920 	struct cfiscsi_session *cs, *tcs;
2921 	struct cfiscsi_softc *softc;
2922 	int cold_reset = 0;
2923 
2924 	request = PRIV_REQUEST(io);
2925 	cs = PDU_SESSION(request);
2926 	bhstmr = (struct iscsi_bhs_task_management_request *)request->ip_bhs;
2927 	KASSERT((bhstmr->bhstmr_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
2928 	    ISCSI_BHS_OPCODE_TASK_REQUEST,
2929 	    ("replying to wrong opcode 0x%x", bhstmr->bhstmr_opcode));
2930 
2931 #if 0
2932 	CFISCSI_SESSION_DEBUG(cs, "initiator task tag 0x%x; referenced task tag 0x%x",
2933 	    bhstmr->bhstmr_initiator_task_tag,
2934 	    bhstmr->bhstmr_referenced_task_tag);
2935 #endif
2936 
2937 	if ((bhstmr->bhstmr_function & ~0x80) ==
2938 	    BHSTMR_FUNCTION_ABORT_TASK) {
2939 		/*
2940 		 * Make sure we no longer wait for Data-Out for this command.
2941 		 */
2942 		CFISCSI_SESSION_LOCK(cs);
2943 		TAILQ_FOREACH_SAFE(cdw,
2944 		    &cs->cs_waiting_for_data_out, cdw_next, tmpcdw) {
2945 			if (bhstmr->bhstmr_referenced_task_tag !=
2946 			    cdw->cdw_initiator_task_tag)
2947 				continue;
2948 
2949 #if 0
2950 			CFISCSI_SESSION_DEBUG(cs, "removing csw for initiator task "
2951 			    "tag 0x%x", bhstmr->bhstmr_initiator_task_tag);
2952 #endif
2953 			TAILQ_REMOVE(&cs->cs_waiting_for_data_out,
2954 			    cdw, cdw_next);
2955 			io->io_hdr.flags &= ~CTL_FLAG_DMA_INPROG;
2956 			cdw->cdw_ctl_io->scsiio.io_hdr.port_status = 43;
2957 			cdw->cdw_ctl_io->scsiio.be_move_done(cdw->cdw_ctl_io);
2958 			cfiscsi_data_wait_free(cs, cdw);
2959 		}
2960 		CFISCSI_SESSION_UNLOCK(cs);
2961 	}
2962 	if ((bhstmr->bhstmr_function & ~0x80) ==
2963 	    BHSTMR_FUNCTION_TARGET_COLD_RESET &&
2964 	    io->io_hdr.status == CTL_SUCCESS)
2965 		cold_reset = 1;
2966 
2967 	response = cfiscsi_pdu_new_response(request, M_WAITOK);
2968 	bhstmr2 = (struct iscsi_bhs_task_management_response *)
2969 	    response->ip_bhs;
2970 	bhstmr2->bhstmr_opcode = ISCSI_BHS_OPCODE_TASK_RESPONSE;
2971 	bhstmr2->bhstmr_flags = 0x80;
2972 	switch (io->taskio.task_status) {
2973 	case CTL_TASK_FUNCTION_COMPLETE:
2974 		bhstmr2->bhstmr_response = BHSTMR_RESPONSE_FUNCTION_COMPLETE;
2975 		break;
2976 	case CTL_TASK_FUNCTION_SUCCEEDED:
2977 		bhstmr2->bhstmr_response = BHSTMR_RESPONSE_FUNCTION_SUCCEEDED;
2978 		break;
2979 	case CTL_TASK_LUN_DOES_NOT_EXIST:
2980 		bhstmr2->bhstmr_response = BHSTMR_RESPONSE_LUN_DOES_NOT_EXIST;
2981 		break;
2982 	case CTL_TASK_FUNCTION_NOT_SUPPORTED:
2983 	default:
2984 		bhstmr2->bhstmr_response = BHSTMR_RESPONSE_FUNCTION_NOT_SUPPORTED;
2985 		break;
2986 	}
2987 	memcpy(bhstmr2->bhstmr_additional_reponse_information,
2988 	    io->taskio.task_resp, sizeof(io->taskio.task_resp));
2989 	bhstmr2->bhstmr_initiator_task_tag = bhstmr->bhstmr_initiator_task_tag;
2990 
2991 	ctl_free_io(io);
2992 	icl_pdu_free(request);
2993 	cfiscsi_pdu_queue(response);
2994 
2995 	if (cold_reset) {
2996 		softc = cs->cs_target->ct_softc;
2997 		mtx_lock(&softc->lock);
2998 		TAILQ_FOREACH(tcs, &softc->sessions, cs_next) {
2999 			if (tcs->cs_target == cs->cs_target)
3000 				cfiscsi_session_terminate(tcs);
3001 		}
3002 		mtx_unlock(&softc->lock);
3003 	}
3004 }
3005 
3006 static void
3007 cfiscsi_done(union ctl_io *io)
3008 {
3009 	struct icl_pdu *request;
3010 	struct cfiscsi_session *cs;
3011 
3012 	KASSERT(((io->io_hdr.status & CTL_STATUS_MASK) != CTL_STATUS_NONE),
3013 		("invalid CTL status %#x", io->io_hdr.status));
3014 
3015 	if (io->io_hdr.io_type == CTL_IO_TASK &&
3016 	    io->taskio.task_action == CTL_TASK_I_T_NEXUS_RESET) {
3017 		/*
3018 		 * Implicit task termination has just completed; nothing to do.
3019 		 */
3020 		cs = PRIV_REQUEST(io);
3021 		cs->cs_tasks_aborted = true;
3022 		refcount_release(&cs->cs_outstanding_ctl_pdus);
3023 		wakeup(__DEVOLATILE(void *, &cs->cs_outstanding_ctl_pdus));
3024 		ctl_free_io(io);
3025 		return;
3026 	}
3027 
3028 	request = PRIV_REQUEST(io);
3029 	cs = PDU_SESSION(request);
3030 
3031 	switch (request->ip_bhs->bhs_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) {
3032 	case ISCSI_BHS_OPCODE_SCSI_COMMAND:
3033 		cfiscsi_scsi_command_done(io);
3034 		break;
3035 	case ISCSI_BHS_OPCODE_TASK_REQUEST:
3036 		cfiscsi_task_management_done(io);
3037 		break;
3038 	default:
3039 		panic("cfiscsi_done called with wrong opcode 0x%x",
3040 		    request->ip_bhs->bhs_opcode);
3041 	}
3042 
3043 	refcount_release(&cs->cs_outstanding_ctl_pdus);
3044 }
3045