xref: /freebsd/sys/dev/iscsi/iscsi.c (revision cc3f4b99653c34ae64f8a1fddea370abefef680e)
1 /*-
2  * Copyright (c) 2012 The FreeBSD Foundation
3  * All rights reserved.
4  *
5  * This software was developed by Edward Tomasz Napierala under sponsorship
6  * from the FreeBSD Foundation.
7  *
8  * Redistribution and use in source and binary forms, with or without
9  * modification, are permitted provided that the following conditions
10  * are met:
11  * 1. Redistributions of source code must retain the above copyright
12  *    notice, this list of conditions and the following disclaimer.
13  * 2. Redistributions in binary form must reproduce the above copyright
14  *    notice, this list of conditions and the following disclaimer in the
15  *    documentation and/or other materials provided with the distribution.
16  *
17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
21  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27  * SUCH DAMAGE.
28  *
29  * $FreeBSD$
30  */
31 
32 #include <sys/param.h>
33 #include <sys/condvar.h>
34 #include <sys/conf.h>
35 #include <sys/endian.h>
36 #include <sys/eventhandler.h>
37 #include <sys/file.h>
38 #include <sys/kernel.h>
39 #include <sys/kthread.h>
40 #include <sys/lock.h>
41 #include <sys/malloc.h>
42 #include <sys/mutex.h>
43 #include <sys/module.h>
44 #include <sys/sysctl.h>
45 #include <sys/systm.h>
46 #include <sys/sx.h>
47 #include <vm/uma.h>
48 
49 #include <cam/cam.h>
50 #include <cam/cam_ccb.h>
51 #include <cam/cam_xpt.h>
52 #include <cam/cam_debug.h>
53 #include <cam/cam_sim.h>
54 #include <cam/cam_xpt_sim.h>
55 #include <cam/cam_xpt_periph.h>
56 #include <cam/cam_periph.h>
57 #include <cam/scsi/scsi_all.h>
58 #include <cam/scsi/scsi_message.h>
59 
60 #include "iscsi_ioctl.h"
61 #include "iscsi.h"
62 #include "icl.h"
63 #include "iscsi_proto.h"
64 
65 #ifdef ICL_KERNEL_PROXY
66 #include <sys/socketvar.h>
67 #endif
68 
69 #ifdef ICL_KERNEL_PROXY
70 FEATURE(iscsi_kernel_proxy, "iSCSI initiator built with ICL_KERNEL_PROXY");
71 #endif
72 
73 /*
74  * XXX: This is global so the iscsi_unload() can access it.
75  * 	Think about how to do this properly.
76  */
77 static struct iscsi_softc	*sc;
78 
79 SYSCTL_NODE(_kern, OID_AUTO, iscsi, CTLFLAG_RD, 0, "iSCSI initiator");
80 static int debug = 1;
81 TUNABLE_INT("kern.iscsi.debug", &debug);
82 SYSCTL_INT(_kern_iscsi, OID_AUTO, debug, CTLFLAG_RWTUN,
83     &debug, 0, "Enable debug messages");
84 static int ping_timeout = 5;
85 TUNABLE_INT("kern.iscsi.ping_timeout", &ping_timeout);
86 SYSCTL_INT(_kern_iscsi, OID_AUTO, ping_timeout, CTLFLAG_RWTUN, &ping_timeout,
87     0, "Timeout for ping (NOP-Out) requests, in seconds");
88 static int iscsid_timeout = 60;
89 TUNABLE_INT("kern.iscsi.iscsid_timeout", &iscsid_timeout);
90 SYSCTL_INT(_kern_iscsi, OID_AUTO, iscsid_timeout, CTLFLAG_RWTUN, &iscsid_timeout,
91     0, "Time to wait for iscsid(8) to handle reconnection, in seconds");
92 static int login_timeout = 60;
93 TUNABLE_INT("kern.iscsi.login_timeout", &login_timeout);
94 SYSCTL_INT(_kern_iscsi, OID_AUTO, login_timeout, CTLFLAG_RWTUN, &login_timeout,
95     0, "Time to wait for iscsid(8) to finish Login Phase, in seconds");
96 static int maxtags = 255;
97 TUNABLE_INT("kern.iscsi.maxtags", &maxtags);
98 SYSCTL_INT(_kern_iscsi, OID_AUTO, maxtags, CTLFLAG_RWTUN, &maxtags,
99     0, "Max number of IO requests queued");
100 static int fail_on_disconnection = 0;
101 TUNABLE_INT("kern.iscsi.fail_on_disconnection", &fail_on_disconnection);
102 SYSCTL_INT(_kern_iscsi, OID_AUTO, fail_on_disconnection, CTLFLAG_RWTUN,
103     &fail_on_disconnection, 0, "Destroy CAM SIM on connection failure");
104 
105 static MALLOC_DEFINE(M_ISCSI, "iSCSI", "iSCSI initiator");
106 static uma_zone_t iscsi_outstanding_zone;
107 
108 #define	CONN_SESSION(X)	((struct iscsi_session *)X->ic_prv0)
109 #define	PDU_SESSION(X)	(CONN_SESSION(X->ip_conn))
110 
111 #define	ISCSI_DEBUG(X, ...)						\
112 	do {								\
113 		if (debug > 1) 						\
114 			printf("%s: " X "\n", __func__, ## __VA_ARGS__);\
115 	} while (0)
116 
117 #define	ISCSI_WARN(X, ...)						\
118 	do {								\
119 		if (debug > 0) {					\
120 			printf("WARNING: %s: " X "\n",			\
121 			    __func__, ## __VA_ARGS__);			\
122 		}							\
123 	} while (0)
124 
125 #define	ISCSI_SESSION_DEBUG(S, X, ...)					\
126 	do {								\
127 		if (debug > 1) {					\
128 			printf("%s: %s (%s): " X "\n",			\
129 			    __func__, S->is_conf.isc_target_addr,	\
130 			    S->is_conf.isc_target, ## __VA_ARGS__);	\
131 		}							\
132 	} while (0)
133 
134 #define	ISCSI_SESSION_WARN(S, X, ...)					\
135 	do {								\
136 		if (debug > 0) {					\
137 			printf("WARNING: %s (%s): " X "\n",		\
138 			    S->is_conf.isc_target_addr,			\
139 			    S->is_conf.isc_target, ## __VA_ARGS__);	\
140 		}							\
141 	} while (0)
142 
143 #define ISCSI_SESSION_LOCK(X)		mtx_lock(&X->is_lock)
144 #define ISCSI_SESSION_UNLOCK(X)		mtx_unlock(&X->is_lock)
145 #define ISCSI_SESSION_LOCK_ASSERT(X)	mtx_assert(&X->is_lock, MA_OWNED)
146 
147 static int	iscsi_ioctl(struct cdev *dev, u_long cmd, caddr_t arg,
148 		    int mode, struct thread *td);
149 
150 static struct cdevsw iscsi_cdevsw = {
151      .d_version = D_VERSION,
152      .d_ioctl   = iscsi_ioctl,
153      .d_name    = "iscsi",
154 };
155 
156 static void	iscsi_pdu_queue_locked(struct icl_pdu *request);
157 static void	iscsi_pdu_queue(struct icl_pdu *request);
158 static void	iscsi_pdu_update_statsn(const struct icl_pdu *response);
159 static void	iscsi_pdu_handle_nop_in(struct icl_pdu *response);
160 static void	iscsi_pdu_handle_scsi_response(struct icl_pdu *response);
161 static void	iscsi_pdu_handle_data_in(struct icl_pdu *response);
162 static void	iscsi_pdu_handle_logout_response(struct icl_pdu *response);
163 static void	iscsi_pdu_handle_r2t(struct icl_pdu *response);
164 static void	iscsi_pdu_handle_async_message(struct icl_pdu *response);
165 static void	iscsi_pdu_handle_reject(struct icl_pdu *response);
166 static void	iscsi_session_reconnect(struct iscsi_session *is);
167 static void	iscsi_session_terminate(struct iscsi_session *is);
168 static void	iscsi_action(struct cam_sim *sim, union ccb *ccb);
169 static void	iscsi_poll(struct cam_sim *sim);
170 static struct iscsi_outstanding	*iscsi_outstanding_find(struct iscsi_session *is,
171 		    uint32_t initiator_task_tag);
172 static int	iscsi_outstanding_add(struct iscsi_session *is,
173 		    uint32_t initiator_task_tag, union ccb *ccb);
174 static void	iscsi_outstanding_remove(struct iscsi_session *is,
175 		    struct iscsi_outstanding *io);
176 
177 static bool
178 iscsi_pdu_prepare(struct icl_pdu *request)
179 {
180 	struct iscsi_session *is;
181 	struct iscsi_bhs_scsi_command *bhssc;
182 
183 	is = PDU_SESSION(request);
184 
185 	ISCSI_SESSION_LOCK_ASSERT(is);
186 
187 	/*
188 	 * We're only using fields common for all the request
189 	 * (initiator -> target) PDUs.
190 	 */
191 	bhssc = (struct iscsi_bhs_scsi_command *)request->ip_bhs;
192 
193 	/*
194 	 * Data-Out PDU does not contain CmdSN.
195 	 */
196 	if (bhssc->bhssc_opcode != ISCSI_BHS_OPCODE_SCSI_DATA_OUT) {
197 		if (is->is_cmdsn > is->is_maxcmdsn &&
198 		    (bhssc->bhssc_opcode & ISCSI_BHS_OPCODE_IMMEDIATE) == 0) {
199 			/*
200 			 * Current MaxCmdSN prevents us from sending any more
201 			 * SCSI Command PDUs to the target; postpone the PDU.
202 			 * It will get resent by either iscsi_pdu_queue(),
203 			 * or by maintenance thread.
204 			 */
205 #if 0
206 			ISCSI_SESSION_DEBUG(is, "postponing send, CmdSN %d, ExpCmdSN %d, MaxCmdSN %d, opcode 0x%x",
207 			    is->is_cmdsn, is->is_expcmdsn, is->is_maxcmdsn, bhssc->bhssc_opcode);
208 #endif
209 			return (true);
210 		}
211 		bhssc->bhssc_cmdsn = htonl(is->is_cmdsn);
212 		if ((bhssc->bhssc_opcode & ISCSI_BHS_OPCODE_IMMEDIATE) == 0)
213 			is->is_cmdsn++;
214 	}
215 	bhssc->bhssc_expstatsn = htonl(is->is_statsn + 1);
216 
217 	return (false);
218 }
219 
220 static void
221 iscsi_session_send_postponed(struct iscsi_session *is)
222 {
223 	struct icl_pdu *request;
224 	bool postpone;
225 
226 	ISCSI_SESSION_LOCK_ASSERT(is);
227 
228 	while (!STAILQ_EMPTY(&is->is_postponed)) {
229 		request = STAILQ_FIRST(&is->is_postponed);
230 		postpone = iscsi_pdu_prepare(request);
231 		if (postpone)
232 			break;
233 		STAILQ_REMOVE_HEAD(&is->is_postponed, ip_next);
234 		icl_pdu_queue(request);
235 	}
236 }
237 
238 static void
239 iscsi_pdu_queue_locked(struct icl_pdu *request)
240 {
241 	struct iscsi_session *is;
242 	bool postpone;
243 
244 	is = PDU_SESSION(request);
245 	ISCSI_SESSION_LOCK_ASSERT(is);
246 	iscsi_session_send_postponed(is);
247 	postpone = iscsi_pdu_prepare(request);
248 	if (postpone) {
249 		STAILQ_INSERT_TAIL(&is->is_postponed, request, ip_next);
250 		return;
251 	}
252 	icl_pdu_queue(request);
253 }
254 
255 static void
256 iscsi_pdu_queue(struct icl_pdu *request)
257 {
258 	struct iscsi_session *is;
259 
260 	is = PDU_SESSION(request);
261 	ISCSI_SESSION_LOCK(is);
262 	iscsi_pdu_queue_locked(request);
263 	ISCSI_SESSION_UNLOCK(is);
264 }
265 
266 static void
267 iscsi_session_logout(struct iscsi_session *is)
268 {
269 	struct icl_pdu *request;
270 	struct iscsi_bhs_logout_request *bhslr;
271 
272 	request = icl_pdu_new_bhs(is->is_conn, M_NOWAIT);
273 	if (request == NULL)
274 		return;
275 
276 	bhslr = (struct iscsi_bhs_logout_request *)request->ip_bhs;
277 	bhslr->bhslr_opcode = ISCSI_BHS_OPCODE_LOGOUT_REQUEST;
278 	bhslr->bhslr_reason = BHSLR_REASON_CLOSE_SESSION;
279 	iscsi_pdu_queue_locked(request);
280 }
281 
282 static void
283 iscsi_session_terminate_tasks(struct iscsi_session *is, bool requeue)
284 {
285 	struct iscsi_outstanding *io, *tmp;
286 
287 	ISCSI_SESSION_LOCK_ASSERT(is);
288 
289 	TAILQ_FOREACH_SAFE(io, &is->is_outstanding, io_next, tmp) {
290 		if (requeue) {
291 			io->io_ccb->ccb_h.status &= ~CAM_SIM_QUEUED;
292 			io->io_ccb->ccb_h.status |= CAM_REQUEUE_REQ;
293 		} else {
294 			io->io_ccb->ccb_h.status = CAM_REQ_ABORTED;
295 		}
296 
297 		if ((io->io_ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
298 			xpt_freeze_devq(io->io_ccb->ccb_h.path, 1);
299 			ISCSI_SESSION_DEBUG(is, "freezing devq");
300 		}
301 		io->io_ccb->ccb_h.status |= CAM_DEV_QFRZN;
302 		xpt_done(io->io_ccb);
303 		iscsi_outstanding_remove(is, io);
304 	}
305 }
306 
307 static void
308 iscsi_session_cleanup(struct iscsi_session *is, bool destroy_sim)
309 {
310 	struct icl_pdu *pdu;
311 
312 	ISCSI_SESSION_LOCK_ASSERT(is);
313 
314 	/*
315 	 * Don't queue any new PDUs.
316 	 */
317 	if (is->is_sim != NULL && is->is_simq_frozen == false) {
318 		ISCSI_SESSION_DEBUG(is, "freezing");
319 		xpt_freeze_simq(is->is_sim, 1);
320 		is->is_simq_frozen = true;
321 	}
322 
323 	/*
324 	 * Remove postponed PDUs.
325 	 */
326 	while (!STAILQ_EMPTY(&is->is_postponed)) {
327 		pdu = STAILQ_FIRST(&is->is_postponed);
328 		STAILQ_REMOVE_HEAD(&is->is_postponed, ip_next);
329 		icl_pdu_free(pdu);
330 	}
331 
332 	if (destroy_sim == false) {
333 		/*
334 		 * Terminate SCSI tasks, asking CAM to requeue them.
335 		 */
336 		iscsi_session_terminate_tasks(is, true);
337 		return;
338 	}
339 
340 	iscsi_session_terminate_tasks(is, false);
341 
342 	if (is->is_sim == NULL)
343 		return;
344 
345 	ISCSI_SESSION_DEBUG(is, "deregistering SIM");
346 	xpt_async(AC_LOST_DEVICE, is->is_path, NULL);
347 
348 	if (is->is_simq_frozen) {
349 		xpt_release_simq(is->is_sim, 1);
350 		is->is_simq_frozen = false;
351 	}
352 
353 	xpt_free_path(is->is_path);
354 	is->is_path = NULL;
355 	xpt_bus_deregister(cam_sim_path(is->is_sim));
356 	cam_sim_free(is->is_sim, TRUE /*free_devq*/);
357 	is->is_sim = NULL;
358 	is->is_devq = NULL;
359 }
360 
361 static void
362 iscsi_maintenance_thread_reconnect(struct iscsi_session *is)
363 {
364 
365 	icl_conn_shutdown(is->is_conn);
366 	icl_conn_close(is->is_conn);
367 
368 	ISCSI_SESSION_LOCK(is);
369 
370 	is->is_connected = false;
371 	is->is_reconnecting = false;
372 	is->is_login_phase = false;
373 
374 #ifdef ICL_KERNEL_PROXY
375 	if (is->is_login_pdu != NULL) {
376 		icl_pdu_free(is->is_login_pdu);
377 		is->is_login_pdu = NULL;
378 	}
379 	cv_signal(&is->is_login_cv);
380 #endif
381 
382 	if (fail_on_disconnection) {
383 		ISCSI_SESSION_DEBUG(is, "connection failed, destroying devices");
384 		iscsi_session_cleanup(is, true);
385 	} else {
386 		iscsi_session_cleanup(is, false);
387 	}
388 
389 	KASSERT(TAILQ_EMPTY(&is->is_outstanding),
390 	    ("destroying session with active tasks"));
391 	KASSERT(STAILQ_EMPTY(&is->is_postponed),
392 	    ("destroying session with postponed PDUs"));
393 
394 	/*
395 	 * Request immediate reconnection from iscsid(8).
396 	 */
397 	//ISCSI_SESSION_DEBUG(is, "waking up iscsid(8)");
398 	is->is_waiting_for_iscsid = true;
399 	strlcpy(is->is_reason, "Waiting for iscsid(8)", sizeof(is->is_reason));
400 	is->is_timeout = 0;
401 	ISCSI_SESSION_UNLOCK(is);
402 	cv_signal(&is->is_softc->sc_cv);
403 }
404 
405 static void
406 iscsi_maintenance_thread_terminate(struct iscsi_session *is)
407 {
408 	struct iscsi_softc *sc;
409 
410 	sc = is->is_softc;
411 	sx_xlock(&sc->sc_lock);
412 	TAILQ_REMOVE(&sc->sc_sessions, is, is_next);
413 	sx_xunlock(&sc->sc_lock);
414 
415 	icl_conn_close(is->is_conn);
416 
417 	ISCSI_SESSION_LOCK(is);
418 
419 	KASSERT(is->is_terminating, ("is_terminating == false"));
420 
421 #ifdef ICL_KERNEL_PROXY
422 	if (is->is_login_pdu != NULL) {
423 		icl_pdu_free(is->is_login_pdu);
424 		is->is_login_pdu = NULL;
425 	}
426 	cv_signal(&is->is_login_cv);
427 #endif
428 
429 	callout_drain(&is->is_callout);
430 
431 	iscsi_session_cleanup(is, true);
432 
433 	KASSERT(TAILQ_EMPTY(&is->is_outstanding),
434 	    ("destroying session with active tasks"));
435 	KASSERT(STAILQ_EMPTY(&is->is_postponed),
436 	    ("destroying session with postponed PDUs"));
437 
438 	ISCSI_SESSION_UNLOCK(is);
439 
440 	icl_conn_free(is->is_conn);
441 	mtx_destroy(&is->is_lock);
442 	cv_destroy(&is->is_maintenance_cv);
443 #ifdef ICL_KERNEL_PROXY
444 	cv_destroy(&is->is_login_cv);
445 #endif
446 	ISCSI_SESSION_DEBUG(is, "terminated");
447 	free(is, M_ISCSI);
448 
449 	/*
450 	 * The iscsi_unload() routine might be waiting.
451 	 */
452 	cv_signal(&sc->sc_cv);
453 }
454 
455 static void
456 iscsi_maintenance_thread(void *arg)
457 {
458 	struct iscsi_session *is;
459 
460 	is = arg;
461 
462 	for (;;) {
463 		ISCSI_SESSION_LOCK(is);
464 		if (is->is_reconnecting == false &&
465 		    is->is_terminating == false &&
466 		    STAILQ_EMPTY(&is->is_postponed))
467 			cv_wait(&is->is_maintenance_cv, &is->is_lock);
468 
469 		if (is->is_reconnecting) {
470 			ISCSI_SESSION_UNLOCK(is);
471 			iscsi_maintenance_thread_reconnect(is);
472 			continue;
473 		}
474 
475 		if (is->is_terminating) {
476 			ISCSI_SESSION_UNLOCK(is);
477 			iscsi_maintenance_thread_terminate(is);
478 			kthread_exit();
479 			return;
480 		}
481 
482 		iscsi_session_send_postponed(is);
483 		ISCSI_SESSION_UNLOCK(is);
484 	}
485 }
486 
487 static void
488 iscsi_session_reconnect(struct iscsi_session *is)
489 {
490 
491 	/*
492 	 * XXX: We can't use locking here, because
493 	 * 	it's being called from various contexts.
494 	 * 	Hope it doesn't break anything.
495 	 */
496 	if (is->is_reconnecting)
497 		return;
498 
499 	is->is_reconnecting = true;
500 	cv_signal(&is->is_maintenance_cv);
501 }
502 
503 static void
504 iscsi_session_terminate(struct iscsi_session *is)
505 {
506 	if (is->is_terminating)
507 		return;
508 
509 	is->is_terminating = true;
510 
511 #if 0
512 	iscsi_session_logout(is);
513 #endif
514 	cv_signal(&is->is_maintenance_cv);
515 }
516 
517 static void
518 iscsi_callout(void *context)
519 {
520 	struct icl_pdu *request;
521 	struct iscsi_bhs_nop_out *bhsno;
522 	struct iscsi_session *is;
523 	bool reconnect_needed = false;
524 
525 	is = context;
526 
527 	if (is->is_terminating)
528 		return;
529 
530 	callout_schedule(&is->is_callout, 1 * hz);
531 
532 	ISCSI_SESSION_LOCK(is);
533 	is->is_timeout++;
534 
535 	if (is->is_waiting_for_iscsid) {
536 		if (is->is_timeout > iscsid_timeout) {
537 			ISCSI_SESSION_WARN(is, "timed out waiting for iscsid(8) "
538 			    "for %d seconds; reconnecting",
539 			    is->is_timeout);
540 			reconnect_needed = true;
541 		}
542 		goto out;
543 	}
544 
545 	if (is->is_login_phase) {
546 		if (is->is_timeout > login_timeout) {
547 			ISCSI_SESSION_WARN(is, "login timed out after %d seconds; "
548 			    "reconnecting", is->is_timeout);
549 			reconnect_needed = true;
550 		}
551 		goto out;
552 	}
553 
554 	if (is->is_timeout >= ping_timeout) {
555 		ISCSI_SESSION_WARN(is, "no ping reply (NOP-In) after %d seconds; "
556 		    "reconnecting", ping_timeout);
557 		reconnect_needed = true;
558 		goto out;
559 	}
560 
561 	ISCSI_SESSION_UNLOCK(is);
562 
563 	/*
564 	 * If the ping was reset less than one second ago - which means
565 	 * that we've received some PDU during the last second - assume
566 	 * the traffic flows correctly and don't bother sending a NOP-Out.
567 	 *
568 	 * (It's 2 - one for one second, and one for incrementing is_timeout
569 	 * earlier in this routine.)
570 	 */
571 	if (is->is_timeout < 2)
572 		return;
573 
574 	request = icl_pdu_new_bhs(is->is_conn, M_NOWAIT);
575 	if (request == NULL) {
576 		ISCSI_SESSION_WARN(is, "failed to allocate PDU");
577 		return;
578 	}
579 	bhsno = (struct iscsi_bhs_nop_out *)request->ip_bhs;
580 	bhsno->bhsno_opcode = ISCSI_BHS_OPCODE_NOP_OUT |
581 	    ISCSI_BHS_OPCODE_IMMEDIATE;
582 	bhsno->bhsno_flags = 0x80;
583 	bhsno->bhsno_target_transfer_tag = 0xffffffff;
584 	iscsi_pdu_queue(request);
585 	return;
586 
587 out:
588 	ISCSI_SESSION_UNLOCK(is);
589 
590 	if (reconnect_needed)
591 		iscsi_session_reconnect(is);
592 }
593 
594 static void
595 iscsi_pdu_update_statsn(const struct icl_pdu *response)
596 {
597 	const struct iscsi_bhs_data_in *bhsdi;
598 	struct iscsi_session *is;
599 	uint32_t expcmdsn, maxcmdsn;
600 
601 	is = PDU_SESSION(response);
602 
603 	ISCSI_SESSION_LOCK_ASSERT(is);
604 
605 	/*
606 	 * We're only using fields common for all the response
607 	 * (target -> initiator) PDUs.
608 	 */
609 	bhsdi = (const struct iscsi_bhs_data_in *)response->ip_bhs;
610 	/*
611 	 * Ok, I lied.  In case of Data-In, "The fields StatSN, Status,
612 	 * and Residual Count only have meaningful content if the S bit
613 	 * is set to 1", so we also need to check the bit specific for
614 	 * Data-In PDU.
615 	 */
616 	if (bhsdi->bhsdi_opcode != ISCSI_BHS_OPCODE_SCSI_DATA_IN ||
617 	    (bhsdi->bhsdi_flags & BHSDI_FLAGS_S) != 0) {
618 		if (ntohl(bhsdi->bhsdi_statsn) < is->is_statsn) {
619 			ISCSI_SESSION_WARN(is,
620 			    "PDU StatSN %d >= session StatSN %d, opcode 0x%x",
621 			    is->is_statsn, ntohl(bhsdi->bhsdi_statsn),
622 			    bhsdi->bhsdi_opcode);
623 		}
624 		is->is_statsn = ntohl(bhsdi->bhsdi_statsn);
625 	}
626 
627 	expcmdsn = ntohl(bhsdi->bhsdi_expcmdsn);
628 	maxcmdsn = ntohl(bhsdi->bhsdi_maxcmdsn);
629 
630 	/*
631 	 * XXX: Compare using Serial Arithmetic Sense.
632 	 */
633 	if (maxcmdsn + 1 < expcmdsn) {
634 		ISCSI_SESSION_DEBUG(is, "PDU MaxCmdSN %d + 1 < PDU ExpCmdSN %d; ignoring",
635 		    maxcmdsn, expcmdsn);
636 	} else {
637 		if (maxcmdsn > is->is_maxcmdsn) {
638 			is->is_maxcmdsn = maxcmdsn;
639 
640 			/*
641 			 * Command window increased; kick the maintanance thread
642 			 * to send out postponed commands.
643 			 */
644 			if (!STAILQ_EMPTY(&is->is_postponed))
645 				cv_signal(&is->is_maintenance_cv);
646 		} else if (maxcmdsn < is->is_maxcmdsn) {
647 			ISCSI_SESSION_DEBUG(is, "PDU MaxCmdSN %d < session MaxCmdSN %d; ignoring",
648 			    maxcmdsn, is->is_maxcmdsn);
649 		}
650 
651 		if (expcmdsn > is->is_expcmdsn) {
652 			is->is_expcmdsn = expcmdsn;
653 		} else if (expcmdsn < is->is_expcmdsn) {
654 			ISCSI_SESSION_DEBUG(is, "PDU ExpCmdSN %d < session ExpCmdSN %d; ignoring",
655 			    expcmdsn, is->is_expcmdsn);
656 		}
657 	}
658 
659 	/*
660 	 * Every incoming PDU - not just NOP-In - resets the ping timer.
661 	 * The purpose of the timeout is to reset the connection when it stalls;
662 	 * we don't want this to happen when NOP-In or NOP-Out ends up delayed
663 	 * in some queue.
664 	 */
665 	is->is_timeout = 0;
666 }
667 
668 static void
669 iscsi_receive_callback(struct icl_pdu *response)
670 {
671 	struct iscsi_session *is;
672 
673 	is = PDU_SESSION(response);
674 
675 	ISCSI_SESSION_LOCK(is);
676 
677 #ifdef ICL_KERNEL_PROXY
678 	if (is->is_login_phase) {
679 		if (is->is_login_pdu == NULL)
680 			is->is_login_pdu = response;
681 		else
682 			icl_pdu_free(response);
683 		ISCSI_SESSION_UNLOCK(is);
684 		cv_signal(&is->is_login_cv);
685 		return;
686 	}
687 #endif
688 
689 	iscsi_pdu_update_statsn(response);
690 
691 	/*
692 	 * The handling routine is responsible for freeing the PDU
693 	 * when it's no longer needed.
694 	 */
695 	switch (response->ip_bhs->bhs_opcode) {
696 	case ISCSI_BHS_OPCODE_NOP_IN:
697 		iscsi_pdu_handle_nop_in(response);
698 		break;
699 	case ISCSI_BHS_OPCODE_SCSI_RESPONSE:
700 		iscsi_pdu_handle_scsi_response(response);
701 		break;
702 	case ISCSI_BHS_OPCODE_SCSI_DATA_IN:
703 		iscsi_pdu_handle_data_in(response);
704 		break;
705 	case ISCSI_BHS_OPCODE_LOGOUT_RESPONSE:
706 		iscsi_pdu_handle_logout_response(response);
707 		break;
708 	case ISCSI_BHS_OPCODE_R2T:
709 		iscsi_pdu_handle_r2t(response);
710 		break;
711 	case ISCSI_BHS_OPCODE_ASYNC_MESSAGE:
712 		iscsi_pdu_handle_async_message(response);
713 		break;
714 	case ISCSI_BHS_OPCODE_REJECT:
715 		iscsi_pdu_handle_reject(response);
716 		break;
717 	default:
718 		ISCSI_SESSION_WARN(is, "received PDU with unsupported "
719 		    "opcode 0x%x; reconnecting",
720 		    response->ip_bhs->bhs_opcode);
721 		iscsi_session_reconnect(is);
722 		icl_pdu_free(response);
723 	}
724 
725 	ISCSI_SESSION_UNLOCK(is);
726 }
727 
728 static void
729 iscsi_error_callback(struct icl_conn *ic)
730 {
731 	struct iscsi_session *is;
732 
733 	is = CONN_SESSION(ic);
734 
735 	ISCSI_SESSION_WARN(is, "connection error; reconnecting");
736 	iscsi_session_reconnect(is);
737 }
738 
739 static void
740 iscsi_pdu_handle_nop_in(struct icl_pdu *response)
741 {
742 	struct iscsi_session *is;
743 	struct iscsi_bhs_nop_out *bhsno;
744 	struct iscsi_bhs_nop_in *bhsni;
745 	struct icl_pdu *request;
746 	void *data = NULL;
747 	size_t datasize;
748 	int error;
749 
750 	is = PDU_SESSION(response);
751 	bhsni = (struct iscsi_bhs_nop_in *)response->ip_bhs;
752 
753 	if (bhsni->bhsni_target_transfer_tag == 0xffffffff) {
754 		/*
755 		 * Nothing to do; iscsi_pdu_update_statsn() already
756 		 * zeroed the timeout.
757 		 */
758 		icl_pdu_free(response);
759 		return;
760 	}
761 
762 	datasize = icl_pdu_data_segment_length(response);
763 	if (datasize > 0) {
764 		data = malloc(datasize, M_ISCSI, M_NOWAIT | M_ZERO);
765 		if (data == NULL) {
766 			ISCSI_SESSION_WARN(is, "failed to allocate memory; "
767 			    "reconnecting");
768 			icl_pdu_free(response);
769 			iscsi_session_reconnect(is);
770 			return;
771 		}
772 		icl_pdu_get_data(response, 0, data, datasize);
773 	}
774 
775 	request = icl_pdu_new_bhs(response->ip_conn, M_NOWAIT);
776 	if (request == NULL) {
777 		ISCSI_SESSION_WARN(is, "failed to allocate memory; "
778 		    "reconnecting");
779 		free(data, M_ISCSI);
780 		icl_pdu_free(response);
781 		iscsi_session_reconnect(is);
782 		return;
783 	}
784 	bhsno = (struct iscsi_bhs_nop_out *)request->ip_bhs;
785 	bhsno->bhsno_opcode = ISCSI_BHS_OPCODE_NOP_OUT |
786 	    ISCSI_BHS_OPCODE_IMMEDIATE;
787 	bhsno->bhsno_flags = 0x80;
788 	bhsno->bhsno_initiator_task_tag = 0xffffffff;
789 	bhsno->bhsno_target_transfer_tag = bhsni->bhsni_target_transfer_tag;
790 	if (datasize > 0) {
791 		error = icl_pdu_append_data(request, data, datasize, M_NOWAIT);
792 		if (error != 0) {
793 			ISCSI_SESSION_WARN(is, "failed to allocate memory; "
794 			    "reconnecting");
795 			free(data, M_ISCSI);
796 			icl_pdu_free(request);
797 			icl_pdu_free(response);
798 			iscsi_session_reconnect(is);
799 			return;
800 		}
801 		free(data, M_ISCSI);
802 	}
803 
804 	icl_pdu_free(response);
805 	iscsi_pdu_queue_locked(request);
806 }
807 
808 static void
809 iscsi_pdu_handle_scsi_response(struct icl_pdu *response)
810 {
811 	struct iscsi_bhs_scsi_response *bhssr;
812 	struct iscsi_outstanding *io;
813 	struct iscsi_session *is;
814 	struct ccb_scsiio *csio;
815 	size_t data_segment_len;
816 	uint16_t sense_len;
817 
818 	is = PDU_SESSION(response);
819 
820 	bhssr = (struct iscsi_bhs_scsi_response *)response->ip_bhs;
821 	io = iscsi_outstanding_find(is, bhssr->bhssr_initiator_task_tag);
822 	if (io == NULL) {
823 		ISCSI_SESSION_WARN(is, "bad itt 0x%x", bhssr->bhssr_initiator_task_tag);
824 		icl_pdu_free(response);
825 		iscsi_session_reconnect(is);
826 		return;
827 	}
828 
829 	if (bhssr->bhssr_response != BHSSR_RESPONSE_COMMAND_COMPLETED) {
830 		ISCSI_SESSION_WARN(is, "service response 0x%x", bhssr->bhssr_response);
831  		if ((io->io_ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
832  			xpt_freeze_devq(io->io_ccb->ccb_h.path, 1);
833 			ISCSI_SESSION_DEBUG(is, "freezing devq");
834 		}
835  		io->io_ccb->ccb_h.status = CAM_REQ_CMP_ERR | CAM_DEV_QFRZN;
836 	} else if (bhssr->bhssr_status == 0) {
837 		io->io_ccb->ccb_h.status = CAM_REQ_CMP;
838 	} else {
839  		if ((io->io_ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
840  			xpt_freeze_devq(io->io_ccb->ccb_h.path, 1);
841 			ISCSI_SESSION_DEBUG(is, "freezing devq");
842 		}
843  		io->io_ccb->ccb_h.status = CAM_SCSI_STATUS_ERROR | CAM_DEV_QFRZN;
844 		io->io_ccb->csio.scsi_status = bhssr->bhssr_status;
845 	}
846 
847 	if (bhssr->bhssr_flags & BHSSR_FLAGS_RESIDUAL_OVERFLOW) {
848 		ISCSI_SESSION_WARN(is, "target indicated residual overflow");
849 		icl_pdu_free(response);
850 		iscsi_session_reconnect(is);
851 		return;
852 	}
853 
854 	csio = &io->io_ccb->csio;
855 
856 	data_segment_len = icl_pdu_data_segment_length(response);
857 	if (data_segment_len > 0) {
858 		if (data_segment_len < sizeof(sense_len)) {
859 			ISCSI_SESSION_WARN(is, "truncated data segment (%zd bytes)",
860 			    data_segment_len);
861 			if ((io->io_ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
862 				xpt_freeze_devq(io->io_ccb->ccb_h.path, 1);
863 				ISCSI_SESSION_DEBUG(is, "freezing devq");
864 			}
865 			io->io_ccb->ccb_h.status = CAM_REQ_CMP_ERR | CAM_DEV_QFRZN;
866 			goto out;
867 		}
868 		icl_pdu_get_data(response, 0, &sense_len, sizeof(sense_len));
869 		sense_len = ntohs(sense_len);
870 #if 0
871 		ISCSI_SESSION_DEBUG(is, "sense_len %d, data len %zd",
872 		    sense_len, data_segment_len);
873 #endif
874 		if (sizeof(sense_len) + sense_len > data_segment_len) {
875 			ISCSI_SESSION_WARN(is, "truncated data segment "
876 			    "(%zd bytes, should be %zd)",
877 			    data_segment_len, sizeof(sense_len) + sense_len);
878 			if ((io->io_ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
879 				xpt_freeze_devq(io->io_ccb->ccb_h.path, 1);
880 				ISCSI_SESSION_DEBUG(is, "freezing devq");
881 			}
882 			io->io_ccb->ccb_h.status = CAM_REQ_CMP_ERR | CAM_DEV_QFRZN;
883 			goto out;
884 		} else if (sizeof(sense_len) + sense_len < data_segment_len)
885 			ISCSI_SESSION_WARN(is, "oversize data segment "
886 			    "(%zd bytes, should be %zd)",
887 			    data_segment_len, sizeof(sense_len) + sense_len);
888 		if (sense_len > csio->sense_len) {
889 			ISCSI_SESSION_DEBUG(is, "truncating sense from %d to %d",
890 			    sense_len, csio->sense_len);
891 			sense_len = csio->sense_len;
892 		}
893 		icl_pdu_get_data(response, sizeof(sense_len), &csio->sense_data, sense_len);
894 		csio->sense_resid = csio->sense_len - sense_len;
895 		io->io_ccb->ccb_h.status |= CAM_AUTOSNS_VALID;
896 	}
897 
898 out:
899 	if (bhssr->bhssr_flags & BHSSR_FLAGS_RESIDUAL_UNDERFLOW)
900 		csio->resid = ntohl(bhssr->bhssr_residual_count);
901 
902 	if ((csio->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_IN) {
903 		KASSERT(io->io_received <= csio->dxfer_len,
904 		    ("io->io_received > csio->dxfer_len"));
905 		if (io->io_received < csio->dxfer_len) {
906 			if (csio->resid != csio->dxfer_len - io->io_received) {
907 				ISCSI_SESSION_WARN(is, "underflow mismatch: "
908 				    "target indicates %d, we calculated %zd",
909 				    csio->resid,
910 				    csio->dxfer_len - io->io_received);
911 			}
912 			csio->resid = csio->dxfer_len - io->io_received;
913 		}
914 	}
915 
916 	xpt_done(io->io_ccb);
917 	iscsi_outstanding_remove(is, io);
918 	icl_pdu_free(response);
919 }
920 
921 static void
922 iscsi_pdu_handle_data_in(struct icl_pdu *response)
923 {
924 	struct iscsi_bhs_data_in *bhsdi;
925 	struct iscsi_outstanding *io;
926 	struct iscsi_session *is;
927 	struct ccb_scsiio *csio;
928 	size_t data_segment_len;
929 
930 	is = PDU_SESSION(response);
931 	bhsdi = (struct iscsi_bhs_data_in *)response->ip_bhs;
932 	io = iscsi_outstanding_find(is, bhsdi->bhsdi_initiator_task_tag);
933 	if (io == NULL) {
934 		ISCSI_SESSION_WARN(is, "bad itt 0x%x", bhsdi->bhsdi_initiator_task_tag);
935 		icl_pdu_free(response);
936 		iscsi_session_reconnect(is);
937 		return;
938 	}
939 
940 	data_segment_len = icl_pdu_data_segment_length(response);
941 	if (data_segment_len == 0) {
942 		/*
943 		 * "The sending of 0 length data segments should be avoided,
944 		 * but initiators and targets MUST be able to properly receive
945 		 * 0 length data segments."
946 		 */
947 		icl_pdu_free(response);
948 		return;
949 	}
950 
951 	/*
952 	 * We need to track this for security reasons - without it, malicious target
953 	 * could respond to SCSI READ without sending Data-In PDUs, which would result
954 	 * in read operation on the initiator side returning random kernel data.
955 	 */
956 	if (ntohl(bhsdi->bhsdi_buffer_offset) != io->io_received) {
957 		ISCSI_SESSION_WARN(is, "data out of order; expected offset %zd, got %zd",
958 		    io->io_received, (size_t)ntohl(bhsdi->bhsdi_buffer_offset));
959 		icl_pdu_free(response);
960 		iscsi_session_reconnect(is);
961 		return;
962 	}
963 
964 	csio = &io->io_ccb->csio;
965 
966 	if (io->io_received + data_segment_len > csio->dxfer_len) {
967 		ISCSI_SESSION_WARN(is, "oversize data segment (%zd bytes "
968 		    "at offset %zd, buffer is %d)",
969 		    data_segment_len, io->io_received, csio->dxfer_len);
970 		icl_pdu_free(response);
971 		iscsi_session_reconnect(is);
972 		return;
973 	}
974 
975 	icl_pdu_get_data(response, 0, csio->data_ptr + io->io_received, data_segment_len);
976 	io->io_received += data_segment_len;
977 
978 	/*
979 	 * XXX: Check DataSN.
980 	 * XXX: Check F.
981 	 */
982 	if ((bhsdi->bhsdi_flags & BHSDI_FLAGS_S) == 0) {
983 		/*
984 		 * Nothing more to do.
985 		 */
986 		icl_pdu_free(response);
987 		return;
988 	}
989 
990 	//ISCSI_SESSION_DEBUG(is, "got S flag; status 0x%x", bhsdi->bhsdi_status);
991 	if (bhsdi->bhsdi_status == 0) {
992 		io->io_ccb->ccb_h.status = CAM_REQ_CMP;
993 	} else {
994 		if ((io->io_ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
995 			xpt_freeze_devq(io->io_ccb->ccb_h.path, 1);
996 			ISCSI_SESSION_DEBUG(is, "freezing devq");
997 		}
998 		io->io_ccb->ccb_h.status = CAM_SCSI_STATUS_ERROR | CAM_DEV_QFRZN;
999 		csio->scsi_status = bhsdi->bhsdi_status;
1000 	}
1001 
1002 	if ((csio->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_IN) {
1003 		KASSERT(io->io_received <= csio->dxfer_len,
1004 		    ("io->io_received > csio->dxfer_len"));
1005 		if (io->io_received < csio->dxfer_len) {
1006 			csio->resid = ntohl(bhsdi->bhsdi_residual_count);
1007 			if (csio->resid != csio->dxfer_len - io->io_received) {
1008 				ISCSI_SESSION_WARN(is, "underflow mismatch: "
1009 				    "target indicates %d, we calculated %zd",
1010 				    csio->resid,
1011 				    csio->dxfer_len - io->io_received);
1012 			}
1013 			csio->resid = csio->dxfer_len - io->io_received;
1014 		}
1015 	}
1016 
1017 	xpt_done(io->io_ccb);
1018 	iscsi_outstanding_remove(is, io);
1019 	icl_pdu_free(response);
1020 }
1021 
1022 static void
1023 iscsi_pdu_handle_logout_response(struct icl_pdu *response)
1024 {
1025 
1026 	ISCSI_SESSION_DEBUG(PDU_SESSION(response), "logout response");
1027 	icl_pdu_free(response);
1028 }
1029 
1030 static void
1031 iscsi_pdu_handle_r2t(struct icl_pdu *response)
1032 {
1033 	struct icl_pdu *request;
1034 	struct iscsi_session *is;
1035 	struct iscsi_bhs_r2t *bhsr2t;
1036 	struct iscsi_bhs_data_out *bhsdo;
1037 	struct iscsi_outstanding *io;
1038 	struct ccb_scsiio *csio;
1039 	size_t off, len, total_len;
1040 	int error;
1041 
1042 	is = PDU_SESSION(response);
1043 
1044 	bhsr2t = (struct iscsi_bhs_r2t *)response->ip_bhs;
1045 	io = iscsi_outstanding_find(is, bhsr2t->bhsr2t_initiator_task_tag);
1046 	if (io == NULL) {
1047 		ISCSI_SESSION_WARN(is, "bad itt 0x%x; reconnecting",
1048 		    bhsr2t->bhsr2t_initiator_task_tag);
1049 		icl_pdu_free(response);
1050 		iscsi_session_reconnect(is);
1051 		return;
1052 	}
1053 
1054 	csio = &io->io_ccb->csio;
1055 
1056 	if ((csio->ccb_h.flags & CAM_DIR_MASK) != CAM_DIR_OUT) {
1057 		ISCSI_SESSION_WARN(is, "received R2T for read command; reconnecting");
1058 		icl_pdu_free(response);
1059 		iscsi_session_reconnect(is);
1060 		return;
1061 	}
1062 
1063 	/*
1064 	 * XXX: Verify R2TSN.
1065 	 */
1066 
1067 	io->io_datasn = 0;
1068 
1069 	off = ntohl(bhsr2t->bhsr2t_buffer_offset);
1070 	if (off > csio->dxfer_len) {
1071 		ISCSI_SESSION_WARN(is, "target requested invalid offset "
1072 		    "%zd, buffer is is %d; reconnecting", off, csio->dxfer_len);
1073 		icl_pdu_free(response);
1074 		iscsi_session_reconnect(is);
1075 		return;
1076 	}
1077 
1078 	total_len = ntohl(bhsr2t->bhsr2t_desired_data_transfer_length);
1079 	if (total_len == 0 || total_len > csio->dxfer_len) {
1080 		ISCSI_SESSION_WARN(is, "target requested invalid length "
1081 		    "%zd, buffer is %d; reconnecting", total_len, csio->dxfer_len);
1082 		icl_pdu_free(response);
1083 		iscsi_session_reconnect(is);
1084 		return;
1085 	}
1086 
1087 	//ISCSI_SESSION_DEBUG(is, "r2t; off %zd, len %zd", off, total_len);
1088 
1089 	for (;;) {
1090 		len = total_len;
1091 
1092 		if (len > is->is_max_data_segment_length)
1093 			len = is->is_max_data_segment_length;
1094 
1095 		if (off + len > csio->dxfer_len) {
1096 			ISCSI_SESSION_WARN(is, "target requested invalid "
1097 			    "length/offset %zd, buffer is %d; reconnecting",
1098 			    off + len, csio->dxfer_len);
1099 			icl_pdu_free(response);
1100 			iscsi_session_reconnect(is);
1101 			return;
1102 		}
1103 
1104 		request = icl_pdu_new_bhs(response->ip_conn, M_NOWAIT);
1105 		if (request == NULL) {
1106 			icl_pdu_free(response);
1107 			iscsi_session_reconnect(is);
1108 			return;
1109 		}
1110 
1111 		bhsdo = (struct iscsi_bhs_data_out *)request->ip_bhs;
1112 		bhsdo->bhsdo_opcode = ISCSI_BHS_OPCODE_SCSI_DATA_OUT;
1113 		bhsdo->bhsdo_lun = bhsr2t->bhsr2t_lun;
1114 		bhsdo->bhsdo_initiator_task_tag =
1115 		    bhsr2t->bhsr2t_initiator_task_tag;
1116 		bhsdo->bhsdo_target_transfer_tag =
1117 		    bhsr2t->bhsr2t_target_transfer_tag;
1118 		bhsdo->bhsdo_datasn = htonl(io->io_datasn++);
1119 		bhsdo->bhsdo_buffer_offset = htonl(off);
1120 		error = icl_pdu_append_data(request, csio->data_ptr + off, len,
1121 		    M_NOWAIT);
1122 		if (error != 0) {
1123 			ISCSI_SESSION_WARN(is, "failed to allocate memory; "
1124 			    "reconnecting");
1125 			icl_pdu_free(request);
1126 			icl_pdu_free(response);
1127 			iscsi_session_reconnect(is);
1128 			return;
1129 		}
1130 
1131 		off += len;
1132 		total_len -= len;
1133 
1134 		if (total_len == 0) {
1135 			bhsdo->bhsdo_flags |= BHSDO_FLAGS_F;
1136 			//ISCSI_SESSION_DEBUG(is, "setting F, off %zd", off);
1137 		} else {
1138 			//ISCSI_SESSION_DEBUG(is, "not finished, off %zd", off);
1139 		}
1140 
1141 		iscsi_pdu_queue_locked(request);
1142 
1143 		if (total_len == 0)
1144 			break;
1145 	}
1146 
1147 	icl_pdu_free(response);
1148 }
1149 
1150 static void
1151 iscsi_pdu_handle_async_message(struct icl_pdu *response)
1152 {
1153 	struct iscsi_bhs_asynchronous_message *bhsam;
1154 	struct iscsi_session *is;
1155 
1156 	is = PDU_SESSION(response);
1157 	bhsam = (struct iscsi_bhs_asynchronous_message *)response->ip_bhs;
1158 	switch (bhsam->bhsam_async_event) {
1159 	case BHSAM_EVENT_TARGET_REQUESTS_LOGOUT:
1160 		ISCSI_SESSION_WARN(is, "target requests logout; removing session");
1161 		iscsi_session_logout(is);
1162 		iscsi_session_terminate(is);
1163 		break;
1164 	case BHSAM_EVENT_TARGET_TERMINATES_CONNECTION:
1165 		ISCSI_SESSION_WARN(is, "target indicates it will drop drop the connection");
1166 		break;
1167 	case BHSAM_EVENT_TARGET_TERMINATES_SESSION:
1168 		ISCSI_SESSION_WARN(is, "target indicates it will drop drop the session");
1169 		break;
1170 	default:
1171 		/*
1172 		 * XXX: Technically, we're obligated to also handle
1173 		 * 	parameter renegotiation.
1174 		 */
1175 		ISCSI_SESSION_WARN(is, "ignoring AsyncEvent %d", bhsam->bhsam_async_event);
1176 		break;
1177 	}
1178 
1179 	icl_pdu_free(response);
1180 }
1181 
1182 static void
1183 iscsi_pdu_handle_reject(struct icl_pdu *response)
1184 {
1185 	struct iscsi_bhs_reject *bhsr;
1186 	struct iscsi_session *is;
1187 
1188 	is = PDU_SESSION(response);
1189 	bhsr = (struct iscsi_bhs_reject *)response->ip_bhs;
1190 	ISCSI_SESSION_WARN(is, "received Reject PDU, reason 0x%x; protocol error?",
1191 	    bhsr->bhsr_reason);
1192 
1193 	icl_pdu_free(response);
1194 }
1195 
1196 static int
1197 iscsi_ioctl_daemon_wait(struct iscsi_softc *sc,
1198     struct iscsi_daemon_request *request)
1199 {
1200 	struct iscsi_session *is;
1201 	int error;
1202 
1203 	sx_slock(&sc->sc_lock);
1204 	for (;;) {
1205 		TAILQ_FOREACH(is, &sc->sc_sessions, is_next) {
1206 			ISCSI_SESSION_LOCK(is);
1207 			if (is->is_waiting_for_iscsid)
1208 				break;
1209 			ISCSI_SESSION_UNLOCK(is);
1210 		}
1211 
1212 		if (is == NULL) {
1213 			/*
1214 			 * No session requires attention from iscsid(8); wait.
1215 			 */
1216 			error = cv_wait_sig(&sc->sc_cv, &sc->sc_lock);
1217 			if (error != 0) {
1218 				sx_sunlock(&sc->sc_lock);
1219 				return (error);
1220 			}
1221 			continue;
1222 		}
1223 
1224 		is->is_waiting_for_iscsid = false;
1225 		is->is_login_phase = true;
1226 		is->is_reason[0] = '\0';
1227 		ISCSI_SESSION_UNLOCK(is);
1228 
1229 		request->idr_session_id = is->is_id;
1230 		memcpy(&request->idr_conf, &is->is_conf,
1231 		    sizeof(request->idr_conf));
1232 
1233 		sx_sunlock(&sc->sc_lock);
1234 		return (0);
1235 	}
1236 }
1237 
1238 static int
1239 iscsi_ioctl_daemon_handoff(struct iscsi_softc *sc,
1240     struct iscsi_daemon_handoff *handoff)
1241 {
1242 	struct iscsi_session *is;
1243 	int error;
1244 
1245 	sx_slock(&sc->sc_lock);
1246 
1247 	/*
1248 	 * Find the session to hand off socket to.
1249 	 */
1250 	TAILQ_FOREACH(is, &sc->sc_sessions, is_next) {
1251 		if (is->is_id == handoff->idh_session_id)
1252 			break;
1253 	}
1254 	if (is == NULL) {
1255 		sx_sunlock(&sc->sc_lock);
1256 		return (ESRCH);
1257 	}
1258 	ISCSI_SESSION_LOCK(is);
1259 	if (is->is_conf.isc_discovery || is->is_terminating) {
1260 		ISCSI_SESSION_UNLOCK(is);
1261 		sx_sunlock(&sc->sc_lock);
1262 		return (EINVAL);
1263 	}
1264 	if (is->is_connected) {
1265 		/*
1266 		 * This might have happened because another iscsid(8)
1267 		 * instance handed off the connection in the meantime.
1268 		 * Just return.
1269 		 */
1270 		ISCSI_SESSION_WARN(is, "handoff on already connected "
1271 		    "session");
1272 		ISCSI_SESSION_UNLOCK(is);
1273 		sx_sunlock(&sc->sc_lock);
1274 		return (EBUSY);
1275 	}
1276 
1277 	strlcpy(is->is_target_alias, handoff->idh_target_alias,
1278 	    sizeof(is->is_target_alias));
1279 	memcpy(is->is_isid, handoff->idh_isid, sizeof(is->is_isid));
1280 	is->is_statsn = handoff->idh_statsn;
1281 	is->is_initial_r2t = handoff->idh_initial_r2t;
1282 	is->is_immediate_data = handoff->idh_immediate_data;
1283 	is->is_max_data_segment_length = handoff->idh_max_data_segment_length;
1284 	is->is_max_burst_length = handoff->idh_max_burst_length;
1285 	is->is_first_burst_length = handoff->idh_first_burst_length;
1286 
1287 	if (handoff->idh_header_digest == ISCSI_DIGEST_CRC32C)
1288 		is->is_conn->ic_header_crc32c = true;
1289 	else
1290 		is->is_conn->ic_header_crc32c = false;
1291 	if (handoff->idh_data_digest == ISCSI_DIGEST_CRC32C)
1292 		is->is_conn->ic_data_crc32c = true;
1293 	else
1294 		is->is_conn->ic_data_crc32c = false;
1295 
1296 	is->is_cmdsn = 0;
1297 	is->is_expcmdsn = 0;
1298 	is->is_maxcmdsn = 0;
1299 	is->is_waiting_for_iscsid = false;
1300 	is->is_login_phase = false;
1301 	is->is_timeout = 0;
1302 	is->is_connected = true;
1303 	is->is_reason[0] = '\0';
1304 
1305 	ISCSI_SESSION_UNLOCK(is);
1306 
1307 #ifdef ICL_KERNEL_PROXY
1308 	if (handoff->idh_socket != 0) {
1309 #endif
1310 		/*
1311 		 * Handoff without using ICL proxy.
1312 		 */
1313 		error = icl_conn_handoff(is->is_conn, handoff->idh_socket);
1314 		if (error != 0) {
1315 			sx_sunlock(&sc->sc_lock);
1316 			iscsi_session_terminate(is);
1317 			return (error);
1318 		}
1319 #ifdef ICL_KERNEL_PROXY
1320 	}
1321 #endif
1322 
1323 	sx_sunlock(&sc->sc_lock);
1324 
1325 	if (is->is_sim != NULL) {
1326 		/*
1327 		 * When reconnecting, there already is SIM allocated for the session.
1328 		 */
1329 		KASSERT(is->is_simq_frozen, ("reconnect without frozen simq"));
1330 		ISCSI_SESSION_LOCK(is);
1331 		ISCSI_SESSION_DEBUG(is, "releasing");
1332 		xpt_release_simq(is->is_sim, 1);
1333 		is->is_simq_frozen = false;
1334 		ISCSI_SESSION_UNLOCK(is);
1335 
1336 	} else {
1337 		ISCSI_SESSION_LOCK(is);
1338 		is->is_devq = cam_simq_alloc(maxtags);
1339 		if (is->is_devq == NULL) {
1340 			ISCSI_SESSION_WARN(is, "failed to allocate simq");
1341 			iscsi_session_terminate(is);
1342 			return (ENOMEM);
1343 		}
1344 
1345 		is->is_sim = cam_sim_alloc(iscsi_action, iscsi_poll, "iscsi",
1346 		    is, is->is_id /* unit */, &is->is_lock,
1347 		    maxtags, maxtags, is->is_devq);
1348 		if (is->is_sim == NULL) {
1349 			ISCSI_SESSION_UNLOCK(is);
1350 			ISCSI_SESSION_WARN(is, "failed to allocate SIM");
1351 			cam_simq_free(is->is_devq);
1352 			iscsi_session_terminate(is);
1353 			return (ENOMEM);
1354 		}
1355 
1356 		error = xpt_bus_register(is->is_sim, NULL, 0);
1357 		if (error != 0) {
1358 			ISCSI_SESSION_UNLOCK(is);
1359 			ISCSI_SESSION_WARN(is, "failed to register bus");
1360 			iscsi_session_terminate(is);
1361 			return (ENOMEM);
1362 		}
1363 
1364 		error = xpt_create_path(&is->is_path, /*periph*/NULL,
1365 		    cam_sim_path(is->is_sim), CAM_TARGET_WILDCARD,
1366 		    CAM_LUN_WILDCARD);
1367 		if (error != CAM_REQ_CMP) {
1368 			ISCSI_SESSION_UNLOCK(is);
1369 			ISCSI_SESSION_WARN(is, "failed to create path");
1370 			iscsi_session_terminate(is);
1371 			return (ENOMEM);
1372 		}
1373 		ISCSI_SESSION_UNLOCK(is);
1374 	}
1375 
1376 	return (0);
1377 }
1378 
1379 static int
1380 iscsi_ioctl_daemon_fail(struct iscsi_softc *sc,
1381     struct iscsi_daemon_fail *fail)
1382 {
1383 	struct iscsi_session *is;
1384 
1385 	sx_slock(&sc->sc_lock);
1386 
1387 	TAILQ_FOREACH(is, &sc->sc_sessions, is_next) {
1388 		if (is->is_id == fail->idf_session_id)
1389 			break;
1390 	}
1391 	if (is == NULL) {
1392 		sx_sunlock(&sc->sc_lock);
1393 		return (ESRCH);
1394 	}
1395 	ISCSI_SESSION_LOCK(is);
1396 	ISCSI_SESSION_DEBUG(is, "iscsid(8) failed: %s",
1397 	    fail->idf_reason);
1398 	strlcpy(is->is_reason, fail->idf_reason, sizeof(is->is_reason));
1399 	//is->is_waiting_for_iscsid = false;
1400 	//is->is_login_phase = true;
1401 	//iscsi_session_reconnect(is);
1402 	ISCSI_SESSION_UNLOCK(is);
1403 	sx_sunlock(&sc->sc_lock);
1404 
1405 	return (0);
1406 }
1407 
1408 #ifdef ICL_KERNEL_PROXY
1409 static int
1410 iscsi_ioctl_daemon_connect(struct iscsi_softc *sc,
1411     struct iscsi_daemon_connect *idc)
1412 {
1413 	struct iscsi_session *is;
1414 	struct sockaddr *from_sa, *to_sa;
1415 	int error;
1416 
1417 	sx_slock(&sc->sc_lock);
1418 	TAILQ_FOREACH(is, &sc->sc_sessions, is_next) {
1419 		if (is->is_id == idc->idc_session_id)
1420 			break;
1421 	}
1422 	if (is == NULL) {
1423 		sx_sunlock(&sc->sc_lock);
1424 		return (ESRCH);
1425 	}
1426 	sx_sunlock(&sc->sc_lock);
1427 
1428 	if (idc->idc_from_addrlen > 0) {
1429 		error = getsockaddr(&from_sa, (void *)idc->idc_from_addr, idc->idc_from_addrlen);
1430 		if (error != 0) {
1431 			ISCSI_SESSION_WARN(is,
1432 			    "getsockaddr failed with error %d", error);
1433 			return (error);
1434 		}
1435 	} else {
1436 		from_sa = NULL;
1437 	}
1438 	error = getsockaddr(&to_sa, (void *)idc->idc_to_addr, idc->idc_to_addrlen);
1439 	if (error != 0) {
1440 		ISCSI_SESSION_WARN(is, "getsockaddr failed with error %d",
1441 		    error);
1442 		free(from_sa, M_SONAME);
1443 		return (error);
1444 	}
1445 
1446 	ISCSI_SESSION_LOCK(is);
1447 	is->is_waiting_for_iscsid = false;
1448 	is->is_login_phase = true;
1449 	is->is_timeout = 0;
1450 	ISCSI_SESSION_UNLOCK(is);
1451 
1452 	error = icl_conn_connect(is->is_conn, idc->idc_iser, idc->idc_domain,
1453 	    idc->idc_socktype, idc->idc_protocol, from_sa, to_sa);
1454 	free(from_sa, M_SONAME);
1455 	free(to_sa, M_SONAME);
1456 
1457 	/*
1458 	 * Digests are always disabled during login phase.
1459 	 */
1460 	is->is_conn->ic_header_crc32c = false;
1461 	is->is_conn->ic_data_crc32c = false;
1462 
1463 	return (error);
1464 }
1465 
1466 static int
1467 iscsi_ioctl_daemon_send(struct iscsi_softc *sc,
1468     struct iscsi_daemon_send *ids)
1469 {
1470 	struct iscsi_session *is;
1471 	struct icl_pdu *ip;
1472 	size_t datalen;
1473 	void *data;
1474 	int error;
1475 
1476 	sx_slock(&sc->sc_lock);
1477 	TAILQ_FOREACH(is, &sc->sc_sessions, is_next) {
1478 		if (is->is_id == ids->ids_session_id)
1479 			break;
1480 	}
1481 	if (is == NULL) {
1482 		sx_sunlock(&sc->sc_lock);
1483 		return (ESRCH);
1484 	}
1485 	sx_sunlock(&sc->sc_lock);
1486 
1487 	if (is->is_login_phase == false)
1488 		return (EBUSY);
1489 
1490 	if (is->is_terminating || is->is_reconnecting)
1491 		return (EIO);
1492 
1493 	datalen = ids->ids_data_segment_len;
1494 	if (datalen > ISCSI_MAX_DATA_SEGMENT_LENGTH)
1495 		return (EINVAL);
1496 	if (datalen > 0) {
1497 		data = malloc(datalen, M_ISCSI, M_WAITOK);
1498 		error = copyin(ids->ids_data_segment, data, datalen);
1499 		if (error != 0) {
1500 			free(data, M_ISCSI);
1501 			return (error);
1502 		}
1503 	}
1504 
1505 	ip = icl_pdu_new_bhs(is->is_conn, M_WAITOK);
1506 	memcpy(ip->ip_bhs, ids->ids_bhs, sizeof(*ip->ip_bhs));
1507 	if (datalen > 0) {
1508 		error = icl_pdu_append_data(ip, data, datalen, M_WAITOK);
1509 		KASSERT(error == 0, ("icl_pdu_append_data(..., M_WAITOK) failed"));
1510 		free(data, M_ISCSI);
1511 	}
1512 	icl_pdu_queue(ip);
1513 
1514 	return (0);
1515 }
1516 
1517 static int
1518 iscsi_ioctl_daemon_receive(struct iscsi_softc *sc,
1519     struct iscsi_daemon_receive *idr)
1520 {
1521 	struct iscsi_session *is;
1522 	struct icl_pdu *ip;
1523 	void *data;
1524 
1525 	sx_slock(&sc->sc_lock);
1526 	TAILQ_FOREACH(is, &sc->sc_sessions, is_next) {
1527 		if (is->is_id == idr->idr_session_id)
1528 			break;
1529 	}
1530 	if (is == NULL) {
1531 		sx_sunlock(&sc->sc_lock);
1532 		return (ESRCH);
1533 	}
1534 	sx_sunlock(&sc->sc_lock);
1535 
1536 	if (is->is_login_phase == false)
1537 		return (EBUSY);
1538 
1539 	ISCSI_SESSION_LOCK(is);
1540 	while (is->is_login_pdu == NULL &&
1541 	    is->is_terminating == false &&
1542 	    is->is_reconnecting == false)
1543 		cv_wait(&is->is_login_cv, &is->is_lock);
1544 	if (is->is_terminating || is->is_reconnecting) {
1545 		ISCSI_SESSION_UNLOCK(is);
1546 		return (EIO);
1547 	}
1548 	ip = is->is_login_pdu;
1549 	is->is_login_pdu = NULL;
1550 	ISCSI_SESSION_UNLOCK(is);
1551 
1552 	if (ip->ip_data_len > idr->idr_data_segment_len) {
1553 		icl_pdu_free(ip);
1554 		return (EMSGSIZE);
1555 	}
1556 
1557 	copyout(ip->ip_bhs, idr->idr_bhs, sizeof(*ip->ip_bhs));
1558 	if (ip->ip_data_len > 0) {
1559 		data = malloc(ip->ip_data_len, M_ISCSI, M_WAITOK);
1560 		icl_pdu_get_data(ip, 0, data, ip->ip_data_len);
1561 		copyout(data, idr->idr_data_segment, ip->ip_data_len);
1562 		free(data, M_ISCSI);
1563 	}
1564 
1565 	icl_pdu_free(ip);
1566 
1567 	return (0);
1568 }
1569 #endif /* ICL_KERNEL_PROXY */
1570 
1571 static void
1572 iscsi_sanitize_session_conf(struct iscsi_session_conf *isc)
1573 {
1574 	/*
1575 	 * Just make sure all the fields are null-terminated.
1576 	 *
1577 	 * XXX: This is not particularly secure.  We should
1578 	 * 	create our own conf and then copy in relevant
1579 	 * 	fields.
1580 	 */
1581 	isc->isc_initiator[ISCSI_NAME_LEN - 1] = '\0';
1582 	isc->isc_initiator_addr[ISCSI_ADDR_LEN - 1] = '\0';
1583 	isc->isc_initiator_alias[ISCSI_ALIAS_LEN - 1] = '\0';
1584 	isc->isc_target[ISCSI_NAME_LEN - 1] = '\0';
1585 	isc->isc_target_addr[ISCSI_ADDR_LEN - 1] = '\0';
1586 	isc->isc_user[ISCSI_NAME_LEN - 1] = '\0';
1587 	isc->isc_secret[ISCSI_SECRET_LEN - 1] = '\0';
1588 	isc->isc_mutual_user[ISCSI_NAME_LEN - 1] = '\0';
1589 	isc->isc_mutual_secret[ISCSI_SECRET_LEN - 1] = '\0';
1590 }
1591 
1592 static int
1593 iscsi_ioctl_session_add(struct iscsi_softc *sc, struct iscsi_session_add *isa)
1594 {
1595 	struct iscsi_session *is;
1596 	const struct iscsi_session *is2;
1597 	int error;
1598 
1599 	iscsi_sanitize_session_conf(&isa->isa_conf);
1600 
1601 	is = malloc(sizeof(*is), M_ISCSI, M_ZERO | M_WAITOK);
1602 	memcpy(&is->is_conf, &isa->isa_conf, sizeof(is->is_conf));
1603 
1604 	if (is->is_conf.isc_initiator[0] == '\0' ||
1605 	    is->is_conf.isc_target_addr[0] == '\0') {
1606 		free(is, M_ISCSI);
1607 		return (EINVAL);
1608 	}
1609 
1610 	if ((is->is_conf.isc_discovery != 0 && is->is_conf.isc_target[0] != 0) ||
1611 	    (is->is_conf.isc_discovery == 0 && is->is_conf.isc_target[0] == 0)) {
1612 		free(is, M_ISCSI);
1613 		return (EINVAL);
1614 	}
1615 
1616 	sx_xlock(&sc->sc_lock);
1617 
1618 	/*
1619 	 * Prevent duplicates.
1620 	 */
1621 	TAILQ_FOREACH(is2, &sc->sc_sessions, is_next) {
1622 		if (!!is->is_conf.isc_discovery !=
1623 		    !!is2->is_conf.isc_discovery)
1624 			continue;
1625 
1626 		if (strcmp(is->is_conf.isc_target_addr,
1627 		    is2->is_conf.isc_target_addr) != 0)
1628 			continue;
1629 
1630 		if (is->is_conf.isc_discovery == 0 &&
1631 		    strcmp(is->is_conf.isc_target,
1632 		    is2->is_conf.isc_target) != 0)
1633 			continue;
1634 
1635 		sx_xunlock(&sc->sc_lock);
1636 		free(is, M_ISCSI);
1637 		return (EBUSY);
1638 	}
1639 
1640 	is->is_conn = icl_conn_new("iscsi", &is->is_lock);
1641 	is->is_conn->ic_receive = iscsi_receive_callback;
1642 	is->is_conn->ic_error = iscsi_error_callback;
1643 	is->is_conn->ic_prv0 = is;
1644 	TAILQ_INIT(&is->is_outstanding);
1645 	STAILQ_INIT(&is->is_postponed);
1646 	mtx_init(&is->is_lock, "iscsi_lock", NULL, MTX_DEF);
1647 	cv_init(&is->is_maintenance_cv, "iscsi_mt");
1648 #ifdef ICL_KERNEL_PROXY
1649 	cv_init(&is->is_login_cv, "iscsi_login");
1650 #endif
1651 
1652 	is->is_softc = sc;
1653 	sc->sc_last_session_id++;
1654 	is->is_id = sc->sc_last_session_id;
1655 	callout_init(&is->is_callout, 1);
1656 	callout_reset(&is->is_callout, 1 * hz, iscsi_callout, is);
1657 	TAILQ_INSERT_TAIL(&sc->sc_sessions, is, is_next);
1658 
1659 	error = kthread_add(iscsi_maintenance_thread, is, NULL, NULL, 0, 0, "iscsimt");
1660 	if (error != 0) {
1661 		ISCSI_SESSION_WARN(is, "kthread_add(9) failed with error %d", error);
1662 		return (error);
1663 	}
1664 
1665 	/*
1666 	 * Trigger immediate reconnection.
1667 	 */
1668 	ISCSI_SESSION_LOCK(is);
1669 	is->is_waiting_for_iscsid = true;
1670 	strlcpy(is->is_reason, "Waiting for iscsid(8)", sizeof(is->is_reason));
1671 	ISCSI_SESSION_UNLOCK(is);
1672 	cv_signal(&sc->sc_cv);
1673 
1674 	sx_xunlock(&sc->sc_lock);
1675 
1676 	return (0);
1677 }
1678 
1679 static bool
1680 iscsi_session_conf_matches(unsigned int id1, const struct iscsi_session_conf *c1,
1681     unsigned int id2, const struct iscsi_session_conf *c2)
1682 {
1683 	if (id2 == 0 && c2->isc_target[0] == '\0' &&
1684 	    c2->isc_target_addr[0] == '\0')
1685 		return (true);
1686 	if (id2 != 0 && id2 == id1)
1687 		return (true);
1688 	if (c2->isc_target[0] != '\0' &&
1689 	    strcmp(c1->isc_target, c2->isc_target) == 0)
1690 		return (true);
1691 	if (c2->isc_target_addr[0] != '\0' &&
1692 	    strcmp(c1->isc_target_addr, c2->isc_target_addr) == 0)
1693 		return (true);
1694 	return (false);
1695 }
1696 
1697 static int
1698 iscsi_ioctl_session_remove(struct iscsi_softc *sc,
1699     struct iscsi_session_remove *isr)
1700 {
1701 	struct iscsi_session *is, *tmp;
1702 	bool found = false;
1703 
1704 	iscsi_sanitize_session_conf(&isr->isr_conf);
1705 
1706 	sx_xlock(&sc->sc_lock);
1707 	TAILQ_FOREACH_SAFE(is, &sc->sc_sessions, is_next, tmp) {
1708 		ISCSI_SESSION_LOCK(is);
1709 		if (iscsi_session_conf_matches(is->is_id, &is->is_conf,
1710 		    isr->isr_session_id, &isr->isr_conf)) {
1711 			found = true;
1712 			iscsi_session_logout(is);
1713 			iscsi_session_terminate(is);
1714 		}
1715 		ISCSI_SESSION_UNLOCK(is);
1716 	}
1717 	sx_xunlock(&sc->sc_lock);
1718 
1719 	if (!found)
1720 		return (ESRCH);
1721 
1722 	return (0);
1723 }
1724 
1725 static int
1726 iscsi_ioctl_session_list(struct iscsi_softc *sc, struct iscsi_session_list *isl)
1727 {
1728 	int error;
1729 	unsigned int i = 0;
1730 	struct iscsi_session *is;
1731 	struct iscsi_session_state iss;
1732 
1733 	sx_slock(&sc->sc_lock);
1734 	TAILQ_FOREACH(is, &sc->sc_sessions, is_next) {
1735 		if (i >= isl->isl_nentries) {
1736 			sx_sunlock(&sc->sc_lock);
1737 			return (EMSGSIZE);
1738 		}
1739 		memset(&iss, 0, sizeof(iss));
1740 		memcpy(&iss.iss_conf, &is->is_conf, sizeof(iss.iss_conf));
1741 		iss.iss_id = is->is_id;
1742 		strlcpy(iss.iss_target_alias, is->is_target_alias, sizeof(iss.iss_target_alias));
1743 		strlcpy(iss.iss_reason, is->is_reason, sizeof(iss.iss_reason));
1744 
1745 		if (is->is_conn->ic_header_crc32c)
1746 			iss.iss_header_digest = ISCSI_DIGEST_CRC32C;
1747 		else
1748 			iss.iss_header_digest = ISCSI_DIGEST_NONE;
1749 
1750 		if (is->is_conn->ic_data_crc32c)
1751 			iss.iss_data_digest = ISCSI_DIGEST_CRC32C;
1752 		else
1753 			iss.iss_data_digest = ISCSI_DIGEST_NONE;
1754 
1755 		iss.iss_max_data_segment_length = is->is_max_data_segment_length;
1756 		iss.iss_immediate_data = is->is_immediate_data;
1757 		iss.iss_connected = is->is_connected;
1758 
1759 		error = copyout(&iss, isl->isl_pstates + i, sizeof(iss));
1760 		if (error != 0) {
1761 			sx_sunlock(&sc->sc_lock);
1762 			return (error);
1763 		}
1764 		i++;
1765 	}
1766 	sx_sunlock(&sc->sc_lock);
1767 
1768 	isl->isl_nentries = i;
1769 
1770 	return (0);
1771 }
1772 
1773 static int
1774 iscsi_ioctl(struct cdev *dev, u_long cmd, caddr_t arg, int mode,
1775     struct thread *td)
1776 {
1777 	struct iscsi_softc *sc;
1778 
1779 	sc = dev->si_drv1;
1780 
1781 	switch (cmd) {
1782 	case ISCSIDWAIT:
1783 		return (iscsi_ioctl_daemon_wait(sc,
1784 		    (struct iscsi_daemon_request *)arg));
1785 	case ISCSIDHANDOFF:
1786 		return (iscsi_ioctl_daemon_handoff(sc,
1787 		    (struct iscsi_daemon_handoff *)arg));
1788 	case ISCSIDFAIL:
1789 		return (iscsi_ioctl_daemon_fail(sc,
1790 		    (struct iscsi_daemon_fail *)arg));
1791 #ifdef ICL_KERNEL_PROXY
1792 	case ISCSIDCONNECT:
1793 		return (iscsi_ioctl_daemon_connect(sc,
1794 		    (struct iscsi_daemon_connect *)arg));
1795 	case ISCSIDSEND:
1796 		return (iscsi_ioctl_daemon_send(sc,
1797 		    (struct iscsi_daemon_send *)arg));
1798 	case ISCSIDRECEIVE:
1799 		return (iscsi_ioctl_daemon_receive(sc,
1800 		    (struct iscsi_daemon_receive *)arg));
1801 #endif /* ICL_KERNEL_PROXY */
1802 	case ISCSISADD:
1803 		return (iscsi_ioctl_session_add(sc,
1804 		    (struct iscsi_session_add *)arg));
1805 	case ISCSISREMOVE:
1806 		return (iscsi_ioctl_session_remove(sc,
1807 		    (struct iscsi_session_remove *)arg));
1808 	case ISCSISLIST:
1809 		return (iscsi_ioctl_session_list(sc,
1810 		    (struct iscsi_session_list *)arg));
1811 	default:
1812 		return (EINVAL);
1813 	}
1814 }
1815 
1816 static struct iscsi_outstanding *
1817 iscsi_outstanding_find(struct iscsi_session *is, uint32_t initiator_task_tag)
1818 {
1819 	struct iscsi_outstanding *io;
1820 
1821 	ISCSI_SESSION_LOCK_ASSERT(is);
1822 
1823 	TAILQ_FOREACH(io, &is->is_outstanding, io_next) {
1824 		if (io->io_initiator_task_tag == initiator_task_tag)
1825 			return (io);
1826 	}
1827 	return (NULL);
1828 }
1829 
1830 static int
1831 iscsi_outstanding_add(struct iscsi_session *is,
1832     uint32_t initiator_task_tag, union ccb *ccb)
1833 {
1834 	struct iscsi_outstanding *io;
1835 
1836 	ISCSI_SESSION_LOCK_ASSERT(is);
1837 
1838 	KASSERT(iscsi_outstanding_find(is, initiator_task_tag) == NULL,
1839 	    ("initiator_task_tag 0x%x already added", initiator_task_tag));
1840 
1841 	io = uma_zalloc(iscsi_outstanding_zone, M_NOWAIT | M_ZERO);
1842 	if (io == NULL) {
1843 		ISCSI_SESSION_WARN(is, "failed to allocate %zd bytes", sizeof(*io));
1844 		return (ENOMEM);
1845 	}
1846 	io->io_initiator_task_tag = initiator_task_tag;
1847 	io->io_ccb = ccb;
1848 	TAILQ_INSERT_TAIL(&is->is_outstanding, io, io_next);
1849 	return (0);
1850 }
1851 
1852 static void
1853 iscsi_outstanding_remove(struct iscsi_session *is, struct iscsi_outstanding *io)
1854 {
1855 
1856 	ISCSI_SESSION_LOCK_ASSERT(is);
1857 
1858 	TAILQ_REMOVE(&is->is_outstanding, io, io_next);
1859 	uma_zfree(iscsi_outstanding_zone, io);
1860 }
1861 
1862 static void
1863 iscsi_action_scsiio(struct iscsi_session *is, union ccb *ccb)
1864 {
1865 	struct icl_pdu *request;
1866 	struct iscsi_bhs_scsi_command *bhssc;
1867 	struct ccb_scsiio *csio;
1868 	size_t len;
1869 	int error;
1870 
1871 	ISCSI_SESSION_LOCK_ASSERT(is);
1872 
1873 #if 0
1874 	KASSERT(is->is_login_phase == false, ("%s called during Login Phase", __func__));
1875 #else
1876 	if (is->is_login_phase) {
1877 		ISCSI_SESSION_DEBUG(is, "called during login phase");
1878 		if ((ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
1879 			xpt_freeze_devq(ccb->ccb_h.path, 1);
1880 			ISCSI_SESSION_DEBUG(is, "freezing devq");
1881 		}
1882 		ccb->ccb_h.status = CAM_REQ_ABORTED | CAM_DEV_QFRZN;
1883 		xpt_done(ccb);
1884 		return;
1885 	}
1886 #endif
1887 
1888 	request = icl_pdu_new_bhs(is->is_conn, M_NOWAIT);
1889 	if (request == NULL) {
1890 		if ((ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
1891 			xpt_freeze_devq(ccb->ccb_h.path, 1);
1892 			ISCSI_SESSION_DEBUG(is, "freezing devq");
1893 		}
1894 		ccb->ccb_h.status = CAM_RESRC_UNAVAIL | CAM_DEV_QFRZN;
1895 		xpt_done(ccb);
1896 		return;
1897 	}
1898 
1899 	csio = &ccb->csio;
1900 	bhssc = (struct iscsi_bhs_scsi_command *)request->ip_bhs;
1901 	bhssc->bhssc_opcode = ISCSI_BHS_OPCODE_SCSI_COMMAND;
1902 	bhssc->bhssc_flags |= BHSSC_FLAGS_F;
1903 	switch (csio->ccb_h.flags & CAM_DIR_MASK) {
1904 	case CAM_DIR_IN:
1905 		bhssc->bhssc_flags |= BHSSC_FLAGS_R;
1906 		break;
1907 	case CAM_DIR_OUT:
1908 		bhssc->bhssc_flags |= BHSSC_FLAGS_W;
1909 		break;
1910 	}
1911 
1912         switch (csio->tag_action) {
1913         case MSG_HEAD_OF_Q_TAG:
1914 		bhssc->bhssc_flags |= BHSSC_FLAGS_ATTR_HOQ;
1915 		break;
1916                 break;
1917         case MSG_ORDERED_Q_TAG:
1918 		bhssc->bhssc_flags |= BHSSC_FLAGS_ATTR_ORDERED;
1919                 break;
1920         case MSG_ACA_TASK:
1921 		bhssc->bhssc_flags |= BHSSC_FLAGS_ATTR_ACA;
1922                 break;
1923         case CAM_TAG_ACTION_NONE:
1924         case MSG_SIMPLE_Q_TAG:
1925         default:
1926 		bhssc->bhssc_flags |= BHSSC_FLAGS_ATTR_SIMPLE;
1927                 break;
1928         }
1929 
1930 	bhssc->bhssc_lun = htobe64(CAM_EXTLUN_BYTE_SWIZZLE(ccb->ccb_h.target_lun));
1931 	bhssc->bhssc_initiator_task_tag = is->is_initiator_task_tag;
1932 	is->is_initiator_task_tag++;
1933 	bhssc->bhssc_expected_data_transfer_length = htonl(csio->dxfer_len);
1934 	KASSERT(csio->cdb_len <= sizeof(bhssc->bhssc_cdb),
1935 	    ("unsupported CDB size %zd", (size_t)csio->cdb_len));
1936 
1937 	if (csio->ccb_h.flags & CAM_CDB_POINTER)
1938 		memcpy(&bhssc->bhssc_cdb, csio->cdb_io.cdb_ptr, csio->cdb_len);
1939 	else
1940 		memcpy(&bhssc->bhssc_cdb, csio->cdb_io.cdb_bytes, csio->cdb_len);
1941 
1942 	error = iscsi_outstanding_add(is, bhssc->bhssc_initiator_task_tag, ccb);
1943 	if (error != 0) {
1944 		icl_pdu_free(request);
1945 		if ((ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
1946 			xpt_freeze_devq(ccb->ccb_h.path, 1);
1947 			ISCSI_SESSION_DEBUG(is, "freezing devq");
1948 		}
1949 		ccb->ccb_h.status = CAM_RESRC_UNAVAIL | CAM_DEV_QFRZN;
1950 		xpt_done(ccb);
1951 		return;
1952 	}
1953 
1954 	if (is->is_immediate_data &&
1955 	    (csio->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_OUT) {
1956 		len = csio->dxfer_len;
1957 		//ISCSI_SESSION_DEBUG(is, "adding %zd of immediate data", len);
1958 		if (len > is->is_first_burst_length) {
1959 			ISCSI_SESSION_DEBUG(is, "len %zd -> %zd", len, is->is_first_burst_length);
1960 			len = is->is_first_burst_length;
1961 		}
1962 
1963 		error = icl_pdu_append_data(request, csio->data_ptr, len, M_NOWAIT);
1964 		if (error != 0) {
1965 			icl_pdu_free(request);
1966 			if ((ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
1967 				xpt_freeze_devq(ccb->ccb_h.path, 1);
1968 				ISCSI_SESSION_DEBUG(is, "freezing devq");
1969 			}
1970 			ccb->ccb_h.status = CAM_RESRC_UNAVAIL | CAM_DEV_QFRZN;
1971 			xpt_done(ccb);
1972 			return;
1973 		}
1974 	}
1975 	iscsi_pdu_queue_locked(request);
1976 }
1977 
1978 static void
1979 iscsi_action(struct cam_sim *sim, union ccb *ccb)
1980 {
1981 	struct iscsi_session *is;
1982 
1983 	is = cam_sim_softc(sim);
1984 
1985 	ISCSI_SESSION_LOCK_ASSERT(is);
1986 
1987 	if (is->is_terminating ||
1988 	    (is->is_connected == false && fail_on_disconnection)) {
1989 		ccb->ccb_h.status = CAM_DEV_NOT_THERE;
1990 		xpt_done(ccb);
1991 		return;
1992 	}
1993 
1994 	switch (ccb->ccb_h.func_code) {
1995 	case XPT_PATH_INQ:
1996 	{
1997 		struct ccb_pathinq *cpi = &ccb->cpi;
1998 
1999 		cpi->version_num = 1;
2000 		cpi->hba_inquiry = PI_TAG_ABLE;
2001 		cpi->target_sprt = 0;
2002 		cpi->hba_misc = PIM_EXTLUNS;
2003 		cpi->hba_eng_cnt = 0;
2004 		cpi->max_target = 0;
2005 		cpi->max_lun = 0;
2006 		cpi->initiator_id = ~0;
2007 		strlcpy(cpi->sim_vid, "FreeBSD", SIM_IDLEN);
2008 		strlcpy(cpi->hba_vid, "iSCSI", HBA_IDLEN);
2009 		strlcpy(cpi->dev_name, cam_sim_name(sim), DEV_IDLEN);
2010 		cpi->unit_number = cam_sim_unit(sim);
2011 		cpi->bus_id = cam_sim_bus(sim);
2012 		cpi->base_transfer_speed = 150000; /* XXX */
2013 		cpi->transport = XPORT_ISCSI;
2014 		cpi->transport_version = 0;
2015 		cpi->protocol = PROTO_SCSI;
2016 		cpi->protocol_version = SCSI_REV_SPC3;
2017 		cpi->maxio = MAXPHYS;
2018 		cpi->ccb_h.status = CAM_REQ_CMP;
2019 		break;
2020 	}
2021 	case XPT_CALC_GEOMETRY:
2022 		cam_calc_geometry(&ccb->ccg, /*extended*/1);
2023 		ccb->ccb_h.status = CAM_REQ_CMP;
2024 		break;
2025 #if 0
2026 	/*
2027 	 * XXX: What's the point?
2028 	 */
2029 	case XPT_RESET_BUS:
2030 	case XPT_ABORT:
2031 	case XPT_TERM_IO:
2032 		ISCSI_SESSION_DEBUG(is, "faking success for reset, abort, or term_io");
2033 		ccb->ccb_h.status = CAM_REQ_CMP;
2034 		break;
2035 #endif
2036 	case XPT_SCSI_IO:
2037 		iscsi_action_scsiio(is, ccb);
2038 		return;
2039 	default:
2040 #if 0
2041 		ISCSI_SESSION_DEBUG(is, "got unsupported code 0x%x", ccb->ccb_h.func_code);
2042 #endif
2043 		ccb->ccb_h.status = CAM_FUNC_NOTAVAIL;
2044 		break;
2045 	}
2046 	xpt_done(ccb);
2047 }
2048 
2049 static void
2050 iscsi_poll(struct cam_sim *sim)
2051 {
2052 
2053 	KASSERT(0, ("%s: you're not supposed to be here", __func__));
2054 }
2055 
2056 static void
2057 iscsi_shutdown(struct iscsi_softc *sc)
2058 {
2059 	struct iscsi_session *is;
2060 
2061 	ISCSI_DEBUG("removing all sessions due to shutdown");
2062 
2063 	sx_slock(&sc->sc_lock);
2064 	TAILQ_FOREACH(is, &sc->sc_sessions, is_next)
2065 		iscsi_session_terminate(is);
2066 	sx_sunlock(&sc->sc_lock);
2067 }
2068 
2069 static int
2070 iscsi_load(void)
2071 {
2072 	int error;
2073 
2074 	sc = malloc(sizeof(*sc), M_ISCSI, M_ZERO | M_WAITOK);
2075 	sx_init(&sc->sc_lock, "iscsi");
2076 	TAILQ_INIT(&sc->sc_sessions);
2077 	cv_init(&sc->sc_cv, "iscsi_cv");
2078 
2079 	iscsi_outstanding_zone = uma_zcreate("iscsi_outstanding",
2080 	    sizeof(struct iscsi_outstanding), NULL, NULL, NULL, NULL,
2081 	    UMA_ALIGN_PTR, 0);
2082 
2083 	error = make_dev_p(MAKEDEV_CHECKNAME, &sc->sc_cdev, &iscsi_cdevsw,
2084 	    NULL, UID_ROOT, GID_WHEEL, 0600, "iscsi");
2085 	if (error != 0) {
2086 		ISCSI_WARN("failed to create device node, error %d", error);
2087 		return (error);
2088 	}
2089 	sc->sc_cdev->si_drv1 = sc;
2090 
2091 	/*
2092 	 * Note that this needs to get run before dashutdown().  Otherwise,
2093 	 * when rebooting with iSCSI session with outstanding requests,
2094 	 * but disconnected, dashutdown() will hang on cam_periph_runccb().
2095 	 */
2096 	sc->sc_shutdown_eh = EVENTHANDLER_REGISTER(shutdown_post_sync,
2097 	    iscsi_shutdown, sc, SHUTDOWN_PRI_FIRST);
2098 
2099 	return (0);
2100 }
2101 
2102 static int
2103 iscsi_unload(void)
2104 {
2105 	struct iscsi_session *is, *tmp;
2106 
2107 	if (sc->sc_cdev != NULL) {
2108 		ISCSI_DEBUG("removing device node");
2109 		destroy_dev(sc->sc_cdev);
2110 		ISCSI_DEBUG("device node removed");
2111 	}
2112 
2113 	if (sc->sc_shutdown_eh != NULL)
2114 		EVENTHANDLER_DEREGISTER(shutdown_post_sync, sc->sc_shutdown_eh);
2115 
2116 	sx_slock(&sc->sc_lock);
2117 	TAILQ_FOREACH_SAFE(is, &sc->sc_sessions, is_next, tmp)
2118 		iscsi_session_terminate(is);
2119 	while(!TAILQ_EMPTY(&sc->sc_sessions)) {
2120 		ISCSI_DEBUG("waiting for sessions to terminate");
2121 		cv_wait(&sc->sc_cv, &sc->sc_lock);
2122 	}
2123 	ISCSI_DEBUG("all sessions terminated");
2124 	sx_sunlock(&sc->sc_lock);
2125 
2126 	uma_zdestroy(iscsi_outstanding_zone);
2127 	sx_destroy(&sc->sc_lock);
2128 	cv_destroy(&sc->sc_cv);
2129 	free(sc, M_ISCSI);
2130 	return (0);
2131 }
2132 
2133 static int
2134 iscsi_quiesce(void)
2135 {
2136 	sx_slock(&sc->sc_lock);
2137 	if (!TAILQ_EMPTY(&sc->sc_sessions)) {
2138 		sx_sunlock(&sc->sc_lock);
2139 		return (EBUSY);
2140 	}
2141 	sx_sunlock(&sc->sc_lock);
2142 	return (0);
2143 }
2144 
2145 static int
2146 iscsi_modevent(module_t mod, int what, void *arg)
2147 {
2148 	int error;
2149 
2150 	switch (what) {
2151 	case MOD_LOAD:
2152 		error = iscsi_load();
2153 		break;
2154 	case MOD_UNLOAD:
2155 		error = iscsi_unload();
2156 		break;
2157 	case MOD_QUIESCE:
2158 		error = iscsi_quiesce();
2159 		break;
2160 	default:
2161 		error = EINVAL;
2162 		break;
2163 	}
2164 	return (error);
2165 }
2166 
2167 moduledata_t iscsi_data = {
2168 	"iscsi",
2169 	iscsi_modevent,
2170 	0
2171 };
2172 
2173 DECLARE_MODULE(iscsi, iscsi_data, SI_SUB_DRIVERS, SI_ORDER_MIDDLE);
2174 MODULE_DEPEND(iscsi, cam, 1, 1, 1);
2175 MODULE_DEPEND(iscsi, icl, 1, 1, 1);
2176