xref: /freebsd/sys/dev/isp/isp_target.c (revision eacee0ff7ec955b32e09515246bd97b6edcd2b0f)
1 /* $FreeBSD$ */
2 /*
3  * Machine and OS Independent Target Mode Code for the Qlogic SCSI/FC adapters.
4  *
5  * Copyright (c) 1999, 2000, 2001 by Matthew Jacob
6  * All rights reserved.
7  * mjacob@feral.com
8  *
9  * Redistribution and use in source and binary forms, with or without
10  * modification, are permitted provided that the following conditions
11  * are met:
12  * 1. Redistributions of source code must retain the above copyright
13  *    notice immediately at the beginning of the file, without modification,
14  *    this list of conditions, and the following disclaimer.
15  * 2. The name of the author may not be used to endorse or promote products
16  *    derived from this software without specific prior written permission.
17  *
18  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
19  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21  * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR
22  * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28  * SUCH DAMAGE.
29  */
30 
31 /*
32  * Bug fixes gratefully acknowledged from:
33  *	Oded Kedem <oded@kashya.com>
34  */
35 /*
36  * Include header file appropriate for platform we're building on.
37  */
38 
39 #ifdef	__NetBSD__
40 #include <dev/ic/isp_netbsd.h>
41 #endif
42 #ifdef	__FreeBSD__
43 #include <dev/isp/isp_freebsd.h>
44 #endif
45 #ifdef	__OpenBSD__
46 #include <dev/ic/isp_openbsd.h>
47 #endif
48 #ifdef	__linux__
49 #include "isp_linux.h"
50 #endif
51 
52 #ifdef	ISP_TARGET_MODE
53 static const char atiocope[] =
54     "ATIO returned for lun %d because it was in the middle of Bus Device Reset "
55     "on bus %d";
56 static const char atior[] =
57     "ATIO returned on for lun %d on from IID %d because a Bus Reset occurred "
58     "on bus %d";
59 
60 static void isp_got_msg(struct ispsoftc *, int, in_entry_t *);
61 static void isp_got_msg_fc(struct ispsoftc *, int, in_fcentry_t *);
62 static void isp_notify_ack(struct ispsoftc *, void *);
63 static void isp_handle_atio(struct ispsoftc *, at_entry_t *);
64 static void isp_handle_atio2(struct ispsoftc *, at2_entry_t *);
65 static void isp_handle_ctio(struct ispsoftc *, ct_entry_t *);
66 static void isp_handle_ctio2(struct ispsoftc *, ct2_entry_t *);
67 
68 /*
69  * The Qlogic driver gets an interrupt to look at response queue entries.
70  * Some of these are status completions for initiatior mode commands, but
71  * if target mode is enabled, we get a whole wad of response queue entries
72  * to be handled here.
73  *
74  * Basically the split into 3 main groups: Lun Enable/Modification responses,
75  * SCSI Command processing, and Immediate Notification events.
76  *
77  * You start by writing a request queue entry to enable target mode (and
78  * establish some resource limitations which you can modify later).
79  * The f/w responds with a LUN ENABLE or LUN MODIFY response with
80  * the status of this action. If the enable was successful, you can expect...
81  *
82  * Response queue entries with SCSI commands encapsulate show up in an ATIO
83  * (Accept Target IO) type- sometimes with enough info to stop the command at
84  * this level. Ultimately the driver has to feed back to the f/w's request
85  * queue a sequence of CTIOs (continue target I/O) that describe data to
86  * be moved and/or status to be sent) and finally finishing with sending
87  * to the f/w's response queue an ATIO which then completes the handshake
88  * with the f/w for that command. There's a lot of variations on this theme,
89  * including flags you can set in the CTIO for the Qlogic 2X00 fibre channel
90  * cards that 'auto-replenish' the f/w's ATIO count, but this is the basic
91  * gist of it.
92  *
93  * The third group that can show up in the response queue are Immediate
94  * Notification events. These include things like notifications of SCSI bus
95  * resets, or Bus Device Reset messages or other messages received. This
96  * a classic oddbins area. It can get  a little weird because you then turn
97  * around and acknowledge the Immediate Notify by writing an entry onto the
98  * request queue and then the f/w turns around and gives you an acknowledgement
99  * to *your* acknowledgement on the response queue (the idea being to let
100  * the f/w tell you when the event is *really* over I guess).
101  *
102  */
103 
104 
105 /*
106  * A new response queue entry has arrived. The interrupt service code
107  * has already swizzled it into the platform dependent from canonical form.
108  *
109  * Because of the way this driver is designed, unfortunately most of the
110  * actual synchronization work has to be done in the platform specific
111  * code- we have no synchroniation primitives in the common code.
112  */
113 
114 int
115 isp_target_notify(struct ispsoftc *isp, void *vptr, u_int16_t *optrp)
116 {
117 	u_int16_t status, seqid;
118 	union {
119 		at_entry_t	*atiop;
120 		at2_entry_t	*at2iop;
121 		ct_entry_t	*ctiop;
122 		ct2_entry_t	*ct2iop;
123 		lun_entry_t	*lunenp;
124 		in_entry_t	*inotp;
125 		in_fcentry_t	*inot_fcp;
126 		na_entry_t	*nackp;
127 		na_fcentry_t	*nack_fcp;
128 		isphdr_t	*hp;
129 		void *		*vp;
130 #define	atiop		unp.atiop
131 #define	at2iop		unp.at2iop
132 #define	ctiop		unp.ctiop
133 #define	ct2iop		unp.ct2iop
134 #define	lunenp		unp.lunenp
135 #define	inotp		unp.inotp
136 #define	inot_fcp	unp.inot_fcp
137 #define	nackp		unp.nackp
138 #define	nack_fcp	unp.nack_fcp
139 #define	hdrp		unp.hp
140 	} unp;
141 	u_int8_t local[QENTRY_LEN];
142 	int bus, type, rval = 0;
143 
144 	type = isp_get_response_type(isp, (isphdr_t *)vptr);
145 	unp.vp = vptr;
146 
147 	ISP_TDQE(isp, "isp_target_notify", (int) *optrp, vptr);
148 
149 	switch(type) {
150 	case RQSTYPE_ATIO:
151 		isp_get_atio(isp, atiop, (at_entry_t *) local);
152 		isp_handle_atio(isp, (at_entry_t *) local);
153 		break;
154 	case RQSTYPE_CTIO:
155 		isp_get_ctio(isp, ctiop, (ct_entry_t *) local);
156 		isp_handle_ctio(isp, (ct_entry_t *) local);
157 		break;
158 	case RQSTYPE_ATIO2:
159 		isp_get_atio2(isp, at2iop, (at2_entry_t *) local);
160 		isp_handle_atio2(isp, (at2_entry_t *) local);
161 		break;
162 	case RQSTYPE_CTIO2:
163 		isp_get_ctio2(isp, ct2iop, (ct2_entry_t *) local);
164 		isp_handle_ctio2(isp, (ct2_entry_t *) local);
165 		break;
166 	case RQSTYPE_ENABLE_LUN:
167 	case RQSTYPE_MODIFY_LUN:
168 		isp_get_enable_lun(isp, lunenp, (lun_entry_t *) local);
169 		(void) isp_async(isp, ISPASYNC_TARGET_ACTION, local);
170 		break;
171 
172 	case RQSTYPE_NOTIFY:
173 		/*
174 		 * Either the ISP received a SCSI message it can't
175 		 * handle, or it's returning an Immed. Notify entry
176 		 * we sent. We can send Immed. Notify entries to
177 		 * increment the firmware's resource count for them
178 		 * (we set this initially in the Enable Lun entry).
179 		 */
180 		bus = 0;
181 		if (IS_FC(isp)) {
182 			isp_get_notify_fc(isp, inot_fcp, (in_fcentry_t *)local);
183 			inot_fcp = (in_fcentry_t *) local;
184 			status = inot_fcp->in_status;
185 			seqid = inot_fcp->in_seqid;
186 		} else {
187 			isp_get_notify(isp, inotp, (in_entry_t *)local);
188 			inotp = (in_entry_t *) local;
189 			status = inotp->in_status & 0xff;
190 			seqid = inotp->in_seqid;
191 			if (IS_DUALBUS(isp)) {
192 				bus = GET_BUS_VAL(inotp->in_iid);
193 				SET_BUS_VAL(inotp->in_iid, 0);
194 			}
195 		}
196 		isp_prt(isp, ISP_LOGTDEBUG0,
197 		    "Immediate Notify On Bus %d, status=0x%x seqid=0x%x",
198 		    bus, status, seqid);
199 
200 		/*
201 		 * ACK it right away.
202 		 */
203 		isp_notify_ack(isp, (status == IN_RESET)? NULL : local);
204 		switch (status) {
205 		case IN_RESET:
206 			(void) isp_async(isp, ISPASYNC_BUS_RESET, &bus);
207 			break;
208 		case IN_MSG_RECEIVED:
209 		case IN_IDE_RECEIVED:
210 			if (IS_FC(isp)) {
211 				isp_got_msg_fc(isp, bus, (in_fcentry_t *)local);
212 			} else {
213 				isp_got_msg(isp, bus, (in_entry_t *)local);
214 			}
215 			break;
216 		case IN_RSRC_UNAVAIL:
217 			isp_prt(isp, ISP_LOGWARN, "Firmware out of ATIOs");
218 			break;
219 		case IN_ABORT_TASK:
220 			isp_prt(isp, ISP_LOGWARN,
221 			    "Abort Task from IID %d RX_ID 0x%x",
222 			    inot_fcp->in_iid, seqid);
223 			(void) isp_async(isp, ISPASYNC_TARGET_ACTION, &bus);
224 			break;
225 		case IN_PORT_LOGOUT:
226 			isp_prt(isp, ISP_LOGWARN,
227 			    "Port Logout for Initiator %d RX_ID 0x%x",
228 			    inot_fcp->in_iid, seqid);
229 			break;
230 		case IN_PORT_CHANGED:
231 			isp_prt(isp, ISP_LOGWARN,
232 			    "Port Changed for Initiator %d RX_ID 0x%x",
233 			    inot_fcp->in_iid, seqid);
234 			break;
235 		case IN_GLOBAL_LOGO:
236 			isp_prt(isp, ISP_LOGWARN, "All ports logged out");
237 			break;
238 		default:
239 			isp_prt(isp, ISP_LOGERR,
240 			    "bad status (0x%x) in isp_target_notify", status);
241 			break;
242 		}
243 		break;
244 
245 	case RQSTYPE_NOTIFY_ACK:
246 		/*
247 		 * The ISP is acknowledging our acknowledgement of an
248 		 * Immediate Notify entry for some asynchronous event.
249 		 */
250 		if (IS_FC(isp)) {
251 			isp_get_notify_ack_fc(isp, nack_fcp,
252 			    (na_fcentry_t *)local);
253 			nack_fcp = (na_fcentry_t *)local;
254 			isp_prt(isp, ISP_LOGTDEBUG1,
255 			    "Notify Ack status=0x%x seqid 0x%x",
256 			    nack_fcp->na_status, nack_fcp->na_seqid);
257 		} else {
258 			isp_get_notify_ack(isp, nackp, (na_entry_t *)local);
259 			nackp = (na_entry_t *)local;
260 			isp_prt(isp, ISP_LOGTDEBUG1,
261 			    "Notify Ack event 0x%x status=0x%x seqid 0x%x",
262 			    nackp->na_event, nackp->na_status, nackp->na_seqid);
263 		}
264 		break;
265 	default:
266 		isp_prt(isp, ISP_LOGERR,
267 		    "Unknown entry type 0x%x in isp_target_notify", type);
268 		rval = -1;
269 		break;
270 	}
271 #undef	atiop
272 #undef	at2iop
273 #undef	ctiop
274 #undef	ct2iop
275 #undef	lunenp
276 #undef	inotp
277 #undef	inot_fcp
278 #undef	nackp
279 #undef	nack_fcp
280 #undef	hdrp
281 	return (rval);
282 }
283 
284 
285 /*
286  * Toggle (on/off) target mode for bus/target/lun
287  *
288  * The caller has checked for overlap and legality.
289  *
290  * Note that not all of bus, target or lun can be paid attention to.
291  * Note also that this action will not be complete until the f/w writes
292  * response entry. The caller is responsible for synchronizing this.
293  */
294 int
295 isp_lun_cmd(struct ispsoftc *isp, int cmd, int bus, int tgt, int lun,
296     int cmd_cnt, int inot_cnt, u_int32_t opaque)
297 {
298 	lun_entry_t el;
299 	u_int16_t nxti, optr;
300 	void *outp;
301 
302 
303 	MEMZERO(&el, sizeof (el));
304 	if (IS_DUALBUS(isp)) {
305 		el.le_rsvd = (bus & 0x1) << 7;
306 	}
307 	el.le_cmd_count = cmd_cnt;
308 	el.le_in_count = inot_cnt;
309 	if (cmd == RQSTYPE_ENABLE_LUN) {
310 		if (IS_SCSI(isp)) {
311 			el.le_flags = LUN_TQAE|LUN_DISAD;
312 			el.le_cdb6len = 12;
313 			el.le_cdb7len = 12;
314 		}
315 	} else if (cmd == -RQSTYPE_ENABLE_LUN) {
316 		cmd = RQSTYPE_ENABLE_LUN;
317 		el.le_cmd_count = 0;
318 		el.le_in_count = 0;
319 	} else if (cmd == -RQSTYPE_MODIFY_LUN) {
320 		cmd = RQSTYPE_MODIFY_LUN;
321 		el.le_ops = LUN_CCDECR | LUN_INDECR;
322 	} else {
323 		el.le_ops = LUN_CCINCR | LUN_ININCR;
324 	}
325 	el.le_header.rqs_entry_type = cmd;
326 	el.le_header.rqs_entry_count = 1;
327 	el.le_reserved = opaque;
328 	if (IS_SCSI(isp)) {
329 		el.le_tgt = tgt;
330 		el.le_lun = lun;
331 	} else if ((FCPARAM(isp)->isp_fwattr & ISP_FW_ATTR_SCCLUN) == 0) {
332 		el.le_lun = lun;
333 	}
334 	el.le_timeout = 2;
335 
336 	if (isp_getrqentry(isp, &nxti, &optr, &outp)) {
337 		isp_prt(isp, ISP_LOGERR,
338 		    "Request Queue Overflow in isp_lun_cmd");
339 		return (-1);
340 	}
341 	ISP_TDQE(isp, "isp_lun_cmd", (int) optr, &el);
342 	isp_put_enable_lun(isp, &el, outp);
343 	ISP_ADD_REQUEST(isp, nxti);
344 	return (0);
345 }
346 
347 
348 int
349 isp_target_put_entry(struct ispsoftc *isp, void *ap)
350 {
351 	void *outp;
352 	u_int16_t nxti, optr;
353 	u_int8_t etype = ((isphdr_t *) ap)->rqs_entry_type;
354 
355 	if (isp_getrqentry(isp, &nxti, &optr, &outp)) {
356 		isp_prt(isp, ISP_LOGWARN,
357 		    "Request Queue Overflow in isp_target_put_entry");
358 		return (-1);
359 	}
360 	switch (etype) {
361 	case RQSTYPE_ATIO:
362 		isp_put_atio(isp, (at_entry_t *) ap, (at_entry_t *) outp);
363 		break;
364 	case RQSTYPE_ATIO2:
365 		isp_put_atio2(isp, (at2_entry_t *) ap, (at2_entry_t *) outp);
366 		break;
367 	case RQSTYPE_CTIO:
368 		isp_put_ctio(isp, (ct_entry_t *) ap, (ct_entry_t *) outp);
369 		break;
370 	case RQSTYPE_CTIO2:
371 		isp_put_ctio2(isp, (ct2_entry_t *) ap, (ct2_entry_t *) outp);
372 		break;
373 	default:
374 		isp_prt(isp, ISP_LOGERR,
375 		    "Unknown type 0x%x in isp_put_entry", etype);
376 		return (-1);
377 	}
378 
379 	ISP_TDQE(isp, "isp_target_put_entry", (int) optr, ap);;
380 	ISP_ADD_REQUEST(isp, nxti);
381 	return (0);
382 }
383 
384 int
385 isp_target_put_atio(struct ispsoftc *isp, void *arg)
386 {
387 	union {
388 		at_entry_t _atio;
389 		at2_entry_t _atio2;
390 	} atun;
391 
392 	MEMZERO(&atun, sizeof atun);
393 	if (IS_FC(isp)) {
394 		at2_entry_t *aep = arg;
395 		atun._atio2.at_header.rqs_entry_type = RQSTYPE_ATIO2;
396 		atun._atio2.at_header.rqs_entry_count = 1;
397 		if (FCPARAM(isp)->isp_fwattr & ISP_FW_ATTR_SCCLUN) {
398 			atun._atio2.at_scclun = (u_int16_t) aep->at_scclun;
399 		} else {
400 			atun._atio2.at_lun = (u_int8_t) aep->at_lun;
401 		}
402 		atun._atio2.at_status = CT_OK;
403 	} else {
404 		at_entry_t *aep = arg;
405 		atun._atio.at_header.rqs_entry_type = RQSTYPE_ATIO;
406 		atun._atio.at_header.rqs_entry_count = 1;
407 		atun._atio.at_handle = aep->at_handle;
408 		atun._atio.at_iid = aep->at_iid;
409 		atun._atio.at_tgt = aep->at_tgt;
410 		atun._atio.at_lun = aep->at_lun;
411 		atun._atio.at_tag_type = aep->at_tag_type;
412 		atun._atio.at_tag_val = aep->at_tag_val;
413 		atun._atio.at_status = (aep->at_flags & AT_TQAE);
414 		atun._atio.at_status |= CT_OK;
415 	}
416 	return (isp_target_put_entry(isp, &atun));
417 }
418 
419 /*
420  * Command completion- both for handling cases of no resources or
421  * no blackhole driver, or other cases where we have to, inline,
422  * finish the command sanely, or for normal command completion.
423  *
424  * The 'completion' code value has the scsi status byte in the low 8 bits.
425  * If status is a CHECK CONDITION and bit 8 is nonzero, then bits 12..15 have
426  * the sense key and  bits 16..23 have the ASCQ and bits 24..31 have the ASC
427  * values.
428  *
429  * NB: the key, asc, ascq, cannot be used for parallel SCSI as it doesn't
430  * NB: inline SCSI sense reporting. As such, we lose this information. XXX.
431  *
432  * For both parallel && fibre channel, we use the feature that does
433  * an automatic resource autoreplenish so we don't have then later do
434  * put of an atio to replenish the f/w's resource count.
435  */
436 
437 int
438 isp_endcmd(struct ispsoftc *isp, void *arg, u_int32_t code, u_int16_t hdl)
439 {
440 	int sts;
441 	union {
442 		ct_entry_t _ctio;
443 		ct2_entry_t _ctio2;
444 	} un;
445 
446 	MEMZERO(&un, sizeof un);
447 	sts = code & 0xff;
448 
449 	if (IS_FC(isp)) {
450 		at2_entry_t *aep = arg;
451 		ct2_entry_t *cto = &un._ctio2;
452 
453 		cto->ct_header.rqs_entry_type = RQSTYPE_CTIO2;
454 		cto->ct_header.rqs_entry_count = 1;
455 		cto->ct_iid = aep->at_iid;
456 		if ((FCPARAM(isp)->isp_fwattr & ISP_FW_ATTR_SCCLUN) == 0) {
457 			cto->ct_lun = aep->at_lun;
458 		}
459 		cto->ct_rxid = aep->at_rxid;
460 		cto->rsp.m1.ct_scsi_status = sts & 0xff;
461 		cto->ct_flags = CT2_SENDSTATUS | CT2_NO_DATA | CT2_FLAG_MODE1;
462 		if (hdl == 0) {
463 			cto->ct_flags |= CT2_CCINCR;
464 		}
465 		if (aep->at_datalen) {
466 			cto->ct_resid = aep->at_datalen;
467 			cto->rsp.m1.ct_scsi_status |= CT2_DATA_UNDER;
468 		}
469 		if ((sts & 0xff) == SCSI_CHECK && (sts & ECMD_SVALID)) {
470 			cto->rsp.m1.ct_resp[0] = 0xf0;
471 			cto->rsp.m1.ct_resp[2] = (code >> 12) & 0xf;
472 			cto->rsp.m1.ct_resp[7] = 8;
473 			cto->rsp.m1.ct_resp[12] = (code >> 24) & 0xff;
474 			cto->rsp.m1.ct_resp[13] = (code >> 16) & 0xff;
475 			cto->rsp.m1.ct_senselen = 16;
476 			cto->rsp.m1.ct_scsi_status |= CT2_SNSLEN_VALID;
477 		}
478 		cto->ct_syshandle = hdl;
479 	} else {
480 		at_entry_t *aep = arg;
481 		ct_entry_t *cto = &un._ctio;
482 
483 		cto->ct_header.rqs_entry_type = RQSTYPE_CTIO;
484 		cto->ct_header.rqs_entry_count = 1;
485 		cto->ct_fwhandle = aep->at_handle;
486 		cto->ct_iid = aep->at_iid;
487 		cto->ct_tgt = aep->at_tgt;
488 		cto->ct_lun = aep->at_lun;
489 		cto->ct_tag_type = aep->at_tag_type;
490 		cto->ct_tag_val = aep->at_tag_val;
491 		if (aep->at_flags & AT_TQAE) {
492 			cto->ct_flags |= CT_TQAE;
493 		}
494 		cto->ct_flags = CT_SENDSTATUS | CT_NO_DATA;
495 		if (hdl == 0) {
496 			cto->ct_flags |= CT_CCINCR;
497 		}
498 		cto->ct_scsi_status = sts;
499 		cto->ct_syshandle = hdl;
500 	}
501 	return (isp_target_put_entry(isp, &un));
502 }
503 
504 void
505 isp_target_async(struct ispsoftc *isp, int bus, int event)
506 {
507 	tmd_event_t evt;
508 	tmd_msg_t msg;
509 
510 	switch (event) {
511 	/*
512 	 * These three we handle here to propagate an effective bus reset
513 	 * upstream, but these do not require any immediate notify actions
514 	 * so we return when done.
515 	 */
516 	case ASYNC_LIP_F8:
517 	case ASYNC_LIP_OCCURRED:
518 	case ASYNC_LOOP_UP:
519 	case ASYNC_LOOP_DOWN:
520 	case ASYNC_LOOP_RESET:
521 	case ASYNC_PTPMODE:
522 		/*
523 		 * These don't require any immediate notify actions. We used
524 		 * treat them like SCSI Bus Resets, but that was just plain
525 		 * wrong. Let the normal CTIO completion report what occurred.
526 		 */
527                 return;
528 
529 	case ASYNC_BUS_RESET:
530 	case ASYNC_TIMEOUT_RESET:
531 		if (IS_FC(isp)) {
532 			return;	/* we'll be getting an inotify instead */
533 		}
534 		evt.ev_bus = bus;
535 		evt.ev_event = event;
536 		(void) isp_async(isp, ISPASYNC_TARGET_EVENT, &evt);
537 		break;
538 	case ASYNC_DEVICE_RESET:
539 		/*
540 		 * Bus Device Reset resets a specific target, so
541 		 * we pass this as a synthesized message.
542 		 */
543 		MEMZERO(&msg, sizeof msg);
544 		if (IS_FC(isp)) {
545 			msg.nt_iid = FCPARAM(isp)->isp_loopid;
546 		} else {
547 			msg.nt_iid = SDPARAM(isp)->isp_initiator_id;
548 		}
549 		msg.nt_bus = bus;
550 		msg.nt_msg[0] = MSG_BUS_DEV_RESET;
551 		(void) isp_async(isp, ISPASYNC_TARGET_MESSAGE, &msg);
552 		break;
553 	default:
554 		isp_prt(isp, ISP_LOGERR,
555 		    "isp_target_async: unknown event 0x%x", event);
556 		break;
557 	}
558 	if (isp->isp_state == ISP_RUNSTATE)
559 		isp_notify_ack(isp, NULL);
560 }
561 
562 
563 /*
564  * Process a received message.
565  * The ISP firmware can handle most messages, there are only
566  * a few that we need to deal with:
567  * - abort: clean up the current command
568  * - abort tag and clear queue
569  */
570 
571 static void
572 isp_got_msg(struct ispsoftc *isp, int bus, in_entry_t *inp)
573 {
574 	u_int8_t status = inp->in_status & ~QLTM_SVALID;
575 
576 	if (status == IN_IDE_RECEIVED || status == IN_MSG_RECEIVED) {
577 		tmd_msg_t msg;
578 
579 		MEMZERO(&msg, sizeof (msg));
580 		msg.nt_bus = bus;
581 		msg.nt_iid = inp->in_iid;
582 		msg.nt_tgt = inp->in_tgt;
583 		msg.nt_lun = inp->in_lun;
584 		msg.nt_tagtype = inp->in_tag_type;
585 		msg.nt_tagval = inp->in_tag_val;
586 		MEMCPY(msg.nt_msg, inp->in_msg, IN_MSGLEN);
587 		(void) isp_async(isp, ISPASYNC_TARGET_MESSAGE, &msg);
588 	} else {
589 		isp_prt(isp, ISP_LOGERR,
590 		    "unknown immediate notify status 0x%x", inp->in_status);
591 	}
592 }
593 
594 /*
595  * Synthesize a message from the task management flags in a FCP_CMND_IU.
596  */
597 static void
598 isp_got_msg_fc(struct ispsoftc *isp, int bus, in_fcentry_t *inp)
599 {
600 	int lun;
601 	static const char f1[] = "%s from iid %d lun %d seq 0x%x";
602 	static const char f2[] =
603 	    "unknown %s 0x%x lun %d iid %d task flags 0x%x seq 0x%x\n";
604 
605 	if (FCPARAM(isp)->isp_fwattr & ISP_FW_ATTR_SCCLUN) {
606 		lun = inp->in_scclun;
607 	} else {
608 		lun = inp->in_lun;
609 	}
610 
611 	if (inp->in_status != IN_MSG_RECEIVED) {
612 		isp_prt(isp, ISP_LOGINFO, f2, "immediate notify status",
613 		    inp->in_status, lun, inp->in_iid,
614 		    inp->in_task_flags,  inp->in_seqid);
615 	} else {
616 		tmd_msg_t msg;
617 
618 		MEMZERO(&msg, sizeof (msg));
619 		msg.nt_bus = bus;
620 		msg.nt_iid = inp->in_iid;
621 		msg.nt_tagval = inp->in_seqid;
622 		msg.nt_lun = lun;
623 
624 		if (inp->in_task_flags & TASK_FLAGS_ABORT_TASK) {
625 			isp_prt(isp, ISP_LOGINFO, f1, "ABORT TASK",
626 			    inp->in_iid, msg.nt_lun, inp->in_seqid);
627 			msg.nt_msg[0] = MSG_ABORT_TAG;
628 		} else if (inp->in_task_flags & TASK_FLAGS_CLEAR_TASK_SET) {
629 			isp_prt(isp, ISP_LOGINFO, f1, "CLEAR TASK SET",
630 			    inp->in_iid, msg.nt_lun, inp->in_seqid);
631 			msg.nt_msg[0] = MSG_CLEAR_QUEUE;
632 		} else if (inp->in_task_flags & TASK_FLAGS_TARGET_RESET) {
633 			isp_prt(isp, ISP_LOGINFO, f1, "TARGET RESET",
634 			    inp->in_iid, msg.nt_lun, inp->in_seqid);
635 			msg.nt_msg[0] = MSG_BUS_DEV_RESET;
636 		} else if (inp->in_task_flags & TASK_FLAGS_CLEAR_ACA) {
637 			isp_prt(isp, ISP_LOGINFO, f1, "CLEAR ACA",
638 			    inp->in_iid, msg.nt_lun, inp->in_seqid);
639 			/* ???? */
640 			msg.nt_msg[0] = MSG_REL_RECOVERY;
641 		} else if (inp->in_task_flags & TASK_FLAGS_TERMINATE_TASK) {
642 			isp_prt(isp, ISP_LOGINFO, f1, "TERMINATE TASK",
643 			    inp->in_iid, msg.nt_lun, inp->in_seqid);
644 			msg.nt_msg[0] = MSG_TERM_IO_PROC;
645 		} else {
646 			isp_prt(isp, ISP_LOGWARN, f2, "task flag",
647 			    inp->in_status, msg.nt_lun, inp->in_iid,
648 			    inp->in_task_flags,  inp->in_seqid);
649 		}
650 		if (msg.nt_msg[0]) {
651 			(void) isp_async(isp, ISPASYNC_TARGET_MESSAGE, &msg);
652 		}
653 	}
654 }
655 
656 static void
657 isp_notify_ack(struct ispsoftc *isp, void *arg)
658 {
659 	char storage[QENTRY_LEN];
660 	u_int16_t nxti, optr;
661 	void *outp;
662 
663 	if (isp_getrqentry(isp, &nxti, &optr, &outp)) {
664 		isp_prt(isp, ISP_LOGWARN,
665 		    "Request Queue Overflow For isp_notify_ack");
666 		return;
667 	}
668 
669 	MEMZERO(storage, QENTRY_LEN);
670 
671 	if (IS_FC(isp)) {
672 		na_fcentry_t *na = (na_fcentry_t *) storage;
673 		if (arg) {
674 			in_fcentry_t *inp = arg;
675 			MEMCPY(storage, arg, sizeof (isphdr_t));
676 			na->na_iid = inp->in_iid;
677 			if (FCPARAM(isp)->isp_fwattr & ISP_FW_ATTR_SCCLUN) {
678 				na->na_lun = inp->in_scclun;
679 			} else {
680 				na->na_lun = inp->in_lun;
681 			}
682 			na->na_task_flags = inp->in_task_flags;
683 			na->na_seqid = inp->in_seqid;
684 			na->na_flags = NAFC_RCOUNT;
685 			na->na_status = inp->in_status;
686 			if (inp->in_status == IN_RESET) {
687 				na->na_flags |= NAFC_RST_CLRD;
688 			}
689 		} else {
690 			na->na_flags = NAFC_RST_CLRD;
691 		}
692 		na->na_header.rqs_entry_type = RQSTYPE_NOTIFY_ACK;
693 		na->na_header.rqs_entry_count = 1;
694 		isp_put_notify_ack_fc(isp, na, (na_fcentry_t *)outp);
695 	} else {
696 		na_entry_t *na = (na_entry_t *) storage;
697 		if (arg) {
698 			in_entry_t *inp = arg;
699 			MEMCPY(storage, arg, sizeof (isphdr_t));
700 			na->na_iid = inp->in_iid;
701 			na->na_lun = inp->in_lun;
702 			na->na_tgt = inp->in_tgt;
703 			na->na_seqid = inp->in_seqid;
704 			if (inp->in_status == IN_RESET) {
705 				na->na_event = NA_RST_CLRD;
706 			}
707 		} else {
708 			na->na_event = NA_RST_CLRD;
709 		}
710 		na->na_header.rqs_entry_type = RQSTYPE_NOTIFY_ACK;
711 		na->na_header.rqs_entry_count = 1;
712 		isp_put_notify_ack(isp, na, (na_entry_t *)outp);
713 	}
714 	ISP_TDQE(isp, "isp_notify_ack", (int) optr, storage);
715 	ISP_ADD_REQUEST(isp, nxti);
716 }
717 
718 static void
719 isp_handle_atio(struct ispsoftc *isp, at_entry_t *aep)
720 {
721 	int lun;
722 	lun = aep->at_lun;
723 	/*
724 	 * The firmware status (except for the QLTM_SVALID bit) indicates
725 	 * why this ATIO was sent to us.
726 	 *
727 	 * If QLTM_SVALID is set, the firware has recommended Sense Data.
728 	 *
729 	 * If the DISCONNECTS DISABLED bit is set in the flags field,
730 	 * we're still connected on the SCSI bus - i.e. the initiator
731 	 * did not set DiscPriv in the identify message. We don't care
732 	 * about this so it's ignored.
733 	 */
734 
735 	switch(aep->at_status & ~QLTM_SVALID) {
736 	case AT_PATH_INVALID:
737 		/*
738 		 * ATIO rejected by the firmware due to disabled lun.
739 		 */
740 		isp_prt(isp, ISP_LOGERR,
741 		    "rejected ATIO for disabled lun %d", lun);
742 		break;
743 	case AT_NOCAP:
744 		/*
745 		 * Requested Capability not available
746 		 * We sent an ATIO that overflowed the firmware's
747 		 * command resource count.
748 		 */
749 		isp_prt(isp, ISP_LOGERR,
750 		    "rejected ATIO for lun %d because of command count"
751 		    " overflow", lun);
752 		break;
753 
754 	case AT_BDR_MSG:
755 		/*
756 		 * If we send an ATIO to the firmware to increment
757 		 * its command resource count, and the firmware is
758 		 * recovering from a Bus Device Reset, it returns
759 		 * the ATIO with this status. We set the command
760 		 * resource count in the Enable Lun entry and do
761 		 * not increment it. Therefore we should never get
762 		 * this status here.
763 		 */
764 		isp_prt(isp, ISP_LOGERR, atiocope, lun,
765 		    GET_BUS_VAL(aep->at_iid));
766 		break;
767 
768 	case AT_CDB:		/* Got a CDB */
769 	case AT_PHASE_ERROR:	/* Bus Phase Sequence Error */
770 		/*
771 		 * Punt to platform specific layer.
772 		 */
773 		(void) isp_async(isp, ISPASYNC_TARGET_ACTION, aep);
774 		break;
775 
776 	case AT_RESET:
777 		/*
778 		 * A bus reset came along an blew away this command. Why
779 		 * they do this in addition the async event code stuff,
780 		 * I dunno.
781 		 *
782 		 * Ignore it because the async event will clear things
783 		 * up for us.
784 		 */
785 		isp_prt(isp, ISP_LOGWARN, atior, lun,
786 		    GET_IID_VAL(aep->at_iid), GET_BUS_VAL(aep->at_iid));
787 		break;
788 
789 
790 	default:
791 		isp_prt(isp, ISP_LOGERR,
792 		    "Unknown ATIO status 0x%x from initiator %d for lun %d",
793 		    aep->at_status, aep->at_iid, lun);
794 		(void) isp_target_put_atio(isp, aep);
795 		break;
796 	}
797 }
798 
799 static void
800 isp_handle_atio2(struct ispsoftc *isp, at2_entry_t *aep)
801 {
802 	int lun;
803 
804 	if (FCPARAM(isp)->isp_fwattr & ISP_FW_ATTR_SCCLUN) {
805 		lun = aep->at_scclun;
806 	} else {
807 		lun = aep->at_lun;
808 	}
809 
810 	/*
811 	 * The firmware status (except for the QLTM_SVALID bit) indicates
812 	 * why this ATIO was sent to us.
813 	 *
814 	 * If QLTM_SVALID is set, the firware has recommended Sense Data.
815 	 *
816 	 * If the DISCONNECTS DISABLED bit is set in the flags field,
817 	 * we're still connected on the SCSI bus - i.e. the initiator
818 	 * did not set DiscPriv in the identify message. We don't care
819 	 * about this so it's ignored.
820 	 */
821 
822 	switch(aep->at_status & ~QLTM_SVALID) {
823 	case AT_PATH_INVALID:
824 		/*
825 		 * ATIO rejected by the firmware due to disabled lun.
826 		 */
827 		isp_prt(isp, ISP_LOGERR,
828 		    "rejected ATIO2 for disabled lun %d", lun);
829 		break;
830 	case AT_NOCAP:
831 		/*
832 		 * Requested Capability not available
833 		 * We sent an ATIO that overflowed the firmware's
834 		 * command resource count.
835 		 */
836 		isp_prt(isp, ISP_LOGERR,
837 		    "rejected ATIO2 for lun %d- command count overflow", lun);
838 		break;
839 
840 	case AT_BDR_MSG:
841 		/*
842 		 * If we send an ATIO to the firmware to increment
843 		 * its command resource count, and the firmware is
844 		 * recovering from a Bus Device Reset, it returns
845 		 * the ATIO with this status. We set the command
846 		 * resource count in the Enable Lun entry and no
847 		 * not increment it. Therefore we should never get
848 		 * this status here.
849 		 */
850 		isp_prt(isp, ISP_LOGERR, atiocope, lun, 0);
851 		break;
852 
853 	case AT_CDB:		/* Got a CDB */
854 		/*
855 		 * Punt to platform specific layer.
856 		 */
857 		(void) isp_async(isp, ISPASYNC_TARGET_ACTION, aep);
858 		break;
859 
860 	case AT_RESET:
861 		/*
862 		 * A bus reset came along an blew away this command. Why
863 		 * they do this in addition the async event code stuff,
864 		 * I dunno.
865 		 *
866 		 * Ignore it because the async event will clear things
867 		 * up for us.
868 		 */
869 		isp_prt(isp, ISP_LOGERR, atior, lun, aep->at_iid, 0);
870 		break;
871 
872 
873 	default:
874 		isp_prt(isp, ISP_LOGERR,
875 		    "Unknown ATIO2 status 0x%x from initiator %d for lun %d",
876 		    aep->at_status, aep->at_iid, lun);
877 		(void) isp_target_put_atio(isp, aep);
878 		break;
879 	}
880 }
881 
882 static void
883 isp_handle_ctio(struct ispsoftc *isp, ct_entry_t *ct)
884 {
885 	void *xs;
886 	int pl = ISP_LOGTDEBUG2;
887 	char *fmsg = NULL;
888 
889 	if (ct->ct_syshandle) {
890 		xs = isp_find_xs(isp, ct->ct_syshandle);
891 		if (xs == NULL)
892 			pl = ISP_LOGALL;
893 	} else {
894 		xs = NULL;
895 	}
896 
897 	switch(ct->ct_status & ~QLTM_SVALID) {
898 	case CT_OK:
899 		/*
900 		 * There are generally 3 possibilities as to why we'd get
901 		 * this condition:
902 		 * 	We disconnected after receiving a CDB.
903 		 * 	We sent or received data.
904 		 * 	We sent status & command complete.
905 		 */
906 
907 		if (ct->ct_flags & CT_SENDSTATUS) {
908 			break;
909 		} else if ((ct->ct_flags & CT_DATAMASK) == CT_NO_DATA) {
910 			/*
911 			 * Nothing to do in this case.
912 			 */
913 			isp_prt(isp, pl, "CTIO- iid %d disconnected OK",
914 			    ct->ct_iid);
915 			return;
916 		}
917 		break;
918 
919 	case CT_BDR_MSG:
920 		/*
921 		 * Bus Device Reset message received or the SCSI Bus has
922 		 * been Reset; the firmware has gone to Bus Free.
923 		 *
924 		 * The firmware generates an async mailbox interupt to
925 		 * notify us of this and returns outstanding CTIOs with this
926 		 * status. These CTIOs are handled in that same way as
927 		 * CT_ABORTED ones, so just fall through here.
928 		 */
929 		fmsg = "Bus Device Reset";
930 		/*FALLTHROUGH*/
931 	case CT_RESET:
932 		if (fmsg == NULL)
933 			fmsg = "Bus Reset";
934 		/*FALLTHROUGH*/
935 	case CT_ABORTED:
936 		/*
937 		 * When an Abort message is received the firmware goes to
938 		 * Bus Free and returns all outstanding CTIOs with the status
939 		 * set, then sends us an Immediate Notify entry.
940 		 */
941 		if (fmsg == NULL)
942 			fmsg = "ABORT TAG message sent by Initiator";
943 
944 		isp_prt(isp, ISP_LOGWARN, "CTIO destroyed by %s", fmsg);
945 		break;
946 
947 	case CT_INVAL:
948 		/*
949 		 * CTIO rejected by the firmware due to disabled lun.
950 		 * "Cannot Happen".
951 		 */
952 		isp_prt(isp, ISP_LOGERR,
953 		    "Firmware rejected CTIO for disabled lun %d",
954 		    ct->ct_lun);
955 		break;
956 
957 	case CT_NOPATH:
958 		/*
959 		 * CTIO rejected by the firmware due "no path for the
960 		 * nondisconnecting nexus specified". This means that
961 		 * we tried to access the bus while a non-disconnecting
962 		 * command is in process.
963 		 */
964 		isp_prt(isp, ISP_LOGERR,
965 		    "Firmware rejected CTIO for bad nexus %d/%d/%d",
966 		    ct->ct_iid, ct->ct_tgt, ct->ct_lun);
967 		break;
968 
969 	case CT_RSELTMO:
970 		fmsg = "Reselection";
971 		/*FALLTHROUGH*/
972 	case CT_TIMEOUT:
973 		if (fmsg == NULL)
974 			fmsg = "Command";
975 		isp_prt(isp, ISP_LOGERR, "Firmware timed out on %s", fmsg);
976 		break;
977 
978 	case	CT_PANIC:
979 		if (fmsg == NULL)
980 			fmsg = "Unrecoverable Error";
981 		/*FALLTHROUGH*/
982 	case CT_ERR:
983 		if (fmsg == NULL)
984 			fmsg = "Completed with Error";
985 		/*FALLTHROUGH*/
986 	case CT_PHASE_ERROR:
987 		if (fmsg == NULL)
988 			fmsg = "Phase Sequence Error";
989 		/*FALLTHROUGH*/
990 	case CT_TERMINATED:
991 		if (fmsg == NULL)
992 			fmsg = "terminated by TERMINATE TRANSFER";
993 		/*FALLTHROUGH*/
994 	case CT_NOACK:
995 		if (fmsg == NULL)
996 			fmsg = "unacknowledged Immediate Notify pending";
997 		isp_prt(isp, ISP_LOGERR, "CTIO returned by f/w- %s", fmsg);
998 		break;
999 	default:
1000 		isp_prt(isp, ISP_LOGERR, "Unknown CTIO status 0x%x",
1001 		    ct->ct_status & ~QLTM_SVALID);
1002 		break;
1003 	}
1004 
1005 	if (xs == NULL) {
1006 		/*
1007 		 * There may be more than one CTIO for a data transfer,
1008 		 * or this may be a status CTIO we're not monitoring.
1009 		 *
1010 		 * The assumption is that they'll all be returned in the
1011 		 * order we got them.
1012 		 */
1013 		if (ct->ct_syshandle == 0) {
1014 			if ((ct->ct_flags & CT_SENDSTATUS) == 0) {
1015 				isp_prt(isp, pl,
1016 				    "intermediate CTIO completed ok");
1017 			} else {
1018 				isp_prt(isp, pl,
1019 				    "unmonitored CTIO completed ok");
1020 			}
1021 		} else {
1022 			isp_prt(isp, pl,
1023 			    "NO xs for CTIO (handle 0x%x) status 0x%x",
1024 			    ct->ct_syshandle, ct->ct_status & ~QLTM_SVALID);
1025 		}
1026 	} else {
1027 		/*
1028 		 * Final CTIO completed. Release DMA resources and
1029 		 * notify platform dependent layers.
1030 		 */
1031 		if ((ct->ct_flags & CT_DATAMASK) != CT_NO_DATA) {
1032 			ISP_DMAFREE(isp, xs, ct->ct_syshandle);
1033 		}
1034 		isp_prt(isp, pl, "final CTIO complete");
1035 		/*
1036 		 * The platform layer will destroy the handle if appropriate.
1037 		 */
1038 		(void) isp_async(isp, ISPASYNC_TARGET_ACTION, ct);
1039 	}
1040 }
1041 
1042 static void
1043 isp_handle_ctio2(struct ispsoftc *isp, ct2_entry_t *ct)
1044 {
1045 	XS_T *xs;
1046 	int pl = ISP_LOGTDEBUG2;
1047 	char *fmsg = NULL;
1048 
1049 	if (ct->ct_syshandle) {
1050 		xs = isp_find_xs(isp, ct->ct_syshandle);
1051 		if (xs == NULL)
1052 			pl = ISP_LOGALL;
1053 	} else {
1054 		xs = NULL;
1055 	}
1056 
1057 	switch(ct->ct_status & ~QLTM_SVALID) {
1058 	case CT_BUS_ERROR:
1059 		isp_prt(isp, ISP_LOGERR, "PCI DMA Bus Error");
1060 		/* FALL Through */
1061 	case CT_DATA_OVER:
1062 	case CT_DATA_UNDER:
1063 	case CT_OK:
1064 		/*
1065 		 * There are generally 2 possibilities as to why we'd get
1066 		 * this condition:
1067 		 * 	We sent or received data.
1068 		 * 	We sent status & command complete.
1069 		 */
1070 
1071 		break;
1072 
1073 	case CT_BDR_MSG:
1074 		/*
1075 		 * Target Reset function received.
1076 		 *
1077 		 * The firmware generates an async mailbox interupt to
1078 		 * notify us of this and returns outstanding CTIOs with this
1079 		 * status. These CTIOs are handled in that same way as
1080 		 * CT_ABORTED ones, so just fall through here.
1081 		 */
1082 		fmsg = "TARGET RESET Task Management Function Received";
1083 		/*FALLTHROUGH*/
1084 	case CT_RESET:
1085 		if (fmsg == NULL)
1086 			fmsg = "LIP Reset";
1087 		/*FALLTHROUGH*/
1088 	case CT_ABORTED:
1089 		/*
1090 		 * When an Abort message is received the firmware goes to
1091 		 * Bus Free and returns all outstanding CTIOs with the status
1092 		 * set, then sends us an Immediate Notify entry.
1093 		 */
1094 		if (fmsg == NULL)
1095 			fmsg = "ABORT Task Management Function Received";
1096 
1097 		isp_prt(isp, ISP_LOGERR, "CTIO2 destroyed by %s", fmsg);
1098 		break;
1099 
1100 	case CT_INVAL:
1101 		/*
1102 		 * CTIO rejected by the firmware - invalid data direction.
1103 		 */
1104 		isp_prt(isp, ISP_LOGERR, "CTIO2 had wrong data directiond");
1105 		break;
1106 
1107 	case CT_RSELTMO:
1108 		fmsg = "failure to reconnect to initiator";
1109 		/*FALLTHROUGH*/
1110 	case CT_TIMEOUT:
1111 		if (fmsg == NULL)
1112 			fmsg = "command";
1113 		isp_prt(isp, ISP_LOGERR, "Firmware timed out on %s", fmsg);
1114 		break;
1115 
1116 	case CT_ERR:
1117 		fmsg = "Completed with Error";
1118 		/*FALLTHROUGH*/
1119 	case CT_LOGOUT:
1120 		if (fmsg == NULL)
1121 			fmsg = "Port Logout";
1122 		/*FALLTHROUGH*/
1123 	case CT_PORTNOTAVAIL:
1124 		if (fmsg == NULL)
1125 			fmsg = "Port not available";
1126 	case CT_PORTCHANGED:
1127 		if (fmsg == NULL)
1128 			fmsg = "Port Changed";
1129 	case CT_NOACK:
1130 		if (fmsg == NULL)
1131 			fmsg = "unacknowledged Immediate Notify pending";
1132 		isp_prt(isp, ISP_LOGERR, "CTIO returned by f/w- %s", fmsg);
1133 		break;
1134 
1135 	case CT_INVRXID:
1136 		/*
1137 		 * CTIO rejected by the firmware because an invalid RX_ID.
1138 		 * Just print a message.
1139 		 */
1140 		isp_prt(isp, ISP_LOGERR,
1141 		    "CTIO2 completed with Invalid RX_ID 0x%x", ct->ct_rxid);
1142 		break;
1143 
1144 	default:
1145 		isp_prt(isp, ISP_LOGERR, "Unknown CTIO2 status 0x%x",
1146 		    ct->ct_status & ~QLTM_SVALID);
1147 		break;
1148 	}
1149 
1150 	if (xs == NULL) {
1151 		/*
1152 		 * There may be more than one CTIO for a data transfer,
1153 		 * or this may be a status CTIO we're not monitoring.
1154 		 *
1155 		 * The assumption is that they'll all be returned in the
1156 		 * order we got them.
1157 		 */
1158 		if (ct->ct_syshandle == 0) {
1159 			if ((ct->ct_flags & CT_SENDSTATUS) == 0) {
1160 				isp_prt(isp, pl,
1161 				    "intermediate CTIO completed ok");
1162 			} else {
1163 				isp_prt(isp, pl,
1164 				    "unmonitored CTIO completed ok");
1165 			}
1166 		} else {
1167 			isp_prt(isp, pl,
1168 			    "NO xs for CTIO (handle 0x%x) status 0x%x",
1169 			    ct->ct_syshandle, ct->ct_status & ~QLTM_SVALID);
1170 		}
1171 	} else {
1172 		if ((ct->ct_flags & CT2_DATAMASK) != CT2_NO_DATA) {
1173 			ISP_DMAFREE(isp, xs, ct->ct_syshandle);
1174 		}
1175 		if (ct->ct_flags & CT_SENDSTATUS) {
1176 			/*
1177 			 * Sent status and command complete.
1178 			 *
1179 			 * We're now really done with this command, so we
1180 			 * punt to the platform dependent layers because
1181 			 * only there can we do the appropriate command
1182 			 * complete thread synchronization.
1183 			 */
1184 			isp_prt(isp, pl, "status CTIO complete");
1185 		} else {
1186 			/*
1187 			 * Final CTIO completed. Release DMA resources and
1188 			 * notify platform dependent layers.
1189 			 */
1190 			isp_prt(isp, pl, "data CTIO complete");
1191 		}
1192 		(void) isp_async(isp, ISPASYNC_TARGET_ACTION, ct);
1193 		/*
1194 		 * The platform layer will destroy the handle if appropriate.
1195 		 */
1196 	}
1197 }
1198 #endif
1199