xref: /freebsd/sys/dev/isp/isp_target.c (revision 9207b4cff7b8d483f4dd3c62266c2b58819eb7f9)
1 /* $FreeBSD$ */
2 /*
3  * Machine and OS Independent Target Mode Code for the Qlogic SCSI/FC adapters.
4  *
5  * Copyright (c) 1999, 2000, 2001 by Matthew Jacob
6  * All rights reserved.
7  * mjacob@feral.com
8  *
9  * Redistribution and use in source and binary forms, with or without
10  * modification, are permitted provided that the following conditions
11  * are met:
12  * 1. Redistributions of source code must retain the above copyright
13  *    notice immediately at the beginning of the file, without modification,
14  *    this list of conditions, and the following disclaimer.
15  * 2. The name of the author may not be used to endorse or promote products
16  *    derived from this software without specific prior written permission.
17  *
18  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
19  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21  * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR
22  * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28  * SUCH DAMAGE.
29  */
30 
31 /*
32  * Include header file appropriate for platform we're building on.
33  */
34 
35 #ifdef	__NetBSD__
36 #include <dev/ic/isp_netbsd.h>
37 #endif
38 #ifdef	__FreeBSD__
39 #include <dev/isp/isp_freebsd.h>
40 #endif
41 #ifdef	__OpenBSD__
42 #include <dev/ic/isp_openbsd.h>
43 #endif
44 #ifdef	__linux__
45 #include "isp_linux.h"
46 #endif
47 
48 #ifdef	ISP_TARGET_MODE
49 static const char atiocope[] =
50     "ATIO returned for lun %d because it was in the middle of Bus Device Reset "
51     "on bus %d";
52 static const char atior[] =
53     "ATIO returned on for lun %d on from IID %d because a Bus Reset occurred "
54     "on bus %d";
55 
56 static void isp_got_msg(struct ispsoftc *, int, in_entry_t *);
57 static void isp_got_msg_fc(struct ispsoftc *, int, in_fcentry_t *);
58 static void isp_notify_ack(struct ispsoftc *, void *);
59 static void isp_handle_atio(struct ispsoftc *, at_entry_t *);
60 static void isp_handle_atio2(struct ispsoftc *, at2_entry_t *);
61 static void isp_handle_ctio(struct ispsoftc *, ct_entry_t *);
62 static void isp_handle_ctio2(struct ispsoftc *, ct2_entry_t *);
63 
64 /*
65  * The Qlogic driver gets an interrupt to look at response queue entries.
66  * Some of these are status completions for initiatior mode commands, but
67  * if target mode is enabled, we get a whole wad of response queue entries
68  * to be handled here.
69  *
70  * Basically the split into 3 main groups: Lun Enable/Modification responses,
71  * SCSI Command processing, and Immediate Notification events.
72  *
73  * You start by writing a request queue entry to enable target mode (and
74  * establish some resource limitations which you can modify later).
75  * The f/w responds with a LUN ENABLE or LUN MODIFY response with
76  * the status of this action. If the enable was successful, you can expect...
77  *
78  * Response queue entries with SCSI commands encapsulate show up in an ATIO
79  * (Accept Target IO) type- sometimes with enough info to stop the command at
80  * this level. Ultimately the driver has to feed back to the f/w's request
81  * queue a sequence of CTIOs (continue target I/O) that describe data to
82  * be moved and/or status to be sent) and finally finishing with sending
83  * to the f/w's response queue an ATIO which then completes the handshake
84  * with the f/w for that command. There's a lot of variations on this theme,
85  * including flags you can set in the CTIO for the Qlogic 2X00 fibre channel
86  * cards that 'auto-replenish' the f/w's ATIO count, but this is the basic
87  * gist of it.
88  *
89  * The third group that can show up in the response queue are Immediate
90  * Notification events. These include things like notifications of SCSI bus
91  * resets, or Bus Device Reset messages or other messages received. This
92  * a classic oddbins area. It can get  a little weird because you then turn
93  * around and acknowledge the Immediate Notify by writing an entry onto the
94  * request queue and then the f/w turns around and gives you an acknowledgement
95  * to *your* acknowledgement on the response queue (the idea being to let
96  * the f/w tell you when the event is *really* over I guess).
97  *
98  */
99 
100 
101 /*
102  * A new response queue entry has arrived. The interrupt service code
103  * has already swizzled it into the platform dependent from canonical form.
104  *
105  * Because of the way this driver is designed, unfortunately most of the
106  * actual synchronization work has to be done in the platform specific
107  * code- we have no synchroniation primitives in the common code.
108  */
109 
110 int
111 isp_target_notify(struct ispsoftc *isp, void *vptr, u_int16_t *optrp)
112 {
113 	u_int16_t status, seqid;
114 	union {
115 		at_entry_t	*atiop;
116 		at2_entry_t	*at2iop;
117 		ct_entry_t	*ctiop;
118 		ct2_entry_t	*ct2iop;
119 		lun_entry_t	*lunenp;
120 		in_entry_t	*inotp;
121 		in_fcentry_t	*inot_fcp;
122 		na_entry_t	*nackp;
123 		na_fcentry_t	*nack_fcp;
124 		isphdr_t	*hp;
125 		void *		*vp;
126 #define	atiop		unp.atiop
127 #define	at2iop		unp.at2iop
128 #define	ctiop		unp.ctiop
129 #define	ct2iop		unp.ct2iop
130 #define	lunenp		unp.lunenp
131 #define	inotp		unp.inotp
132 #define	inot_fcp	unp.inot_fcp
133 #define	nackp		unp.nackp
134 #define	nack_fcp	unp.nack_fcp
135 #define	hdrp		unp.hp
136 	} unp;
137 	u_int8_t local[QENTRY_LEN];
138 	int bus, type, rval = 0;
139 
140 	type = isp_get_response_type(isp, (isphdr_t *)vptr);
141 	unp.vp = vptr;
142 
143 	ISP_TDQE(isp, "isp_target_notify", (int) *optrp, vptr);
144 
145 	switch(type) {
146 	case RQSTYPE_ATIO:
147 		isp_get_atio(isp, atiop, (at_entry_t *) local);
148 		isp_handle_atio(isp, (at_entry_t *) local);
149 		break;
150 	case RQSTYPE_CTIO:
151 		isp_get_ctio(isp, ctiop, (ct_entry_t *) local);
152 		isp_handle_ctio(isp, (ct_entry_t *) local);
153 		break;
154 	case RQSTYPE_ATIO2:
155 		isp_get_atio2(isp, at2iop, (at2_entry_t *) local);
156 		isp_handle_atio2(isp, (at2_entry_t *) local);
157 		break;
158 	case RQSTYPE_CTIO2:
159 		isp_get_ctio2(isp, ct2iop, (ct2_entry_t *) local);
160 		isp_handle_ctio2(isp, (ct2_entry_t *) local);
161 		break;
162 	case RQSTYPE_ENABLE_LUN:
163 	case RQSTYPE_MODIFY_LUN:
164 		isp_get_enable_lun(isp, lunenp, (lun_entry_t *) local);
165 		(void) isp_async(isp, ISPASYNC_TARGET_ACTION, local);
166 		break;
167 
168 	case RQSTYPE_NOTIFY:
169 		/*
170 		 * Either the ISP received a SCSI message it can't
171 		 * handle, or it's returning an Immed. Notify entry
172 		 * we sent. We can send Immed. Notify entries to
173 		 * increment the firmware's resource count for them
174 		 * (we set this initially in the Enable Lun entry).
175 		 */
176 		bus = 0;
177 		if (IS_FC(isp)) {
178 			isp_get_notify_fc(isp, inot_fcp, (in_fcentry_t *)local);
179 			inot_fcp = (in_fcentry_t *) local;
180 			status = inot_fcp->in_status;
181 			seqid = inot_fcp->in_seqid;
182 		} else {
183 			isp_get_notify(isp, inotp, (in_entry_t *)local);
184 			inotp = (in_entry_t *) local;
185 			status = inotp->in_status & 0xff;
186 			seqid = inotp->in_seqid;
187 			if (IS_DUALBUS(isp)) {
188 				bus = GET_BUS_VAL(inotp->in_iid);
189 				SET_BUS_VAL(inotp->in_iid, 0);
190 			}
191 		}
192 		isp_prt(isp, ISP_LOGTDEBUG0,
193 		    "Immediate Notify On Bus %d, status=0x%x seqid=0x%x",
194 		    bus, status, seqid);
195 
196 		/*
197 		 * ACK it right away.
198 		 */
199 		isp_notify_ack(isp, (status == IN_RESET)? NULL : local);
200 		switch (status) {
201 		case IN_RESET:
202 			(void) isp_async(isp, ISPASYNC_BUS_RESET, &bus);
203 			break;
204 		case IN_MSG_RECEIVED:
205 		case IN_IDE_RECEIVED:
206 			if (IS_FC(isp)) {
207 				isp_got_msg_fc(isp, bus, (in_fcentry_t *)local);
208 			} else {
209 				isp_got_msg(isp, bus, (in_entry_t *)local);
210 			}
211 			break;
212 		case IN_RSRC_UNAVAIL:
213 			isp_prt(isp, ISP_LOGWARN, "Firmware out of ATIOs");
214 			break;
215 		case IN_ABORT_TASK:
216 			isp_prt(isp, ISP_LOGWARN,
217 			    "Abort Task from IID %d RX_ID 0x%x",
218 			    inot_fcp->in_iid, seqid);
219 			(void) isp_async(isp, ISPASYNC_TARGET_ACTION, &bus);
220 			break;
221 		case IN_PORT_LOGOUT:
222 			isp_prt(isp, ISP_LOGWARN,
223 			    "Port Logout for Initiator %d RX_ID 0x%x",
224 			    inot_fcp->in_iid, seqid);
225 			break;
226 		case IN_PORT_CHANGED:
227 			isp_prt(isp, ISP_LOGWARN,
228 			    "Port Changed for Initiator %d RX_ID 0x%x",
229 			    inot_fcp->in_iid, seqid);
230 			break;
231 		case IN_GLOBAL_LOGO:
232 			isp_prt(isp, ISP_LOGWARN, "All ports logged out");
233 			break;
234 		default:
235 			isp_prt(isp, ISP_LOGERR,
236 			    "bad status (0x%x) in isp_target_notify", status);
237 			break;
238 		}
239 		break;
240 
241 	case RQSTYPE_NOTIFY_ACK:
242 		/*
243 		 * The ISP is acknowledging our acknowledgement of an
244 		 * Immediate Notify entry for some asynchronous event.
245 		 */
246 		if (IS_FC(isp)) {
247 			isp_get_notify_ack_fc(isp, nack_fcp,
248 			    (na_fcentry_t *)local);
249 			nack_fcp = (na_fcentry_t *)local;
250 			isp_prt(isp, ISP_LOGTDEBUG1,
251 			    "Notify Ack status=0x%x seqid 0x%x",
252 			    nack_fcp->na_status, nack_fcp->na_seqid);
253 		} else {
254 			isp_get_notify_ack(isp, nackp, (na_entry_t *)local);
255 			nackp = (na_entry_t *)local;
256 			isp_prt(isp, ISP_LOGTDEBUG1,
257 			    "Notify Ack event 0x%x status=0x%x seqid 0x%x",
258 			    nackp->na_event, nackp->na_status, nackp->na_seqid);
259 		}
260 		break;
261 	default:
262 		isp_prt(isp, ISP_LOGERR,
263 		    "Unknown entry type 0x%x in isp_target_notify", type);
264 		rval = -1;
265 		break;
266 	}
267 #undef	atiop
268 #undef	at2iop
269 #undef	ctiop
270 #undef	ct2iop
271 #undef	lunenp
272 #undef	inotp
273 #undef	inot_fcp
274 #undef	nackp
275 #undef	nack_fcp
276 #undef	hdrp
277 	return (rval);
278 }
279 
280 
281 /*
282  * Toggle (on/off) target mode for bus/target/lun
283  *
284  * The caller has checked for overlap and legality.
285  *
286  * Note that not all of bus, target or lun can be paid attention to.
287  * Note also that this action will not be complete until the f/w writes
288  * response entry. The caller is responsible for synchronizing this.
289  */
290 int
291 isp_lun_cmd(struct ispsoftc *isp, int cmd, int bus, int tgt, int lun,
292     int cmd_cnt, int inot_cnt, u_int32_t opaque)
293 {
294 	lun_entry_t el;
295 	u_int16_t nxti, optr;
296 	void *outp;
297 
298 
299 	MEMZERO(&el, sizeof (el));
300 	if (IS_DUALBUS(isp)) {
301 		el.le_rsvd = (bus & 0x1) << 7;
302 	}
303 	el.le_cmd_count = cmd_cnt;
304 	el.le_in_count = inot_cnt;
305 	if (cmd == RQSTYPE_ENABLE_LUN) {
306 		if (IS_SCSI(isp)) {
307 			el.le_flags = LUN_TQAE|LUN_DISAD;
308 			el.le_cdb6len = 12;
309 			el.le_cdb7len = 12;
310 		}
311 	} else if (cmd == -RQSTYPE_ENABLE_LUN) {
312 		cmd = RQSTYPE_ENABLE_LUN;
313 		el.le_cmd_count = 0;
314 		el.le_in_count = 0;
315 	} else if (cmd == -RQSTYPE_MODIFY_LUN) {
316 		cmd = RQSTYPE_MODIFY_LUN;
317 		el.le_ops = LUN_CCDECR | LUN_INDECR;
318 	} else {
319 		el.le_ops = LUN_CCINCR | LUN_ININCR;
320 	}
321 	el.le_header.rqs_entry_type = cmd;
322 	el.le_header.rqs_entry_count = 1;
323 	el.le_reserved = opaque;
324 	if (IS_SCSI(isp)) {
325 		el.le_tgt = tgt;
326 		el.le_lun = lun;
327 	} else if ((FCPARAM(isp)->isp_fwattr & ISP_FW_ATTR_SCCLUN) == 0) {
328 		el.le_lun = lun;
329 	}
330 	el.le_timeout = 2;
331 
332 	if (isp_getrqentry(isp, &nxti, &optr, &outp)) {
333 		isp_prt(isp, ISP_LOGERR,
334 		    "Request Queue Overflow in isp_lun_cmd");
335 		return (-1);
336 	}
337 	ISP_TDQE(isp, "isp_lun_cmd", (int) optr, &el);
338 	isp_put_enable_lun(isp, &el, outp);
339 	ISP_ADD_REQUEST(isp, nxti);
340 	return (0);
341 }
342 
343 
344 int
345 isp_target_put_entry(struct ispsoftc *isp, void *ap)
346 {
347 	void *outp;
348 	u_int16_t nxti, optr;
349 	u_int8_t etype = ((isphdr_t *) ap)->rqs_entry_type;
350 
351 	if (isp_getrqentry(isp, &nxti, &optr, &outp)) {
352 		isp_prt(isp, ISP_LOGWARN,
353 		    "Request Queue Overflow in isp_target_put_entry");
354 		return (-1);
355 	}
356 	switch (etype) {
357 	case RQSTYPE_ATIO:
358 		isp_put_atio(isp, (at_entry_t *) ap, (at_entry_t *) outp);
359 		break;
360 	case RQSTYPE_ATIO2:
361 		isp_put_atio2(isp, (at2_entry_t *) ap, (at2_entry_t *) outp);
362 		break;
363 	case RQSTYPE_CTIO:
364 		isp_put_ctio(isp, (ct_entry_t *) ap, (ct_entry_t *) outp);
365 		break;
366 	case RQSTYPE_CTIO2:
367 		isp_put_ctio2(isp, (ct2_entry_t *) ap, (ct2_entry_t *) outp);
368 		break;
369 	default:
370 		isp_prt(isp, ISP_LOGERR,
371 		    "Unknown type 0x%x in isp_put_entry", etype);
372 		return (-1);
373 	}
374 
375 	ISP_TDQE(isp, "isp_target_put_entry", (int) optr, ap);;
376 	ISP_ADD_REQUEST(isp, nxti);
377 	return (0);
378 }
379 
380 int
381 isp_target_put_atio(struct ispsoftc *isp, void *arg)
382 {
383 	union {
384 		at_entry_t _atio;
385 		at2_entry_t _atio2;
386 	} atun;
387 
388 	MEMZERO(&atun, sizeof atun);
389 	if (IS_FC(isp)) {
390 		at2_entry_t *aep = arg;
391 		atun._atio2.at_header.rqs_entry_type = RQSTYPE_ATIO2;
392 		atun._atio2.at_header.rqs_entry_count = 1;
393 		if (FCPARAM(isp)->isp_fwattr & ISP_FW_ATTR_SCCLUN) {
394 			atun._atio2.at_scclun = (u_int16_t) aep->at_scclun;
395 		} else {
396 			atun._atio2.at_lun = (u_int8_t) aep->at_lun;
397 		}
398 		atun._atio2.at_status = CT_OK;
399 	} else {
400 		at_entry_t *aep = arg;
401 		atun._atio.at_header.rqs_entry_type = RQSTYPE_ATIO;
402 		atun._atio.at_header.rqs_entry_count = 1;
403 		atun._atio.at_handle = aep->at_handle;
404 		atun._atio.at_iid = aep->at_iid;
405 		atun._atio.at_tgt = aep->at_tgt;
406 		atun._atio.at_lun = aep->at_lun;
407 		atun._atio.at_tag_type = aep->at_tag_type;
408 		atun._atio.at_tag_val = aep->at_tag_val;
409 		atun._atio.at_status = (aep->at_flags & AT_TQAE);
410 		atun._atio.at_status |= CT_OK;
411 	}
412 	return (isp_target_put_entry(isp, &atun));
413 }
414 
415 /*
416  * Command completion- both for handling cases of no resources or
417  * no blackhole driver, or other cases where we have to, inline,
418  * finish the command sanely, or for normal command completion.
419  *
420  * The 'completion' code value has the scsi status byte in the low 8 bits.
421  * If status is a CHECK CONDITION and bit 8 is nonzero, then bits 12..15 have
422  * the sense key and  bits 16..23 have the ASCQ and bits 24..31 have the ASC
423  * values.
424  *
425  * NB: the key, asc, ascq, cannot be used for parallel SCSI as it doesn't
426  * NB: inline SCSI sense reporting. As such, we lose this information. XXX.
427  *
428  * For both parallel && fibre channel, we use the feature that does
429  * an automatic resource autoreplenish so we don't have then later do
430  * put of an atio to replenish the f/w's resource count.
431  */
432 
433 int
434 isp_endcmd(struct ispsoftc *isp, void *arg, u_int32_t code, u_int16_t hdl)
435 {
436 	int sts;
437 	union {
438 		ct_entry_t _ctio;
439 		ct2_entry_t _ctio2;
440 	} un;
441 
442 	MEMZERO(&un, sizeof un);
443 	sts = code & 0xff;
444 
445 	if (IS_FC(isp)) {
446 		at2_entry_t *aep = arg;
447 		ct2_entry_t *cto = &un._ctio2;
448 
449 		cto->ct_header.rqs_entry_type = RQSTYPE_CTIO2;
450 		cto->ct_header.rqs_entry_count = 1;
451 		cto->ct_iid = aep->at_iid;
452 		if ((FCPARAM(isp)->isp_fwattr & ISP_FW_ATTR_SCCLUN) == 0) {
453 			cto->ct_lun = aep->at_lun;
454 		}
455 		cto->ct_rxid = aep->at_rxid;
456 		cto->rsp.m1.ct_scsi_status = sts & 0xff;
457 		cto->ct_flags = CT2_SENDSTATUS | CT2_NO_DATA | CT2_FLAG_MODE1;
458 		if (hdl == 0) {
459 			cto->ct_flags |= CT2_CCINCR;
460 		}
461 		if (aep->at_datalen) {
462 			cto->ct_resid = aep->at_datalen;
463 			cto->rsp.m1.ct_scsi_status |= CT2_DATA_UNDER;
464 		}
465 		if ((sts & 0xff) == SCSI_CHECK && (sts & ECMD_SVALID)) {
466 			cto->rsp.m1.ct_resp[0] = 0xf0;
467 			cto->rsp.m1.ct_resp[2] = (code >> 12) & 0xf;
468 			cto->rsp.m1.ct_resp[7] = 8;
469 			cto->rsp.m1.ct_resp[12] = (code >> 24) & 0xff;
470 			cto->rsp.m1.ct_resp[13] = (code >> 16) & 0xff;
471 			cto->rsp.m1.ct_senselen = 16;
472 			cto->rsp.m1.ct_scsi_status |= CT2_SNSLEN_VALID;
473 		}
474 		cto->ct_syshandle = hdl;
475 	} else {
476 		at_entry_t *aep = arg;
477 		ct_entry_t *cto = &un._ctio;
478 
479 		cto->ct_header.rqs_entry_type = RQSTYPE_CTIO;
480 		cto->ct_header.rqs_entry_count = 1;
481 		cto->ct_fwhandle = aep->at_handle;
482 		cto->ct_iid = aep->at_iid;
483 		cto->ct_tgt = aep->at_tgt;
484 		cto->ct_lun = aep->at_lun;
485 		cto->ct_tag_type = aep->at_tag_type;
486 		cto->ct_tag_val = aep->at_tag_val;
487 		if (aep->at_flags & AT_TQAE) {
488 			cto->ct_flags |= CT_TQAE;
489 		}
490 		cto->ct_flags = CT_SENDSTATUS | CT_NO_DATA;
491 		if (hdl == 0) {
492 			cto->ct_flags |= CT_CCINCR;
493 		}
494 		cto->ct_scsi_status = sts;
495 		cto->ct_syshandle = hdl;
496 	}
497 	return (isp_target_put_entry(isp, &un));
498 }
499 
500 void
501 isp_target_async(struct ispsoftc *isp, int bus, int event)
502 {
503 	tmd_event_t evt;
504 	tmd_msg_t msg;
505 
506 	switch (event) {
507 	/*
508 	 * These three we handle here to propagate an effective bus reset
509 	 * upstream, but these do not require any immediate notify actions
510 	 * so we return when done.
511 	 */
512 	case ASYNC_LIP_F8:
513 	case ASYNC_LIP_OCCURRED:
514 	case ASYNC_LOOP_UP:
515 	case ASYNC_LOOP_DOWN:
516 	case ASYNC_LOOP_RESET:
517 	case ASYNC_PTPMODE:
518 		/*
519 		 * These don't require any immediate notify actions. We used
520 		 * treat them like SCSI Bus Resets, but that was just plain
521 		 * wrong. Let the normal CTIO completion report what occurred.
522 		 */
523                 return;
524 
525 	case ASYNC_BUS_RESET:
526 	case ASYNC_TIMEOUT_RESET:
527 		if (IS_FC(isp)) {
528 			return;	/* we'll be getting an inotify instead */
529 		}
530 		evt.ev_bus = bus;
531 		evt.ev_event = event;
532 		(void) isp_async(isp, ISPASYNC_TARGET_EVENT, &evt);
533 		break;
534 	case ASYNC_DEVICE_RESET:
535 		/*
536 		 * Bus Device Reset resets a specific target, so
537 		 * we pass this as a synthesized message.
538 		 */
539 		MEMZERO(&msg, sizeof msg);
540 		if (IS_FC(isp)) {
541 			msg.nt_iid = FCPARAM(isp)->isp_loopid;
542 		} else {
543 			msg.nt_iid = SDPARAM(isp)->isp_initiator_id;
544 		}
545 		msg.nt_bus = bus;
546 		msg.nt_msg[0] = MSG_BUS_DEV_RESET;
547 		(void) isp_async(isp, ISPASYNC_TARGET_MESSAGE, &msg);
548 		break;
549 	default:
550 		isp_prt(isp, ISP_LOGERR,
551 		    "isp_target_async: unknown event 0x%x", event);
552 		break;
553 	}
554 	if (isp->isp_state == ISP_RUNSTATE)
555 		isp_notify_ack(isp, NULL);
556 }
557 
558 
559 /*
560  * Process a received message.
561  * The ISP firmware can handle most messages, there are only
562  * a few that we need to deal with:
563  * - abort: clean up the current command
564  * - abort tag and clear queue
565  */
566 
567 static void
568 isp_got_msg(struct ispsoftc *isp, int bus, in_entry_t *inp)
569 {
570 	u_int8_t status = inp->in_status & ~QLTM_SVALID;
571 
572 	if (status == IN_IDE_RECEIVED || status == IN_MSG_RECEIVED) {
573 		tmd_msg_t msg;
574 
575 		MEMZERO(&msg, sizeof (msg));
576 		msg.nt_bus = bus;
577 		msg.nt_iid = inp->in_iid;
578 		msg.nt_tgt = inp->in_tgt;
579 		msg.nt_lun = inp->in_lun;
580 		msg.nt_tagtype = inp->in_tag_type;
581 		msg.nt_tagval = inp->in_tag_val;
582 		MEMCPY(msg.nt_msg, inp->in_msg, IN_MSGLEN);
583 		(void) isp_async(isp, ISPASYNC_TARGET_MESSAGE, &msg);
584 	} else {
585 		isp_prt(isp, ISP_LOGERR,
586 		    "unknown immediate notify status 0x%x", inp->in_status);
587 	}
588 }
589 
590 /*
591  * Synthesize a message from the task management flags in a FCP_CMND_IU.
592  */
593 static void
594 isp_got_msg_fc(struct ispsoftc *isp, int bus, in_fcentry_t *inp)
595 {
596 	int lun;
597 	static const char f1[] = "%s from iid %d lun %d seq 0x%x";
598 	static const char f2[] =
599 	    "unknown %s 0x%x lun %d iid %d task flags 0x%x seq 0x%x\n";
600 
601 	if (FCPARAM(isp)->isp_fwattr & ISP_FW_ATTR_SCCLUN) {
602 		lun = inp->in_scclun;
603 	} else {
604 		lun = inp->in_lun;
605 	}
606 
607 	if (inp->in_status != IN_MSG_RECEIVED) {
608 		isp_prt(isp, ISP_LOGINFO, f2, "immediate notify status",
609 		    inp->in_status, lun, inp->in_iid,
610 		    inp->in_task_flags,  inp->in_seqid);
611 	} else {
612 		tmd_msg_t msg;
613 
614 		MEMZERO(&msg, sizeof (msg));
615 		msg.nt_bus = bus;
616 		msg.nt_iid = inp->in_iid;
617 		msg.nt_tagval = inp->in_seqid;
618 		msg.nt_lun = lun;
619 
620 		if (inp->in_task_flags & TASK_FLAGS_ABORT_TASK) {
621 			isp_prt(isp, ISP_LOGINFO, f1, "ABORT TASK",
622 			    inp->in_iid, msg.nt_lun, inp->in_seqid);
623 			msg.nt_msg[0] = MSG_ABORT_TAG;
624 		} else if (inp->in_task_flags & TASK_FLAGS_CLEAR_TASK_SET) {
625 			isp_prt(isp, ISP_LOGINFO, f1, "CLEAR TASK SET",
626 			    inp->in_iid, msg.nt_lun, inp->in_seqid);
627 			msg.nt_msg[0] = MSG_CLEAR_QUEUE;
628 		} else if (inp->in_task_flags & TASK_FLAGS_TARGET_RESET) {
629 			isp_prt(isp, ISP_LOGINFO, f1, "TARGET RESET",
630 			    inp->in_iid, msg.nt_lun, inp->in_seqid);
631 			msg.nt_msg[0] = MSG_BUS_DEV_RESET;
632 		} else if (inp->in_task_flags & TASK_FLAGS_CLEAR_ACA) {
633 			isp_prt(isp, ISP_LOGINFO, f1, "CLEAR ACA",
634 			    inp->in_iid, msg.nt_lun, inp->in_seqid);
635 			/* ???? */
636 			msg.nt_msg[0] = MSG_REL_RECOVERY;
637 		} else if (inp->in_task_flags & TASK_FLAGS_TERMINATE_TASK) {
638 			isp_prt(isp, ISP_LOGINFO, f1, "TERMINATE TASK",
639 			    inp->in_iid, msg.nt_lun, inp->in_seqid);
640 			msg.nt_msg[0] = MSG_TERM_IO_PROC;
641 		} else {
642 			isp_prt(isp, ISP_LOGWARN, f2, "task flag",
643 			    inp->in_status, msg.nt_lun, inp->in_iid,
644 			    inp->in_task_flags,  inp->in_seqid);
645 		}
646 		if (msg.nt_msg[0]) {
647 			(void) isp_async(isp, ISPASYNC_TARGET_MESSAGE, &msg);
648 		}
649 	}
650 }
651 
652 static void
653 isp_notify_ack(struct ispsoftc *isp, void *arg)
654 {
655 	char storage[QENTRY_LEN];
656 	u_int16_t nxti, optr;
657 	void *outp;
658 
659 	if (isp_getrqentry(isp, &nxti, &optr, &outp)) {
660 		isp_prt(isp, ISP_LOGWARN,
661 		    "Request Queue Overflow For isp_notify_ack");
662 		return;
663 	}
664 
665 	MEMZERO(storage, QENTRY_LEN);
666 
667 	if (IS_FC(isp)) {
668 		na_fcentry_t *na = (na_fcentry_t *) storage;
669 		if (arg) {
670 			in_fcentry_t *inp = arg;
671 			MEMCPY(storage, arg, sizeof (isphdr_t));
672 			na->na_iid = inp->in_iid;
673 			if (FCPARAM(isp)->isp_fwattr & ISP_FW_ATTR_SCCLUN) {
674 				na->na_lun = inp->in_scclun;
675 			} else {
676 				na->na_lun = inp->in_lun;
677 			}
678 			na->na_task_flags = inp->in_task_flags;
679 			na->na_seqid = inp->in_seqid;
680 			na->na_flags = NAFC_RCOUNT;
681 			if (inp->in_status == IN_RESET) {
682 				na->na_flags |= NAFC_RST_CLRD;
683 			}
684 		} else {
685 			na->na_flags = NAFC_RST_CLRD;
686 		}
687 		na->na_header.rqs_entry_type = RQSTYPE_NOTIFY_ACK;
688 		na->na_header.rqs_entry_count = 1;
689 		isp_put_notify_ack_fc(isp, na, (na_fcentry_t *)outp);
690 	} else {
691 		na_entry_t *na = (na_entry_t *) storage;
692 		if (arg) {
693 			in_entry_t *inp = arg;
694 			MEMCPY(storage, arg, sizeof (isphdr_t));
695 			na->na_iid = inp->in_iid;
696 			na->na_lun = inp->in_lun;
697 			na->na_tgt = inp->in_tgt;
698 			na->na_seqid = inp->in_seqid;
699 			if (inp->in_status == IN_RESET) {
700 				na->na_event = NA_RST_CLRD;
701 			}
702 		} else {
703 			na->na_event = NA_RST_CLRD;
704 		}
705 		na->na_header.rqs_entry_type = RQSTYPE_NOTIFY_ACK;
706 		na->na_header.rqs_entry_count = 1;
707 		isp_put_notify_ack(isp, na, (na_entry_t *)outp);
708 	}
709 	ISP_TDQE(isp, "isp_notify_ack", (int) optr, storage);
710 	ISP_ADD_REQUEST(isp, nxti);
711 }
712 
713 static void
714 isp_handle_atio(struct ispsoftc *isp, at_entry_t *aep)
715 {
716 	int lun;
717 	lun = aep->at_lun;
718 	/*
719 	 * The firmware status (except for the QLTM_SVALID bit) indicates
720 	 * why this ATIO was sent to us.
721 	 *
722 	 * If QLTM_SVALID is set, the firware has recommended Sense Data.
723 	 *
724 	 * If the DISCONNECTS DISABLED bit is set in the flags field,
725 	 * we're still connected on the SCSI bus - i.e. the initiator
726 	 * did not set DiscPriv in the identify message. We don't care
727 	 * about this so it's ignored.
728 	 */
729 
730 	switch(aep->at_status & ~QLTM_SVALID) {
731 	case AT_PATH_INVALID:
732 		/*
733 		 * ATIO rejected by the firmware due to disabled lun.
734 		 */
735 		isp_prt(isp, ISP_LOGERR,
736 		    "rejected ATIO for disabled lun %d", lun);
737 		break;
738 	case AT_NOCAP:
739 		/*
740 		 * Requested Capability not available
741 		 * We sent an ATIO that overflowed the firmware's
742 		 * command resource count.
743 		 */
744 		isp_prt(isp, ISP_LOGERR,
745 		    "rejected ATIO for lun %d because of command count"
746 		    " overflow", lun);
747 		break;
748 
749 	case AT_BDR_MSG:
750 		/*
751 		 * If we send an ATIO to the firmware to increment
752 		 * its command resource count, and the firmware is
753 		 * recovering from a Bus Device Reset, it returns
754 		 * the ATIO with this status. We set the command
755 		 * resource count in the Enable Lun entry and do
756 		 * not increment it. Therefore we should never get
757 		 * this status here.
758 		 */
759 		isp_prt(isp, ISP_LOGERR, atiocope, lun,
760 		    GET_BUS_VAL(aep->at_iid));
761 		break;
762 
763 	case AT_CDB:		/* Got a CDB */
764 	case AT_PHASE_ERROR:	/* Bus Phase Sequence Error */
765 		/*
766 		 * Punt to platform specific layer.
767 		 */
768 		(void) isp_async(isp, ISPASYNC_TARGET_ACTION, aep);
769 		break;
770 
771 	case AT_RESET:
772 		/*
773 		 * A bus reset came along an blew away this command. Why
774 		 * they do this in addition the async event code stuff,
775 		 * I dunno.
776 		 *
777 		 * Ignore it because the async event will clear things
778 		 * up for us.
779 		 */
780 		isp_prt(isp, ISP_LOGWARN, atior, lun,
781 		    GET_IID_VAL(aep->at_iid), GET_BUS_VAL(aep->at_iid));
782 		break;
783 
784 
785 	default:
786 		isp_prt(isp, ISP_LOGERR,
787 		    "Unknown ATIO status 0x%x from initiator %d for lun %d",
788 		    aep->at_status, aep->at_iid, lun);
789 		(void) isp_target_put_atio(isp, aep);
790 		break;
791 	}
792 }
793 
794 static void
795 isp_handle_atio2(struct ispsoftc *isp, at2_entry_t *aep)
796 {
797 	int lun;
798 
799 	if (FCPARAM(isp)->isp_fwattr & ISP_FW_ATTR_SCCLUN) {
800 		lun = aep->at_scclun;
801 	} else {
802 		lun = aep->at_lun;
803 	}
804 
805 	/*
806 	 * The firmware status (except for the QLTM_SVALID bit) indicates
807 	 * why this ATIO was sent to us.
808 	 *
809 	 * If QLTM_SVALID is set, the firware has recommended Sense Data.
810 	 *
811 	 * If the DISCONNECTS DISABLED bit is set in the flags field,
812 	 * we're still connected on the SCSI bus - i.e. the initiator
813 	 * did not set DiscPriv in the identify message. We don't care
814 	 * about this so it's ignored.
815 	 */
816 
817 	switch(aep->at_status & ~QLTM_SVALID) {
818 	case AT_PATH_INVALID:
819 		/*
820 		 * ATIO rejected by the firmware due to disabled lun.
821 		 */
822 		isp_prt(isp, ISP_LOGERR,
823 		    "rejected ATIO2 for disabled lun %d", lun);
824 		break;
825 	case AT_NOCAP:
826 		/*
827 		 * Requested Capability not available
828 		 * We sent an ATIO that overflowed the firmware's
829 		 * command resource count.
830 		 */
831 		isp_prt(isp, ISP_LOGERR,
832 		    "rejected ATIO2 for lun %d- command count overflow", lun);
833 		break;
834 
835 	case AT_BDR_MSG:
836 		/*
837 		 * If we send an ATIO to the firmware to increment
838 		 * its command resource count, and the firmware is
839 		 * recovering from a Bus Device Reset, it returns
840 		 * the ATIO with this status. We set the command
841 		 * resource count in the Enable Lun entry and no
842 		 * not increment it. Therefore we should never get
843 		 * this status here.
844 		 */
845 		isp_prt(isp, ISP_LOGERR, atiocope, lun, 0);
846 		break;
847 
848 	case AT_CDB:		/* Got a CDB */
849 		/*
850 		 * Punt to platform specific layer.
851 		 */
852 		(void) isp_async(isp, ISPASYNC_TARGET_ACTION, aep);
853 		break;
854 
855 	case AT_RESET:
856 		/*
857 		 * A bus reset came along an blew away this command. Why
858 		 * they do this in addition the async event code stuff,
859 		 * I dunno.
860 		 *
861 		 * Ignore it because the async event will clear things
862 		 * up for us.
863 		 */
864 		isp_prt(isp, ISP_LOGERR, atior, lun, aep->at_iid, 0);
865 		break;
866 
867 
868 	default:
869 		isp_prt(isp, ISP_LOGERR,
870 		    "Unknown ATIO2 status 0x%x from initiator %d for lun %d",
871 		    aep->at_status, aep->at_iid, lun);
872 		(void) isp_target_put_atio(isp, aep);
873 		break;
874 	}
875 }
876 
877 static void
878 isp_handle_ctio(struct ispsoftc *isp, ct_entry_t *ct)
879 {
880 	void *xs;
881 	int pl = ISP_LOGTDEBUG2;
882 	char *fmsg = NULL;
883 
884 	if (ct->ct_syshandle) {
885 		xs = isp_find_xs(isp, ct->ct_syshandle);
886 		if (xs == NULL)
887 			pl = ISP_LOGALL;
888 	} else {
889 		xs = NULL;
890 	}
891 
892 	switch(ct->ct_status & ~QLTM_SVALID) {
893 	case CT_OK:
894 		/*
895 		 * There are generally 3 possibilities as to why we'd get
896 		 * this condition:
897 		 * 	We disconnected after receiving a CDB.
898 		 * 	We sent or received data.
899 		 * 	We sent status & command complete.
900 		 */
901 
902 		if (ct->ct_flags & CT_SENDSTATUS) {
903 			break;
904 		} else if ((ct->ct_flags & CT_DATAMASK) == CT_NO_DATA) {
905 			/*
906 			 * Nothing to do in this case.
907 			 */
908 			isp_prt(isp, pl, "CTIO- iid %d disconnected OK",
909 			    ct->ct_iid);
910 			return;
911 		}
912 		break;
913 
914 	case CT_BDR_MSG:
915 		/*
916 		 * Bus Device Reset message received or the SCSI Bus has
917 		 * been Reset; the firmware has gone to Bus Free.
918 		 *
919 		 * The firmware generates an async mailbox interupt to
920 		 * notify us of this and returns outstanding CTIOs with this
921 		 * status. These CTIOs are handled in that same way as
922 		 * CT_ABORTED ones, so just fall through here.
923 		 */
924 		fmsg = "Bus Device Reset";
925 		/*FALLTHROUGH*/
926 	case CT_RESET:
927 		if (fmsg == NULL)
928 			fmsg = "Bus Reset";
929 		/*FALLTHROUGH*/
930 	case CT_ABORTED:
931 		/*
932 		 * When an Abort message is received the firmware goes to
933 		 * Bus Free and returns all outstanding CTIOs with the status
934 		 * set, then sends us an Immediate Notify entry.
935 		 */
936 		if (fmsg == NULL)
937 			fmsg = "ABORT TAG message sent by Initiator";
938 
939 		isp_prt(isp, ISP_LOGWARN, "CTIO destroyed by %s", fmsg);
940 		break;
941 
942 	case CT_INVAL:
943 		/*
944 		 * CTIO rejected by the firmware due to disabled lun.
945 		 * "Cannot Happen".
946 		 */
947 		isp_prt(isp, ISP_LOGERR,
948 		    "Firmware rejected CTIO for disabled lun %d",
949 		    ct->ct_lun);
950 		break;
951 
952 	case CT_NOPATH:
953 		/*
954 		 * CTIO rejected by the firmware due "no path for the
955 		 * nondisconnecting nexus specified". This means that
956 		 * we tried to access the bus while a non-disconnecting
957 		 * command is in process.
958 		 */
959 		isp_prt(isp, ISP_LOGERR,
960 		    "Firmware rejected CTIO for bad nexus %d/%d/%d",
961 		    ct->ct_iid, ct->ct_tgt, ct->ct_lun);
962 		break;
963 
964 	case CT_RSELTMO:
965 		fmsg = "Reselection";
966 		/*FALLTHROUGH*/
967 	case CT_TIMEOUT:
968 		if (fmsg == NULL)
969 			fmsg = "Command";
970 		isp_prt(isp, ISP_LOGERR, "Firmware timed out on %s", fmsg);
971 		break;
972 
973 	case	CT_PANIC:
974 		if (fmsg == NULL)
975 			fmsg = "Unrecoverable Error";
976 		/*FALLTHROUGH*/
977 	case CT_ERR:
978 		if (fmsg == NULL)
979 			fmsg = "Completed with Error";
980 		/*FALLTHROUGH*/
981 	case CT_PHASE_ERROR:
982 		if (fmsg == NULL)
983 			fmsg = "Phase Sequence Error";
984 		/*FALLTHROUGH*/
985 	case CT_TERMINATED:
986 		if (fmsg == NULL)
987 			fmsg = "terminated by TERMINATE TRANSFER";
988 		/*FALLTHROUGH*/
989 	case CT_NOACK:
990 		if (fmsg == NULL)
991 			fmsg = "unacknowledged Immediate Notify pending";
992 		isp_prt(isp, ISP_LOGERR, "CTIO returned by f/w- %s", fmsg);
993 		break;
994 	default:
995 		isp_prt(isp, ISP_LOGERR, "Unknown CTIO status 0x%x",
996 		    ct->ct_status & ~QLTM_SVALID);
997 		break;
998 	}
999 
1000 	if (xs == NULL) {
1001 		/*
1002 		 * There may be more than one CTIO for a data transfer,
1003 		 * or this may be a status CTIO we're not monitoring.
1004 		 *
1005 		 * The assumption is that they'll all be returned in the
1006 		 * order we got them.
1007 		 */
1008 		if (ct->ct_syshandle == 0) {
1009 			if ((ct->ct_flags & CT_SENDSTATUS) == 0) {
1010 				isp_prt(isp, pl,
1011 				    "intermediate CTIO completed ok");
1012 			} else {
1013 				isp_prt(isp, pl,
1014 				    "unmonitored CTIO completed ok");
1015 			}
1016 		} else {
1017 			isp_prt(isp, pl,
1018 			    "NO xs for CTIO (handle 0x%x) status 0x%x",
1019 			    ct->ct_syshandle, ct->ct_status & ~QLTM_SVALID);
1020 		}
1021 	} else {
1022 		/*
1023 		 * Final CTIO completed. Release DMA resources and
1024 		 * notify platform dependent layers.
1025 		 */
1026 		if ((ct->ct_flags & CT_DATAMASK) != CT_NO_DATA) {
1027 			ISP_DMAFREE(isp, xs, ct->ct_syshandle);
1028 		}
1029 		isp_prt(isp, pl, "final CTIO complete");
1030 		/*
1031 		 * The platform layer will destroy the handle if appropriate.
1032 		 */
1033 		(void) isp_async(isp, ISPASYNC_TARGET_ACTION, ct);
1034 	}
1035 }
1036 
1037 static void
1038 isp_handle_ctio2(struct ispsoftc *isp, ct2_entry_t *ct)
1039 {
1040 	XS_T *xs;
1041 	int pl = ISP_LOGTDEBUG2;
1042 	char *fmsg = NULL;
1043 
1044 	if (ct->ct_syshandle) {
1045 		xs = isp_find_xs(isp, ct->ct_syshandle);
1046 		if (xs == NULL)
1047 			pl = ISP_LOGALL;
1048 	} else {
1049 		xs = NULL;
1050 	}
1051 
1052 	switch(ct->ct_status & ~QLTM_SVALID) {
1053 	case CT_BUS_ERROR:
1054 		isp_prt(isp, ISP_LOGERR, "PCI DMA Bus Error");
1055 		/* FALL Through */
1056 	case CT_DATA_OVER:
1057 	case CT_DATA_UNDER:
1058 	case CT_OK:
1059 		/*
1060 		 * There are generally 2 possibilities as to why we'd get
1061 		 * this condition:
1062 		 * 	We sent or received data.
1063 		 * 	We sent status & command complete.
1064 		 */
1065 
1066 		break;
1067 
1068 	case CT_BDR_MSG:
1069 		/*
1070 		 * Target Reset function received.
1071 		 *
1072 		 * The firmware generates an async mailbox interupt to
1073 		 * notify us of this and returns outstanding CTIOs with this
1074 		 * status. These CTIOs are handled in that same way as
1075 		 * CT_ABORTED ones, so just fall through here.
1076 		 */
1077 		fmsg = "TARGET RESET Task Management Function Received";
1078 		/*FALLTHROUGH*/
1079 	case CT_RESET:
1080 		if (fmsg == NULL)
1081 			fmsg = "LIP Reset";
1082 		/*FALLTHROUGH*/
1083 	case CT_ABORTED:
1084 		/*
1085 		 * When an Abort message is received the firmware goes to
1086 		 * Bus Free and returns all outstanding CTIOs with the status
1087 		 * set, then sends us an Immediate Notify entry.
1088 		 */
1089 		if (fmsg == NULL)
1090 			fmsg = "ABORT Task Management Function Received";
1091 
1092 		isp_prt(isp, ISP_LOGERR, "CTIO2 destroyed by %s", fmsg);
1093 		break;
1094 
1095 	case CT_INVAL:
1096 		/*
1097 		 * CTIO rejected by the firmware - invalid data direction.
1098 		 */
1099 		isp_prt(isp, ISP_LOGERR, "CTIO2 had wrong data directiond");
1100 		break;
1101 
1102 	case CT_RSELTMO:
1103 		fmsg = "failure to reconnect to initiator";
1104 		/*FALLTHROUGH*/
1105 	case CT_TIMEOUT:
1106 		if (fmsg == NULL)
1107 			fmsg = "command";
1108 		isp_prt(isp, ISP_LOGERR, "Firmware timed out on %s", fmsg);
1109 		break;
1110 
1111 	case CT_ERR:
1112 		fmsg = "Completed with Error";
1113 		/*FALLTHROUGH*/
1114 	case CT_LOGOUT:
1115 		if (fmsg == NULL)
1116 			fmsg = "Port Logout";
1117 		/*FALLTHROUGH*/
1118 	case CT_PORTNOTAVAIL:
1119 		if (fmsg == NULL)
1120 			fmsg = "Port not available";
1121 	case CT_PORTCHANGED:
1122 		if (fmsg == NULL)
1123 			fmsg = "Port Changed";
1124 	case CT_NOACK:
1125 		if (fmsg == NULL)
1126 			fmsg = "unacknowledged Immediate Notify pending";
1127 		isp_prt(isp, ISP_LOGERR, "CTIO returned by f/w- %s", fmsg);
1128 		break;
1129 
1130 	case CT_INVRXID:
1131 		/*
1132 		 * CTIO rejected by the firmware because an invalid RX_ID.
1133 		 * Just print a message.
1134 		 */
1135 		isp_prt(isp, ISP_LOGERR,
1136 		    "CTIO2 completed with Invalid RX_ID 0x%x", ct->ct_rxid);
1137 		break;
1138 
1139 	default:
1140 		isp_prt(isp, ISP_LOGERR, "Unknown CTIO2 status 0x%x",
1141 		    ct->ct_status & ~QLTM_SVALID);
1142 		break;
1143 	}
1144 
1145 	if (xs == NULL) {
1146 		/*
1147 		 * There may be more than one CTIO for a data transfer,
1148 		 * or this may be a status CTIO we're not monitoring.
1149 		 *
1150 		 * The assumption is that they'll all be returned in the
1151 		 * order we got them.
1152 		 */
1153 		if (ct->ct_syshandle == 0) {
1154 			if ((ct->ct_flags & CT_SENDSTATUS) == 0) {
1155 				isp_prt(isp, pl,
1156 				    "intermediate CTIO completed ok");
1157 			} else {
1158 				isp_prt(isp, pl,
1159 				    "unmonitored CTIO completed ok");
1160 			}
1161 		} else {
1162 			isp_prt(isp, pl,
1163 			    "NO xs for CTIO (handle 0x%x) status 0x%x",
1164 			    ct->ct_syshandle, ct->ct_status & ~QLTM_SVALID);
1165 		}
1166 	} else {
1167 		if ((ct->ct_flags & CT2_DATAMASK) != CT2_NO_DATA) {
1168 			ISP_DMAFREE(isp, xs, ct->ct_syshandle);
1169 		}
1170 		if (ct->ct_flags & CT_SENDSTATUS) {
1171 			/*
1172 			 * Sent status and command complete.
1173 			 *
1174 			 * We're now really done with this command, so we
1175 			 * punt to the platform dependent layers because
1176 			 * only there can we do the appropriate command
1177 			 * complete thread synchronization.
1178 			 */
1179 			isp_prt(isp, pl, "status CTIO complete");
1180 		} else {
1181 			/*
1182 			 * Final CTIO completed. Release DMA resources and
1183 			 * notify platform dependent layers.
1184 			 */
1185 			isp_prt(isp, pl, "data CTIO complete");
1186 		}
1187 		(void) isp_async(isp, ISPASYNC_TARGET_ACTION, ct);
1188 		/*
1189 		 * The platform layer will destroy the handle if appropriate.
1190 		 */
1191 	}
1192 }
1193 #endif
1194