xref: /illumos-gate/usr/src/uts/common/io/scsi/adapters/pmcs/pmcs_scsa.c (revision 5084e753b79a753c8b532c06eb3ad1d025e8e472)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright (c) 2009, 2010, Oracle and/or its affiliates. All rights reserved.
23  */
24 /*
25  * SCSI (SCSA) midlayer interface for PMC drier.
26  */
27 
28 #include <sys/scsi/adapters/pmcs/pmcs.h>
29 
30 extern scsi_lun_t scsi_lun64_to_lun(scsi_lun64_t lun64);
31 
32 static int pmcs_scsa_tran_tgt_init(dev_info_t *, dev_info_t *,
33     scsi_hba_tran_t *, struct scsi_device *);
34 static void pmcs_scsa_tran_tgt_free(dev_info_t *, dev_info_t *,
35     scsi_hba_tran_t *, struct scsi_device *);
36 static int pmcs_scsa_start(struct scsi_address *, struct scsi_pkt *);
37 static int pmcs_scsa_abort(struct scsi_address *, struct scsi_pkt *);
38 static int pmcs_scsa_reset(struct scsi_address *, int);
39 static int pmcs_scsi_reset_notify(struct scsi_address *, int,
40     void (*)(caddr_t), caddr_t);
41 static int pmcs_scsa_getcap(struct scsi_address *, char *, int);
42 static int pmcs_scsa_setcap(struct scsi_address *, char *, int, int);
43 static int pmcs_scsa_setup_pkt(struct scsi_pkt *, int (*)(caddr_t), caddr_t);
44 static void pmcs_scsa_teardown_pkt(struct scsi_pkt *);
45 
46 static int pmcs_smp_init(dev_info_t *, dev_info_t *, smp_hba_tran_t *,
47     smp_device_t *);
48 static void pmcs_smp_free(dev_info_t *, dev_info_t *, smp_hba_tran_t *,
49     smp_device_t *);
50 static int pmcs_smp_start(struct smp_pkt *);
51 
52 static int pmcs_scsi_quiesce(dev_info_t *);
53 static int pmcs_scsi_unquiesce(dev_info_t *);
54 
55 static int pmcs_cap(struct scsi_address *, char *, int, int, int);
56 static pmcs_xscsi_t *
57     pmcs_addr2xp(struct scsi_address *, uint64_t *, pmcs_cmd_t *);
58 static int pmcs_SAS_run(pmcs_cmd_t *, pmcwork_t *);
59 static void pmcs_SAS_done(pmcs_hw_t *, pmcwork_t *, uint32_t *);
60 
61 static int pmcs_SATA_run(pmcs_cmd_t *, pmcwork_t *);
62 static void pmcs_SATA_done(pmcs_hw_t *, pmcwork_t *, uint32_t *);
63 static uint8_t pmcs_SATA_rwparm(uint8_t *, uint32_t *, uint64_t *, uint64_t);
64 
65 static void pmcs_ioerror(pmcs_hw_t *, pmcs_dtype_t pmcs_dtype,
66     pmcwork_t *, uint32_t *, uint32_t);
67 
68 
69 int
70 pmcs_scsa_init(pmcs_hw_t *pwp, const ddi_dma_attr_t *ap)
71 {
72 	scsi_hba_tran_t *tran;
73 	ddi_dma_attr_t pmcs_scsa_dattr;
74 	int flags;
75 
76 	(void) memcpy(&pmcs_scsa_dattr, ap, sizeof (ddi_dma_attr_t));
77 	pmcs_scsa_dattr.dma_attr_sgllen =
78 	    ((PMCS_SGL_NCHUNKS - 1) * (PMCS_MAX_CHUNKS - 1)) + PMCS_SGL_NCHUNKS;
79 	pmcs_scsa_dattr.dma_attr_flags = DDI_DMA_RELAXED_ORDERING;
80 	pmcs_scsa_dattr.dma_attr_flags |= DDI_DMA_FLAGERR;
81 
82 	/*
83 	 * Allocate a transport structure
84 	 */
85 	tran = scsi_hba_tran_alloc(pwp->dip, SCSI_HBA_CANSLEEP);
86 	if (tran == NULL) {
87 		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
88 		    "scsi_hba_tran_alloc failed");
89 		return (DDI_FAILURE);
90 	}
91 
92 	tran->tran_hba_private		= pwp;
93 	tran->tran_tgt_init		= pmcs_scsa_tran_tgt_init;
94 	tran->tran_tgt_free		= pmcs_scsa_tran_tgt_free;
95 	tran->tran_start		= pmcs_scsa_start;
96 	tran->tran_abort		= pmcs_scsa_abort;
97 	tran->tran_reset		= pmcs_scsa_reset;
98 	tran->tran_reset_notify		= pmcs_scsi_reset_notify;
99 	tran->tran_getcap		= pmcs_scsa_getcap;
100 	tran->tran_setcap		= pmcs_scsa_setcap;
101 	tran->tran_setup_pkt		= pmcs_scsa_setup_pkt;
102 	tran->tran_teardown_pkt		= pmcs_scsa_teardown_pkt;
103 	tran->tran_quiesce		= pmcs_scsi_quiesce;
104 	tran->tran_unquiesce		= pmcs_scsi_unquiesce;
105 	tran->tran_interconnect_type	= INTERCONNECT_SAS;
106 	tran->tran_hba_len		= sizeof (pmcs_cmd_t);
107 
108 	/*
109 	 * Attach this instance of the hba
110 	 */
111 
112 	flags = SCSI_HBA_TRAN_SCB | SCSI_HBA_TRAN_CDB | SCSI_HBA_ADDR_COMPLEX |
113 	    SCSI_HBA_TRAN_PHCI | SCSI_HBA_HBA;
114 
115 	if (scsi_hba_attach_setup(pwp->dip, &pmcs_scsa_dattr, tran, flags)) {
116 		scsi_hba_tran_free(tran);
117 		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
118 		    "scsi_hba_attach failed");
119 		return (DDI_FAILURE);
120 	}
121 	pwp->tran = tran;
122 
123 	/*
124 	 * Attach the SMP part of this hba
125 	 */
126 	pwp->smp_tran = smp_hba_tran_alloc(pwp->dip);
127 	ASSERT(pwp->smp_tran != NULL);
128 	pwp->smp_tran->smp_tran_hba_private = pwp;
129 	pwp->smp_tran->smp_tran_init = pmcs_smp_init;
130 	pwp->smp_tran->smp_tran_free = pmcs_smp_free;
131 	pwp->smp_tran->smp_tran_start = pmcs_smp_start;
132 
133 	if (smp_hba_attach_setup(pwp->dip, pwp->smp_tran) != DDI_SUCCESS) {
134 		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
135 		    "smp_hba_attach failed");
136 		smp_hba_tran_free(pwp->smp_tran);
137 		pwp->smp_tran = NULL;
138 		scsi_hba_tran_free(tran);
139 		return (DDI_FAILURE);
140 	}
141 
142 	return (DDI_SUCCESS);
143 }
144 
145 /*
146  * SCSA entry points
147  */
148 
149 static int
150 pmcs_scsa_tran_tgt_init(dev_info_t *hba_dip, dev_info_t *tgt_dip,
151     scsi_hba_tran_t *tran, struct scsi_device *sd)
152 {
153 	pmcs_hw_t	*pwp = NULL;
154 	int		rval;
155 	char		*variant_prop = "sata";
156 	char		*tgt_port = NULL, *ua = NULL;
157 	pmcs_xscsi_t	*tgt = NULL;
158 	pmcs_iport_t	*iport;
159 	pmcs_lun_t	*lun = NULL;
160 	pmcs_phy_t	*phyp = NULL;
161 	uint64_t	lun_num;
162 	boolean_t	got_scratch = B_FALSE;
163 
164 	/*
165 	 * First, make sure we're an iport and get the pointer to the HBA
166 	 * node's softstate
167 	 */
168 	if (scsi_hba_iport_unit_address(hba_dip) == NULL) {
169 		pmcs_prt(TRAN2PMC(tran), PMCS_PRT_DEBUG_CONFIG, NULL, NULL,
170 		    "%s: We don't enumerate devices on the HBA node", __func__);
171 		goto tgt_init_fail;
172 	}
173 
174 	pwp = ITRAN2PMC(tran);
175 	iport = ITRAN2IPORT(tran);
176 
177 	/*
178 	 * Get the unit-address
179 	 */
180 	ua = scsi_device_unit_address(sd);
181 	if (ua == NULL) {
182 		pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, NULL, NULL,
183 		    "%s: Couldn't get UA", __func__);
184 		pwp = NULL;
185 		goto tgt_init_fail;
186 	}
187 	pmcs_prt(pwp, PMCS_PRT_DEBUG3, NULL, NULL,
188 	    "got ua '%s'", ua);
189 
190 	/*
191 	 * Get the target address
192 	 */
193 	rval = scsi_device_prop_lookup_string(sd, SCSI_DEVICE_PROP_PATH,
194 	    SCSI_ADDR_PROP_TARGET_PORT, &tgt_port);
195 	if (rval != DDI_PROP_SUCCESS) {
196 		pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, NULL, NULL,
197 		    "Couldn't get target UA");
198 		pwp = NULL;
199 		goto tgt_init_fail;
200 	}
201 	pmcs_prt(pwp, PMCS_PRT_DEBUG3, NULL, NULL,
202 	    "got tgt_port '%s'", tgt_port);
203 
204 	/*
205 	 * Validate that this tran_tgt_init is for an active iport.
206 	 */
207 	if (iport->ua_state == UA_INACTIVE) {
208 		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
209 		    "%s: Got tran_tgt_init on inactive iport for '%s'",
210 		    __func__, tgt_port);
211 		pwp = NULL;
212 		goto tgt_init_fail;
213 	}
214 
215 	/*
216 	 * Since we're going to wait for scratch, be sure to acquire it while
217 	 * we're not holding any other locks
218 	 */
219 	(void) pmcs_acquire_scratch(pwp, B_TRUE);
220 	got_scratch = B_TRUE;
221 
222 	mutex_enter(&pwp->lock);
223 
224 	/*
225 	 * See if there's already a target softstate.  If not, allocate one.
226 	 */
227 	tgt = pmcs_get_target(iport, tgt_port, B_TRUE);
228 
229 	if (tgt == NULL) {
230 		pmcs_prt(pwp, PMCS_PRT_DEBUG2, NULL, NULL, "%s: "
231 		    "No tgt for tgt_port (%s)", __func__, tgt_port);
232 		goto tgt_init_fail;
233 	}
234 
235 	phyp = tgt->phy;
236 	if (!IS_ROOT_PHY(phyp)) {
237 		pmcs_inc_phy_ref_count(phyp);
238 	}
239 	ASSERT(mutex_owned(&phyp->phy_lock));
240 
241 	pmcs_prt(pwp, PMCS_PRT_DEBUG2, phyp, tgt, "@%s tgt = 0x%p, dip = 0x%p",
242 	    ua, (void *)tgt, (void *)tgt_dip);
243 
244 	/* Now get the lun */
245 	lun_num = scsi_device_prop_get_int64(sd, SCSI_DEVICE_PROP_PATH,
246 	    SCSI_ADDR_PROP_LUN64, SCSI_LUN64_ILLEGAL);
247 	if (lun_num == SCSI_LUN64_ILLEGAL) {
248 		pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, phyp, tgt,
249 		    "No LUN for tgt %p", (void *)tgt);
250 		goto tgt_init_fail;
251 	}
252 
253 	pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, phyp, tgt, "%s: @%s tgt 0x%p phy "
254 	    "0x%p (%s)", __func__, ua, (void *)tgt, (void *)phyp, phyp->path);
255 
256 	mutex_enter(&tgt->statlock);
257 	tgt->dtype = phyp->dtype;
258 	if (tgt->dtype != SAS && tgt->dtype != SATA) {
259 		pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, phyp, tgt,
260 		    "PHY 0x%p went away?", (void *)phyp);
261 		goto tgt_init_fail;
262 	}
263 
264 	/* We don't support SATA devices at LUN > 0. */
265 	if ((tgt->dtype == SATA) && (lun_num > 0)) {
266 		pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, phyp, tgt,
267 		    "%s: No support for SATA devices at LUN > 0 "
268 		    "(target = 0x%p)", __func__, (void *)tgt);
269 		goto tgt_init_fail;
270 	}
271 
272 	/*
273 	 * Allocate LU soft state. We use ddi_soft_state_bystr_zalloc instead
274 	 * of kmem_alloc because ddi_soft_state_bystr_zalloc allows us to
275 	 * verify that the framework never tries to initialize two scsi_device
276 	 * structures with the same unit-address at the same time.
277 	 */
278 	if (ddi_soft_state_bystr_zalloc(tgt->lun_sstate, ua) != DDI_SUCCESS) {
279 		pmcs_prt(pwp, PMCS_PRT_DEBUG2, phyp, tgt,
280 		    "Couldn't allocate LU soft state");
281 		goto tgt_init_fail;
282 	}
283 
284 	lun = ddi_soft_state_bystr_get(tgt->lun_sstate, ua);
285 	if (lun == NULL) {
286 		pmcs_prt(pwp, PMCS_PRT_DEBUG2, phyp, tgt,
287 		    "Couldn't get LU soft state");
288 		goto tgt_init_fail;
289 	}
290 	scsi_device_hba_private_set(sd, lun);
291 	lun->lun_num = lun_num;
292 
293 	/* convert the scsi_lun64_t value to SCSI standard form */
294 	lun->scsi_lun = scsi_lun64_to_lun(lun_num);
295 
296 	ASSERT(strlen(ua) < (PMCS_MAX_UA_SIZE - 1));
297 	bcopy(ua, lun->unit_address, strnlen(ua, PMCS_MAX_UA_SIZE - 1));
298 
299 	lun->target = tgt;
300 
301 	/*
302 	 * If this is the first tran_tgt_init, add this target to our list
303 	 */
304 	if (tgt->target_num == PMCS_INVALID_TARGET_NUM) {
305 		int target;
306 		for (target = 0; target < pwp->max_dev; target++) {
307 			if (pwp->targets[target] != NULL) {
308 				continue;
309 			}
310 
311 			pwp->targets[target] = tgt;
312 			tgt->target_num = (uint16_t)target;
313 			break;
314 		}
315 
316 		if (target == pwp->max_dev) {
317 			pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, phyp, tgt,
318 			    "Target list full.");
319 			goto tgt_init_fail;
320 		}
321 	}
322 
323 	tgt->dip = sd->sd_dev;
324 	lun->sd = sd;
325 	list_insert_tail(&tgt->lun_list, lun);
326 
327 	if (!pmcs_assign_device(pwp, tgt)) {
328 		pmcs_release_scratch(pwp);
329 		pwp->targets[tgt->target_num] = NULL;
330 		tgt->target_num = PMCS_INVALID_TARGET_NUM;
331 		tgt->phy = NULL;
332 		pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, phyp, tgt,
333 		    "%s: pmcs_assign_device failed for target 0x%p",
334 		    __func__, (void *)tgt);
335 		goto tgt_init_fail;
336 	}
337 
338 	pmcs_release_scratch(pwp);
339 	tgt->ref_count++;
340 
341 	(void) scsi_device_prop_update_int(sd, SCSI_DEVICE_PROP_PATH,
342 	    SCSI_ADDR_PROP_TARGET, (uint32_t)(tgt->target_num));
343 
344 	/* SM-HBA */
345 	if (tgt->dtype == SATA) {
346 		/* TCR in PSARC/1997/281 opinion */
347 		(void) scsi_device_prop_update_string(sd,
348 		    SCSI_DEVICE_PROP_PATH, "variant", variant_prop);
349 	}
350 
351 	tgt->phy_addressable = PMCS_PHY_ADDRESSABLE(phyp);
352 
353 	if (tgt->phy_addressable) {
354 		(void) scsi_device_prop_update_int(sd, SCSI_DEVICE_PROP_PATH,
355 		    SCSI_ADDR_PROP_SATA_PHY, phyp->phynum);
356 	}
357 
358 	/* SM-HBA */
359 	(void) pmcs_smhba_set_scsi_device_props(pwp, phyp, sd);
360 	/*
361 	 * Make sure attached port and target port pm props are updated
362 	 * By passing in 0s, we're not actually updating any values, but
363 	 * the properties should now get updated on the node.
364 	 */
365 
366 	mutex_exit(&tgt->statlock);
367 	pmcs_update_phy_pm_props(phyp, 0, 0, B_TRUE);
368 	pmcs_unlock_phy(phyp);
369 	mutex_exit(&pwp->lock);
370 	scsi_device_prop_free(sd, SCSI_DEVICE_PROP_PATH, tgt_port);
371 	return (DDI_SUCCESS);
372 
373 tgt_init_fail:
374 	scsi_device_hba_private_set(sd, NULL);
375 	if (got_scratch) {
376 		pmcs_release_scratch(pwp);
377 	}
378 	if (lun) {
379 		list_remove(&tgt->lun_list, lun);
380 		ddi_soft_state_bystr_free(tgt->lun_sstate, ua);
381 	}
382 	if (phyp) {
383 		mutex_exit(&tgt->statlock);
384 		pmcs_unlock_phy(phyp);
385 		/*
386 		 * phyp's ref count was incremented in pmcs_new_tport.
387 		 * We're failing configuration, we now need to decrement it.
388 		 */
389 		if (!IS_ROOT_PHY(phyp)) {
390 			pmcs_dec_phy_ref_count(phyp);
391 		}
392 		phyp->target = NULL;
393 	}
394 	if (tgt && tgt->ref_count == 0) {
395 		ddi_soft_state_bystr_free(iport->tgt_sstate, tgt_port);
396 	}
397 	if (pwp) {
398 		mutex_exit(&pwp->lock);
399 		pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, phyp, tgt,
400 		    "%s: failed for @%s tgt 0x%p phy 0x%p", __func__, ua,
401 		    (void *)tgt, (void *)phyp);
402 	}
403 	if (tgt_port) {
404 		scsi_device_prop_free(sd, SCSI_DEVICE_PROP_PATH, tgt_port);
405 	}
406 	return (DDI_FAILURE);
407 }
408 
409 static void
410 pmcs_scsa_tran_tgt_free(dev_info_t *hba_dip, dev_info_t *tgt_dip,
411     scsi_hba_tran_t *tran, struct scsi_device *sd)
412 {
413 	_NOTE(ARGUNUSED(hba_dip, tgt_dip));
414 	pmcs_hw_t	*pwp;
415 	pmcs_lun_t	*lun;
416 	pmcs_xscsi_t	*target;
417 	char		*unit_address;
418 	pmcs_phy_t	*phyp;
419 
420 	if (scsi_hba_iport_unit_address(hba_dip) == NULL) {
421 		pwp = TRAN2PMC(tran);
422 		pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, NULL, NULL,
423 		    "%s: We don't enumerate devices on the HBA node", __func__);
424 		return;
425 	}
426 
427 	lun = (pmcs_lun_t *)scsi_device_hba_private_get(sd);
428 
429 	ASSERT((lun != NULL) && (lun->target != NULL));
430 	ASSERT(lun->target->ref_count > 0);
431 
432 	target = lun->target;
433 	unit_address = lun->unit_address;
434 	list_remove(&target->lun_list, lun);
435 
436 	pwp = ITRAN2PMC(tran);
437 	mutex_enter(&pwp->lock);
438 	phyp = target->phy;
439 	if (phyp) {
440 		mutex_enter(&phyp->phy_lock);
441 	}
442 	mutex_enter(&target->statlock);
443 
444 	pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, phyp, target,
445 	    "%s: for @%s tgt 0x%p phy 0x%p", __func__, unit_address,
446 	    (void *)target, (void *)phyp);
447 	ddi_soft_state_bystr_free(lun->target->lun_sstate, unit_address);
448 
449 	if (target->recover_wait) {
450 		mutex_exit(&target->statlock);
451 		if (phyp) {
452 			mutex_exit(&phyp->phy_lock);
453 		}
454 		mutex_exit(&pwp->lock);
455 		pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, phyp, target, "%s: "
456 		    "Target 0x%p in device state recovery, fail tran_tgt_free",
457 		    __func__, (void *)target);
458 		return;
459 	}
460 
461 	/*
462 	 * If this target still has a PHY pointer and that PHY's target pointer
463 	 * has been cleared, then that PHY has been reaped. In that case, there
464 	 * would be no need to decrement the reference count
465 	 */
466 	if (phyp && !IS_ROOT_PHY(phyp) && phyp->target) {
467 		pmcs_dec_phy_ref_count(phyp);
468 	}
469 
470 	if (--target->ref_count == 0) {
471 		/*
472 		 * Remove this target from our list.  The target soft
473 		 * state will remain, and the device will remain registered
474 		 * with the hardware unless/until we're told the device
475 		 * physically went away.
476 		 */
477 		pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, phyp, target,
478 		    "%s: Free target 0x%p (vtgt %d)", __func__, (void *)target,
479 		    target->target_num);
480 		pwp->targets[target->target_num] = NULL;
481 		target->target_num = PMCS_INVALID_TARGET_NUM;
482 		/* If the PHY has a pointer to this target, clear it */
483 		if (phyp && (phyp->target == target)) {
484 			phyp->target = NULL;
485 		}
486 		target->phy = NULL;
487 		if (phyp) {
488 			mutex_exit(&phyp->phy_lock);
489 		}
490 		pmcs_destroy_target(target);
491 	} else {
492 		mutex_exit(&target->statlock);
493 		if (phyp) {
494 			mutex_exit(&phyp->phy_lock);
495 		}
496 	}
497 
498 	mutex_exit(&pwp->lock);
499 }
500 
501 static int
502 pmcs_scsa_start(struct scsi_address *ap, struct scsi_pkt *pkt)
503 {
504 	pmcs_cmd_t *sp = PKT2CMD(pkt);
505 	pmcs_hw_t *pwp = ADDR2PMC(ap);
506 	pmcs_xscsi_t *xp;
507 	boolean_t blocked;
508 	uint32_t hba_state;
509 
510 	pmcs_prt(pwp, PMCS_PRT_DEBUG2, NULL, NULL,
511 	    "%s: pkt %p sd %p cdb0=0x%02x dl=%lu", __func__, (void *)pkt,
512 	    (void *)scsi_address_device(&pkt->pkt_address),
513 	    pkt->pkt_cdbp[0] & 0xff, pkt->pkt_dma_len);
514 
515 	if (pkt->pkt_flags & FLAG_NOINTR) {
516 		pmcs_prt(pwp, PMCS_PRT_DEBUG3, NULL, NULL,
517 		    "%s: nointr pkt", __func__);
518 		return (TRAN_BADPKT);
519 	}
520 
521 	sp->cmd_tag = 0;
522 	pkt->pkt_state = pkt->pkt_statistics = 0;
523 	pkt->pkt_reason = CMD_INCOMPLETE;
524 
525 	mutex_enter(&pwp->lock);
526 	hba_state = pwp->state;
527 	blocked = pwp->blocked;
528 	mutex_exit(&pwp->lock);
529 
530 	if (hba_state != STATE_RUNNING) {
531 		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
532 		    "%s: hba dead", __func__);
533 		return (TRAN_FATAL_ERROR);
534 	}
535 
536 	xp = pmcs_addr2xp(ap, NULL, sp);
537 	if (xp == NULL) {
538 		pmcs_prt(pwp, PMCS_PRT_DEBUG2, NULL, NULL,
539 		    "%s: dropping due to null target", __func__);
540 		goto dead_target;
541 	}
542 	ASSERT(mutex_owned(&xp->statlock));
543 
544 	/*
545 	 * First, check to see if the device is gone.
546 	 */
547 	if (xp->dev_gone) {
548 		xp->actv_pkts++;
549 		mutex_exit(&xp->statlock);
550 		pmcs_prt(pwp, PMCS_PRT_DEBUG3, NULL, xp,
551 		    "%s: dropping due to dead target 0x%p",
552 		    __func__, (void *)xp);
553 		goto dead_target;
554 	}
555 
556 	/*
557 	 * If we're blocked (quiesced) just return.
558 	 */
559 	if (blocked) {
560 		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
561 		    "%s: hba blocked", __func__);
562 		xp->actv_pkts++;
563 		mutex_exit(&xp->statlock);
564 		mutex_enter(&xp->wqlock);
565 		STAILQ_INSERT_TAIL(&xp->wq, sp, cmd_next);
566 		mutex_exit(&xp->wqlock);
567 		return (TRAN_ACCEPT);
568 	}
569 
570 	/*
571 	 * If we're draining or resetting, queue and return.
572 	 */
573 	if (xp->draining || xp->resetting || xp->recover_wait) {
574 		xp->actv_pkts++;
575 		mutex_exit(&xp->statlock);
576 		mutex_enter(&xp->wqlock);
577 		STAILQ_INSERT_TAIL(&xp->wq, sp, cmd_next);
578 		mutex_exit(&xp->wqlock);
579 		pmcs_prt(pwp, PMCS_PRT_DEBUG1, NULL, xp,
580 		    "%s: draining/resetting/recovering (cnt %u)",
581 		    __func__, xp->actv_cnt);
582 		/*
583 		 * By the time we get here, draining or
584 		 * resetting may have come and gone, not
585 		 * yet noticing that we had put something
586 		 * on the wait queue, so schedule a worker
587 		 * to look at this later.
588 		 */
589 		SCHEDULE_WORK(pwp, PMCS_WORK_RUN_QUEUES);
590 		return (TRAN_ACCEPT);
591 	}
592 
593 	xp->actv_pkts++;
594 	mutex_exit(&xp->statlock);
595 
596 	/*
597 	 * Queue this command to the tail of the wait queue.
598 	 * This keeps us getting commands out of order.
599 	 */
600 	mutex_enter(&xp->wqlock);
601 	STAILQ_INSERT_TAIL(&xp->wq, sp, cmd_next);
602 	mutex_exit(&xp->wqlock);
603 
604 	/*
605 	 * Now run the queue for this device.
606 	 */
607 	(void) pmcs_scsa_wq_run_one(pwp, xp);
608 
609 	return (TRAN_ACCEPT);
610 
611 dead_target:
612 	pkt->pkt_state = STATE_GOT_BUS;
613 	pkt->pkt_reason = CMD_DEV_GONE;
614 	mutex_enter(&pwp->cq_lock);
615 	STAILQ_INSERT_TAIL(&pwp->cq, sp, cmd_next);
616 	PMCS_CQ_RUN_LOCKED(pwp);
617 	mutex_exit(&pwp->cq_lock);
618 	return (TRAN_ACCEPT);
619 }
620 
621 /* Return code 1 = Success */
622 static int
623 pmcs_scsa_abort(struct scsi_address *ap, struct scsi_pkt *pkt)
624 {
625 	pmcs_hw_t *pwp = ADDR2PMC(ap);
626 	pmcs_cmd_t *sp = NULL;
627 	pmcs_xscsi_t *xp = NULL;
628 	pmcs_phy_t *pptr = NULL;
629 	pmcs_lun_t *pmcs_lun = (pmcs_lun_t *)
630 	    scsi_device_hba_private_get(scsi_address_device(ap));
631 	uint32_t tag;
632 	uint64_t lun;
633 	pmcwork_t *pwrk;
634 
635 	mutex_enter(&pwp->lock);
636 	if (pwp->state != STATE_RUNNING) {
637 		mutex_exit(&pwp->lock);
638 		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
639 		    "%s: hba dead", __func__);
640 		return (0);
641 	}
642 	mutex_exit(&pwp->lock);
643 
644 	if (pkt == NULL) {
645 		if (pmcs_lun == NULL) {
646 			pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL, "%s: "
647 			    "No pmcs_lun_t struct to do ABORT_ALL", __func__);
648 			return (0);
649 		}
650 		xp = pmcs_lun->target;
651 		if (xp != NULL) {
652 			pptr = xp->phy;
653 		}
654 		if (pptr == NULL) {
655 			pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, xp, "%s: pkt is "
656 			    "NULL. No tgt/phy to do ABORT_ALL", __func__);
657 			return (0);
658 		}
659 		pmcs_lock_phy(pptr);
660 		if (pmcs_abort(pwp, pptr, 0, 1, 0)) {
661 			pptr->abort_pending = 1;
662 			SCHEDULE_WORK(pwp, PMCS_WORK_ABORT_HANDLE);
663 		}
664 		pmcs_unlock_phy(pptr);
665 		return (1);
666 	}
667 
668 	sp = PKT2CMD(pkt);
669 	xp = sp->cmd_target;
670 
671 	if (sp->cmd_lun) {
672 		lun = sp->cmd_lun->lun_num;
673 	} else {
674 		lun = 0;
675 	}
676 	if (xp == NULL) {
677 		return (0);
678 	}
679 
680 	/*
681 	 * See if we have a real work structure associated with this cmd.
682 	 */
683 	pwrk = pmcs_tag2wp(pwp, sp->cmd_tag, B_FALSE);
684 	if (pwrk && pwrk->arg == sp) {
685 		tag = pwrk->htag;
686 		pptr = pwrk->phy;
687 		pwrk->timer = 0;	/* we don't time this here */
688 		ASSERT(pwrk->state == PMCS_WORK_STATE_ONCHIP);
689 		mutex_exit(&pwrk->lock);
690 		pmcs_lock_phy(pptr);
691 		if (pptr->dtype == SAS) {
692 			if (pmcs_ssp_tmf(pwp, pptr, SAS_ABORT_TASK, tag, lun,
693 			    NULL)) {
694 				pptr->abort_pending = 1;
695 				pmcs_unlock_phy(pptr);
696 				SCHEDULE_WORK(pwp, PMCS_WORK_ABORT_HANDLE);
697 				return (0);
698 			}
699 		} else {
700 			/*
701 			 * XXX: Was the command that was active an
702 			 * NCQ I/O command?
703 			 */
704 			pptr->need_rl_ext = 1;
705 			if (pmcs_sata_abort_ncq(pwp, pptr)) {
706 				pptr->abort_pending = 1;
707 				pmcs_unlock_phy(pptr);
708 				SCHEDULE_WORK(pwp, PMCS_WORK_ABORT_HANDLE);
709 				return (0);
710 			}
711 		}
712 		pptr->abort_pending = 1;
713 		pmcs_unlock_phy(pptr);
714 		SCHEDULE_WORK(pwp, PMCS_WORK_ABORT_HANDLE);
715 		return (1);
716 	}
717 	if (pwrk) {
718 		mutex_exit(&pwrk->lock);
719 	}
720 	/*
721 	 * Okay, those weren't the droids we were looking for.
722 	 * See if the command is on any of the wait queues.
723 	 */
724 	mutex_enter(&xp->wqlock);
725 	sp = NULL;
726 	STAILQ_FOREACH(sp, &xp->wq, cmd_next) {
727 		if (sp == PKT2CMD(pkt)) {
728 			STAILQ_REMOVE(&xp->wq, sp, pmcs_cmd, cmd_next);
729 			break;
730 		}
731 	}
732 	mutex_exit(&xp->wqlock);
733 	if (sp) {
734 		pkt->pkt_reason = CMD_ABORTED;
735 		pkt->pkt_statistics |= STAT_ABORTED;
736 		mutex_enter(&pwp->cq_lock);
737 		STAILQ_INSERT_TAIL(&pwp->cq, sp, cmd_next);
738 		PMCS_CQ_RUN_LOCKED(pwp);
739 		mutex_exit(&pwp->cq_lock);
740 		return (1);
741 	}
742 	return (0);
743 }
744 
745 /*
746  * SCSA reset functions
747  */
748 static int
749 pmcs_scsa_reset(struct scsi_address *ap, int level)
750 {
751 	pmcs_hw_t *pwp = ADDR2PMC(ap);
752 	pmcs_phy_t *pptr;
753 	pmcs_xscsi_t *xp;
754 	uint64_t lun = (uint64_t)-1, *lp = NULL;
755 	int rval;
756 
757 	mutex_enter(&pwp->lock);
758 	if (pwp->state != STATE_RUNNING) {
759 		mutex_exit(&pwp->lock);
760 		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
761 		    "%s: hba dead", __func__);
762 		return (0);
763 	}
764 	mutex_exit(&pwp->lock);
765 
766 	switch (level)  {
767 	case RESET_ALL:
768 		rval = 0;
769 		break;
770 	case RESET_LUN:
771 		/*
772 		 * Point lp at lun so that pmcs_addr2xp
773 		 * will fill out the 64 bit lun number.
774 		 */
775 		lp = &lun;
776 		/* FALLTHROUGH */
777 	case RESET_TARGET:
778 		xp = pmcs_addr2xp(ap, lp, NULL);
779 		if (xp == NULL) {
780 			pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
781 			    "%s: no xp found for this scsi address", __func__);
782 			return (0);
783 		}
784 
785 		if (xp->dev_gone) {
786 			mutex_exit(&xp->statlock);
787 			pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, xp,
788 			    "%s: Target 0x%p has gone away", __func__,
789 			    (void *)xp);
790 			return (0);
791 		}
792 
793 		/*
794 		 * If we're already performing this action, or if device
795 		 * state recovery is already running, just return failure.
796 		 */
797 		if (xp->resetting || xp->recover_wait) {
798 			mutex_exit(&xp->statlock);
799 			return (0);
800 		}
801 		xp->reset_wait = 0;
802 		xp->reset_success = 0;
803 		xp->resetting = 1;
804 		pptr = xp->phy;
805 		mutex_exit(&xp->statlock);
806 
807 		if (pmcs_reset_dev(pwp, pptr, lun)) {
808 			rval = 0;
809 		} else {
810 			rval = 1;
811 		}
812 
813 		mutex_enter(&xp->statlock);
814 		if (rval == 1) {
815 			xp->reset_success = 1;
816 		}
817 		if (xp->reset_wait) {
818 			xp->reset_wait = 0;
819 			cv_signal(&xp->reset_cv);
820 		}
821 		xp->resetting = 0;
822 		mutex_exit(&xp->statlock);
823 		SCHEDULE_WORK(pwp, PMCS_WORK_RUN_QUEUES);
824 		break;
825 	default:
826 		rval = 0;
827 		break;
828 	}
829 
830 	return (rval);
831 }
832 
833 static int
834 pmcs_scsi_reset_notify(struct scsi_address *ap, int flag,
835     void (*callback)(caddr_t), caddr_t arg)
836 {
837 	pmcs_hw_t *pwp = ADDR2PMC(ap);
838 	return (scsi_hba_reset_notify_setup(ap, flag, callback, arg,
839 	    &pwp->lock, &pwp->reset_notify_listf));
840 }
841 
842 
843 static int
844 pmcs_cap(struct scsi_address *ap, char *cap, int val, int tonly, int set)
845 {
846 	_NOTE(ARGUNUSED(val, tonly));
847 	int cidx, rval = 0;
848 	pmcs_xscsi_t *xp;
849 
850 	cidx = scsi_hba_lookup_capstr(cap);
851 	if (cidx == -1) {
852 		return (-1);
853 	}
854 
855 	xp = pmcs_addr2xp(ap, NULL, NULL);
856 	if (xp == NULL) {
857 		return (-1);
858 	}
859 
860 	switch (cidx) {
861 	case SCSI_CAP_DMA_MAX:
862 	case SCSI_CAP_INITIATOR_ID:
863 		if (set == 0) {
864 			rval = INT_MAX;	/* argh */
865 		}
866 		break;
867 	case SCSI_CAP_DISCONNECT:
868 	case SCSI_CAP_SYNCHRONOUS:
869 	case SCSI_CAP_WIDE_XFER:
870 	case SCSI_CAP_PARITY:
871 	case SCSI_CAP_ARQ:
872 	case SCSI_CAP_UNTAGGED_QING:
873 		if (set == 0) {
874 			rval = 1;
875 		}
876 		break;
877 
878 	case SCSI_CAP_TAGGED_QING:
879 		rval = 1;
880 		break;
881 
882 	case SCSI_CAP_MSG_OUT:
883 	case SCSI_CAP_RESET_NOTIFICATION:
884 	case SCSI_CAP_QFULL_RETRIES:
885 	case SCSI_CAP_QFULL_RETRY_INTERVAL:
886 		break;
887 	case SCSI_CAP_SCSI_VERSION:
888 		if (set == 0) {
889 			rval = SCSI_VERSION_3;
890 		}
891 		break;
892 	case SCSI_CAP_INTERCONNECT_TYPE:
893 		if (set) {
894 			break;
895 		}
896 		if (xp->phy_addressable) {
897 			rval = INTERCONNECT_SATA;
898 		} else {
899 			rval = INTERCONNECT_SAS;
900 		}
901 		break;
902 	case SCSI_CAP_CDB_LEN:
903 		if (set == 0) {
904 			rval = 16;
905 		}
906 		break;
907 	case SCSI_CAP_LUN_RESET:
908 		if (set) {
909 			break;
910 		}
911 		if (xp->dtype == SATA) {
912 			rval = 0;
913 		} else {
914 			rval = 1;
915 		}
916 		break;
917 	default:
918 		rval = -1;
919 		break;
920 	}
921 	mutex_exit(&xp->statlock);
922 	pmcs_prt(ADDR2PMC(ap), PMCS_PRT_DEBUG3, NULL, NULL,
923 	    "%s: cap %s val %d set %d rval %d",
924 	    __func__, cap, val, set, rval);
925 	return (rval);
926 }
927 
928 /*
929  * Returns with statlock held if the xp is found.
930  * Fills in pmcs_cmd_t with values if pmcs_cmd_t pointer non-NULL.
931  */
932 static pmcs_xscsi_t *
933 pmcs_addr2xp(struct scsi_address *ap, uint64_t *lp, pmcs_cmd_t *sp)
934 {
935 	pmcs_xscsi_t *xp;
936 	pmcs_lun_t *lun = (pmcs_lun_t *)
937 	    scsi_device_hba_private_get(scsi_address_device(ap));
938 
939 	if ((lun == NULL) || (lun->target == NULL)) {
940 		return (NULL);
941 	}
942 	xp = lun->target;
943 	mutex_enter(&xp->statlock);
944 
945 	if (xp->dev_gone || (xp->phy == NULL)) {
946 		/*
947 		 * This may be a retried packet, so it's possible cmd_target
948 		 * and cmd_lun may still be populated.  Clear them.
949 		 */
950 		if (sp != NULL) {
951 			sp->cmd_target = NULL;
952 			sp->cmd_lun = NULL;
953 		}
954 		mutex_exit(&xp->statlock);
955 		return (NULL);
956 	}
957 
958 	if (sp != NULL) {
959 		sp->cmd_target = xp;
960 		sp->cmd_lun = lun;
961 	}
962 	if (lp) {
963 		*lp = lun->lun_num;
964 	}
965 	return (xp);
966 }
967 
968 static int
969 pmcs_scsa_getcap(struct scsi_address *ap, char *cap, int whom)
970 {
971 	int r;
972 	if (cap == NULL) {
973 		return (-1);
974 	}
975 	r = pmcs_cap(ap, cap, 0, whom, 0);
976 	return (r);
977 }
978 
979 static int
980 pmcs_scsa_setcap(struct scsi_address *ap, char *cap, int value, int whom)
981 {
982 	int r;
983 	if (cap == NULL) {
984 		return (-1);
985 	}
986 	r = pmcs_cap(ap, cap, value, whom, 1);
987 	return (r);
988 }
989 
990 static int
991 pmcs_scsa_setup_pkt(struct scsi_pkt *pkt, int (*callback)(caddr_t),
992     caddr_t cbarg)
993 {
994 	_NOTE(ARGUNUSED(callback, cbarg));
995 	pmcs_cmd_t *sp = pkt->pkt_ha_private;
996 
997 	bzero(sp, sizeof (pmcs_cmd_t));
998 	sp->cmd_pkt = pkt;
999 	return (0);
1000 }
1001 
1002 static void
1003 pmcs_scsa_teardown_pkt(struct scsi_pkt *pkt)
1004 {
1005 	pmcs_cmd_t *sp = pkt->pkt_ha_private;
1006 	sp->cmd_target = NULL;
1007 	sp->cmd_lun = NULL;
1008 }
1009 
1010 static int
1011 pmcs_smp_start(struct smp_pkt *smp_pkt)
1012 {
1013 	struct pmcwork *pwrk;
1014 	pmcs_iport_t *iport;
1015 	const uint_t rdoff = SAS_SMP_MAX_PAYLOAD;
1016 	uint32_t msg[PMCS_MSG_SIZE], *ptr, htag, status;
1017 	uint64_t wwn;
1018 	pmcs_hw_t *pwp;
1019 	pmcs_phy_t *pptr;
1020 	pmcs_xscsi_t *xp;
1021 	uint_t reqsz, rspsz, will_retry;
1022 	int result;
1023 
1024 	pwp = smp_pkt->smp_pkt_address->smp_a_hba_tran->smp_tran_hba_private;
1025 	bcopy(smp_pkt->smp_pkt_address->smp_a_wwn, &wwn, SAS_WWN_BYTE_SIZE);
1026 
1027 	pmcs_prt(pwp, PMCS_PRT_DEBUG1, NULL, NULL,
1028 	    "%s: starting for wwn 0x%" PRIx64, __func__, wwn);
1029 
1030 	will_retry = smp_pkt->smp_pkt_will_retry;
1031 
1032 	(void) pmcs_acquire_scratch(pwp, B_TRUE);
1033 	reqsz = smp_pkt->smp_pkt_reqsize;
1034 	if (reqsz > SAS_SMP_MAX_PAYLOAD) {
1035 		reqsz = SAS_SMP_MAX_PAYLOAD;
1036 	}
1037 	(void) memcpy(pwp->scratch, smp_pkt->smp_pkt_req, reqsz);
1038 
1039 	rspsz = smp_pkt->smp_pkt_rspsize;
1040 	if (rspsz > SAS_SMP_MAX_PAYLOAD) {
1041 		rspsz = SAS_SMP_MAX_PAYLOAD;
1042 	}
1043 
1044 	/*
1045 	 * The request size from the SMP driver always includes 4 bytes
1046 	 * for the CRC. The PMCS chip, however, doesn't want to see those
1047 	 * counts as part of the transfer size.
1048 	 */
1049 	reqsz -= 4;
1050 
1051 	pptr = pmcs_find_phy_by_wwn(pwp, wwn);
1052 	/* PHY is now locked */
1053 	if (pptr == NULL || pptr->dtype != EXPANDER) {
1054 		if (pptr) {
1055 			pmcs_unlock_phy(pptr);
1056 		}
1057 		pmcs_release_scratch(pwp);
1058 		pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, NULL,
1059 		    "%s: could not find phy", __func__);
1060 		smp_pkt->smp_pkt_reason = ENXIO;
1061 		return (DDI_FAILURE);
1062 	}
1063 
1064 	if ((pptr->iport == NULL) || !pptr->valid_device_id) {
1065 		pmcs_unlock_phy(pptr);
1066 		pmcs_release_scratch(pwp);
1067 		pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, pptr->target,
1068 		    "%s: Can't reach PHY %s", __func__, pptr->path);
1069 		smp_pkt->smp_pkt_reason = ENXIO;
1070 		return (DDI_FAILURE);
1071 	}
1072 
1073 	pwrk = pmcs_gwork(pwp, PMCS_TAG_TYPE_WAIT, pptr);
1074 	if (pwrk == NULL) {
1075 		pmcs_unlock_phy(pptr);
1076 		pmcs_release_scratch(pwp);
1077 		pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, NULL,
1078 		    "%s: could not get work structure", __func__);
1079 		smp_pkt->smp_pkt_reason = will_retry ? EAGAIN : EBUSY;
1080 		return (DDI_FAILURE);
1081 	}
1082 
1083 	pwrk->arg = msg;
1084 	pwrk->dtype = EXPANDER;
1085 	mutex_enter(&pwp->iqp_lock[PMCS_IQ_OTHER]);
1086 	ptr = GET_IQ_ENTRY(pwp, PMCS_IQ_OTHER);
1087 	if (ptr == NULL) {
1088 		pmcs_pwork(pwp, pwrk);
1089 		mutex_exit(&pwp->iqp_lock[PMCS_IQ_OTHER]);
1090 		pmcs_unlock_phy(pptr);
1091 		pmcs_release_scratch(pwp);
1092 		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
1093 		    "%s: could not get IQ entry", __func__);
1094 		smp_pkt->smp_pkt_reason = will_retry ? EAGAIN :EBUSY;
1095 		return (DDI_FAILURE);
1096 	}
1097 	msg[0] = LE_32(PMCS_HIPRI(pwp, PMCS_OQ_GENERAL, PMCIN_SMP_REQUEST));
1098 	msg[1] = LE_32(pwrk->htag);
1099 	msg[2] = LE_32(pptr->device_id);
1100 	msg[3] = LE_32(SMP_INDIRECT_RESPONSE | SMP_INDIRECT_REQUEST);
1101 	msg[8] = LE_32(DWORD0(pwp->scratch_dma));
1102 	msg[9] = LE_32(DWORD1(pwp->scratch_dma));
1103 	msg[10] = LE_32(reqsz);
1104 	msg[11] = 0;
1105 	msg[12] = LE_32(DWORD0(pwp->scratch_dma+rdoff));
1106 	msg[13] = LE_32(DWORD1(pwp->scratch_dma+rdoff));
1107 	msg[14] = LE_32(rspsz);
1108 	msg[15] = 0;
1109 
1110 	COPY_MESSAGE(ptr, msg, PMCS_MSG_SIZE);
1111 
1112 	pmcs_hold_iport(pptr->iport);
1113 	iport = pptr->iport;
1114 	pmcs_smp_acquire(iport);
1115 	pwrk->state = PMCS_WORK_STATE_ONCHIP;
1116 	htag = pwrk->htag;
1117 	INC_IQ_ENTRY(pwp, PMCS_IQ_OTHER);
1118 	pmcs_unlock_phy(pptr);
1119 	WAIT_FOR(pwrk, smp_pkt->smp_pkt_timeout * 1000, result);
1120 	pmcs_pwork(pwp, pwrk);
1121 	pmcs_lock_phy(pptr);
1122 	if (result) {
1123 		pmcs_timed_out(pwp, htag, __func__);
1124 		if (pmcs_abort(pwp, pptr, htag, 0, 0)) {
1125 			pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, pptr, pptr->target,
1126 			    "%s: Unable to issue SMP ABORT for htag 0x%08x",
1127 			    __func__, htag);
1128 		} else {
1129 			pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, pptr, pptr->target,
1130 			    "%s: Issuing SMP ABORT for htag 0x%08x",
1131 			    __func__, htag);
1132 		}
1133 		pmcs_smp_release(iport);
1134 		pmcs_rele_iport(iport);
1135 		pmcs_unlock_phy(pptr);
1136 		pmcs_release_scratch(pwp);
1137 		smp_pkt->smp_pkt_reason = ETIMEDOUT;
1138 		return (DDI_FAILURE);
1139 	}
1140 	pmcs_smp_release(iport);
1141 	pmcs_rele_iport(iport);
1142 	status = LE_32(msg[2]);
1143 	if (status == PMCOUT_STATUS_OVERFLOW) {
1144 		status = PMCOUT_STATUS_OK;
1145 		smp_pkt->smp_pkt_reason = EOVERFLOW;
1146 	}
1147 	if (status != PMCOUT_STATUS_OK) {
1148 		const char *emsg = pmcs_status_str(status);
1149 		if (emsg == NULL) {
1150 			pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, pptr, pptr->target,
1151 			    "SMP operation failed (0x%x)", status);
1152 		} else {
1153 			pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, pptr, pptr->target,
1154 			    "SMP operation failed (%s)", emsg);
1155 		}
1156 
1157 		if ((status == PMCOUT_STATUS_ERROR_HW_TIMEOUT) ||
1158 		    (status == PMCOUT_STATUS_IO_XFER_OPEN_RETRY_TIMEOUT)) {
1159 			smp_pkt->smp_pkt_reason =
1160 			    will_retry ? EAGAIN : ETIMEDOUT;
1161 			result = DDI_FAILURE;
1162 		} else if (status ==
1163 		    PMCOUT_STATUS_OPEN_CNX_ERROR_IT_NEXUS_LOSS) {
1164 			xp = pptr->target;
1165 			if (xp == NULL) {
1166 				smp_pkt->smp_pkt_reason = EIO;
1167 				result = DDI_FAILURE;
1168 				goto out;
1169 			}
1170 			if (xp->dev_state !=
1171 			    PMCS_DEVICE_STATE_NON_OPERATIONAL) {
1172 				xp->dev_state =
1173 				    PMCS_DEVICE_STATE_NON_OPERATIONAL;
1174 				pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, xp->phy,
1175 				    xp, "%s: Got _IT_NEXUS_LOSS SMP status. "
1176 				    "Tgt(0x%p) dev_state set to "
1177 				    "_NON_OPERATIONAL", __func__,
1178 				    (void *)xp);
1179 			}
1180 			/* ABORT any pending commands related to this device */
1181 			if (pmcs_abort(pwp, pptr, pptr->device_id, 1, 1) != 0) {
1182 				pptr->abort_pending = 1;
1183 				smp_pkt->smp_pkt_reason = EIO;
1184 				result = DDI_FAILURE;
1185 			}
1186 		} else {
1187 			smp_pkt->smp_pkt_reason = will_retry ? EAGAIN : EIO;
1188 			result = DDI_FAILURE;
1189 		}
1190 	} else {
1191 		(void) memcpy(smp_pkt->smp_pkt_rsp,
1192 		    &((uint8_t *)pwp->scratch)[rdoff], rspsz);
1193 		if (smp_pkt->smp_pkt_reason == EOVERFLOW) {
1194 			result = DDI_FAILURE;
1195 		} else {
1196 			result = DDI_SUCCESS;
1197 		}
1198 	}
1199 out:
1200 	pmcs_prt(pwp, PMCS_PRT_DEBUG1, pptr, pptr->target,
1201 	    "%s: done for wwn 0x%" PRIx64, __func__, wwn);
1202 
1203 	pmcs_unlock_phy(pptr);
1204 	pmcs_release_scratch(pwp);
1205 	return (result);
1206 }
1207 
1208 static int
1209 pmcs_smp_init(dev_info_t *self, dev_info_t *child,
1210     smp_hba_tran_t *tran, smp_device_t *smp_sd)
1211 {
1212 	_NOTE(ARGUNUSED(tran, smp_sd));
1213 	pmcs_iport_t *iport;
1214 	pmcs_hw_t *pwp;
1215 	pmcs_xscsi_t *tgt;
1216 	pmcs_phy_t *phy, *pphy;
1217 	uint64_t wwn;
1218 	char *addr, *tgt_port;
1219 	int ua_form = 1;
1220 
1221 	iport = ddi_get_soft_state(pmcs_iport_softstate,
1222 	    ddi_get_instance(self));
1223 	ASSERT(iport);
1224 	if (iport == NULL)
1225 		return (DDI_FAILURE);
1226 	pwp = iport->pwp;
1227 	ASSERT(pwp);
1228 	if (pwp == NULL)
1229 		return (DDI_FAILURE);
1230 
1231 	/* Get "target-port" prop from devinfo node */
1232 	if (ddi_prop_lookup_string(DDI_DEV_T_ANY, child,
1233 	    DDI_PROP_DONTPASS | DDI_PROP_NOTPROM,
1234 	    SCSI_ADDR_PROP_TARGET_PORT, &tgt_port) != DDI_SUCCESS) {
1235 		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL, "%s: Failed to "
1236 		    "lookup prop ("SCSI_ADDR_PROP_TARGET_PORT")", __func__);
1237 		/* Dont fail _smp_init() because we couldnt get/set a prop */
1238 		return (DDI_SUCCESS);
1239 	}
1240 
1241 	/*
1242 	 * Validate that this tran_tgt_init is for an active iport.
1243 	 */
1244 	if (iport->ua_state == UA_INACTIVE) {
1245 		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
1246 		    "%s: Init on inactive iport for '%s'", __func__, tgt_port);
1247 		ddi_prop_free(tgt_port);
1248 		return (DDI_FAILURE);
1249 	}
1250 
1251 	mutex_enter(&pwp->lock);
1252 
1253 	/* Retrieve softstate using unit-address */
1254 	tgt = pmcs_get_target(iport, tgt_port, B_TRUE);
1255 	if (tgt == NULL) {
1256 		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
1257 		    "%s: tgt softstate not found", __func__);
1258 		ddi_prop_free(tgt_port);
1259 		mutex_exit(&pwp->lock);
1260 		return (DDI_FAILURE);
1261 	}
1262 
1263 	pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, NULL, tgt, "%s: %s (%s)",
1264 	    __func__, ddi_get_name(child), tgt_port);
1265 
1266 	mutex_enter(&tgt->statlock);
1267 	phy = tgt->phy;
1268 	ASSERT(mutex_owned(&phy->phy_lock));
1269 
1270 	if (IS_ROOT_PHY(phy)) {
1271 		/* Expander attached to HBA - don't ref_count it */
1272 		wwn = pwp->sas_wwns[0];
1273 	} else {
1274 		pmcs_inc_phy_ref_count(phy);
1275 
1276 		/*
1277 		 * Parent (in topology) is also an expander
1278 		 * Now that we've increased the ref count on phy, it's OK
1279 		 * to drop the lock so we can acquire the parent's lock.
1280 		 */
1281 		pphy = phy->parent;
1282 		mutex_exit(&tgt->statlock);
1283 		pmcs_unlock_phy(phy);
1284 		pmcs_lock_phy(pphy);
1285 		wwn = pmcs_barray2wwn(pphy->sas_address);
1286 		pmcs_unlock_phy(pphy);
1287 		pmcs_lock_phy(phy);
1288 		mutex_enter(&tgt->statlock);
1289 	}
1290 
1291 	/*
1292 	 * If this is the 1st smp_init, add this to our list.
1293 	 */
1294 	if (tgt->target_num == PMCS_INVALID_TARGET_NUM) {
1295 		int target;
1296 		for (target = 0; target < pwp->max_dev; target++) {
1297 			if (pwp->targets[target] != NULL) {
1298 				continue;
1299 			}
1300 
1301 			pwp->targets[target] = tgt;
1302 			tgt->target_num = (uint16_t)target;
1303 			tgt->assigned = 1;
1304 			tgt->dev_state = PMCS_DEVICE_STATE_OPERATIONAL;
1305 			break;
1306 		}
1307 
1308 		if (target == pwp->max_dev) {
1309 			pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, NULL, NULL,
1310 			    "Target list full.");
1311 			goto smp_init_fail;
1312 		}
1313 	}
1314 
1315 	if (!pmcs_assign_device(pwp, tgt)) {
1316 		pwp->targets[tgt->target_num] = NULL;
1317 		pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, NULL, tgt,
1318 		    "%s: pmcs_assign_device failed for target 0x%p",
1319 		    __func__, (void *)tgt);
1320 		goto smp_init_fail;
1321 	}
1322 
1323 	/*
1324 	 * Update the attached port and target port pm properties
1325 	 */
1326 	tgt->smpd = smp_sd;
1327 
1328 	pmcs_unlock_phy(phy);
1329 	mutex_exit(&pwp->lock);
1330 
1331 	tgt->ref_count++;
1332 	tgt->dtype = phy->dtype;
1333 	mutex_exit(&tgt->statlock);
1334 
1335 	pmcs_update_phy_pm_props(phy, 0, 0, B_TRUE);
1336 
1337 	addr = scsi_wwn_to_wwnstr(wwn, ua_form, NULL);
1338 	if (smp_device_prop_update_string(smp_sd, SCSI_ADDR_PROP_ATTACHED_PORT,
1339 	    addr) != DDI_SUCCESS) {
1340 		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL, "%s: Failed to set "
1341 		    "prop ("SCSI_ADDR_PROP_ATTACHED_PORT")", __func__);
1342 	}
1343 	(void) scsi_free_wwnstr(addr);
1344 	ddi_prop_free(tgt_port);
1345 	return (DDI_SUCCESS);
1346 
1347 smp_init_fail:
1348 	tgt->phy = NULL;
1349 	tgt->target_num = PMCS_INVALID_TARGET_NUM;
1350 	phy->target = NULL;
1351 	if (!IS_ROOT_PHY(phy)) {
1352 		pmcs_dec_phy_ref_count(phy);
1353 	}
1354 	mutex_exit(&tgt->statlock);
1355 	pmcs_unlock_phy(phy);
1356 	mutex_exit(&pwp->lock);
1357 	ddi_soft_state_bystr_free(iport->tgt_sstate, tgt->unit_address);
1358 	ddi_prop_free(tgt_port);
1359 	return (DDI_FAILURE);
1360 }
1361 
1362 static void
1363 pmcs_smp_free(dev_info_t *self, dev_info_t *child,
1364     smp_hba_tran_t *tran, smp_device_t *smp)
1365 {
1366 	_NOTE(ARGUNUSED(tran, smp));
1367 	pmcs_iport_t *iport;
1368 	pmcs_hw_t *pwp;
1369 	pmcs_xscsi_t *tgt;
1370 	pmcs_phy_t *phyp;
1371 	char *tgt_port;
1372 
1373 	iport = ddi_get_soft_state(pmcs_iport_softstate,
1374 	    ddi_get_instance(self));
1375 	ASSERT(iport);
1376 	if (iport == NULL)
1377 		return;
1378 
1379 	pwp = iport->pwp;
1380 	if (pwp == NULL)
1381 		return;
1382 	ASSERT(pwp);
1383 
1384 	/* Get "target-port" prop from devinfo node */
1385 	if (ddi_prop_lookup_string(DDI_DEV_T_ANY, child,
1386 	    DDI_PROP_DONTPASS | DDI_PROP_NOTPROM,
1387 	    SCSI_ADDR_PROP_TARGET_PORT, &tgt_port) != DDI_SUCCESS) {
1388 		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL, "%s: Failed to "
1389 		    "lookup prop ("SCSI_ADDR_PROP_TARGET_PORT")", __func__);
1390 		return;
1391 	}
1392 
1393 	/* Retrieve softstate using unit-address */
1394 	mutex_enter(&pwp->lock);
1395 	tgt = ddi_soft_state_bystr_get(iport->tgt_sstate, tgt_port);
1396 	pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, NULL, tgt, "%s: %s (%s)", __func__,
1397 	    ddi_get_name(child), tgt_port);
1398 	ddi_prop_free(tgt_port);
1399 
1400 	if (tgt == NULL) {
1401 		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
1402 		    "%s: tgt softstate not found", __func__);
1403 		mutex_exit(&pwp->lock);
1404 		return;
1405 	}
1406 
1407 	phyp = tgt->phy;
1408 	if (phyp) {
1409 		mutex_enter(&phyp->phy_lock);
1410 		if (!IS_ROOT_PHY(phyp)) {
1411 			pmcs_dec_phy_ref_count(phyp);
1412 		}
1413 	}
1414 	mutex_enter(&tgt->statlock);
1415 
1416 	if (--tgt->ref_count == 0) {
1417 		/*
1418 		 * Remove this target from our list. The softstate
1419 		 * will remain, and the device will remain registered
1420 		 * with the hardware unless/until we're told that the
1421 		 * device physically went away.
1422 		 */
1423 		pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, NULL, tgt,
1424 		    "Removing target 0x%p (vtgt %d) from target list",
1425 		    (void *)tgt, tgt->target_num);
1426 		pwp->targets[tgt->target_num] = NULL;
1427 		tgt->target_num = PMCS_INVALID_TARGET_NUM;
1428 		/* If the PHY has a pointer to this target, clear it */
1429 		if (phyp && (phyp->target == tgt)) {
1430 			phyp->target = NULL;
1431 		}
1432 		tgt->phy = NULL;
1433 		pmcs_destroy_target(tgt);
1434 	} else {
1435 		mutex_exit(&tgt->statlock);
1436 	}
1437 
1438 	if (phyp) {
1439 		mutex_exit(&phyp->phy_lock);
1440 	}
1441 	mutex_exit(&pwp->lock);
1442 }
1443 
1444 static int
1445 pmcs_scsi_quiesce(dev_info_t *dip)
1446 {
1447 	pmcs_hw_t *pwp;
1448 	int totactive = -1;
1449 	pmcs_xscsi_t *xp;
1450 	uint16_t target;
1451 
1452 	if (ddi_get_soft_state(pmcs_iport_softstate, ddi_get_instance(dip)))
1453 		return (0);		/* iport */
1454 
1455 	pwp  = ddi_get_soft_state(pmcs_softc_state, ddi_get_instance(dip));
1456 	if (pwp == NULL) {
1457 		return (-1);
1458 	}
1459 	mutex_enter(&pwp->lock);
1460 	if (pwp->state != STATE_RUNNING) {
1461 		mutex_exit(&pwp->lock);
1462 		return (-1);
1463 	}
1464 
1465 	pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL, "%s called", __func__);
1466 	pwp->quiesced = pwp->blocked = 1;
1467 	while (totactive) {
1468 		totactive = 0;
1469 		for (target = 0; target < pwp->max_dev; target++) {
1470 			xp = pwp->targets[target];
1471 			if (xp == NULL) {
1472 				continue;
1473 			}
1474 			mutex_enter(&xp->statlock);
1475 			if (xp->actv_cnt) {
1476 				totactive += xp->actv_cnt;
1477 				xp->draining = 1;
1478 			}
1479 			mutex_exit(&xp->statlock);
1480 		}
1481 		if (totactive) {
1482 			cv_wait(&pwp->drain_cv, &pwp->lock);
1483 		}
1484 		/*
1485 		 * The pwp->blocked may have been reset. e.g a SCSI bus reset
1486 		 */
1487 		pwp->blocked = 1;
1488 	}
1489 
1490 	for (target = 0; target < pwp->max_dev; target++) {
1491 		xp = pwp->targets[target];
1492 		if (xp == NULL) {
1493 			continue;
1494 		}
1495 		mutex_enter(&xp->statlock);
1496 		xp->draining = 0;
1497 		mutex_exit(&xp->statlock);
1498 	}
1499 
1500 	mutex_exit(&pwp->lock);
1501 	if (totactive == 0) {
1502 		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, xp,
1503 		    "%s drain complete", __func__);
1504 	}
1505 	return (0);
1506 }
1507 
1508 static int
1509 pmcs_scsi_unquiesce(dev_info_t *dip)
1510 {
1511 	pmcs_hw_t *pwp;
1512 
1513 	if (ddi_get_soft_state(pmcs_iport_softstate, ddi_get_instance(dip)))
1514 		return (0);		/* iport */
1515 
1516 	pwp  = ddi_get_soft_state(pmcs_softc_state, ddi_get_instance(dip));
1517 	if (pwp == NULL) {
1518 		return (-1);
1519 	}
1520 	mutex_enter(&pwp->lock);
1521 	if (pwp->state != STATE_RUNNING) {
1522 		mutex_exit(&pwp->lock);
1523 		return (-1);
1524 	}
1525 	pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL, "%s called", __func__);
1526 	pwp->blocked = pwp->quiesced = 0;
1527 	mutex_exit(&pwp->lock);
1528 
1529 	/*
1530 	 * Run all pending commands.
1531 	 */
1532 	pmcs_scsa_wq_run(pwp);
1533 
1534 	/*
1535 	 * Complete all completed commands.
1536 	 * This also unlocks us.
1537 	 */
1538 	PMCS_CQ_RUN(pwp);
1539 	return (0);
1540 }
1541 
1542 /*
1543  * Start commands for a particular device
1544  * If the actual start of a command fails, return B_FALSE.  Any other result
1545  * is a B_TRUE return.
1546  */
1547 boolean_t
1548 pmcs_scsa_wq_run_one(pmcs_hw_t *pwp, pmcs_xscsi_t *xp)
1549 {
1550 	pmcs_cmd_t *sp;
1551 	pmcs_phy_t *phyp;
1552 	pmcwork_t *pwrk;
1553 	boolean_t run_one, blocked;
1554 	int rval;
1555 
1556 	/*
1557 	 * First, check to see if we're blocked or resource limited
1558 	 */
1559 	mutex_enter(&pwp->lock);
1560 	blocked = pwp->blocked;
1561 	/*
1562 	 * If resource_limited is set, we're resource constrained and
1563 	 * we will run only one work request for this target.
1564 	 */
1565 	run_one = pwp->resource_limited;
1566 	mutex_exit(&pwp->lock);
1567 
1568 	if (blocked) {
1569 		/* Queues will get restarted when we get unblocked */
1570 		return (B_TRUE);
1571 	}
1572 
1573 	/*
1574 	 * Might as well verify the queue is not empty before moving on
1575 	 */
1576 	mutex_enter(&xp->wqlock);
1577 	if (STAILQ_EMPTY(&xp->wq)) {
1578 		mutex_exit(&xp->wqlock);
1579 		return (B_TRUE);
1580 	}
1581 	mutex_exit(&xp->wqlock);
1582 
1583 	/*
1584 	 * If we're draining or resetting, just reschedule work queue and bail.
1585 	 */
1586 	mutex_enter(&xp->statlock);
1587 	if (xp->draining || xp->resetting || xp->special_running ||
1588 	    xp->special_needed) {
1589 		mutex_exit(&xp->statlock);
1590 		SCHEDULE_WORK(pwp, PMCS_WORK_RUN_QUEUES);
1591 		return (B_TRUE);
1592 	}
1593 
1594 	/*
1595 	 * Next, check to see if the target is gone.
1596 	 */
1597 	if (xp->dev_gone) {
1598 		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, xp,
1599 		    "%s: Flushing wait queue for dead tgt 0x%p", __func__,
1600 		    (void *)xp);
1601 		pmcs_flush_target_queues(pwp, xp, PMCS_TGT_WAIT_QUEUE);
1602 		mutex_exit(&xp->statlock);
1603 		return (B_TRUE);
1604 	}
1605 
1606 	/*
1607 	 * Increment the PHY's ref_count now so we know it won't go away
1608 	 * after we drop the target lock.  Drop it before returning.  If the
1609 	 * PHY dies, the commands we attempt to send will fail, but at least
1610 	 * we know we have a real PHY pointer.
1611 	 */
1612 	phyp = xp->phy;
1613 	pmcs_inc_phy_ref_count(phyp);
1614 	mutex_exit(&xp->statlock);
1615 
1616 	mutex_enter(&xp->wqlock);
1617 	while ((sp = STAILQ_FIRST(&xp->wq)) != NULL) {
1618 		pwrk = pmcs_gwork(pwp, PMCS_TAG_TYPE_CBACK, phyp);
1619 		if (pwrk == NULL) {
1620 			mutex_exit(&xp->wqlock);
1621 			mutex_enter(&pwp->lock);
1622 			if (pwp->resource_limited == 0) {
1623 				pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
1624 				    "%s: out of work structures", __func__);
1625 			}
1626 			pwp->resource_limited = 1;
1627 			SCHEDULE_WORK(pwp, PMCS_WORK_RUN_QUEUES);
1628 			mutex_exit(&pwp->lock);
1629 			return (B_FALSE);
1630 		}
1631 		STAILQ_REMOVE_HEAD(&xp->wq, cmd_next);
1632 		mutex_exit(&xp->wqlock);
1633 
1634 		pwrk->xp = xp;
1635 		pwrk->arg = sp;
1636 		pwrk->timer = 0;
1637 		sp->cmd_tag = pwrk->htag;
1638 
1639 		pwrk->dtype = xp->dtype;
1640 
1641 		if (xp->dtype == SAS) {
1642 			pwrk->ptr = (void *) pmcs_SAS_done;
1643 			if ((rval = pmcs_SAS_run(sp, pwrk)) != 0) {
1644 				if (rval != PMCS_WQ_RUN_FAIL_RES_CMP) {
1645 					sp->cmd_tag = NULL;
1646 				}
1647 				pmcs_dec_phy_ref_count(phyp);
1648 				pmcs_pwork(pwp, pwrk);
1649 				SCHEDULE_WORK(pwp, PMCS_WORK_RUN_QUEUES);
1650 				if (rval == PMCS_WQ_RUN_FAIL_RES) {
1651 					return (B_FALSE);
1652 				} else {
1653 					return (B_TRUE);
1654 				}
1655 			}
1656 		} else {
1657 			ASSERT(xp->dtype == SATA);
1658 			pwrk->ptr = (void *) pmcs_SATA_done;
1659 			if ((rval = pmcs_SATA_run(sp, pwrk)) != 0) {
1660 				sp->cmd_tag = NULL;
1661 				pmcs_dec_phy_ref_count(phyp);
1662 				pmcs_pwork(pwp, pwrk);
1663 				SCHEDULE_WORK(pwp, PMCS_WORK_RUN_QUEUES);
1664 				if (rval == PMCS_WQ_RUN_FAIL_RES) {
1665 					return (B_FALSE);
1666 				} else {
1667 					return (B_TRUE);
1668 				}
1669 			}
1670 		}
1671 
1672 		if (run_one) {
1673 			goto wq_out;
1674 		}
1675 		mutex_enter(&xp->wqlock);
1676 	}
1677 
1678 	mutex_exit(&xp->wqlock);
1679 
1680 wq_out:
1681 	pmcs_dec_phy_ref_count(phyp);
1682 	return (B_TRUE);
1683 }
1684 
1685 /*
1686  * Start commands for all devices.
1687  */
1688 void
1689 pmcs_scsa_wq_run(pmcs_hw_t *pwp)
1690 {
1691 	pmcs_xscsi_t *xp;
1692 	uint16_t target_start, target;
1693 	boolean_t	rval = B_TRUE;
1694 
1695 	mutex_enter(&pwp->lock);
1696 	target_start = pwp->last_wq_dev;
1697 	target = target_start;
1698 
1699 	do {
1700 		xp = pwp->targets[target];
1701 		if ((xp == NULL) || (STAILQ_EMPTY(&xp->wq))) {
1702 			if (++target == pwp->max_dev) {
1703 				target = 0;
1704 			}
1705 			continue;
1706 		}
1707 
1708 		mutex_exit(&pwp->lock);
1709 		rval = pmcs_scsa_wq_run_one(pwp, xp);
1710 		mutex_enter(&pwp->lock);
1711 
1712 		if (rval == B_FALSE) {
1713 			break;
1714 		}
1715 
1716 		if (++target == pwp->max_dev) {
1717 			target = 0;
1718 		}
1719 	} while (target != target_start);
1720 
1721 	if (rval) {
1722 		/*
1723 		 * If we were resource limited, but apparently are not now,
1724 		 * reschedule the work queues anyway.
1725 		 */
1726 		if (pwp->resource_limited) {
1727 			SCHEDULE_WORK(pwp, PMCS_WORK_RUN_QUEUES);
1728 		}
1729 		pwp->resource_limited = 0; /* Not resource-constrained */
1730 	} else {
1731 		/*
1732 		 * Give everybody a chance, and reschedule to run the queues
1733 		 * again as long as we're limited.
1734 		 */
1735 		pwp->resource_limited = 1;
1736 		SCHEDULE_WORK(pwp, PMCS_WORK_RUN_QUEUES);
1737 	}
1738 
1739 	pwp->last_wq_dev = target;
1740 	mutex_exit(&pwp->lock);
1741 }
1742 
1743 /*
1744  * Pull the completion queue, drop the lock and complete all elements.
1745  */
1746 
1747 void
1748 pmcs_scsa_cq_run(void *arg)
1749 {
1750 	pmcs_cq_thr_info_t *cqti = (pmcs_cq_thr_info_t *)arg;
1751 	pmcs_hw_t *pwp = cqti->cq_pwp;
1752 	pmcs_cmd_t *sp, *nxt;
1753 	struct scsi_pkt *pkt;
1754 	pmcs_xscsi_t *tgt;
1755 	pmcs_iocomp_cb_t *ioccb, *ioccb_next;
1756 	pmcs_cb_t callback;
1757 
1758 	DTRACE_PROBE1(pmcs__scsa__cq__run__start, pmcs_cq_thr_info_t *, cqti);
1759 
1760 	mutex_enter(&pwp->cq_lock);
1761 
1762 	while (!pwp->cq_info.cq_stop) {
1763 		/*
1764 		 * First, check the I/O completion callback queue.
1765 		 */
1766 		ioccb = pwp->iocomp_cb_head;
1767 		pwp->iocomp_cb_head = NULL;
1768 		pwp->iocomp_cb_tail = NULL;
1769 		mutex_exit(&pwp->cq_lock);
1770 
1771 		while (ioccb) {
1772 			/*
1773 			 * Grab the lock on the work structure. The callback
1774 			 * routine is responsible for clearing it.
1775 			 */
1776 			mutex_enter(&ioccb->pwrk->lock);
1777 			ioccb_next = ioccb->next;
1778 			callback = (pmcs_cb_t)ioccb->pwrk->ptr;
1779 			(*callback)(pwp, ioccb->pwrk,
1780 			    (uint32_t *)((void *)ioccb->iomb));
1781 			kmem_cache_free(pwp->iocomp_cb_cache, ioccb);
1782 			ioccb = ioccb_next;
1783 		}
1784 
1785 		/*
1786 		 * Next, run the completion queue
1787 		 */
1788 		mutex_enter(&pwp->cq_lock);
1789 		sp = STAILQ_FIRST(&pwp->cq);
1790 		STAILQ_INIT(&pwp->cq);
1791 		mutex_exit(&pwp->cq_lock);
1792 
1793 		DTRACE_PROBE1(pmcs__scsa__cq__run__start__loop,
1794 		    pmcs_cq_thr_info_t *, cqti);
1795 
1796 		if (sp && pmcs_check_acc_dma_handle(pwp)) {
1797 			ddi_fm_service_impact(pwp->dip, DDI_SERVICE_UNAFFECTED);
1798 		}
1799 
1800 		while (sp) {
1801 			nxt = STAILQ_NEXT(sp, cmd_next);
1802 			pkt = CMD2PKT(sp);
1803 			tgt = sp->cmd_target;
1804 			pmcs_prt(pwp, PMCS_PRT_DEBUG3, NULL, tgt,
1805 			    "%s: calling completion on %p for tgt %p", __func__,
1806 			    (void *)sp, (void *)tgt);
1807 			if (tgt) {
1808 				mutex_enter(&tgt->statlock);
1809 				ASSERT(tgt->actv_pkts != 0);
1810 				tgt->actv_pkts--;
1811 				mutex_exit(&tgt->statlock);
1812 			}
1813 			scsi_hba_pkt_comp(pkt);
1814 			sp = nxt;
1815 		}
1816 
1817 		DTRACE_PROBE1(pmcs__scsa__cq__run__end__loop,
1818 		    pmcs_cq_thr_info_t *, cqti);
1819 
1820 		/*
1821 		 * Check if there are more completions to do.  If so, and we've
1822 		 * not been told to stop, skip the wait and cycle through again.
1823 		 */
1824 		mutex_enter(&pwp->cq_lock);
1825 		if ((pwp->iocomp_cb_head == NULL) && STAILQ_EMPTY(&pwp->cq) &&
1826 		    !pwp->cq_info.cq_stop) {
1827 			mutex_exit(&pwp->cq_lock);
1828 			mutex_enter(&cqti->cq_thr_lock);
1829 			cv_wait(&cqti->cq_cv, &cqti->cq_thr_lock);
1830 			mutex_exit(&cqti->cq_thr_lock);
1831 			mutex_enter(&pwp->cq_lock);
1832 		}
1833 	}
1834 
1835 	mutex_exit(&pwp->cq_lock);
1836 	DTRACE_PROBE1(pmcs__scsa__cq__run__stop, pmcs_cq_thr_info_t *, cqti);
1837 	thread_exit();
1838 }
1839 
1840 /*
1841  * Run a SAS command.  Called with pwrk->lock held, returns unlocked.
1842  */
1843 static int
1844 pmcs_SAS_run(pmcs_cmd_t *sp, pmcwork_t *pwrk)
1845 {
1846 	pmcs_hw_t *pwp = CMD2PMC(sp);
1847 	struct scsi_pkt *pkt = CMD2PKT(sp);
1848 	pmcs_xscsi_t *xp = pwrk->xp;
1849 	uint32_t iq, *ptr;
1850 	sas_ssp_cmd_iu_t sc;
1851 
1852 	ASSERT(xp != NULL);
1853 	mutex_enter(&xp->statlock);
1854 	if (!xp->assigned) {
1855 		mutex_exit(&xp->statlock);
1856 		return (PMCS_WQ_RUN_FAIL_OTHER);
1857 	}
1858 	if ((xp->actv_cnt >= xp->qdepth) || xp->recover_wait) {
1859 		mutex_exit(&xp->statlock);
1860 		mutex_enter(&xp->wqlock);
1861 		STAILQ_INSERT_HEAD(&xp->wq, sp, cmd_next);
1862 		mutex_exit(&xp->wqlock);
1863 		return (PMCS_WQ_RUN_FAIL_OTHER);
1864 	}
1865 	GET_IO_IQ_ENTRY(pwp, ptr, pwrk->phy->device_id, iq);
1866 	if (ptr == NULL) {
1867 		mutex_exit(&xp->statlock);
1868 		/*
1869 		 * This is a temporary failure not likely to unblocked by
1870 		 * commands completing as the test for scheduling the
1871 		 * restart of work is a per-device test.
1872 		 */
1873 		mutex_enter(&xp->wqlock);
1874 		STAILQ_INSERT_HEAD(&xp->wq, sp, cmd_next);
1875 		mutex_exit(&xp->wqlock);
1876 		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, xp,
1877 		    "%s: Failed to get IO IQ entry for tgt %d",
1878 		    __func__, xp->target_num);
1879 		return (PMCS_WQ_RUN_FAIL_RES);
1880 
1881 	}
1882 
1883 	ptr[0] =
1884 	    LE_32(PMCS_IOMB_IN_SAS(PMCS_OQ_IODONE, PMCIN_SSP_INI_IO_START));
1885 	ptr[1] = LE_32(pwrk->htag);
1886 	ptr[2] = LE_32(pwrk->phy->device_id);
1887 	ptr[3] = LE_32(pkt->pkt_dma_len);
1888 	if (ptr[3]) {
1889 		ASSERT(pkt->pkt_numcookies);
1890 		if (pkt->pkt_dma_flags & DDI_DMA_READ) {
1891 			ptr[4] = LE_32(PMCIN_DATADIR_2_INI);
1892 		} else {
1893 			ptr[4] = LE_32(PMCIN_DATADIR_2_DEV);
1894 		}
1895 		if (pmcs_dma_load(pwp, sp, ptr)) {
1896 			mutex_exit(&pwp->iqp_lock[iq]);
1897 			mutex_exit(&xp->statlock);
1898 			mutex_enter(&xp->wqlock);
1899 			if (STAILQ_EMPTY(&xp->wq)) {
1900 				STAILQ_INSERT_HEAD(&xp->wq, sp, cmd_next);
1901 				mutex_exit(&xp->wqlock);
1902 				return (PMCS_WQ_RUN_FAIL_RES);
1903 			} else {
1904 				mutex_exit(&xp->wqlock);
1905 				CMD2PKT(sp)->pkt_scbp[0] = STATUS_QFULL;
1906 				CMD2PKT(sp)->pkt_reason = CMD_CMPLT;
1907 				CMD2PKT(sp)->pkt_state |= STATE_GOT_BUS |
1908 				    STATE_GOT_TARGET | STATE_SENT_CMD |
1909 				    STATE_GOT_STATUS;
1910 				sp->cmd_tag = NULL;
1911 				mutex_enter(&pwp->cq_lock);
1912 				STAILQ_INSERT_TAIL(&pwp->cq, sp, cmd_next);
1913 				PMCS_CQ_RUN_LOCKED(pwp);
1914 				mutex_exit(&pwp->cq_lock);
1915 				pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, xp,
1916 				    "%s: Failed to dma_load for tgt %d (QF)",
1917 				    __func__, xp->target_num);
1918 				return (PMCS_WQ_RUN_FAIL_RES_CMP);
1919 			}
1920 		}
1921 	} else {
1922 		ptr[4] = LE_32(PMCIN_DATADIR_NONE);
1923 		CLEAN_MESSAGE(ptr, 12);
1924 	}
1925 	xp->actv_cnt++;
1926 	if (xp->actv_cnt > xp->maxdepth) {
1927 		xp->maxdepth = xp->actv_cnt;
1928 		pmcs_prt(pwp, PMCS_PRT_DEBUG2, pwrk->phy, xp, "%s: max depth "
1929 		    "now %u", pwrk->phy->path, xp->maxdepth);
1930 	}
1931 	mutex_exit(&xp->statlock);
1932 
1933 
1934 #ifdef	DEBUG
1935 	/*
1936 	 * Generate a PMCOUT_STATUS_XFER_CMD_FRAME_ISSUED
1937 	 * event when this goes out on the wire.
1938 	 */
1939 	ptr[4] |= PMCIN_MESSAGE_REPORT;
1940 #endif
1941 	/*
1942 	 * Fill in the SSP IU
1943 	 */
1944 
1945 	bzero(&sc, sizeof (sas_ssp_cmd_iu_t));
1946 	bcopy((uint8_t *)&sp->cmd_lun->scsi_lun, sc.lun, sizeof (scsi_lun_t));
1947 
1948 	switch (pkt->pkt_flags & FLAG_TAGMASK) {
1949 	case FLAG_HTAG:
1950 		sc.task_attribute = SAS_CMD_TASK_ATTR_HEAD;
1951 		break;
1952 	case FLAG_OTAG:
1953 		sc.task_attribute = SAS_CMD_TASK_ATTR_ORDERED;
1954 		break;
1955 	case FLAG_STAG:
1956 	default:
1957 		sc.task_attribute = SAS_CMD_TASK_ATTR_SIMPLE;
1958 		break;
1959 	}
1960 	(void) memcpy(sc.cdb, pkt->pkt_cdbp,
1961 	    min(SCSA_CDBLEN(sp), sizeof (sc.cdb)));
1962 	(void) memcpy(&ptr[5], &sc, sizeof (sas_ssp_cmd_iu_t));
1963 	pwrk->state = PMCS_WORK_STATE_ONCHIP;
1964 	mutex_exit(&pwrk->lock);
1965 	pmcs_prt(pwp, PMCS_PRT_DEBUG2, NULL, NULL,
1966 	    "%s: giving pkt %p (tag %x) to the hardware", __func__,
1967 	    (void *)pkt, pwrk->htag);
1968 #ifdef DEBUG
1969 	pmcs_print_entry(pwp, PMCS_PRT_DEBUG3, "SAS INI Message", ptr);
1970 #endif
1971 	mutex_enter(&xp->aqlock);
1972 	STAILQ_INSERT_TAIL(&xp->aq, sp, cmd_next);
1973 	mutex_exit(&xp->aqlock);
1974 	INC_IQ_ENTRY(pwp, iq);
1975 	mutex_enter(&pwrk->lock);
1976 	pwrk->timer = US2WT(CMD2PKT(sp)->pkt_time * 1000000);
1977 	if (pwrk->timer == 0) {
1978 		pwrk->timer = US2WT(1000000);
1979 	}
1980 	mutex_exit(&pwrk->lock);
1981 
1982 	/*
1983 	 * If we just submitted the last command queued from device state
1984 	 * recovery, clear the wq_recovery_tail pointer.
1985 	 */
1986 	mutex_enter(&xp->wqlock);
1987 	if (xp->wq_recovery_tail == sp) {
1988 		xp->wq_recovery_tail = NULL;
1989 	}
1990 	mutex_exit(&xp->wqlock);
1991 
1992 	return (PMCS_WQ_RUN_SUCCESS);
1993 }
1994 
1995 /*
1996  * Complete a SAS command
1997  *
1998  * Called with pwrk lock held.
1999  * The free of pwrk releases the lock.
2000  */
2001 
2002 static void
2003 pmcs_SAS_done(pmcs_hw_t *pwp, pmcwork_t *pwrk, uint32_t *msg)
2004 {
2005 	pmcs_cmd_t *sp = pwrk->arg;
2006 	pmcs_phy_t *pptr = pwrk->phy;
2007 	pmcs_xscsi_t *xp = pwrk->xp;
2008 	struct scsi_pkt *pkt = CMD2PKT(sp);
2009 	int dead;
2010 	uint32_t sts;
2011 	boolean_t aborted = B_FALSE;
2012 	boolean_t do_ds_recovery = B_FALSE;
2013 
2014 	ASSERT(xp != NULL);
2015 	ASSERT(sp != NULL);
2016 	ASSERT(pptr != NULL);
2017 
2018 	DTRACE_PROBE4(pmcs__io__done, uint64_t, pkt->pkt_dma_len, int,
2019 	    (pkt->pkt_dma_flags & DDI_DMA_READ) != 0, hrtime_t, pwrk->start,
2020 	    hrtime_t, gethrtime());
2021 
2022 	dead = pwrk->dead;
2023 
2024 	if (msg) {
2025 		sts = LE_32(msg[2]);
2026 	} else {
2027 		sts = 0;
2028 	}
2029 
2030 	if (dead != 0) {
2031 		pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, xp, "%s: dead cmd tag "
2032 		    "0x%x for %s", __func__, pwrk->htag, pptr->path);
2033 		goto out;
2034 	}
2035 
2036 	if (sts == PMCOUT_STATUS_ABORTED) {
2037 		aborted = B_TRUE;
2038 	}
2039 
2040 	if (pwrk->state == PMCS_WORK_STATE_TIMED_OUT) {
2041 		pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, xp,
2042 		    "%s: cmd 0x%p (tag 0x%x) timed out for %s",
2043 		    __func__, (void *)sp, pwrk->htag, pptr->path);
2044 		CMD2PKT(sp)->pkt_scbp[0] = STATUS_GOOD;
2045 		CMD2PKT(sp)->pkt_state |= STATE_GOT_BUS | STATE_GOT_TARGET |
2046 		    STATE_SENT_CMD;
2047 		CMD2PKT(sp)->pkt_statistics |= STAT_TIMEOUT;
2048 		goto out;
2049 	}
2050 
2051 	/*
2052 	 * If the status isn't okay but not underflow,
2053 	 * step to the side and parse the (possible) error.
2054 	 */
2055 #ifdef DEBUG
2056 	if (msg) {
2057 		pmcs_print_entry(pwp, PMCS_PRT_DEBUG3, "Outbound Message", msg);
2058 	}
2059 #endif
2060 	if (!msg) {
2061 		goto out;
2062 	}
2063 
2064 	switch (sts) {
2065 	case PMCOUT_STATUS_OPEN_CNX_ERROR_IT_NEXUS_LOSS:
2066 	case PMCOUT_STATUS_IO_DS_NON_OPERATIONAL:
2067 	case PMCOUT_STATUS_IO_DS_IN_RECOVERY:
2068 		pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, xp,
2069 		    "%s: PHY %s requires DS recovery (status=%d)",
2070 		    __func__, pptr->path, sts);
2071 		do_ds_recovery = B_TRUE;
2072 		break;
2073 	case PMCOUT_STATUS_UNDERFLOW:
2074 		(void) pmcs_set_resid(pkt, pkt->pkt_dma_len, LE_32(msg[3]));
2075 		pmcs_prt(pwp, PMCS_PRT_DEBUG_UNDERFLOW, NULL, NULL,
2076 		    "%s: underflow %u for cdb 0x%x",
2077 		    __func__, LE_32(msg[3]), pkt->pkt_cdbp[0] & 0xff);
2078 		sts = PMCOUT_STATUS_OK;
2079 		msg[3] = 0;
2080 		break;
2081 	case PMCOUT_STATUS_OK:
2082 		pkt->pkt_resid = 0;
2083 		break;
2084 	}
2085 
2086 	if (sts != PMCOUT_STATUS_OK) {
2087 		pmcs_ioerror(pwp, SAS, pwrk, msg, sts);
2088 	} else {
2089 		if (msg[3]) {
2090 			uint8_t local[PMCS_QENTRY_SIZE << 1], *xd;
2091 			sas_ssp_rsp_iu_t *rptr = (void *)local;
2092 			const int lim =
2093 			    (PMCS_QENTRY_SIZE << 1) - SAS_RSP_HDR_SIZE;
2094 			static const uint8_t ssp_rsp_evec[] = {
2095 				0x58, 0x61, 0x56, 0x72, 0x00
2096 			};
2097 
2098 			/*
2099 			 * Transform the the first part of the response
2100 			 * to host canonical form. This gives us enough
2101 			 * information to figure out what to do with the
2102 			 * rest (which remains unchanged in the incoming
2103 			 * message which can be up to two queue entries
2104 			 * in length).
2105 			 */
2106 			pmcs_endian_transform(pwp, local, &msg[5],
2107 			    ssp_rsp_evec);
2108 			xd = (uint8_t *)(&msg[5]);
2109 			xd += SAS_RSP_HDR_SIZE;
2110 
2111 			if (rptr->datapres == SAS_RSP_DATAPRES_RESPONSE_DATA) {
2112 				if (rptr->response_data_length != 4) {
2113 					pmcs_print_entry(pwp, PMCS_PRT_DEBUG,
2114 					    "Bad SAS RESPONSE DATA LENGTH",
2115 					    msg);
2116 					pkt->pkt_reason = CMD_TRAN_ERR;
2117 					goto out;
2118 				}
2119 				(void) memcpy(&sts, xd, sizeof (uint32_t));
2120 				sts = BE_32(sts);
2121 				/*
2122 				 * The only response code we should legally get
2123 				 * here is an INVALID FRAME response code.
2124 				 */
2125 				if (sts == SAS_RSP_INVALID_FRAME) {
2126 					pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, xp,
2127 					    "%s: pkt %p tgt %u path %s "
2128 					    "completed: INVALID FRAME response",
2129 					    __func__, (void *)pkt,
2130 					    xp->target_num, pptr->path);
2131 				} else {
2132 					pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, xp,
2133 					    "%s: pkt %p tgt %u path %s "
2134 					    "completed: illegal response 0x%x",
2135 					    __func__, (void *)pkt,
2136 					    xp->target_num, pptr->path, sts);
2137 				}
2138 				pkt->pkt_reason = CMD_TRAN_ERR;
2139 				goto out;
2140 			}
2141 			if (rptr->datapres == SAS_RSP_DATAPRES_SENSE_DATA) {
2142 				uint32_t slen;
2143 				slen = rptr->sense_data_length;
2144 				if (slen > lim) {
2145 					slen = lim;
2146 				}
2147 				pmcs_latch_status(pwp, sp, rptr->status, xd,
2148 				    slen, pptr->path);
2149 			} else if (rptr->datapres == SAS_RSP_DATAPRES_NO_DATA) {
2150 				pmcout_ssp_comp_t *sspcp;
2151 				sspcp = (pmcout_ssp_comp_t *)msg;
2152 				uint32_t *residp;
2153 				/*
2154 				 * This is the case for a plain SCSI status.
2155 				 * Note: If RESC_V is set and we're here, there
2156 				 * is a residual.  We need to find it and update
2157 				 * the packet accordingly.
2158 				 */
2159 				pmcs_latch_status(pwp, sp, rptr->status, NULL,
2160 				    0, pptr->path);
2161 
2162 				if (sspcp->resc_v) {
2163 					/*
2164 					 * Point residual to the SSP_RESP_IU
2165 					 */
2166 					residp = (uint32_t *)(sspcp + 1);
2167 					/*
2168 					 * param contains the number of bytes
2169 					 * between where the SSP_RESP_IU may
2170 					 * or may not be and the residual.
2171 					 * Increment residp by the appropriate
2172 					 * number of words: (param+resc_pad)/4).
2173 					 */
2174 					residp += (LE_32(sspcp->param) +
2175 					    sspcp->resc_pad) /
2176 					    sizeof (uint32_t);
2177 					pmcs_prt(pwp, PMCS_PRT_DEBUG_UNDERFLOW,
2178 					    pptr, xp, "%s: tgt 0x%p "
2179 					    "residual %d for pkt 0x%p",
2180 					    __func__, (void *) xp, *residp,
2181 					    (void *) pkt);
2182 					ASSERT(LE_32(*residp) <=
2183 					    pkt->pkt_dma_len);
2184 					(void) pmcs_set_resid(pkt,
2185 					    pkt->pkt_dma_len, LE_32(*residp));
2186 				}
2187 			} else {
2188 				pmcs_print_entry(pwp, PMCS_PRT_DEBUG,
2189 				    "illegal SAS response", msg);
2190 				pkt->pkt_reason = CMD_TRAN_ERR;
2191 				goto out;
2192 			}
2193 		} else {
2194 			pmcs_latch_status(pwp, sp, STATUS_GOOD, NULL, 0,
2195 			    pptr->path);
2196 		}
2197 		if (pkt->pkt_dma_len) {
2198 			pkt->pkt_state |= STATE_XFERRED_DATA;
2199 		}
2200 	}
2201 	pmcs_prt(pwp, PMCS_PRT_DEBUG2, pptr, xp,
2202 	    "%s: pkt %p tgt %u done reason=%x state=%x resid=%ld status=%x",
2203 	    __func__, (void *)pkt, xp->target_num, pkt->pkt_reason,
2204 	    pkt->pkt_state, pkt->pkt_resid, pkt->pkt_scbp[0]);
2205 
2206 	if (pwrk->state == PMCS_WORK_STATE_ABORTED) {
2207 		pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, xp,
2208 		    "%s: scsi_pkt 0x%p aborted for PHY %s; work = 0x%p",
2209 		    __func__, (void *)pkt, pptr->path, (void *)pwrk);
2210 		aborted = B_TRUE;
2211 	}
2212 
2213 out:
2214 	pmcs_dma_unload(pwp, sp);
2215 	mutex_enter(&xp->statlock);
2216 
2217 	/*
2218 	 * If the device no longer has a PHY pointer, clear the PHY pointer
2219 	 * from the work structure before we free it.  Otherwise, pmcs_pwork
2220 	 * may decrement the ref_count on a PHY that's been freed.
2221 	 */
2222 	if (xp->phy == NULL) {
2223 		pwrk->phy = NULL;
2224 	}
2225 
2226 	/*
2227 	 * We may arrive here due to a command timing out, which in turn
2228 	 * could be addressed in a different context.  So, free the work
2229 	 * back, but only after confirming it's not already been freed
2230 	 * elsewhere.
2231 	 */
2232 	if (pwrk->htag != PMCS_TAG_FREE) {
2233 		pmcs_pwork(pwp, pwrk);
2234 	}
2235 
2236 	/*
2237 	 * If the device is gone, we only put this command on the completion
2238 	 * queue if the work structure is not marked dead.  If it's marked
2239 	 * dead, it will already have been put there.
2240 	 */
2241 	if (xp->dev_gone) {
2242 		mutex_exit(&xp->statlock);
2243 		if (!dead) {
2244 			mutex_enter(&xp->aqlock);
2245 			STAILQ_REMOVE(&xp->aq, sp, pmcs_cmd, cmd_next);
2246 			mutex_exit(&xp->aqlock);
2247 			pmcs_prt(pwp, PMCS_PRT_DEBUG3, pptr, xp,
2248 			    "%s: Removing cmd 0x%p (htag 0x%x) from aq",
2249 			    __func__, (void *)sp, sp->cmd_tag);
2250 			mutex_enter(&pwp->cq_lock);
2251 			STAILQ_INSERT_TAIL(&pwp->cq, sp, cmd_next);
2252 			PMCS_CQ_RUN_LOCKED(pwp);
2253 			mutex_exit(&pwp->cq_lock);
2254 			pmcs_prt(pwp, PMCS_PRT_DEBUG2, pptr, xp,
2255 			    "%s: Completing command for dead target 0x%p",
2256 			    __func__, (void *)xp);
2257 		}
2258 		return;
2259 	}
2260 
2261 	ASSERT(xp->actv_cnt > 0);
2262 	if (--(xp->actv_cnt) == 0) {
2263 		if (xp->draining) {
2264 			pmcs_prt(pwp, PMCS_PRT_DEBUG1, pptr, xp,
2265 			    "%s: waking up drain waiters", __func__);
2266 			cv_signal(&pwp->drain_cv);
2267 		}
2268 	}
2269 	mutex_exit(&xp->statlock);
2270 
2271 	/*
2272 	 * If the status is other than OK, determine if it's something that
2273 	 * is worth re-attempting enumeration.  If so, mark the PHY.
2274 	 */
2275 	if (sts != PMCOUT_STATUS_OK) {
2276 		pmcs_status_disposition(pptr, sts);
2277 	}
2278 
2279 	if (dead == 0) {
2280 #ifdef	DEBUG
2281 		pmcs_cmd_t *wp;
2282 		mutex_enter(&xp->aqlock);
2283 		STAILQ_FOREACH(wp, &xp->aq, cmd_next) {
2284 			if (wp == sp) {
2285 				break;
2286 			}
2287 		}
2288 		ASSERT(wp != NULL);
2289 #else
2290 		mutex_enter(&xp->aqlock);
2291 #endif
2292 		pmcs_prt(pwp, PMCS_PRT_DEBUG3, pptr, xp,
2293 		    "%s: Removing cmd 0x%p (htag 0x%x) from aq", __func__,
2294 		    (void *)sp, sp->cmd_tag);
2295 		STAILQ_REMOVE(&xp->aq, sp, pmcs_cmd, cmd_next);
2296 		if (aborted) {
2297 			pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, xp,
2298 			    "%s: Aborted cmd for tgt 0x%p, signaling waiters",
2299 			    __func__, (void *)xp);
2300 			cv_signal(&xp->abort_cv);
2301 		}
2302 		mutex_exit(&xp->aqlock);
2303 	}
2304 
2305 	/*
2306 	 * If do_ds_recovery is set, we need to initiate device state
2307 	 * recovery.  In this case, we put this I/O back on the head of
2308 	 * the wait queue to run again after recovery is complete
2309 	 */
2310 	if (do_ds_recovery) {
2311 		mutex_enter(&xp->statlock);
2312 		pmcs_start_dev_state_recovery(xp, pptr);
2313 		mutex_exit(&xp->statlock);
2314 		pmcs_prt(pwp, PMCS_PRT_DEBUG1, pptr, xp, "%s: Putting cmd 0x%p "
2315 		    "back on wq during recovery for tgt 0x%p", __func__,
2316 		    (void *)sp, (void *)xp);
2317 		mutex_enter(&xp->wqlock);
2318 		if (xp->wq_recovery_tail == NULL) {
2319 			STAILQ_INSERT_HEAD(&xp->wq, sp, cmd_next);
2320 		} else {
2321 			/*
2322 			 * If there are other I/Os waiting at the head due to
2323 			 * device state recovery, add this one in the right spot
2324 			 * to maintain proper order.
2325 			 */
2326 			STAILQ_INSERT_AFTER(&xp->wq, xp->wq_recovery_tail, sp,
2327 			    cmd_next);
2328 		}
2329 		xp->wq_recovery_tail = sp;
2330 		mutex_exit(&xp->wqlock);
2331 	} else {
2332 		/*
2333 		 * If we're not initiating device state recovery and this
2334 		 * command was not "dead", put it on the completion queue
2335 		 */
2336 		if (!dead) {
2337 			mutex_enter(&pwp->cq_lock);
2338 			STAILQ_INSERT_TAIL(&pwp->cq, sp, cmd_next);
2339 			PMCS_CQ_RUN_LOCKED(pwp);
2340 			mutex_exit(&pwp->cq_lock);
2341 		}
2342 	}
2343 }
2344 
2345 /*
2346  * Run a SATA command (normal reads and writes),
2347  * or block and schedule a SATL interpretation
2348  * of the command.
2349  *
2350  * Called with pwrk lock held, returns unlocked.
2351  */
2352 
2353 static int
2354 pmcs_SATA_run(pmcs_cmd_t *sp, pmcwork_t *pwrk)
2355 {
2356 	pmcs_hw_t *pwp = CMD2PMC(sp);
2357 	struct scsi_pkt *pkt = CMD2PKT(sp);
2358 	pmcs_xscsi_t *xp;
2359 	uint8_t cdb_base, asc, tag;
2360 	uint32_t *ptr, iq, nblk, i, mtype;
2361 	fis_t fis;
2362 	size_t amt;
2363 	uint64_t lba;
2364 
2365 	xp = pwrk->xp;
2366 	ASSERT(xp != NULL);
2367 
2368 	/*
2369 	 * First, see if this is just a plain read/write command.
2370 	 * If not, we have to queue it up for processing, block
2371 	 * any additional commands from coming in, and wake up
2372 	 * the thread that will process this command.
2373 	 */
2374 	cdb_base = pkt->pkt_cdbp[0] & 0x1f;
2375 	if (cdb_base != SCMD_READ && cdb_base != SCMD_WRITE) {
2376 		pmcs_prt(pwp, PMCS_PRT_DEBUG1, NULL, NULL,
2377 		    "%s: special SATA cmd %p", __func__, (void *)sp);
2378 
2379 		ASSERT(xp->phy != NULL);
2380 		pmcs_pwork(pwp, pwrk);
2381 		pmcs_lock_phy(xp->phy);
2382 		mutex_enter(&xp->statlock);
2383 		xp->special_needed = 1; /* Set the special_needed flag */
2384 		STAILQ_INSERT_TAIL(&xp->sq, sp, cmd_next);
2385 		if (pmcs_run_sata_special(pwp, xp)) {
2386 			SCHEDULE_WORK(pwp, PMCS_WORK_SATA_RUN);
2387 		}
2388 		mutex_exit(&xp->statlock);
2389 		pmcs_unlock_phy(xp->phy);
2390 
2391 		return (PMCS_WQ_RUN_SUCCESS);
2392 	}
2393 
2394 	pmcs_prt(pwp, PMCS_PRT_DEBUG2, NULL, NULL, "%s: regular cmd", __func__);
2395 
2396 	mutex_enter(&xp->statlock);
2397 	if (!xp->assigned) {
2398 		mutex_exit(&xp->statlock);
2399 		return (PMCS_WQ_RUN_FAIL_OTHER);
2400 	}
2401 	if (xp->special_running || xp->special_needed || xp->recover_wait) {
2402 		mutex_exit(&xp->statlock);
2403 		mutex_enter(&xp->wqlock);
2404 		STAILQ_INSERT_HEAD(&xp->wq, sp, cmd_next);
2405 		mutex_exit(&xp->wqlock);
2406 		/*
2407 		 * By the time we get here the special
2408 		 * commands running or waiting to be run
2409 		 * may have come and gone, so kick our
2410 		 * worker to run the waiting queues
2411 		 * just in case.
2412 		 */
2413 		SCHEDULE_WORK(pwp, PMCS_WORK_RUN_QUEUES);
2414 		return (PMCS_WQ_RUN_FAIL_OTHER);
2415 	}
2416 	lba = xp->capacity;
2417 	mutex_exit(&xp->statlock);
2418 
2419 	/*
2420 	 * Extract data length and lba parameters out of the command. The
2421 	 * function pmcs_SATA_rwparm returns a non-zero ASC value if the CDB
2422 	 * values are considered illegal.
2423 	 */
2424 	asc = pmcs_SATA_rwparm(pkt->pkt_cdbp, &nblk, &lba, lba);
2425 	if (asc) {
2426 		uint8_t sns[18];
2427 		bzero(sns, sizeof (sns));
2428 		sns[0] = 0xf0;
2429 		sns[2] = 0x5;
2430 		sns[12] = asc;
2431 		pmcs_latch_status(pwp, sp, STATUS_CHECK, sns, sizeof (sns),
2432 		    pwrk->phy->path);
2433 		pmcs_pwork(pwp, pwrk);
2434 		mutex_enter(&pwp->cq_lock);
2435 		STAILQ_INSERT_TAIL(&pwp->cq, sp, cmd_next);
2436 		PMCS_CQ_RUN_LOCKED(pwp);
2437 		mutex_exit(&pwp->cq_lock);
2438 		return (PMCS_WQ_RUN_SUCCESS);
2439 	}
2440 
2441 	/*
2442 	 * If the command decodes as not moving any data, complete it here.
2443 	 */
2444 	amt = nblk;
2445 	amt <<= 9;
2446 	amt = pmcs_set_resid(pkt, amt, nblk << 9);
2447 	if (amt == 0) {
2448 		pmcs_latch_status(pwp, sp, STATUS_GOOD, NULL, 0,
2449 		    pwrk->phy->path);
2450 		pmcs_pwork(pwp, pwrk);
2451 		mutex_enter(&pwp->cq_lock);
2452 		STAILQ_INSERT_TAIL(&pwp->cq, sp, cmd_next);
2453 		PMCS_CQ_RUN_LOCKED(pwp);
2454 		mutex_exit(&pwp->cq_lock);
2455 		return (PMCS_WQ_RUN_SUCCESS);
2456 	}
2457 
2458 	/*
2459 	 * Get an inbound queue entry for this I/O
2460 	 */
2461 	GET_IO_IQ_ENTRY(pwp, ptr, xp->phy->device_id, iq);
2462 	if (ptr == NULL) {
2463 		/*
2464 		 * This is a temporary failure not likely to unblocked by
2465 		 * commands completing as the test for scheduling the
2466 		 * restart of work is a per-device test.
2467 		 */
2468 		mutex_enter(&xp->wqlock);
2469 		STAILQ_INSERT_HEAD(&xp->wq, sp, cmd_next);
2470 		mutex_exit(&xp->wqlock);
2471 		pmcs_dma_unload(pwp, sp);
2472 		SCHEDULE_WORK(pwp, PMCS_WORK_RUN_QUEUES);
2473 		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, xp,
2474 		    "%s: Failed to get IO IQ entry for tgt %d",
2475 		    __func__, xp->target_num);
2476 		return (PMCS_WQ_RUN_FAIL_RES);
2477 	}
2478 
2479 	/*
2480 	 * Get a tag.  At this point, hold statlock until the tagmap is
2481 	 * updated (just prior to sending the cmd to the hardware).
2482 	 */
2483 	mutex_enter(&xp->statlock);
2484 	for (tag = 0; tag < xp->qdepth; tag++) {
2485 		if ((xp->tagmap & (1 << tag)) == 0) {
2486 			break;
2487 		}
2488 	}
2489 
2490 	if (tag == xp->qdepth) {
2491 		mutex_exit(&xp->statlock);
2492 		mutex_exit(&pwp->iqp_lock[iq]);
2493 		mutex_enter(&xp->wqlock);
2494 		STAILQ_INSERT_HEAD(&xp->wq, sp, cmd_next);
2495 		mutex_exit(&xp->wqlock);
2496 		return (PMCS_WQ_RUN_FAIL_OTHER);
2497 	}
2498 
2499 	sp->cmd_satltag = (uint8_t)tag;
2500 
2501 	/*
2502 	 * Set up the command
2503 	 */
2504 	bzero(fis, sizeof (fis));
2505 	ptr[0] =
2506 	    LE_32(PMCS_IOMB_IN_SAS(PMCS_OQ_IODONE, PMCIN_SATA_HOST_IO_START));
2507 	ptr[1] = LE_32(pwrk->htag);
2508 	ptr[2] = LE_32(pwrk->phy->device_id);
2509 	ptr[3] = LE_32(amt);
2510 
2511 	if (xp->ncq) {
2512 		mtype = SATA_PROTOCOL_FPDMA | (tag << 16);
2513 		fis[0] = ((nblk & 0xff) << 24) | (C_BIT << 8) | FIS_REG_H2DEV;
2514 		if (cdb_base == SCMD_READ) {
2515 			fis[0] |= (READ_FPDMA_QUEUED << 16);
2516 		} else {
2517 			fis[0] |= (WRITE_FPDMA_QUEUED << 16);
2518 		}
2519 		fis[1] = (FEATURE_LBA << 24) | (lba & 0xffffff);
2520 		fis[2] = ((nblk & 0xff00) << 16) | ((lba >> 24) & 0xffffff);
2521 		fis[3] = tag << 3;
2522 	} else {
2523 		int op;
2524 		fis[0] = (C_BIT << 8) | FIS_REG_H2DEV;
2525 		if (xp->pio) {
2526 			mtype = SATA_PROTOCOL_PIO;
2527 			if (cdb_base == SCMD_READ) {
2528 				op = READ_SECTORS_EXT;
2529 			} else {
2530 				op = WRITE_SECTORS_EXT;
2531 			}
2532 		} else {
2533 			mtype = SATA_PROTOCOL_DMA;
2534 			if (cdb_base == SCMD_READ) {
2535 				op = READ_DMA_EXT;
2536 			} else {
2537 				op = WRITE_DMA_EXT;
2538 			}
2539 		}
2540 		fis[0] |= (op << 16);
2541 		fis[1] = (FEATURE_LBA << 24) | (lba & 0xffffff);
2542 		fis[2] = (lba >> 24) & 0xffffff;
2543 		fis[3] = nblk;
2544 	}
2545 
2546 	if (cdb_base == SCMD_READ) {
2547 		ptr[4] = LE_32(mtype | PMCIN_DATADIR_2_INI);
2548 	} else {
2549 		ptr[4] = LE_32(mtype | PMCIN_DATADIR_2_DEV);
2550 	}
2551 #ifdef	DEBUG
2552 	/*
2553 	 * Generate a PMCOUT_STATUS_XFER_CMD_FRAME_ISSUED
2554 	 * event when this goes out on the wire.
2555 	 */
2556 	ptr[4] |= PMCIN_MESSAGE_REPORT;
2557 #endif
2558 	for (i = 0; i < (sizeof (fis_t))/(sizeof (uint32_t)); i++) {
2559 		ptr[i+5] = LE_32(fis[i]);
2560 	}
2561 	if (pmcs_dma_load(pwp, sp, ptr)) {
2562 		mutex_exit(&xp->statlock);
2563 		mutex_exit(&pwp->iqp_lock[iq]);
2564 		mutex_enter(&xp->wqlock);
2565 		STAILQ_INSERT_HEAD(&xp->wq, sp, cmd_next);
2566 		mutex_exit(&xp->wqlock);
2567 		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, xp,
2568 		    "%s: Failed to dma_load for tgt %d",
2569 		    __func__, xp->target_num);
2570 		return (PMCS_WQ_RUN_FAIL_RES);
2571 
2572 	}
2573 
2574 	pwrk->state = PMCS_WORK_STATE_ONCHIP;
2575 	mutex_exit(&pwrk->lock);
2576 	xp->tagmap |= (1 << tag);
2577 	xp->actv_cnt++;
2578 	if (xp->actv_cnt > xp->maxdepth) {
2579 		xp->maxdepth = xp->actv_cnt;
2580 		pmcs_prt(pwp, PMCS_PRT_DEBUG2, pwrk->phy, xp,
2581 		    "%s: max depth now %u", pwrk->phy->path, xp->maxdepth);
2582 	}
2583 	mutex_exit(&xp->statlock);
2584 	mutex_enter(&xp->aqlock);
2585 	STAILQ_INSERT_TAIL(&xp->aq, sp, cmd_next);
2586 	mutex_exit(&xp->aqlock);
2587 	pmcs_prt(pwp, PMCS_PRT_DEBUG2, NULL, NULL,
2588 	    "%s: giving pkt %p to hardware", __func__, (void *)pkt);
2589 #ifdef DEBUG
2590 	pmcs_print_entry(pwp, PMCS_PRT_DEBUG3, "SATA INI Message", ptr);
2591 #endif
2592 	INC_IQ_ENTRY(pwp, iq);
2593 	mutex_enter(&pwrk->lock);
2594 	pwrk->timer = US2WT(CMD2PKT(sp)->pkt_time * 1000000);
2595 	if (pwrk->timer == 0) {
2596 		pwrk->timer = US2WT(1000000);
2597 	}
2598 	mutex_exit(&pwrk->lock);
2599 
2600 	return (PMCS_WQ_RUN_SUCCESS);
2601 }
2602 
2603 /*
2604  * Complete a SATA command.  Called with pwrk lock held.
2605  */
2606 void
2607 pmcs_SATA_done(pmcs_hw_t *pwp, pmcwork_t *pwrk, uint32_t *msg)
2608 {
2609 	pmcs_cmd_t *sp = pwrk->arg;
2610 	struct scsi_pkt *pkt = CMD2PKT(sp);
2611 	pmcs_phy_t *pptr = pwrk->phy;
2612 	int dead;
2613 	uint32_t sts;
2614 	pmcs_xscsi_t *xp;
2615 	boolean_t aborted = B_FALSE;
2616 
2617 	xp = pwrk->xp;
2618 	ASSERT(xp != NULL);
2619 
2620 	DTRACE_PROBE4(pmcs__io__done, uint64_t, pkt->pkt_dma_len, int,
2621 	    (pkt->pkt_dma_flags & DDI_DMA_READ) != 0, hrtime_t, pwrk->start,
2622 	    hrtime_t, gethrtime());
2623 
2624 	dead = pwrk->dead;
2625 
2626 	if (msg) {
2627 		sts = LE_32(msg[2]);
2628 	} else {
2629 		sts = 0;
2630 	}
2631 
2632 	if (dead != 0) {
2633 		pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, xp, "%s: dead cmd tag "
2634 		    "0x%x for %s", __func__, pwrk->htag, pptr->path);
2635 		goto out;
2636 	}
2637 	if ((pwrk->state == PMCS_WORK_STATE_TIMED_OUT) &&
2638 	    (sts != PMCOUT_STATUS_ABORTED)) {
2639 		pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, xp,
2640 		    "%s: cmd 0x%p (tag 0x%x) timed out for %s",
2641 		    __func__, (void *)sp, pwrk->htag, pptr->path);
2642 		CMD2PKT(sp)->pkt_scbp[0] = STATUS_GOOD;
2643 		/* pkt_reason already set to CMD_TIMEOUT */
2644 		ASSERT(CMD2PKT(sp)->pkt_reason == CMD_TIMEOUT);
2645 		CMD2PKT(sp)->pkt_state |= STATE_GOT_BUS | STATE_GOT_TARGET |
2646 		    STATE_SENT_CMD;
2647 		CMD2PKT(sp)->pkt_statistics |= STAT_TIMEOUT;
2648 		goto out;
2649 	}
2650 
2651 	pmcs_prt(pwp, PMCS_PRT_DEBUG2, pptr, xp, "%s: pkt %p tgt %u done",
2652 	    __func__, (void *)pkt, xp->target_num);
2653 
2654 	/*
2655 	 * If the status isn't okay but not underflow,
2656 	 * step to the side and parse the (possible) error.
2657 	 */
2658 #ifdef DEBUG
2659 	if (msg) {
2660 		pmcs_print_entry(pwp, PMCS_PRT_DEBUG3, "Outbound Message", msg);
2661 	}
2662 #endif
2663 	if (!msg) {
2664 		goto out;
2665 	}
2666 
2667 	/*
2668 	 * If the status isn't okay or we got a FIS response of some kind,
2669 	 * step to the side and parse the (possible) error.
2670 	 */
2671 	if ((sts != PMCOUT_STATUS_OK) || (LE_32(msg[3]) != 0)) {
2672 		if (sts == PMCOUT_STATUS_IO_DS_NON_OPERATIONAL) {
2673 			mutex_exit(&pwrk->lock);
2674 			pmcs_lock_phy(pptr);
2675 			mutex_enter(&xp->statlock);
2676 			if ((xp->resetting == 0) && (xp->reset_success != 0) &&
2677 			    (xp->reset_wait == 0)) {
2678 				mutex_exit(&xp->statlock);
2679 				if (pmcs_reset_phy(pwp, pptr,
2680 				    PMCS_PHYOP_LINK_RESET) != 0) {
2681 					pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, xp,
2682 					    "%s: PHY (%s) Local Control/Link "
2683 					    "Reset FAILED as part of error "
2684 					    "recovery", __func__, pptr->path);
2685 				}
2686 				mutex_enter(&xp->statlock);
2687 			}
2688 			mutex_exit(&xp->statlock);
2689 			pmcs_unlock_phy(pptr);
2690 			mutex_enter(&pwrk->lock);
2691 		}
2692 		pmcs_ioerror(pwp, SATA, pwrk, msg, sts);
2693 	} else {
2694 		pmcs_latch_status(pwp, sp, STATUS_GOOD, NULL, 0,
2695 		    pwrk->phy->path);
2696 		pkt->pkt_state |= STATE_XFERRED_DATA;
2697 		pkt->pkt_resid = 0;
2698 	}
2699 
2700 	pmcs_prt(pwp, PMCS_PRT_DEBUG2, pptr, xp,
2701 	    "%s: pkt %p tgt %u done reason=%x state=%x resid=%ld status=%x",
2702 	    __func__, (void *)pkt, xp->target_num, pkt->pkt_reason,
2703 	    pkt->pkt_state, pkt->pkt_resid, pkt->pkt_scbp[0]);
2704 
2705 	if (pwrk->state == PMCS_WORK_STATE_ABORTED) {
2706 		pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, xp,
2707 		    "%s: scsi_pkt 0x%p aborted for PHY %s; work = 0x%p",
2708 		    __func__, (void *)pkt, pptr->path, (void *)pwrk);
2709 		aborted = B_TRUE;
2710 	}
2711 
2712 out:
2713 	pmcs_dma_unload(pwp, sp);
2714 	mutex_enter(&xp->statlock);
2715 	xp->tagmap &= ~(1 << sp->cmd_satltag);
2716 
2717 	/*
2718 	 * If the device no longer has a PHY pointer, clear the PHY pointer
2719 	 * from the work structure before we free it.  Otherwise, pmcs_pwork
2720 	 * may decrement the ref_count on a PHY that's been freed.
2721 	 */
2722 	if (xp->phy == NULL) {
2723 		pwrk->phy = NULL;
2724 	}
2725 
2726 	/*
2727 	 * We may arrive here due to a command timing out, which in turn
2728 	 * could be addressed in a different context.  So, free the work
2729 	 * back, but only after confirming it's not already been freed
2730 	 * elsewhere.
2731 	 */
2732 	if (pwrk->htag != PMCS_TAG_FREE) {
2733 		pmcs_pwork(pwp, pwrk);
2734 	}
2735 
2736 	if (xp->dev_gone) {
2737 		mutex_exit(&xp->statlock);
2738 		if (!dead) {
2739 			mutex_enter(&xp->aqlock);
2740 			STAILQ_REMOVE(&xp->aq, sp, pmcs_cmd, cmd_next);
2741 			mutex_exit(&xp->aqlock);
2742 			pmcs_prt(pwp, PMCS_PRT_DEBUG3, pptr, xp,
2743 			    "%s: Removing cmd 0x%p (htag 0x%x) from aq",
2744 			    __func__, (void *)sp, sp->cmd_tag);
2745 			mutex_enter(&pwp->cq_lock);
2746 			STAILQ_INSERT_TAIL(&pwp->cq, sp, cmd_next);
2747 			PMCS_CQ_RUN_LOCKED(pwp);
2748 			mutex_exit(&pwp->cq_lock);
2749 			pmcs_prt(pwp, PMCS_PRT_DEBUG2, pptr, xp,
2750 			    "%s: Completing command for dead target 0x%p",
2751 			    __func__, (void *)xp);
2752 		}
2753 		return;
2754 	}
2755 
2756 	ASSERT(xp->actv_cnt > 0);
2757 	if (--(xp->actv_cnt) == 0) {
2758 		if (xp->draining) {
2759 			pmcs_prt(pwp, PMCS_PRT_DEBUG1, pptr, xp,
2760 			    "%s: waking up drain waiters", __func__);
2761 			cv_signal(&pwp->drain_cv);
2762 		} else if (xp->special_needed) {
2763 			SCHEDULE_WORK(pwp, PMCS_WORK_SATA_RUN);
2764 		}
2765 	}
2766 	mutex_exit(&xp->statlock);
2767 
2768 	/*
2769 	 * If the status is other than OK, determine if it's something that
2770 	 * is worth re-attempting enumeration.  If so, mark the PHY.
2771 	 */
2772 	if (sts != PMCOUT_STATUS_OK) {
2773 		pmcs_status_disposition(pptr, sts);
2774 	}
2775 
2776 	if (dead == 0) {
2777 #ifdef	DEBUG
2778 		pmcs_cmd_t *wp;
2779 		mutex_enter(&xp->aqlock);
2780 		STAILQ_FOREACH(wp, &xp->aq, cmd_next) {
2781 			if (wp == sp) {
2782 				break;
2783 			}
2784 		}
2785 		ASSERT(wp != NULL);
2786 #else
2787 		mutex_enter(&xp->aqlock);
2788 #endif
2789 		STAILQ_REMOVE(&xp->aq, sp, pmcs_cmd, cmd_next);
2790 		if (aborted) {
2791 			pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, xp,
2792 			    "%s: Aborted cmd for tgt 0x%p, signaling waiters",
2793 			    __func__, (void *)xp);
2794 			cv_signal(&xp->abort_cv);
2795 		}
2796 		mutex_exit(&xp->aqlock);
2797 		mutex_enter(&pwp->cq_lock);
2798 		STAILQ_INSERT_TAIL(&pwp->cq, sp, cmd_next);
2799 		PMCS_CQ_RUN_LOCKED(pwp);
2800 		mutex_exit(&pwp->cq_lock);
2801 	}
2802 }
2803 
2804 static uint8_t
2805 pmcs_SATA_rwparm(uint8_t *cdb, uint32_t *xfr, uint64_t *lba, uint64_t lbamax)
2806 {
2807 	uint8_t asc = 0;
2808 	switch (cdb[0]) {
2809 	case SCMD_READ_G5:
2810 	case SCMD_WRITE_G5:
2811 		*xfr =
2812 		    (((uint32_t)cdb[10]) <<  24) |
2813 		    (((uint32_t)cdb[11]) <<  16) |
2814 		    (((uint32_t)cdb[12]) <<   8) |
2815 		    ((uint32_t)cdb[13]);
2816 		*lba =
2817 		    (((uint64_t)cdb[2]) << 56) |
2818 		    (((uint64_t)cdb[3]) << 48) |
2819 		    (((uint64_t)cdb[4]) << 40) |
2820 		    (((uint64_t)cdb[5]) << 32) |
2821 		    (((uint64_t)cdb[6]) << 24) |
2822 		    (((uint64_t)cdb[7]) << 16) |
2823 		    (((uint64_t)cdb[8]) <<  8) |
2824 		    ((uint64_t)cdb[9]);
2825 		/* Check for illegal bits */
2826 		if (cdb[15]) {
2827 			asc = 0x24;	/* invalid field in cdb */
2828 		}
2829 		break;
2830 	case SCMD_READ_G4:
2831 	case SCMD_WRITE_G4:
2832 		*xfr =
2833 		    (((uint32_t)cdb[6]) <<  16) |
2834 		    (((uint32_t)cdb[7]) <<   8) |
2835 		    ((uint32_t)cdb[8]);
2836 		*lba =
2837 		    (((uint32_t)cdb[2]) << 24) |
2838 		    (((uint32_t)cdb[3]) << 16) |
2839 		    (((uint32_t)cdb[4]) <<  8) |
2840 		    ((uint32_t)cdb[5]);
2841 		/* Check for illegal bits */
2842 		if (cdb[11]) {
2843 			asc = 0x24;	/* invalid field in cdb */
2844 		}
2845 		break;
2846 	case SCMD_READ_G1:
2847 	case SCMD_WRITE_G1:
2848 		*xfr = (((uint32_t)cdb[7]) <<  8) | ((uint32_t)cdb[8]);
2849 		*lba =
2850 		    (((uint32_t)cdb[2]) << 24) |
2851 		    (((uint32_t)cdb[3]) << 16) |
2852 		    (((uint32_t)cdb[4]) <<  8) |
2853 		    ((uint32_t)cdb[5]);
2854 		/* Check for illegal bits */
2855 		if (cdb[9]) {
2856 			asc = 0x24;	/* invalid field in cdb */
2857 		}
2858 		break;
2859 	case SCMD_READ:
2860 	case SCMD_WRITE:
2861 		*xfr = cdb[4];
2862 		if (*xfr == 0) {
2863 			*xfr = 256;
2864 		}
2865 		*lba =
2866 		    (((uint32_t)cdb[1] & 0x1f) << 16) |
2867 		    (((uint32_t)cdb[2]) << 8) |
2868 		    ((uint32_t)cdb[3]);
2869 		/* Check for illegal bits */
2870 		if (cdb[5]) {
2871 			asc = 0x24;	/* invalid field in cdb */
2872 		}
2873 		break;
2874 	}
2875 
2876 	if (asc == 0) {
2877 		if ((*lba + *xfr) > lbamax) {
2878 			asc = 0x21;	/* logical block out of range */
2879 		}
2880 	}
2881 	return (asc);
2882 }
2883 
2884 /*
2885  * Called with pwrk lock held.
2886  */
2887 static void
2888 pmcs_ioerror(pmcs_hw_t *pwp, pmcs_dtype_t t, pmcwork_t *pwrk, uint32_t *w,
2889     uint32_t status)
2890 {
2891 	static uint8_t por[] = {
2892 	    0xf0, 0x0, 0x6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x28
2893 	};
2894 	static uint8_t parity[] = {
2895 	    0xf0, 0x0, 0xb, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x47, 5
2896 	};
2897 	const char *msg;
2898 	char buf[20];
2899 	pmcs_cmd_t *sp = pwrk->arg;
2900 	pmcs_phy_t *phyp = pwrk->phy;
2901 	struct scsi_pkt *pkt = CMD2PKT(sp);
2902 	uint32_t resid;
2903 
2904 	ASSERT(w != NULL);
2905 	resid = LE_32(w[3]);
2906 
2907 	msg = pmcs_status_str(status);
2908 	if (msg == NULL) {
2909 		(void) snprintf(buf, sizeof (buf), "Error 0x%x", status);
2910 		msg = buf;
2911 	}
2912 
2913 	if (status != PMCOUT_STATUS_OK) {
2914 		pmcs_prt(pwp, PMCS_PRT_DEBUG1, phyp, NULL,
2915 		    "%s: device %s tag 0x%x status %s @ %llu", __func__,
2916 		    phyp->path, pwrk->htag, msg,
2917 		    (unsigned long long)gethrtime());
2918 	}
2919 
2920 	pkt->pkt_reason = CMD_CMPLT;		/* default reason */
2921 
2922 	switch (status) {
2923 	case PMCOUT_STATUS_OK:
2924 		if (t == SATA) {
2925 			int i;
2926 			fis_t fis;
2927 			for (i = 0; i < sizeof (fis) / sizeof (fis[0]); i++) {
2928 				fis[i] = LE_32(w[4+i]);
2929 			}
2930 			if ((fis[0] & 0xff) != FIS_REG_D2H) {
2931 				pmcs_prt(pwp, PMCS_PRT_DEBUG, phyp, NULL,
2932 				    "unexpected fis code 0x%x", fis[0] & 0xff);
2933 			} else {
2934 				pmcs_prt(pwp, PMCS_PRT_DEBUG, phyp, NULL,
2935 				    "FIS ERROR");
2936 				pmcs_fis_dump(pwp, fis);
2937 			}
2938 			pkt->pkt_reason = CMD_TRAN_ERR;
2939 			break;
2940 		}
2941 		pmcs_latch_status(pwp, sp, STATUS_GOOD, NULL, 0, phyp->path);
2942 		break;
2943 
2944 	case PMCOUT_STATUS_ABORTED:
2945 		/*
2946 		 * Command successfully aborted.
2947 		 */
2948 		if (phyp->dead) {
2949 			pkt->pkt_reason = CMD_DEV_GONE;
2950 			pkt->pkt_state = STATE_GOT_BUS;
2951 		} else if (pwrk->ssp_event != 0) {
2952 			pkt->pkt_reason = CMD_TRAN_ERR;
2953 			pkt->pkt_state = STATE_GOT_BUS;
2954 		} else if (pwrk->state == PMCS_WORK_STATE_TIMED_OUT) {
2955 			pkt->pkt_reason = CMD_TIMEOUT;
2956 			pkt->pkt_statistics |= STAT_TIMEOUT;
2957 			pkt->pkt_state = STATE_GOT_BUS | STATE_GOT_TARGET |
2958 			    STATE_SENT_CMD;
2959 		} else {
2960 			pkt->pkt_reason = CMD_ABORTED;
2961 			pkt->pkt_statistics |= STAT_ABORTED;
2962 			pkt->pkt_state = STATE_GOT_BUS | STATE_GOT_TARGET |
2963 			    STATE_SENT_CMD;
2964 		}
2965 
2966 		/*
2967 		 * PMCS_WORK_STATE_TIMED_OUT doesn't need to be preserved past
2968 		 * this point, so go ahead and mark it as aborted.
2969 		 */
2970 		pwrk->state = PMCS_WORK_STATE_ABORTED;
2971 		break;
2972 
2973 	case PMCOUT_STATUS_UNDERFLOW:
2974 		/*
2975 		 * This will only get called for SATA
2976 		 */
2977 		pkt->pkt_resid = resid;
2978 		if (pkt->pkt_dma_len < pkt->pkt_resid) {
2979 			(void) pmcs_set_resid(pkt, pkt->pkt_dma_len, resid);
2980 		}
2981 		pmcs_latch_status(pwp, sp, STATUS_GOOD, NULL, 0, phyp->path);
2982 		break;
2983 
2984 	case PMCOUT_STATUS_NO_DEVICE:
2985 	case PMCOUT_STATUS_XFER_ERROR_SATA_LINK_TIMEOUT:
2986 		pkt->pkt_reason = CMD_DEV_GONE;
2987 		break;
2988 
2989 	case PMCOUT_STATUS_OPEN_CNX_ERROR_WRONG_DESTINATION:
2990 		/*
2991 		 * Need to do rediscovery. We probably have
2992 		 * the wrong device (disk swap), so kill
2993 		 * this one.
2994 		 */
2995 	case PMCOUT_STATUS_OPEN_CNX_PROTOCOL_NOT_SUPPORTED:
2996 	case PMCOUT_STATUS_OPEN_CNX_ERROR_ZONE_VIOLATION:
2997 	case PMCOUT_STATUS_OPEN_CNX_ERROR_CONNECTION_RATE_NOT_SUPPORTED:
2998 	case PMCOUT_STATUS_OPEN_CNX_ERROR_UNKNOWN_ERROR:
2999 		/*
3000 		 * Need to do rediscovery.
3001 		 */
3002 		if (!phyp->dead) {
3003 			mutex_exit(&pwrk->lock);
3004 			pmcs_lock_phy(pwrk->phy);
3005 			pmcs_kill_changed(pwp, pwrk->phy, 0);
3006 			pmcs_unlock_phy(pwrk->phy);
3007 			mutex_enter(&pwrk->lock);
3008 			pkt->pkt_reason = CMD_INCOMPLETE;
3009 			pkt->pkt_state = STATE_GOT_BUS;
3010 		} else {
3011 			pkt->pkt_reason = CMD_DEV_GONE;
3012 		}
3013 		break;
3014 
3015 	case PMCOUT_STATUS_OPEN_CNX_ERROR_BREAK:
3016 	case PMCOUT_STATUS_OPEN_CNX_ERROR_IT_NEXUS_LOSS:
3017 	case PMCOUT_STATUS_OPENCNX_ERROR_BAD_DESTINATION:
3018 	case PMCOUT_STATUS_IO_XFER_ERROR_NAK_RECEIVED:
3019 		/* cmd is pending on the target */
3020 	case PMCOUT_STATUS_XFER_ERROR_OFFSET_MISMATCH:
3021 	case PMCOUT_STATUS_XFER_ERROR_REJECTED_NCQ_MODE:
3022 		/* transitory - commands sent while in NCQ failure mode */
3023 	case PMCOUT_STATUS_XFER_ERROR_ABORTED_NCQ_MODE:
3024 		/* NCQ failure */
3025 	case PMCOUT_STATUS_IO_PORT_IN_RESET:
3026 	case PMCOUT_STATUS_XFER_ERR_BREAK:
3027 	case PMCOUT_STATUS_XFER_ERR_PHY_NOT_READY:
3028 		pkt->pkt_reason = CMD_INCOMPLETE;
3029 		pkt->pkt_state = STATE_GOT_BUS;
3030 		break;
3031 
3032 	case PMCOUT_STATUS_IO_XFER_OPEN_RETRY_TIMEOUT:
3033 		pmcs_prt(pwp, PMCS_PRT_DEBUG, phyp, phyp->target,
3034 		    "STATUS_BUSY for htag 0x%08x", sp->cmd_tag);
3035 		pmcs_latch_status(pwp, sp, STATUS_BUSY, NULL, 0, phyp->path);
3036 		break;
3037 
3038 	case PMCOUT_STATUS_OPEN_CNX_ERROR_STP_RESOURCES_BUSY:
3039 		/* synthesize a RESERVATION CONFLICT */
3040 		pmcs_prt(pwp, PMCS_PRT_DEBUG, phyp, phyp->target,
3041 		    "%s: Potential affiliation active on 0x%" PRIx64, __func__,
3042 		    pmcs_barray2wwn(phyp->sas_address));
3043 		pmcs_latch_status(pwp, sp, STATUS_RESERVATION_CONFLICT, NULL,
3044 		    0, phyp->path);
3045 		break;
3046 
3047 	case PMCOUT_STATUS_XFER_ERROR_ABORTED_DUE_TO_SRST:
3048 		/* synthesize a power-on/reset */
3049 		pmcs_latch_status(pwp, sp, STATUS_CHECK, por, sizeof (por),
3050 		    phyp->path);
3051 		break;
3052 
3053 	case PMCOUT_STATUS_XFER_ERROR_UNEXPECTED_PHASE:
3054 	case PMCOUT_STATUS_XFER_ERROR_RDY_OVERRUN:
3055 	case PMCOUT_STATUS_XFER_ERROR_RDY_NOT_EXPECTED:
3056 	case PMCOUT_STATUS_XFER_ERROR_CMD_ISSUE_ACK_NAK_TIMEOUT:
3057 	case PMCOUT_STATUS_XFER_ERROR_CMD_ISSUE_BREAK_BEFORE_ACK_NACK:
3058 	case PMCOUT_STATUS_XFER_ERROR_CMD_ISSUE_PHY_DOWN_BEFORE_ACK_NAK:
3059 		/* synthesize a PARITY ERROR */
3060 		pmcs_latch_status(pwp, sp, STATUS_CHECK, parity,
3061 		    sizeof (parity), phyp->path);
3062 		break;
3063 
3064 	case PMCOUT_STATUS_IO_XFER_ERROR_DMA:
3065 	case PMCOUT_STATUS_IO_NOT_VALID:
3066 	case PMCOUT_STATUS_PROG_ERROR:
3067 	case PMCOUT_STATUS_XFER_ERROR_PEER_ABORTED:
3068 	case PMCOUT_STATUS_XFER_ERROR_SATA: /* non-NCQ failure */
3069 	default:
3070 		pkt->pkt_reason = CMD_TRAN_ERR;
3071 		break;
3072 	}
3073 }
3074 
3075 /*
3076  * Latch up SCSI status
3077  */
3078 
3079 void
3080 pmcs_latch_status(pmcs_hw_t *pwp, pmcs_cmd_t *sp, uint8_t status,
3081     uint8_t *snsp, size_t snslen, char *path)
3082 {
3083 	static const char c1[] =
3084 	    "%s: Status Byte 0x%02x for CDB0=0x%02x (%02x %02x %02x) "
3085 	    "HTAG 0x%x @ %llu";
3086 	static const char c2[] =
3087 	    "%s: Status Byte 0x%02x for CDB0=0x%02x HTAG 0x%x @ %llu";
3088 
3089 	CMD2PKT(sp)->pkt_state |= STATE_GOT_BUS | STATE_GOT_TARGET |
3090 	    STATE_SENT_CMD | STATE_GOT_STATUS;
3091 	CMD2PKT(sp)->pkt_scbp[0] = status;
3092 
3093 	if (status == STATUS_CHECK && snsp &&
3094 	    (size_t)SCSA_STSLEN(sp) >= sizeof (struct scsi_arq_status)) {
3095 		struct scsi_arq_status *aqp =
3096 		    (void *) CMD2PKT(sp)->pkt_scbp;
3097 		size_t amt = sizeof (struct scsi_extended_sense);
3098 		uint8_t key = scsi_sense_key(snsp);
3099 		uint8_t asc = scsi_sense_asc(snsp);
3100 		uint8_t ascq = scsi_sense_ascq(snsp);
3101 		if (amt > snslen) {
3102 			amt = snslen;
3103 		}
3104 		pmcs_prt(pwp, PMCS_PRT_DEBUG_SCSI_STATUS, NULL, NULL, c1, path,
3105 		    status, CMD2PKT(sp)->pkt_cdbp[0] & 0xff, key, asc, ascq,
3106 		    sp->cmd_tag, (unsigned long long)gethrtime());
3107 		CMD2PKT(sp)->pkt_state |= STATE_ARQ_DONE;
3108 		(*(uint8_t *)&aqp->sts_rqpkt_status) = STATUS_GOOD;
3109 		aqp->sts_rqpkt_statistics = 0;
3110 		aqp->sts_rqpkt_reason = CMD_CMPLT;
3111 		aqp->sts_rqpkt_state = STATE_GOT_BUS |
3112 		    STATE_GOT_TARGET | STATE_SENT_CMD |
3113 		    STATE_XFERRED_DATA | STATE_GOT_STATUS;
3114 		(void) memcpy(&aqp->sts_sensedata, snsp, amt);
3115 		if (aqp->sts_sensedata.es_class != CLASS_EXTENDED_SENSE) {
3116 			aqp->sts_rqpkt_reason = CMD_TRAN_ERR;
3117 			aqp->sts_rqpkt_state = 0;
3118 			aqp->sts_rqpkt_resid =
3119 			    sizeof (struct scsi_extended_sense);
3120 		} else {
3121 			aqp->sts_rqpkt_resid =
3122 			    sizeof (struct scsi_extended_sense) - amt;
3123 		}
3124 	} else if (status) {
3125 		pmcs_prt(pwp, PMCS_PRT_DEBUG_SCSI_STATUS, NULL, NULL, c2,
3126 		    path, status, CMD2PKT(sp)->pkt_cdbp[0] & 0xff,
3127 		    sp->cmd_tag, (unsigned long long)gethrtime());
3128 	}
3129 
3130 	CMD2PKT(sp)->pkt_reason = CMD_CMPLT;
3131 }
3132 
3133 /*
3134  * Calculate and set packet residual and return the amount
3135  * left over after applying various filters.
3136  */
3137 size_t
3138 pmcs_set_resid(struct scsi_pkt *pkt, size_t amt, uint32_t cdbamt)
3139 {
3140 	pkt->pkt_resid = cdbamt;
3141 	if (amt > pkt->pkt_resid) {
3142 		amt = pkt->pkt_resid;
3143 	}
3144 	if (amt > pkt->pkt_dma_len) {
3145 		amt = pkt->pkt_dma_len;
3146 	}
3147 	return (amt);
3148 }
3149 
3150 /*
3151  * Return the existing target softstate (unlocked) if there is one.  If so,
3152  * the PHY is locked and that lock must be freed by the caller after the
3153  * target/PHY linkage is established.  If there isn't one, and alloc_tgt is
3154  * TRUE, then allocate one.
3155  */
3156 pmcs_xscsi_t *
3157 pmcs_get_target(pmcs_iport_t *iport, char *tgt_port, boolean_t alloc_tgt)
3158 {
3159 	pmcs_hw_t *pwp = iport->pwp;
3160 	pmcs_phy_t *phyp;
3161 	pmcs_xscsi_t *tgt;
3162 	uint64_t wwn;
3163 	char unit_address[PMCS_MAX_UA_SIZE];
3164 	int ua_form = 1;
3165 
3166 	/*
3167 	 * Find the PHY for this target
3168 	 */
3169 	phyp = pmcs_find_phy_by_sas_address(pwp, iport, NULL, tgt_port);
3170 	if (phyp == NULL) {
3171 		pmcs_prt(pwp, PMCS_PRT_DEBUG3, NULL, NULL,
3172 		    "%s: No PHY for target @ %s", __func__, tgt_port);
3173 		return (NULL);
3174 	}
3175 
3176 	tgt = ddi_soft_state_bystr_get(iport->tgt_sstate, tgt_port);
3177 
3178 	if (tgt) {
3179 		mutex_enter(&tgt->statlock);
3180 		/*
3181 		 * There's already a target.  Check its PHY pointer to see
3182 		 * if we need to clear the old linkages
3183 		 */
3184 		if (tgt->phy && (tgt->phy != phyp)) {
3185 			pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, phyp, tgt,
3186 			    "%s: Target PHY updated from %p to %p", __func__,
3187 			    (void *)tgt->phy, (void *)phyp);
3188 			if (!IS_ROOT_PHY(tgt->phy)) {
3189 				pmcs_dec_phy_ref_count(tgt->phy);
3190 				pmcs_inc_phy_ref_count(phyp);
3191 			}
3192 			tgt->phy->target = NULL;
3193 		}
3194 
3195 		/*
3196 		 * If this target has no PHY pointer and alloc_tgt is FALSE,
3197 		 * that implies we expect the target to already exist.  This
3198 		 * implies that there has already been a tran_tgt_init on at
3199 		 * least one LU.
3200 		 */
3201 		if ((tgt->phy == NULL) && !alloc_tgt) {
3202 			pmcs_prt(pwp, PMCS_PRT_DEBUG, phyp, tgt,
3203 			    "%s: Establish linkage from new PHY to old target @"
3204 			    "%s", __func__, tgt->unit_address);
3205 			for (int idx = 0; idx < tgt->ref_count; idx++) {
3206 				pmcs_inc_phy_ref_count(phyp);
3207 			}
3208 		}
3209 
3210 		/*
3211 		 * Set this target pointer back up, since it's been
3212 		 * through pmcs_clear_xp().
3213 		 */
3214 		tgt->dev_gone = 0;
3215 		tgt->assigned = 1;
3216 		tgt->dtype = phyp->dtype;
3217 		tgt->dev_state = PMCS_DEVICE_STATE_OPERATIONAL;
3218 		tgt->phy = phyp;
3219 		phyp->target = tgt;
3220 
3221 		mutex_exit(&tgt->statlock);
3222 		return (tgt);
3223 	}
3224 
3225 	/*
3226 	 * Make sure the PHY we found is on the correct iport
3227 	 */
3228 	if (phyp->iport != iport) {
3229 		pmcs_prt(pwp, PMCS_PRT_DEBUG, phyp, NULL,
3230 		    "%s: No target at %s on this iport", __func__, tgt_port);
3231 		pmcs_unlock_phy(phyp);
3232 		return (NULL);
3233 	}
3234 
3235 	/*
3236 	 * If this was just a lookup (i.e. alloc_tgt is false), return now.
3237 	 */
3238 	if (alloc_tgt == B_FALSE) {
3239 		pmcs_unlock_phy(phyp);
3240 		return (NULL);
3241 	}
3242 
3243 	/*
3244 	 * Allocate the new softstate
3245 	 */
3246 	wwn = pmcs_barray2wwn(phyp->sas_address);
3247 	(void) scsi_wwn_to_wwnstr(wwn, ua_form, unit_address);
3248 
3249 	if (ddi_soft_state_bystr_zalloc(iport->tgt_sstate, unit_address) !=
3250 	    DDI_SUCCESS) {
3251 		pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, phyp, tgt,
3252 		    "%s: Couldn't alloc softstate for device at %s",
3253 		    __func__, unit_address);
3254 		pmcs_unlock_phy(phyp);
3255 		return (NULL);
3256 	}
3257 
3258 	tgt = ddi_soft_state_bystr_get(iport->tgt_sstate, unit_address);
3259 	ASSERT(tgt != NULL);
3260 	STAILQ_INIT(&tgt->wq);
3261 	STAILQ_INIT(&tgt->aq);
3262 	STAILQ_INIT(&tgt->sq);
3263 	mutex_init(&tgt->statlock, NULL, MUTEX_DRIVER,
3264 	    DDI_INTR_PRI(pwp->intr_pri));
3265 	mutex_init(&tgt->wqlock, NULL, MUTEX_DRIVER,
3266 	    DDI_INTR_PRI(pwp->intr_pri));
3267 	mutex_init(&tgt->aqlock, NULL, MUTEX_DRIVER,
3268 	    DDI_INTR_PRI(pwp->intr_pri));
3269 	cv_init(&tgt->reset_cv, NULL, CV_DRIVER, NULL);
3270 	cv_init(&tgt->abort_cv, NULL, CV_DRIVER, NULL);
3271 	list_create(&tgt->lun_list, sizeof (pmcs_lun_t),
3272 	    offsetof(pmcs_lun_t, lun_list_next));
3273 	tgt->qdepth = 1;
3274 	tgt->target_num = PMCS_INVALID_TARGET_NUM;
3275 	bcopy(unit_address, tgt->unit_address, PMCS_MAX_UA_SIZE);
3276 	tgt->pwp = pwp;
3277 	tgt->ua = strdup(iport->ua);
3278 	tgt->phy = phyp;
3279 	ASSERT((phyp->target == NULL) || (phyp->target == tgt));
3280 	if (phyp->target == NULL) {
3281 		phyp->target = tgt;
3282 	}
3283 
3284 	/*
3285 	 * Don't allocate LUN softstate for SMP targets
3286 	 */
3287 	if (phyp->dtype == EXPANDER) {
3288 		return (tgt);
3289 	}
3290 
3291 	if (ddi_soft_state_bystr_init(&tgt->lun_sstate,
3292 	    sizeof (pmcs_lun_t), PMCS_LUN_SSTATE_SZ) != 0) {
3293 		pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, phyp, tgt,
3294 		    "%s: LUN soft_state_bystr_init failed", __func__);
3295 		ddi_soft_state_bystr_free(iport->tgt_sstate, tgt_port);
3296 		pmcs_unlock_phy(phyp);
3297 		return (NULL);
3298 	}
3299 
3300 	return (tgt);
3301 }
3302