xref: /illumos-gate/usr/src/uts/common/io/scsi/adapters/pmcs/pmcs_subr.c (revision 0d166b18feda26f6f45f5be1c0c8c5e539b90e6c)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  *
21  *
22  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 /*
27  * This file contains various support routines.
28  */
29 
30 #include <sys/scsi/adapters/pmcs/pmcs.h>
31 
32 /*
33  * Local static data
34  */
35 static int tgtmap_usec = MICROSEC;
36 
37 /*
38  * SAS Topology Configuration
39  */
40 static void pmcs_new_tport(pmcs_hw_t *, pmcs_phy_t *);
41 static void pmcs_configure_expander(pmcs_hw_t *, pmcs_phy_t *, pmcs_iport_t *);
42 
43 static boolean_t pmcs_check_expanders(pmcs_hw_t *, pmcs_phy_t *);
44 static void pmcs_check_expander(pmcs_hw_t *, pmcs_phy_t *);
45 static void pmcs_clear_expander(pmcs_hw_t *, pmcs_phy_t *, int);
46 
47 static int pmcs_expander_get_nphy(pmcs_hw_t *, pmcs_phy_t *);
48 static int pmcs_expander_content_discover(pmcs_hw_t *, pmcs_phy_t *,
49     pmcs_phy_t *);
50 
51 static int pmcs_smp_function_result(pmcs_hw_t *, smp_response_frame_t *);
52 static boolean_t pmcs_validate_devid(pmcs_phy_t *, pmcs_phy_t *, uint32_t);
53 static void pmcs_clear_phys(pmcs_hw_t *, pmcs_phy_t *);
54 static int pmcs_configure_new_devices(pmcs_hw_t *, pmcs_phy_t *);
55 static boolean_t pmcs_report_observations(pmcs_hw_t *);
56 static boolean_t pmcs_report_iport_observations(pmcs_hw_t *, pmcs_iport_t *,
57     pmcs_phy_t *);
58 static pmcs_phy_t *pmcs_find_phy_needing_work(pmcs_hw_t *, pmcs_phy_t *);
59 static int pmcs_kill_devices(pmcs_hw_t *, pmcs_phy_t *);
60 static void pmcs_lock_phy_impl(pmcs_phy_t *, int);
61 static void pmcs_unlock_phy_impl(pmcs_phy_t *, int);
62 static pmcs_phy_t *pmcs_clone_phy(pmcs_phy_t *);
63 static boolean_t pmcs_configure_phy(pmcs_hw_t *, pmcs_phy_t *);
64 static void pmcs_reap_dead_phy(pmcs_phy_t *);
65 static pmcs_iport_t *pmcs_get_iport_by_ua(pmcs_hw_t *, char *);
66 static boolean_t pmcs_phy_target_match(pmcs_phy_t *);
67 
68 /*
69  * Often used strings
70  */
71 const char pmcs_nowrk[] = "%s: unable to get work structure";
72 const char pmcs_nomsg[] = "%s: unable to get Inbound Message entry";
73 const char pmcs_timeo[] = "%s: command timed out";
74 
75 extern const ddi_dma_attr_t pmcs_dattr;
76 
77 /*
78  * Some Initial setup steps.
79  */
80 
81 int
82 pmcs_setup(pmcs_hw_t *pwp)
83 {
84 	uint32_t barval = pwp->mpibar;
85 	uint32_t i, scratch, regbar, regoff, barbar, baroff;
86 	uint32_t new_ioq_depth, ferr = 0;
87 
88 	/*
89 	 * Check current state. If we're not at READY state,
90 	 * we can't go further.
91 	 */
92 	scratch = pmcs_rd_msgunit(pwp, PMCS_MSGU_SCRATCH1);
93 	if ((scratch & PMCS_MSGU_AAP_STATE_MASK) == PMCS_MSGU_AAP_STATE_ERROR) {
94 		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
95 		    "%s: AAP Error State (0x%x)",
96 		    __func__, pmcs_rd_msgunit(pwp, PMCS_MSGU_SCRATCH1) &
97 		    PMCS_MSGU_AAP_ERROR_MASK);
98 		pmcs_fm_ereport(pwp, DDI_FM_DEVICE_INVAL_STATE);
99 		ddi_fm_service_impact(pwp->dip, DDI_SERVICE_LOST);
100 		return (-1);
101 	}
102 	if ((scratch & PMCS_MSGU_AAP_STATE_MASK) != PMCS_MSGU_AAP_STATE_READY) {
103 		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
104 		    "%s: AAP unit not ready (state 0x%x)",
105 		    __func__, scratch & PMCS_MSGU_AAP_STATE_MASK);
106 		pmcs_fm_ereport(pwp, DDI_FM_DEVICE_INVAL_STATE);
107 		ddi_fm_service_impact(pwp->dip, DDI_SERVICE_LOST);
108 		return (-1);
109 	}
110 
111 	/*
112 	 * Read the offset from the Message Unit scratchpad 0 register.
113 	 * This allows us to read the MPI Configuration table.
114 	 *
115 	 * Check its signature for validity.
116 	 */
117 	baroff = barval;
118 	barbar = barval >> PMCS_MSGU_MPI_BAR_SHIFT;
119 	baroff &= PMCS_MSGU_MPI_OFFSET_MASK;
120 
121 	regoff = pmcs_rd_msgunit(pwp, PMCS_MSGU_SCRATCH0);
122 	regbar = regoff >> PMCS_MSGU_MPI_BAR_SHIFT;
123 	regoff &= PMCS_MSGU_MPI_OFFSET_MASK;
124 
125 	if (regoff > baroff) {
126 		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
127 		    "%s: bad MPI Table Length (register offset=0x%08x, "
128 		    "passed offset=0x%08x)", __func__, regoff, baroff);
129 		return (-1);
130 	}
131 	if (regbar != barbar) {
132 		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
133 		    "%s: bad MPI BAR (register BAROFF=0x%08x, "
134 		    "passed BAROFF=0x%08x)", __func__, regbar, barbar);
135 		return (-1);
136 	}
137 	pwp->mpi_offset = regoff;
138 	if (pmcs_rd_mpi_tbl(pwp, PMCS_MPI_AS) != PMCS_SIGNATURE) {
139 		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
140 		    "%s: Bad MPI Configuration Table Signature 0x%x", __func__,
141 		    pmcs_rd_mpi_tbl(pwp, PMCS_MPI_AS));
142 		return (-1);
143 	}
144 
145 	if (pmcs_rd_mpi_tbl(pwp, PMCS_MPI_IR) != PMCS_MPI_REVISION1) {
146 		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
147 		    "%s: Bad MPI Configuration Revision 0x%x", __func__,
148 		    pmcs_rd_mpi_tbl(pwp, PMCS_MPI_IR));
149 		return (-1);
150 	}
151 
152 	/*
153 	 * Generate offsets for the General System, Inbound Queue Configuration
154 	 * and Outbound Queue configuration tables. This way the macros to
155 	 * access those tables will work correctly.
156 	 */
157 	pwp->mpi_gst_offset =
158 	    pwp->mpi_offset + pmcs_rd_mpi_tbl(pwp, PMCS_MPI_GSTO);
159 	pwp->mpi_iqc_offset =
160 	    pwp->mpi_offset + pmcs_rd_mpi_tbl(pwp, PMCS_MPI_IQCTO);
161 	pwp->mpi_oqc_offset =
162 	    pwp->mpi_offset + pmcs_rd_mpi_tbl(pwp, PMCS_MPI_OQCTO);
163 
164 	pwp->fw = pmcs_rd_mpi_tbl(pwp, PMCS_MPI_FW);
165 
166 	pwp->max_cmd = pmcs_rd_mpi_tbl(pwp, PMCS_MPI_MOIO);
167 	pwp->max_dev = pmcs_rd_mpi_tbl(pwp, PMCS_MPI_INFO0) >> 16;
168 
169 	pwp->max_iq = PMCS_MNIQ(pmcs_rd_mpi_tbl(pwp, PMCS_MPI_INFO1));
170 	pwp->max_oq = PMCS_MNOQ(pmcs_rd_mpi_tbl(pwp, PMCS_MPI_INFO1));
171 	pwp->nphy = PMCS_NPHY(pmcs_rd_mpi_tbl(pwp, PMCS_MPI_INFO1));
172 	if (pwp->max_iq <= PMCS_NIQ) {
173 		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
174 		    "%s: not enough Inbound Queues supported "
175 		    "(need %d, max_oq=%d)", __func__, pwp->max_iq, PMCS_NIQ);
176 		return (-1);
177 	}
178 	if (pwp->max_oq <= PMCS_NOQ) {
179 		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
180 		    "%s: not enough Outbound Queues supported "
181 		    "(need %d, max_oq=%d)", __func__, pwp->max_oq, PMCS_NOQ);
182 		return (-1);
183 	}
184 	if (pwp->nphy == 0) {
185 		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
186 		    "%s: zero phys reported", __func__);
187 		return (-1);
188 	}
189 	if (PMCS_HPIQ(pmcs_rd_mpi_tbl(pwp, PMCS_MPI_INFO1))) {
190 		pwp->hipri_queue = (1 << PMCS_IQ_OTHER);
191 	}
192 
193 
194 	for (i = 0; i < pwp->nphy; i++) {
195 		PMCS_MPI_EVQSET(pwp, PMCS_OQ_EVENTS, i);
196 		PMCS_MPI_NCQSET(pwp, PMCS_OQ_EVENTS, i);
197 	}
198 
199 	pmcs_wr_mpi_tbl(pwp, PMCS_MPI_INFO2,
200 	    (PMCS_OQ_EVENTS << GENERAL_EVENT_OQ_SHIFT) |
201 	    (PMCS_OQ_EVENTS << DEVICE_HANDLE_REMOVED_SHIFT));
202 
203 	/*
204 	 * Verify that ioq_depth is valid (> 0 and not so high that it
205 	 * would cause us to overrun the chip with commands).
206 	 */
207 	if (pwp->ioq_depth == 0) {
208 		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
209 		    "%s: I/O queue depth set to 0. Setting to %d",
210 		    __func__, PMCS_NQENTRY);
211 		pwp->ioq_depth = PMCS_NQENTRY;
212 	}
213 
214 	if (pwp->ioq_depth < PMCS_MIN_NQENTRY) {
215 		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
216 		    "%s: I/O queue depth set too low (%d). Setting to %d",
217 		    __func__, pwp->ioq_depth, PMCS_MIN_NQENTRY);
218 		pwp->ioq_depth = PMCS_MIN_NQENTRY;
219 	}
220 
221 	if (pwp->ioq_depth > (pwp->max_cmd / (PMCS_IO_IQ_MASK + 1))) {
222 		new_ioq_depth = pwp->max_cmd / (PMCS_IO_IQ_MASK + 1);
223 		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
224 		    "%s: I/O queue depth set too high (%d). Setting to %d",
225 		    __func__, pwp->ioq_depth, new_ioq_depth);
226 		pwp->ioq_depth = new_ioq_depth;
227 	}
228 
229 	/*
230 	 * Allocate consistent memory for OQs and IQs.
231 	 */
232 	pwp->iqp_dma_attr = pwp->oqp_dma_attr = pmcs_dattr;
233 	pwp->iqp_dma_attr.dma_attr_align =
234 	    pwp->oqp_dma_attr.dma_attr_align = PMCS_QENTRY_SIZE;
235 
236 	/*
237 	 * The Rev C chip has the ability to do PIO to or from consistent
238 	 * memory anywhere in a 64 bit address space, but the firmware is
239 	 * not presently set up to do so.
240 	 */
241 	pwp->iqp_dma_attr.dma_attr_addr_hi =
242 	    pwp->oqp_dma_attr.dma_attr_addr_hi = 0x000000FFFFFFFFFFull;
243 
244 	for (i = 0; i < PMCS_NIQ; i++) {
245 		if (pmcs_dma_setup(pwp, &pwp->iqp_dma_attr,
246 		    &pwp->iqp_acchdls[i],
247 		    &pwp->iqp_handles[i], PMCS_QENTRY_SIZE * pwp->ioq_depth,
248 		    (caddr_t *)&pwp->iqp[i], &pwp->iqaddr[i]) == B_FALSE) {
249 			pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
250 			    "Failed to setup DMA for iqp[%d]", i);
251 			return (-1);
252 		}
253 		bzero(pwp->iqp[i], PMCS_QENTRY_SIZE * pwp->ioq_depth);
254 	}
255 
256 	for (i = 0; i < PMCS_NOQ; i++) {
257 		if (pmcs_dma_setup(pwp, &pwp->oqp_dma_attr,
258 		    &pwp->oqp_acchdls[i],
259 		    &pwp->oqp_handles[i], PMCS_QENTRY_SIZE * pwp->ioq_depth,
260 		    (caddr_t *)&pwp->oqp[i], &pwp->oqaddr[i]) == B_FALSE) {
261 			pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
262 			    "Failed to setup DMA for oqp[%d]", i);
263 			return (-1);
264 		}
265 		bzero(pwp->oqp[i], PMCS_QENTRY_SIZE * pwp->ioq_depth);
266 	}
267 
268 	/*
269 	 * Install the IQ and OQ addresses (and null out the rest).
270 	 */
271 	for (i = 0; i < pwp->max_iq; i++) {
272 		pwp->iqpi_offset[i] = pmcs_rd_iqc_tbl(pwp, PMCS_IQPIOFFX(i));
273 		if (i < PMCS_NIQ) {
274 			if (i != PMCS_IQ_OTHER) {
275 				pmcs_wr_iqc_tbl(pwp, PMCS_IQC_PARMX(i),
276 				    pwp->ioq_depth | (PMCS_QENTRY_SIZE << 16));
277 			} else {
278 				pmcs_wr_iqc_tbl(pwp, PMCS_IQC_PARMX(i),
279 				    (1 << 30) | pwp->ioq_depth |
280 				    (PMCS_QENTRY_SIZE << 16));
281 			}
282 			pmcs_wr_iqc_tbl(pwp, PMCS_IQBAHX(i),
283 			    DWORD1(pwp->iqaddr[i]));
284 			pmcs_wr_iqc_tbl(pwp, PMCS_IQBALX(i),
285 			    DWORD0(pwp->iqaddr[i]));
286 			pmcs_wr_iqc_tbl(pwp, PMCS_IQCIBAHX(i),
287 			    DWORD1(pwp->ciaddr+IQ_OFFSET(i)));
288 			pmcs_wr_iqc_tbl(pwp, PMCS_IQCIBALX(i),
289 			    DWORD0(pwp->ciaddr+IQ_OFFSET(i)));
290 		} else {
291 			pmcs_wr_iqc_tbl(pwp, PMCS_IQC_PARMX(i), 0);
292 			pmcs_wr_iqc_tbl(pwp, PMCS_IQBAHX(i), 0);
293 			pmcs_wr_iqc_tbl(pwp, PMCS_IQBALX(i), 0);
294 			pmcs_wr_iqc_tbl(pwp, PMCS_IQCIBAHX(i), 0);
295 			pmcs_wr_iqc_tbl(pwp, PMCS_IQCIBALX(i), 0);
296 		}
297 	}
298 
299 	for (i = 0; i < pwp->max_oq; i++) {
300 		pwp->oqci_offset[i] = pmcs_rd_oqc_tbl(pwp, PMCS_OQCIOFFX(i));
301 		if (i < PMCS_NOQ) {
302 			pmcs_wr_oqc_tbl(pwp, PMCS_OQC_PARMX(i), pwp->ioq_depth |
303 			    (PMCS_QENTRY_SIZE << 16) | OQIEX);
304 			pmcs_wr_oqc_tbl(pwp, PMCS_OQBAHX(i),
305 			    DWORD1(pwp->oqaddr[i]));
306 			pmcs_wr_oqc_tbl(pwp, PMCS_OQBALX(i),
307 			    DWORD0(pwp->oqaddr[i]));
308 			pmcs_wr_oqc_tbl(pwp, PMCS_OQPIBAHX(i),
309 			    DWORD1(pwp->ciaddr+OQ_OFFSET(i)));
310 			pmcs_wr_oqc_tbl(pwp, PMCS_OQPIBALX(i),
311 			    DWORD0(pwp->ciaddr+OQ_OFFSET(i)));
312 			pmcs_wr_oqc_tbl(pwp, PMCS_OQIPARM(i),
313 			    pwp->oqvec[i] << 24);
314 			pmcs_wr_oqc_tbl(pwp, PMCS_OQDICX(i), 0);
315 		} else {
316 			pmcs_wr_oqc_tbl(pwp, PMCS_OQC_PARMX(i), 0);
317 			pmcs_wr_oqc_tbl(pwp, PMCS_OQBAHX(i), 0);
318 			pmcs_wr_oqc_tbl(pwp, PMCS_OQBALX(i), 0);
319 			pmcs_wr_oqc_tbl(pwp, PMCS_OQPIBAHX(i), 0);
320 			pmcs_wr_oqc_tbl(pwp, PMCS_OQPIBALX(i), 0);
321 			pmcs_wr_oqc_tbl(pwp, PMCS_OQIPARM(i), 0);
322 			pmcs_wr_oqc_tbl(pwp, PMCS_OQDICX(i), 0);
323 		}
324 	}
325 
326 	/*
327 	 * Set up logging, if defined.
328 	 */
329 	if (pwp->fwlog) {
330 		uint64_t logdma = pwp->fwaddr;
331 		pmcs_wr_mpi_tbl(pwp, PMCS_MPI_MELBAH, DWORD1(logdma));
332 		pmcs_wr_mpi_tbl(pwp, PMCS_MPI_MELBAL, DWORD0(logdma));
333 		pmcs_wr_mpi_tbl(pwp, PMCS_MPI_MELBS, PMCS_FWLOG_SIZE >> 1);
334 		pmcs_wr_mpi_tbl(pwp, PMCS_MPI_MELSEV, pwp->fwlog);
335 		logdma += (PMCS_FWLOG_SIZE >> 1);
336 		pmcs_wr_mpi_tbl(pwp, PMCS_MPI_IELBAH, DWORD1(logdma));
337 		pmcs_wr_mpi_tbl(pwp, PMCS_MPI_IELBAL, DWORD0(logdma));
338 		pmcs_wr_mpi_tbl(pwp, PMCS_MPI_IELBS, PMCS_FWLOG_SIZE >> 1);
339 		pmcs_wr_mpi_tbl(pwp, PMCS_MPI_IELSEV, pwp->fwlog);
340 	}
341 
342 	/*
343 	 * Interrupt vectors, outbound queues, and odb_auto_clear
344 	 *
345 	 * MSI/MSI-X:
346 	 * If we got 4 interrupt vectors, we'll assign one to each outbound
347 	 * queue as well as the fatal interrupt, and auto clear can be set
348 	 * for each.
349 	 *
350 	 * If we only got 2 vectors, one will be used for I/O completions
351 	 * and the other for the other two vectors.  In this case, auto_
352 	 * clear can only be set for I/Os, which is fine.  The fatal
353 	 * interrupt will be mapped to the PMCS_FATAL_INTERRUPT bit, which
354 	 * is not an interrupt vector.
355 	 *
356 	 * MSI/MSI-X/INT-X:
357 	 * If we only got 1 interrupt vector, auto_clear must be set to 0,
358 	 * and again the fatal interrupt will be mapped to the
359 	 * PMCS_FATAL_INTERRUPT bit (again, not an interrupt vector).
360 	 */
361 
362 	switch (pwp->int_type) {
363 	case PMCS_INT_MSIX:
364 	case PMCS_INT_MSI:
365 		switch (pwp->intr_cnt) {
366 		case 1:
367 			pmcs_wr_mpi_tbl(pwp, PMCS_MPI_FERR, PMCS_FERRIE |
368 			    (PMCS_FATAL_INTERRUPT << PMCS_FERIV_SHIFT));
369 			pwp->odb_auto_clear = 0;
370 			break;
371 		case 2:
372 			pmcs_wr_mpi_tbl(pwp, PMCS_MPI_FERR, PMCS_FERRIE |
373 			    (PMCS_FATAL_INTERRUPT << PMCS_FERIV_SHIFT));
374 			pwp->odb_auto_clear = (1 << PMCS_FATAL_INTERRUPT) |
375 			    (1 << PMCS_MSIX_IODONE);
376 			break;
377 		case 4:
378 			pmcs_wr_mpi_tbl(pwp, PMCS_MPI_FERR, PMCS_FERRIE |
379 			    (PMCS_MSIX_FATAL << PMCS_FERIV_SHIFT));
380 			pwp->odb_auto_clear = (1 << PMCS_MSIX_FATAL) |
381 			    (1 << PMCS_MSIX_GENERAL) | (1 << PMCS_MSIX_IODONE) |
382 			    (1 << PMCS_MSIX_EVENTS);
383 			break;
384 		}
385 		break;
386 
387 	case PMCS_INT_FIXED:
388 		pmcs_wr_mpi_tbl(pwp, PMCS_MPI_FERR,
389 		    PMCS_FERRIE | (PMCS_FATAL_INTERRUPT << PMCS_FERIV_SHIFT));
390 		pwp->odb_auto_clear = 0;
391 		break;
392 	}
393 
394 	/*
395 	 * Enable Interrupt Reassertion
396 	 * Default Delay 1000us
397 	 */
398 	ferr = pmcs_rd_mpi_tbl(pwp, PMCS_MPI_FERR);
399 	if ((ferr & PMCS_MPI_IRAE) == 0) {
400 		ferr &= ~(PMCS_MPI_IRAU | PMCS_MPI_IRAD_MASK);
401 		pmcs_wr_mpi_tbl(pwp, PMCS_MPI_FERR, ferr | PMCS_MPI_IRAE);
402 	}
403 
404 	pmcs_wr_topunit(pwp, PMCS_OBDB_AUTO_CLR, pwp->odb_auto_clear);
405 	pwp->mpi_table_setup = 1;
406 	return (0);
407 }
408 
409 /*
410  * Start the Message Passing protocol with the PMC chip.
411  */
412 int
413 pmcs_start_mpi(pmcs_hw_t *pwp)
414 {
415 	int i;
416 
417 	pmcs_wr_msgunit(pwp, PMCS_MSGU_IBDB, PMCS_MSGU_IBDB_MPIINI);
418 	for (i = 0; i < 1000; i++) {
419 		if ((pmcs_rd_msgunit(pwp, PMCS_MSGU_IBDB) &
420 		    PMCS_MSGU_IBDB_MPIINI) == 0) {
421 			break;
422 		}
423 		drv_usecwait(1000);
424 	}
425 	if (pmcs_rd_msgunit(pwp, PMCS_MSGU_IBDB) & PMCS_MSGU_IBDB_MPIINI) {
426 		return (-1);
427 	}
428 	drv_usecwait(500000);
429 
430 	/*
431 	 * Check to make sure we got to INIT state.
432 	 */
433 	if (PMCS_MPI_S(pmcs_rd_gst_tbl(pwp, PMCS_GST_BASE)) !=
434 	    PMCS_MPI_STATE_INIT) {
435 		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
436 		    "%s: MPI launch failed (GST 0x%x DBCLR 0x%x)", __func__,
437 		    pmcs_rd_gst_tbl(pwp, PMCS_GST_BASE),
438 		    pmcs_rd_msgunit(pwp, PMCS_MSGU_IBDB_CLEAR));
439 		return (-1);
440 	}
441 	return (0);
442 }
443 
444 /*
445  * Stop the Message Passing protocol with the PMC chip.
446  */
447 int
448 pmcs_stop_mpi(pmcs_hw_t *pwp)
449 {
450 	int i;
451 
452 	for (i = 0; i < pwp->max_iq; i++) {
453 		pmcs_wr_iqc_tbl(pwp, PMCS_IQC_PARMX(i), 0);
454 		pmcs_wr_iqc_tbl(pwp, PMCS_IQBAHX(i), 0);
455 		pmcs_wr_iqc_tbl(pwp, PMCS_IQBALX(i), 0);
456 		pmcs_wr_iqc_tbl(pwp, PMCS_IQCIBAHX(i), 0);
457 		pmcs_wr_iqc_tbl(pwp, PMCS_IQCIBALX(i), 0);
458 	}
459 	for (i = 0; i < pwp->max_oq; i++) {
460 		pmcs_wr_oqc_tbl(pwp, PMCS_OQC_PARMX(i), 0);
461 		pmcs_wr_oqc_tbl(pwp, PMCS_OQBAHX(i), 0);
462 		pmcs_wr_oqc_tbl(pwp, PMCS_OQBALX(i), 0);
463 		pmcs_wr_oqc_tbl(pwp, PMCS_OQPIBAHX(i), 0);
464 		pmcs_wr_oqc_tbl(pwp, PMCS_OQPIBALX(i), 0);
465 		pmcs_wr_oqc_tbl(pwp, PMCS_OQIPARM(i), 0);
466 		pmcs_wr_oqc_tbl(pwp, PMCS_OQDICX(i), 0);
467 	}
468 	pmcs_wr_mpi_tbl(pwp, PMCS_MPI_FERR, 0);
469 	pmcs_wr_msgunit(pwp, PMCS_MSGU_IBDB, PMCS_MSGU_IBDB_MPICTU);
470 	for (i = 0; i < 2000; i++) {
471 		if ((pmcs_rd_msgunit(pwp, PMCS_MSGU_IBDB) &
472 		    PMCS_MSGU_IBDB_MPICTU) == 0) {
473 			break;
474 		}
475 		drv_usecwait(1000);
476 	}
477 	if (pmcs_rd_msgunit(pwp, PMCS_MSGU_IBDB) & PMCS_MSGU_IBDB_MPICTU) {
478 		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
479 		    "%s: MPI stop failed", __func__);
480 		return (-1);
481 	}
482 	return (0);
483 }
484 
485 /*
486  * Do a sequence of ECHO messages to test for MPI functionality,
487  * all inbound and outbound queue functionality and interrupts.
488  */
489 int
490 pmcs_echo_test(pmcs_hw_t *pwp)
491 {
492 	echo_test_t fred;
493 	struct pmcwork *pwrk;
494 	uint32_t *msg, count;
495 	int iqe = 0, iqo = 0, result, rval = 0;
496 	int iterations;
497 	hrtime_t echo_start, echo_end, echo_total;
498 
499 	ASSERT(pwp->max_cmd > 0);
500 
501 	/*
502 	 * We want iterations to be max_cmd * 3 to ensure that we run the
503 	 * echo test enough times to iterate through every inbound queue
504 	 * at least twice.
505 	 */
506 	iterations = pwp->max_cmd * 3;
507 
508 	echo_total = 0;
509 	count = 0;
510 
511 	while (count < iterations) {
512 		pwrk = pmcs_gwork(pwp, PMCS_TAG_TYPE_WAIT, NULL);
513 		if (pwrk == NULL) {
514 			pmcs_prt(pwp, PMCS_PRT_ERR, NULL, NULL,
515 			    pmcs_nowrk, __func__);
516 			rval = -1;
517 			break;
518 		}
519 
520 		mutex_enter(&pwp->iqp_lock[iqe]);
521 		msg = GET_IQ_ENTRY(pwp, iqe);
522 		if (msg == NULL) {
523 			mutex_exit(&pwp->iqp_lock[iqe]);
524 			pmcs_pwork(pwp, pwrk);
525 			pmcs_prt(pwp, PMCS_PRT_ERR, NULL, NULL,
526 			    pmcs_nomsg, __func__);
527 			rval = -1;
528 			break;
529 		}
530 
531 		bzero(msg, PMCS_QENTRY_SIZE);
532 
533 		if (iqe == PMCS_IQ_OTHER) {
534 			/* This is on the high priority queue */
535 			msg[0] = LE_32(PMCS_HIPRI(pwp, iqo, PMCIN_ECHO));
536 		} else {
537 			msg[0] = LE_32(PMCS_IOMB_IN_SAS(iqo, PMCIN_ECHO));
538 		}
539 		msg[1] = LE_32(pwrk->htag);
540 		fred.signature = 0xdeadbeef;
541 		fred.count = count;
542 		fred.ptr = &count;
543 		(void) memcpy(&msg[2], &fred, sizeof (fred));
544 		pwrk->state = PMCS_WORK_STATE_ONCHIP;
545 
546 		INC_IQ_ENTRY(pwp, iqe);
547 
548 		echo_start = gethrtime();
549 		DTRACE_PROBE2(pmcs__echo__test__wait__start,
550 		    hrtime_t, echo_start, uint32_t, pwrk->htag);
551 
552 		if (++iqe == PMCS_NIQ) {
553 			iqe = 0;
554 		}
555 		if (++iqo == PMCS_NOQ) {
556 			iqo = 0;
557 		}
558 
559 		WAIT_FOR(pwrk, 250, result);
560 
561 		echo_end = gethrtime();
562 		DTRACE_PROBE2(pmcs__echo__test__wait__end,
563 		    hrtime_t, echo_end, int, result);
564 
565 		echo_total += (echo_end - echo_start);
566 
567 		pmcs_pwork(pwp, pwrk);
568 		if (result) {
569 			pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
570 			    "%s: command timed out on echo test #%d",
571 			    __func__, count);
572 			rval = -1;
573 			break;
574 		}
575 	}
576 
577 	/*
578 	 * The intr_threshold is adjusted by PMCS_INTR_THRESHOLD in order to
579 	 * remove the overhead of things like the delay in getting signaled
580 	 * for completion.
581 	 */
582 	if (echo_total != 0) {
583 		pwp->io_intr_coal.intr_latency =
584 		    (echo_total / iterations) / 2;
585 		pwp->io_intr_coal.intr_threshold =
586 		    PMCS_INTR_THRESHOLD(PMCS_QUANTUM_TIME_USECS * 1000 /
587 		    pwp->io_intr_coal.intr_latency);
588 	}
589 
590 	return (rval);
591 }
592 
593 /*
594  * Start the (real) phys
595  */
596 int
597 pmcs_start_phy(pmcs_hw_t *pwp, int phynum, int linkmode, int speed)
598 {
599 	int result;
600 	uint32_t *msg;
601 	struct pmcwork *pwrk;
602 	pmcs_phy_t *pptr;
603 	sas_identify_af_t sap;
604 
605 	mutex_enter(&pwp->lock);
606 	pptr = pwp->root_phys + phynum;
607 	if (pptr == NULL) {
608 		mutex_exit(&pwp->lock);
609 		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
610 		    "%s: cannot find port %d", __func__, phynum);
611 		return (0);
612 	}
613 
614 	pmcs_lock_phy(pptr);
615 	mutex_exit(&pwp->lock);
616 
617 	pwrk = pmcs_gwork(pwp, PMCS_TAG_TYPE_WAIT, pptr);
618 	if (pwrk == NULL) {
619 		pmcs_unlock_phy(pptr);
620 		pmcs_prt(pwp, PMCS_PRT_ERR, pptr, NULL, pmcs_nowrk, __func__);
621 		return (-1);
622 	}
623 
624 	mutex_enter(&pwp->iqp_lock[PMCS_IQ_OTHER]);
625 	msg = GET_IQ_ENTRY(pwp, PMCS_IQ_OTHER);
626 
627 	if (msg == NULL) {
628 		mutex_exit(&pwp->iqp_lock[PMCS_IQ_OTHER]);
629 		pmcs_unlock_phy(pptr);
630 		pmcs_pwork(pwp, pwrk);
631 		pmcs_prt(pwp, PMCS_PRT_ERR, pptr, NULL, pmcs_nomsg, __func__);
632 		return (-1);
633 	}
634 	msg[0] = LE_32(PMCS_HIPRI(pwp, PMCS_OQ_EVENTS, PMCIN_PHY_START));
635 	msg[1] = LE_32(pwrk->htag);
636 	msg[2] = LE_32(linkmode | speed | phynum);
637 	bzero(&sap, sizeof (sap));
638 	sap.device_type = SAS_IF_DTYPE_ENDPOINT;
639 	sap.ssp_ini_port = 1;
640 
641 	if (pwp->separate_ports) {
642 		pmcs_wwn2barray(pwp->sas_wwns[phynum], sap.sas_address);
643 	} else {
644 		pmcs_wwn2barray(pwp->sas_wwns[0], sap.sas_address);
645 	}
646 
647 	ASSERT(phynum < SAS2_PHYNUM_MAX);
648 	sap.phy_identifier = phynum & SAS2_PHYNUM_MASK;
649 	(void) memcpy(&msg[3], &sap, sizeof (sas_identify_af_t));
650 	pwrk->state = PMCS_WORK_STATE_ONCHIP;
651 	INC_IQ_ENTRY(pwp, PMCS_IQ_OTHER);
652 
653 	pptr->state.prog_min_rate = (lowbit((ulong_t)speed) - 1);
654 	pptr->state.prog_max_rate = (highbit((ulong_t)speed) - 1);
655 	pptr->state.hw_min_rate = PMCS_HW_MIN_LINK_RATE;
656 	pptr->state.hw_max_rate = PMCS_HW_MAX_LINK_RATE;
657 
658 	pmcs_unlock_phy(pptr);
659 	WAIT_FOR(pwrk, 1000, result);
660 	pmcs_pwork(pwp, pwrk);
661 
662 	if (result) {
663 		pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, NULL, pmcs_timeo, __func__);
664 	} else {
665 		mutex_enter(&pwp->lock);
666 		pwp->phys_started |= (1 << phynum);
667 		mutex_exit(&pwp->lock);
668 	}
669 
670 	return (0);
671 }
672 
673 int
674 pmcs_start_phys(pmcs_hw_t *pwp)
675 {
676 	int i;
677 
678 	for (i = 0; i < pwp->nphy; i++) {
679 		if ((pwp->phyid_block_mask & (1 << i)) == 0) {
680 			if (pmcs_start_phy(pwp, i,
681 			    (pwp->phymode << PHY_MODE_SHIFT),
682 			    pwp->physpeed << PHY_LINK_SHIFT)) {
683 				return (-1);
684 			}
685 			if (pmcs_clear_diag_counters(pwp, i)) {
686 				pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
687 				    "%s: failed to reset counters on PHY (%d)",
688 				    __func__, i);
689 			}
690 		}
691 	}
692 	return (0);
693 }
694 
695 /*
696  * Called with PHY locked
697  */
698 int
699 pmcs_reset_phy(pmcs_hw_t *pwp, pmcs_phy_t *pptr, uint8_t type)
700 {
701 	uint32_t *msg;
702 	uint32_t iomb[(PMCS_QENTRY_SIZE << 1) >> 2];
703 	const char *mbar;
704 	uint32_t amt;
705 	uint32_t pdevid;
706 	uint32_t stsoff;
707 	uint32_t status;
708 	int result, level, phynum;
709 	struct pmcwork *pwrk;
710 	uint32_t htag;
711 
712 	ASSERT(mutex_owned(&pptr->phy_lock));
713 
714 	bzero(iomb, PMCS_QENTRY_SIZE);
715 	phynum = pptr->phynum;
716 	level = pptr->level;
717 	if (level > 0) {
718 		pdevid = pptr->parent->device_id;
719 	}
720 
721 	pwrk = pmcs_gwork(pwp, PMCS_TAG_TYPE_WAIT, pptr);
722 
723 	if (pwrk == NULL) {
724 		pmcs_prt(pwp, PMCS_PRT_ERR, pptr, NULL, pmcs_nowrk, __func__);
725 		return (ENOMEM);
726 	}
727 
728 	pwrk->arg = iomb;
729 
730 	/*
731 	 * If level > 0, we need to issue an SMP_REQUEST with a PHY_CONTROL
732 	 * function to do either a link reset or hard reset.  If level == 0,
733 	 * then we do a LOCAL_PHY_CONTROL IOMB to do link/hard reset to the
734 	 * root (local) PHY
735 	 */
736 	if (level) {
737 		stsoff = 2;
738 		iomb[0] = LE_32(PMCS_HIPRI(pwp, PMCS_OQ_GENERAL,
739 		    PMCIN_SMP_REQUEST));
740 		iomb[1] = LE_32(pwrk->htag);
741 		iomb[2] = LE_32(pdevid);
742 		iomb[3] = LE_32(40 << SMP_REQUEST_LENGTH_SHIFT);
743 		/*
744 		 * Send SMP PHY CONTROL/HARD or LINK RESET
745 		 */
746 		iomb[4] = BE_32(0x40910000);
747 		iomb[5] = 0;
748 
749 		if (type == PMCS_PHYOP_HARD_RESET) {
750 			mbar = "SMP PHY CONTROL/HARD RESET";
751 			iomb[6] = BE_32((phynum << 24) |
752 			    (PMCS_PHYOP_HARD_RESET << 16));
753 		} else {
754 			mbar = "SMP PHY CONTROL/LINK RESET";
755 			iomb[6] = BE_32((phynum << 24) |
756 			    (PMCS_PHYOP_LINK_RESET << 16));
757 		}
758 		pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, NULL,
759 		    "%s: sending %s to %s for phy 0x%x",
760 		    __func__, mbar, pptr->parent->path, pptr->phynum);
761 		amt = 7;
762 	} else {
763 		/*
764 		 * Unlike most other Outbound messages, status for
765 		 * a local phy operation is in DWORD 3.
766 		 */
767 		stsoff = 3;
768 		iomb[0] = LE_32(PMCS_HIPRI(pwp, PMCS_OQ_GENERAL,
769 		    PMCIN_LOCAL_PHY_CONTROL));
770 		iomb[1] = LE_32(pwrk->htag);
771 		if (type == PMCS_PHYOP_LINK_RESET) {
772 			mbar = "LOCAL PHY LINK RESET";
773 			iomb[2] = LE_32((PMCS_PHYOP_LINK_RESET << 8) | phynum);
774 		} else {
775 			mbar = "LOCAL PHY HARD RESET";
776 			iomb[2] = LE_32((PMCS_PHYOP_HARD_RESET << 8) | phynum);
777 		}
778 		pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, NULL,
779 		    "%s: sending %s to %s", __func__, mbar, pptr->path);
780 		amt = 3;
781 	}
782 
783 	mutex_enter(&pwp->iqp_lock[PMCS_IQ_OTHER]);
784 	msg = GET_IQ_ENTRY(pwp, PMCS_IQ_OTHER);
785 	if (msg == NULL) {
786 		mutex_exit(&pwp->iqp_lock[PMCS_IQ_OTHER]);
787 		pmcs_pwork(pwp, pwrk);
788 		pmcs_prt(pwp, PMCS_PRT_ERR, pptr, NULL, pmcs_nomsg, __func__);
789 		return (ENOMEM);
790 	}
791 	COPY_MESSAGE(msg, iomb, amt);
792 	htag = pwrk->htag;
793 
794 	/* SMP serialization */
795 	pmcs_smp_acquire(pptr->iport);
796 
797 	pwrk->state = PMCS_WORK_STATE_ONCHIP;
798 	INC_IQ_ENTRY(pwp, PMCS_IQ_OTHER);
799 
800 	pmcs_unlock_phy(pptr);
801 	WAIT_FOR(pwrk, 1000, result);
802 	pmcs_pwork(pwp, pwrk);
803 	pmcs_lock_phy(pptr);
804 
805 	/* SMP serialization */
806 	pmcs_smp_release(pptr->iport);
807 
808 	if (result) {
809 		pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, NULL, pmcs_timeo, __func__);
810 
811 		if (pmcs_abort(pwp, pptr, htag, 0, 0)) {
812 			pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, pptr, NULL,
813 			    "%s: Unable to issue SMP abort for htag 0x%08x",
814 			    __func__, htag);
815 		} else {
816 			pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, pptr, NULL,
817 			    "%s: Issuing SMP ABORT for htag 0x%08x",
818 			    __func__, htag);
819 		}
820 		return (EIO);
821 	}
822 	status = LE_32(iomb[stsoff]);
823 
824 	if (status != PMCOUT_STATUS_OK) {
825 		char buf[32];
826 		const char *es =  pmcs_status_str(status);
827 		if (es == NULL) {
828 			(void) snprintf(buf, sizeof (buf), "Status 0x%x",
829 			    status);
830 			es = buf;
831 		}
832 		pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, NULL,
833 		    "%s: %s action returned %s for %s", __func__, mbar, es,
834 		    pptr->path);
835 		return (EIO);
836 	}
837 
838 	return (0);
839 }
840 
841 /*
842  * Stop the (real) phys.  No PHY or softstate locks are required as this only
843  * happens during detach.
844  */
845 void
846 pmcs_stop_phy(pmcs_hw_t *pwp, int phynum)
847 {
848 	int result;
849 	pmcs_phy_t *pptr;
850 	uint32_t *msg;
851 	struct pmcwork *pwrk;
852 
853 	pptr =  pwp->root_phys + phynum;
854 	if (pptr == NULL) {
855 		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
856 		    "%s: unable to find port %d", __func__, phynum);
857 		return;
858 	}
859 
860 	if (pwp->phys_started & (1 << phynum)) {
861 		pwrk = pmcs_gwork(pwp, PMCS_TAG_TYPE_WAIT, pptr);
862 
863 		if (pwrk == NULL) {
864 			pmcs_prt(pwp, PMCS_PRT_ERR, pptr, NULL,
865 			    pmcs_nowrk, __func__);
866 			return;
867 		}
868 
869 		mutex_enter(&pwp->iqp_lock[PMCS_IQ_OTHER]);
870 		msg = GET_IQ_ENTRY(pwp, PMCS_IQ_OTHER);
871 
872 		if (msg == NULL) {
873 			mutex_exit(&pwp->iqp_lock[PMCS_IQ_OTHER]);
874 			pmcs_pwork(pwp, pwrk);
875 			pmcs_prt(pwp, PMCS_PRT_ERR, pptr, NULL,
876 			    pmcs_nomsg, __func__);
877 			return;
878 		}
879 
880 		msg[0] = LE_32(PMCS_HIPRI(pwp, PMCS_OQ_EVENTS, PMCIN_PHY_STOP));
881 		msg[1] = LE_32(pwrk->htag);
882 		msg[2] = LE_32(phynum);
883 		pwrk->state = PMCS_WORK_STATE_ONCHIP;
884 		/*
885 		 * Make this unconfigured now.
886 		 */
887 		INC_IQ_ENTRY(pwp, PMCS_IQ_OTHER);
888 		WAIT_FOR(pwrk, 1000, result);
889 
890 		pmcs_pwork(pwp, pwrk);
891 		if (result) {
892 			pmcs_prt(pwp, PMCS_PRT_DEBUG,
893 			    pptr, NULL, pmcs_timeo, __func__);
894 		}
895 
896 		pwp->phys_started &= ~(1 << phynum);
897 	}
898 
899 	pptr->configured = 0;
900 }
901 
902 /*
903  * No locks should be required as this is only called during detach
904  */
905 void
906 pmcs_stop_phys(pmcs_hw_t *pwp)
907 {
908 	int i;
909 	for (i = 0; i < pwp->nphy; i++) {
910 		if ((pwp->phyid_block_mask & (1 << i)) == 0) {
911 			pmcs_stop_phy(pwp, i);
912 		}
913 	}
914 }
915 
916 /*
917  * Run SAS_DIAG_EXECUTE with cmd and cmd_desc passed.
918  * 	ERR_CNT_RESET: return status of cmd
919  *	DIAG_REPORT_GET: return value of the counter
920  */
921 int
922 pmcs_sas_diag_execute(pmcs_hw_t *pwp, uint32_t cmd, uint32_t cmd_desc,
923     uint8_t phynum)
924 {
925 	uint32_t htag, *ptr, status, msg[PMCS_MSG_SIZE << 1];
926 	int result;
927 	struct pmcwork *pwrk;
928 
929 	pwrk = pmcs_gwork(pwp, PMCS_TAG_TYPE_WAIT, NULL);
930 	if (pwrk == NULL) {
931 		pmcs_prt(pwp, PMCS_PRT_ERR, NULL, NULL, pmcs_nowrk, __func__);
932 		return (DDI_FAILURE);
933 	}
934 	pwrk->arg = msg;
935 	htag = pwrk->htag;
936 	msg[0] = LE_32(PMCS_HIPRI(pwp, PMCS_OQ_EVENTS, PMCIN_SAS_DIAG_EXECUTE));
937 	msg[1] = LE_32(htag);
938 	msg[2] = LE_32((cmd << PMCS_DIAG_CMD_SHIFT) |
939 	    (cmd_desc << PMCS_DIAG_CMD_DESC_SHIFT) | phynum);
940 
941 	mutex_enter(&pwp->iqp_lock[PMCS_IQ_OTHER]);
942 	ptr = GET_IQ_ENTRY(pwp, PMCS_IQ_OTHER);
943 	if (ptr == NULL) {
944 		mutex_exit(&pwp->iqp_lock[PMCS_IQ_OTHER]);
945 		pmcs_pwork(pwp, pwrk);
946 		pmcs_prt(pwp, PMCS_PRT_ERR, NULL, NULL, pmcs_nomsg, __func__);
947 		return (DDI_FAILURE);
948 	}
949 	COPY_MESSAGE(ptr, msg, 3);
950 	pwrk->state = PMCS_WORK_STATE_ONCHIP;
951 	INC_IQ_ENTRY(pwp, PMCS_IQ_OTHER);
952 
953 	WAIT_FOR(pwrk, 1000, result);
954 
955 	pmcs_pwork(pwp, pwrk);
956 
957 	if (result) {
958 		pmcs_timed_out(pwp, htag, __func__);
959 		return (DDI_FAILURE);
960 	}
961 
962 	status = LE_32(msg[3]);
963 
964 	/* Return for counter reset */
965 	if (cmd == PMCS_ERR_CNT_RESET)
966 		return (status);
967 
968 	/* Return for counter value */
969 	if (status) {
970 		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
971 		    "%s: failed, status (0x%x)", __func__, status);
972 		return (DDI_FAILURE);
973 	}
974 	return (LE_32(msg[4]));
975 }
976 
977 /* Get the current value of the counter for desc on phynum and return it. */
978 int
979 pmcs_get_diag_report(pmcs_hw_t *pwp, uint32_t desc, uint8_t phynum)
980 {
981 	return (pmcs_sas_diag_execute(pwp, PMCS_DIAG_REPORT_GET, desc, phynum));
982 }
983 
984 /* Clear all of the counters for phynum. Returns the status of the command. */
985 int
986 pmcs_clear_diag_counters(pmcs_hw_t *pwp, uint8_t phynum)
987 {
988 	uint32_t	cmd = PMCS_ERR_CNT_RESET;
989 	uint32_t	cmd_desc;
990 
991 	cmd_desc = PMCS_INVALID_DWORD_CNT;
992 	if (pmcs_sas_diag_execute(pwp, cmd, cmd_desc, phynum))
993 		return (DDI_FAILURE);
994 
995 	cmd_desc = PMCS_DISPARITY_ERR_CNT;
996 	if (pmcs_sas_diag_execute(pwp, cmd, cmd_desc, phynum))
997 		return (DDI_FAILURE);
998 
999 	cmd_desc = PMCS_LOST_DWORD_SYNC_CNT;
1000 	if (pmcs_sas_diag_execute(pwp, cmd, cmd_desc, phynum))
1001 		return (DDI_FAILURE);
1002 
1003 	cmd_desc = PMCS_RESET_FAILED_CNT;
1004 	if (pmcs_sas_diag_execute(pwp, cmd, cmd_desc, phynum))
1005 		return (DDI_FAILURE);
1006 
1007 	return (DDI_SUCCESS);
1008 }
1009 
1010 /*
1011  * Get firmware timestamp
1012  */
1013 int
1014 pmcs_get_time_stamp(pmcs_hw_t *pwp, uint64_t *ts)
1015 {
1016 	uint32_t htag, *ptr, msg[PMCS_MSG_SIZE << 1];
1017 	int result;
1018 	struct pmcwork *pwrk;
1019 
1020 	pwrk = pmcs_gwork(pwp, PMCS_TAG_TYPE_WAIT, NULL);
1021 	if (pwrk == NULL) {
1022 		pmcs_prt(pwp, PMCS_PRT_ERR, NULL, NULL, pmcs_nowrk, __func__);
1023 		return (-1);
1024 	}
1025 	pwrk->arg = msg;
1026 	htag = pwrk->htag;
1027 	msg[0] = LE_32(PMCS_HIPRI(pwp, PMCS_OQ_EVENTS, PMCIN_GET_TIME_STAMP));
1028 	msg[1] = LE_32(pwrk->htag);
1029 
1030 	mutex_enter(&pwp->iqp_lock[PMCS_IQ_OTHER]);
1031 	ptr = GET_IQ_ENTRY(pwp, PMCS_IQ_OTHER);
1032 	if (ptr == NULL) {
1033 		mutex_exit(&pwp->iqp_lock[PMCS_IQ_OTHER]);
1034 		pmcs_pwork(pwp, pwrk);
1035 		pmcs_prt(pwp, PMCS_PRT_ERR, NULL, NULL, pmcs_nomsg, __func__);
1036 		return (-1);
1037 	}
1038 	COPY_MESSAGE(ptr, msg, 2);
1039 	pwrk->state = PMCS_WORK_STATE_ONCHIP;
1040 	INC_IQ_ENTRY(pwp, PMCS_IQ_OTHER);
1041 
1042 	WAIT_FOR(pwrk, 1000, result);
1043 
1044 	pmcs_pwork(pwp, pwrk);
1045 
1046 	if (result) {
1047 		pmcs_timed_out(pwp, htag, __func__);
1048 		return (-1);
1049 	}
1050 	*ts = LE_32(msg[2]) | (((uint64_t)LE_32(msg[3])) << 32);
1051 	return (0);
1052 }
1053 
1054 /*
1055  * Dump all pertinent registers
1056  */
1057 
1058 void
1059 pmcs_register_dump(pmcs_hw_t *pwp)
1060 {
1061 	int i;
1062 	uint32_t val;
1063 
1064 	pmcs_prt(pwp, PMCS_PRT_INFO, NULL, NULL, "pmcs%d: Register dump start",
1065 	    ddi_get_instance(pwp->dip));
1066 	pmcs_prt(pwp, PMCS_PRT_INFO, NULL, NULL,
1067 	    "OBDB (intr): 0x%08x (mask): 0x%08x (clear): 0x%08x",
1068 	    pmcs_rd_msgunit(pwp, PMCS_MSGU_OBDB),
1069 	    pmcs_rd_msgunit(pwp, PMCS_MSGU_OBDB_MASK),
1070 	    pmcs_rd_msgunit(pwp, PMCS_MSGU_OBDB_CLEAR));
1071 	pmcs_prt(pwp, PMCS_PRT_INFO, NULL, NULL, "SCRATCH0: 0x%08x",
1072 	    pmcs_rd_msgunit(pwp, PMCS_MSGU_SCRATCH0));
1073 	pmcs_prt(pwp, PMCS_PRT_INFO, NULL, NULL, "SCRATCH1: 0x%08x",
1074 	    pmcs_rd_msgunit(pwp, PMCS_MSGU_SCRATCH1));
1075 	pmcs_prt(pwp, PMCS_PRT_INFO, NULL, NULL, "SCRATCH2: 0x%08x",
1076 	    pmcs_rd_msgunit(pwp, PMCS_MSGU_SCRATCH2));
1077 	pmcs_prt(pwp, PMCS_PRT_INFO, NULL, NULL, "SCRATCH3: 0x%08x",
1078 	    pmcs_rd_msgunit(pwp, PMCS_MSGU_SCRATCH3));
1079 	for (i = 0; i < PMCS_NIQ; i++) {
1080 		pmcs_prt(pwp, PMCS_PRT_INFO, NULL, NULL, "IQ %d: CI %u PI %u",
1081 		    i, pmcs_rd_iqci(pwp, i), pmcs_rd_iqpi(pwp, i));
1082 	}
1083 	for (i = 0; i < PMCS_NOQ; i++) {
1084 		pmcs_prt(pwp, PMCS_PRT_INFO, NULL, NULL, "OQ %d: CI %u PI %u",
1085 		    i, pmcs_rd_oqci(pwp, i), pmcs_rd_oqpi(pwp, i));
1086 	}
1087 	val = pmcs_rd_gst_tbl(pwp, PMCS_GST_BASE);
1088 	pmcs_prt(pwp, PMCS_PRT_INFO, NULL, NULL,
1089 	    "GST TABLE BASE: 0x%08x (STATE=0x%x QF=%d GSTLEN=%d HMI_ERR=0x%x)",
1090 	    val, PMCS_MPI_S(val), PMCS_QF(val), PMCS_GSTLEN(val) * 4,
1091 	    PMCS_HMI_ERR(val));
1092 	pmcs_prt(pwp, PMCS_PRT_INFO, NULL, NULL, "GST TABLE IQFRZ0: 0x%08x",
1093 	    pmcs_rd_gst_tbl(pwp, PMCS_GST_IQFRZ0));
1094 	pmcs_prt(pwp, PMCS_PRT_INFO, NULL, NULL, "GST TABLE IQFRZ1: 0x%08x",
1095 	    pmcs_rd_gst_tbl(pwp, PMCS_GST_IQFRZ1));
1096 	pmcs_prt(pwp, PMCS_PRT_INFO, NULL, NULL, "GST TABLE MSGU TICK: 0x%08x",
1097 	    pmcs_rd_gst_tbl(pwp, PMCS_GST_MSGU_TICK));
1098 	pmcs_prt(pwp, PMCS_PRT_INFO, NULL, NULL, "GST TABLE IOP TICK: 0x%08x",
1099 	    pmcs_rd_gst_tbl(pwp, PMCS_GST_IOP_TICK));
1100 	for (i = 0; i < pwp->nphy; i++) {
1101 		uint32_t rerrf, pinfo, started = 0, link = 0;
1102 		pinfo = pmcs_rd_gst_tbl(pwp, PMCS_GST_PHY_INFO(i));
1103 		if (pinfo & 1) {
1104 			started = 1;
1105 			link = pinfo & 2;
1106 		}
1107 		rerrf = pmcs_rd_gst_tbl(pwp, PMCS_GST_RERR_INFO(i));
1108 		pmcs_prt(pwp, PMCS_PRT_INFO, NULL, NULL,
1109 		    "GST TABLE PHY%d STARTED=%d LINK=%d RERR=0x%08x",
1110 		    i, started, link, rerrf);
1111 	}
1112 	pmcs_prt(pwp, PMCS_PRT_INFO, NULL, NULL, "pmcs%d: Register dump end",
1113 	    ddi_get_instance(pwp->dip));
1114 }
1115 
1116 /*
1117  * Handle SATA Abort and other error processing
1118  */
1119 int
1120 pmcs_abort_handler(pmcs_hw_t *pwp)
1121 {
1122 	pmcs_phy_t *pptr, *pnext, *pnext_uplevel[PMCS_MAX_XPND];
1123 	int r, level = 0;
1124 
1125 	pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL, "%s", __func__);
1126 
1127 	mutex_enter(&pwp->lock);
1128 	pptr = pwp->root_phys;
1129 	mutex_exit(&pwp->lock);
1130 
1131 	while (pptr) {
1132 		/*
1133 		 * XXX: Need to make sure this doesn't happen
1134 		 * XXX: when non-NCQ commands are running.
1135 		 */
1136 		pmcs_lock_phy(pptr);
1137 		if (pptr->need_rl_ext) {
1138 			ASSERT(pptr->dtype == SATA);
1139 			if (pmcs_acquire_scratch(pwp, B_FALSE)) {
1140 				goto next_phy;
1141 			}
1142 			r = pmcs_sata_abort_ncq(pwp, pptr);
1143 			pmcs_release_scratch(pwp);
1144 			if (r == ENOMEM) {
1145 				goto next_phy;
1146 			}
1147 			if (r) {
1148 				r = pmcs_reset_phy(pwp, pptr,
1149 				    PMCS_PHYOP_LINK_RESET);
1150 				if (r == ENOMEM) {
1151 					goto next_phy;
1152 				}
1153 				/* what if other failures happened? */
1154 				pptr->abort_pending = 1;
1155 				pptr->abort_sent = 0;
1156 			}
1157 		}
1158 		if (pptr->abort_pending == 0 || pptr->abort_sent) {
1159 			goto next_phy;
1160 		}
1161 		pptr->abort_pending = 0;
1162 		if (pmcs_abort(pwp, pptr, pptr->device_id, 1, 1) == ENOMEM) {
1163 			pptr->abort_pending = 1;
1164 			goto next_phy;
1165 		}
1166 		pptr->abort_sent = 1;
1167 
1168 next_phy:
1169 		if (pptr->children) {
1170 			pnext = pptr->children;
1171 			pnext_uplevel[level++] = pptr->sibling;
1172 		} else {
1173 			pnext = pptr->sibling;
1174 			while ((pnext == NULL) && (level > 0)) {
1175 				pnext = pnext_uplevel[--level];
1176 			}
1177 		}
1178 
1179 		pmcs_unlock_phy(pptr);
1180 		pptr = pnext;
1181 	}
1182 
1183 	return (0);
1184 }
1185 
1186 /*
1187  * Register a device (get a device handle for it).
1188  * Called with PHY lock held.
1189  */
1190 int
1191 pmcs_register_device(pmcs_hw_t *pwp, pmcs_phy_t *pptr)
1192 {
1193 	struct pmcwork *pwrk;
1194 	int result = 0;
1195 	uint32_t *msg;
1196 	uint32_t tmp, status;
1197 	uint32_t iomb[(PMCS_QENTRY_SIZE << 1) >> 2];
1198 
1199 	mutex_enter(&pwp->iqp_lock[PMCS_IQ_OTHER]);
1200 	msg = GET_IQ_ENTRY(pwp, PMCS_IQ_OTHER);
1201 
1202 	if (msg == NULL ||
1203 	    (pwrk = pmcs_gwork(pwp, PMCS_TAG_TYPE_WAIT, pptr)) == NULL) {
1204 		mutex_exit(&pwp->iqp_lock[PMCS_IQ_OTHER]);
1205 		result = ENOMEM;
1206 		goto out;
1207 	}
1208 
1209 	pwrk->arg = iomb;
1210 	pwrk->dtype = pptr->dtype;
1211 
1212 	msg[1] = LE_32(pwrk->htag);
1213 	msg[0] = LE_32(PMCS_HIPRI(pwp, PMCS_OQ_GENERAL, PMCIN_REGISTER_DEVICE));
1214 	tmp = PMCS_DEVREG_TLR |
1215 	    (pptr->link_rate << PMCS_DEVREG_LINK_RATE_SHIFT);
1216 	if (IS_ROOT_PHY(pptr)) {
1217 		msg[2] = LE_32(pptr->portid |
1218 		    (pptr->phynum << PMCS_PHYID_SHIFT));
1219 	} else {
1220 		msg[2] = LE_32(pptr->portid);
1221 	}
1222 	if (pptr->dtype == SATA) {
1223 		if (IS_ROOT_PHY(pptr)) {
1224 			tmp |= PMCS_DEVREG_TYPE_SATA_DIRECT;
1225 		} else {
1226 			tmp |= PMCS_DEVREG_TYPE_SATA;
1227 		}
1228 	} else {
1229 		tmp |= PMCS_DEVREG_TYPE_SAS;
1230 	}
1231 	msg[3] = LE_32(tmp);
1232 	msg[4] = LE_32(PMCS_DEVREG_IT_NEXUS_TIMEOUT);
1233 	(void) memcpy(&msg[5], pptr->sas_address, 8);
1234 
1235 	CLEAN_MESSAGE(msg, 7);
1236 	pwrk->state = PMCS_WORK_STATE_ONCHIP;
1237 	INC_IQ_ENTRY(pwp, PMCS_IQ_OTHER);
1238 
1239 	pmcs_unlock_phy(pptr);
1240 	WAIT_FOR(pwrk, 250, result);
1241 	pmcs_lock_phy(pptr);
1242 	pmcs_pwork(pwp, pwrk);
1243 
1244 	if (result) {
1245 		pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, NULL, pmcs_timeo, __func__);
1246 		result = ETIMEDOUT;
1247 		goto out;
1248 	}
1249 	status = LE_32(iomb[2]);
1250 	tmp = LE_32(iomb[3]);
1251 	switch (status) {
1252 	case PMCS_DEVREG_OK:
1253 	case PMCS_DEVREG_DEVICE_ALREADY_REGISTERED:
1254 	case PMCS_DEVREG_PHY_ALREADY_REGISTERED:
1255 		if (pmcs_validate_devid(pwp->root_phys, pptr, tmp) == B_FALSE) {
1256 			result = EEXIST;
1257 			goto out;
1258 		} else if (status != PMCS_DEVREG_OK) {
1259 			if (tmp == 0xffffffff) {	/* F/W bug */
1260 				pmcs_prt(pwp, PMCS_PRT_INFO, pptr, NULL,
1261 				    "%s: phy %s already has bogus devid 0x%x",
1262 				    __func__, pptr->path, tmp);
1263 				result = EIO;
1264 				goto out;
1265 			} else {
1266 				pmcs_prt(pwp, PMCS_PRT_INFO, pptr, NULL,
1267 				    "%s: phy %s already has a device id 0x%x",
1268 				    __func__, pptr->path, tmp);
1269 			}
1270 		}
1271 		break;
1272 	default:
1273 		pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, NULL,
1274 		    "%s: status 0x%x when trying to register device %s",
1275 		    __func__, status, pptr->path);
1276 		result = EIO;
1277 		goto out;
1278 	}
1279 	pptr->device_id = tmp;
1280 	pptr->valid_device_id = 1;
1281 	pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, pptr, NULL, "Phy %s/" SAS_ADDR_FMT
1282 	    " registered with device_id 0x%x (portid %d)", pptr->path,
1283 	    SAS_ADDR_PRT(pptr->sas_address), tmp, pptr->portid);
1284 out:
1285 	return (result);
1286 }
1287 
1288 /*
1289  * Deregister a device (remove a device handle).
1290  * Called with PHY locked.
1291  */
1292 void
1293 pmcs_deregister_device(pmcs_hw_t *pwp, pmcs_phy_t *pptr)
1294 {
1295 	struct pmcwork *pwrk;
1296 	uint32_t msg[PMCS_MSG_SIZE], *ptr, status;
1297 	uint32_t iomb[(PMCS_QENTRY_SIZE << 1) >> 2];
1298 	int result;
1299 
1300 	pwrk = pmcs_gwork(pwp, PMCS_TAG_TYPE_WAIT, pptr);
1301 	if (pwrk == NULL) {
1302 		return;
1303 	}
1304 
1305 	pwrk->arg = iomb;
1306 	pwrk->dtype = pptr->dtype;
1307 	mutex_enter(&pwp->iqp_lock[PMCS_IQ_OTHER]);
1308 	ptr = GET_IQ_ENTRY(pwp, PMCS_IQ_OTHER);
1309 	if (ptr == NULL) {
1310 		mutex_exit(&pwp->iqp_lock[PMCS_IQ_OTHER]);
1311 		pmcs_pwork(pwp, pwrk);
1312 		return;
1313 	}
1314 	msg[0] = LE_32(PMCS_HIPRI(pwp, PMCS_OQ_GENERAL,
1315 	    PMCIN_DEREGISTER_DEVICE_HANDLE));
1316 	msg[1] = LE_32(pwrk->htag);
1317 	msg[2] = LE_32(pptr->device_id);
1318 	pwrk->state = PMCS_WORK_STATE_ONCHIP;
1319 	COPY_MESSAGE(ptr, msg, 3);
1320 	INC_IQ_ENTRY(pwp, PMCS_IQ_OTHER);
1321 
1322 	pmcs_unlock_phy(pptr);
1323 	WAIT_FOR(pwrk, 250, result);
1324 	pmcs_pwork(pwp, pwrk);
1325 	pmcs_lock_phy(pptr);
1326 
1327 	if (result) {
1328 		pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, NULL, pmcs_timeo, __func__);
1329 		return;
1330 	}
1331 	status = LE_32(iomb[2]);
1332 	if (status != PMCOUT_STATUS_OK) {
1333 		pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, NULL,
1334 		    "%s: status 0x%x when trying to deregister device %s",
1335 		    __func__, status, pptr->path);
1336 	} else {
1337 		pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, NULL,
1338 		    "%s: device %s deregistered", __func__, pptr->path);
1339 		pptr->valid_device_id = 0;
1340 		pptr->device_id = PMCS_INVALID_DEVICE_ID;
1341 	}
1342 }
1343 
1344 /*
1345  * Deregister all registered devices.
1346  */
1347 void
1348 pmcs_deregister_devices(pmcs_hw_t *pwp, pmcs_phy_t *phyp)
1349 {
1350 	/*
1351 	 * Start at the maximum level and walk back to level 0.  This only
1352 	 * gets done during detach after all threads and timers have been
1353 	 * destroyed, so there's no need to hold the softstate or PHY lock.
1354 	 */
1355 	while (phyp) {
1356 		if (phyp->children) {
1357 			pmcs_deregister_devices(pwp, phyp->children);
1358 		}
1359 		if (phyp->valid_device_id) {
1360 			pmcs_deregister_device(pwp, phyp);
1361 		}
1362 		phyp = phyp->sibling;
1363 	}
1364 }
1365 
1366 /*
1367  * Perform a 'soft' reset on the PMC chip
1368  */
1369 int
1370 pmcs_soft_reset(pmcs_hw_t *pwp, boolean_t no_restart)
1371 {
1372 	uint32_t s2, sfrbits, gsm, rapchk, wapchk, wdpchk, spc, tsmode;
1373 	pmcs_phy_t *pptr;
1374 	char *msg = NULL;
1375 	int i;
1376 
1377 	/*
1378 	 * Disable interrupts
1379 	 */
1380 	pmcs_wr_msgunit(pwp, PMCS_MSGU_OBDB_MASK, 0xffffffff);
1381 	pmcs_wr_msgunit(pwp, PMCS_MSGU_OBDB_CLEAR, 0xffffffff);
1382 
1383 	pmcs_prt(pwp, PMCS_PRT_INFO, NULL, NULL, "%s", __func__);
1384 
1385 	if (pwp->locks_initted) {
1386 		mutex_enter(&pwp->lock);
1387 	}
1388 	pwp->blocked = 1;
1389 
1390 	/*
1391 	 * Step 1
1392 	 */
1393 	s2 = pmcs_rd_msgunit(pwp, PMCS_MSGU_SCRATCH2);
1394 	if ((s2 & PMCS_MSGU_HOST_SOFT_RESET_READY) == 0) {
1395 		pmcs_wr_gsm_reg(pwp, RB6_ACCESS, RB6_NMI_SIGNATURE);
1396 		pmcs_wr_gsm_reg(pwp, RB6_ACCESS, RB6_NMI_SIGNATURE);
1397 		for (i = 0; i < 100; i++) {
1398 			s2 = pmcs_rd_msgunit(pwp, PMCS_MSGU_SCRATCH2) &
1399 			    PMCS_MSGU_HOST_SOFT_RESET_READY;
1400 			if (s2) {
1401 				break;
1402 			}
1403 			drv_usecwait(10000);
1404 		}
1405 		s2 = pmcs_rd_msgunit(pwp, PMCS_MSGU_SCRATCH2) &
1406 		    PMCS_MSGU_HOST_SOFT_RESET_READY;
1407 		if (s2 == 0) {
1408 			pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
1409 			    "%s: PMCS_MSGU_HOST_SOFT_RESET_READY never came "
1410 			    "ready", __func__);
1411 			pmcs_register_dump(pwp);
1412 			if ((pmcs_rd_msgunit(pwp, PMCS_MSGU_SCRATCH1) &
1413 			    PMCS_MSGU_CPU_SOFT_RESET_READY) == 0 ||
1414 			    (pmcs_rd_msgunit(pwp, PMCS_MSGU_SCRATCH2) &
1415 			    PMCS_MSGU_CPU_SOFT_RESET_READY) == 0) {
1416 				pwp->state = STATE_DEAD;
1417 				pwp->blocked = 0;
1418 				if (pwp->locks_initted) {
1419 					mutex_exit(&pwp->lock);
1420 				}
1421 				return (-1);
1422 			}
1423 		}
1424 	}
1425 
1426 	/*
1427 	 * Step 2
1428 	 */
1429 	pmcs_wr_gsm_reg(pwp, NMI_EN_VPE0_IOP, 0);
1430 	drv_usecwait(10);
1431 	pmcs_wr_gsm_reg(pwp, NMI_EN_VPE0_AAP1, 0);
1432 	drv_usecwait(10);
1433 	pmcs_wr_topunit(pwp, PMCS_EVENT_INT_ENABLE, 0);
1434 	drv_usecwait(10);
1435 	pmcs_wr_topunit(pwp, PMCS_EVENT_INT_STAT,
1436 	    pmcs_rd_topunit(pwp, PMCS_EVENT_INT_STAT));
1437 	drv_usecwait(10);
1438 	pmcs_wr_topunit(pwp, PMCS_ERROR_INT_ENABLE, 0);
1439 	drv_usecwait(10);
1440 	pmcs_wr_topunit(pwp, PMCS_ERROR_INT_STAT,
1441 	    pmcs_rd_topunit(pwp, PMCS_ERROR_INT_STAT));
1442 	drv_usecwait(10);
1443 
1444 	sfrbits = pmcs_rd_msgunit(pwp, PMCS_MSGU_SCRATCH1) &
1445 	    PMCS_MSGU_AAP_SFR_PROGRESS;
1446 	sfrbits ^= PMCS_MSGU_AAP_SFR_PROGRESS;
1447 	pmcs_prt(pwp, PMCS_PRT_DEBUG2, NULL, NULL, "PMCS_MSGU_HOST_SCRATCH0 "
1448 	    "%08x -> %08x", pmcs_rd_msgunit(pwp, PMCS_MSGU_HOST_SCRATCH0),
1449 	    HST_SFT_RESET_SIG);
1450 	pmcs_wr_msgunit(pwp, PMCS_MSGU_HOST_SCRATCH0, HST_SFT_RESET_SIG);
1451 
1452 	/*
1453 	 * Step 3
1454 	 */
1455 	gsm = pmcs_rd_gsm_reg(pwp, GSM_CFG_AND_RESET);
1456 	pmcs_prt(pwp, PMCS_PRT_DEBUG2, NULL, NULL, "GSM %08x -> %08x", gsm,
1457 	    gsm & ~PMCS_SOFT_RESET_BITS);
1458 	pmcs_wr_gsm_reg(pwp, GSM_CFG_AND_RESET, gsm & ~PMCS_SOFT_RESET_BITS);
1459 
1460 	/*
1461 	 * Step 4
1462 	 */
1463 	rapchk = pmcs_rd_gsm_reg(pwp, READ_ADR_PARITY_CHK_EN);
1464 	pmcs_prt(pwp, PMCS_PRT_DEBUG2, NULL, NULL, "READ_ADR_PARITY_CHK_EN "
1465 	    "%08x -> %08x", rapchk, 0);
1466 	pmcs_wr_gsm_reg(pwp, READ_ADR_PARITY_CHK_EN, 0);
1467 	wapchk = pmcs_rd_gsm_reg(pwp, WRITE_ADR_PARITY_CHK_EN);
1468 	pmcs_prt(pwp, PMCS_PRT_DEBUG2, NULL, NULL, "WRITE_ADR_PARITY_CHK_EN "
1469 	    "%08x -> %08x", wapchk, 0);
1470 	pmcs_wr_gsm_reg(pwp, WRITE_ADR_PARITY_CHK_EN, 0);
1471 	wdpchk = pmcs_rd_gsm_reg(pwp, WRITE_DATA_PARITY_CHK_EN);
1472 	pmcs_prt(pwp, PMCS_PRT_DEBUG2, NULL, NULL, "WRITE_DATA_PARITY_CHK_EN "
1473 	    "%08x -> %08x", wdpchk, 0);
1474 	pmcs_wr_gsm_reg(pwp, WRITE_DATA_PARITY_CHK_EN, 0);
1475 
1476 	/*
1477 	 * Step 5
1478 	 */
1479 	drv_usecwait(100);
1480 
1481 	/*
1482 	 * Step 5.5 (Temporary workaround for 1.07.xx Beta)
1483 	 */
1484 	tsmode = pmcs_rd_gsm_reg(pwp, PMCS_GPIO_TRISTATE_MODE_ADDR);
1485 	pmcs_prt(pwp, PMCS_PRT_DEBUG2, NULL, NULL, "GPIO TSMODE %08x -> %08x",
1486 	    tsmode, tsmode & ~(PMCS_GPIO_TSMODE_BIT0|PMCS_GPIO_TSMODE_BIT1));
1487 	pmcs_wr_gsm_reg(pwp, PMCS_GPIO_TRISTATE_MODE_ADDR,
1488 	    tsmode & ~(PMCS_GPIO_TSMODE_BIT0|PMCS_GPIO_TSMODE_BIT1));
1489 	drv_usecwait(10);
1490 
1491 	/*
1492 	 * Step 6
1493 	 */
1494 	spc = pmcs_rd_topunit(pwp, PMCS_SPC_RESET);
1495 	pmcs_prt(pwp, PMCS_PRT_DEBUG2, NULL, NULL, "SPC_RESET %08x -> %08x",
1496 	    spc, spc & ~(PCS_IOP_SS_RSTB|PCS_AAP1_SS_RSTB));
1497 	pmcs_wr_topunit(pwp, PMCS_SPC_RESET,
1498 	    spc & ~(PCS_IOP_SS_RSTB|PCS_AAP1_SS_RSTB));
1499 	drv_usecwait(10);
1500 
1501 	/*
1502 	 * Step 7
1503 	 */
1504 	spc = pmcs_rd_topunit(pwp, PMCS_SPC_RESET);
1505 	pmcs_prt(pwp, PMCS_PRT_DEBUG2, NULL, NULL, "SPC_RESET %08x -> %08x",
1506 	    spc, spc & ~(BDMA_CORE_RSTB|OSSP_RSTB));
1507 	pmcs_wr_topunit(pwp, PMCS_SPC_RESET, spc & ~(BDMA_CORE_RSTB|OSSP_RSTB));
1508 
1509 	/*
1510 	 * Step 8
1511 	 */
1512 	drv_usecwait(100);
1513 
1514 	/*
1515 	 * Step 9
1516 	 */
1517 	spc = pmcs_rd_topunit(pwp, PMCS_SPC_RESET);
1518 	pmcs_prt(pwp, PMCS_PRT_DEBUG2, NULL, NULL, "SPC_RESET %08x -> %08x",
1519 	    spc, spc | (BDMA_CORE_RSTB|OSSP_RSTB));
1520 	pmcs_wr_topunit(pwp, PMCS_SPC_RESET, spc | (BDMA_CORE_RSTB|OSSP_RSTB));
1521 
1522 	/*
1523 	 * Step 10
1524 	 */
1525 	drv_usecwait(100);
1526 
1527 	/*
1528 	 * Step 11
1529 	 */
1530 	gsm = pmcs_rd_gsm_reg(pwp, GSM_CFG_AND_RESET);
1531 	pmcs_prt(pwp, PMCS_PRT_DEBUG2, NULL, NULL, "GSM %08x -> %08x", gsm,
1532 	    gsm | PMCS_SOFT_RESET_BITS);
1533 	pmcs_wr_gsm_reg(pwp, GSM_CFG_AND_RESET, gsm | PMCS_SOFT_RESET_BITS);
1534 	drv_usecwait(10);
1535 
1536 	/*
1537 	 * Step 12
1538 	 */
1539 	pmcs_prt(pwp, PMCS_PRT_DEBUG2, NULL, NULL, "READ_ADR_PARITY_CHK_EN "
1540 	    "%08x -> %08x", pmcs_rd_gsm_reg(pwp, READ_ADR_PARITY_CHK_EN),
1541 	    rapchk);
1542 	pmcs_wr_gsm_reg(pwp, READ_ADR_PARITY_CHK_EN, rapchk);
1543 	drv_usecwait(10);
1544 	pmcs_prt(pwp, PMCS_PRT_DEBUG2, NULL, NULL, "WRITE_ADR_PARITY_CHK_EN "
1545 	    "%08x -> %08x", pmcs_rd_gsm_reg(pwp, WRITE_ADR_PARITY_CHK_EN),
1546 	    wapchk);
1547 	pmcs_wr_gsm_reg(pwp, WRITE_ADR_PARITY_CHK_EN, wapchk);
1548 	drv_usecwait(10);
1549 	pmcs_prt(pwp, PMCS_PRT_DEBUG2, NULL, NULL, "WRITE_DATA_PARITY_CHK_EN "
1550 	    "%08x -> %08x", pmcs_rd_gsm_reg(pwp, WRITE_DATA_PARITY_CHK_EN),
1551 	    wapchk);
1552 	pmcs_wr_gsm_reg(pwp, WRITE_DATA_PARITY_CHK_EN, wdpchk);
1553 	drv_usecwait(10);
1554 
1555 	/*
1556 	 * Step 13
1557 	 */
1558 	spc = pmcs_rd_topunit(pwp, PMCS_SPC_RESET);
1559 	pmcs_prt(pwp, PMCS_PRT_DEBUG2, NULL, NULL, "SPC_RESET %08x -> %08x",
1560 	    spc, spc | (PCS_IOP_SS_RSTB|PCS_AAP1_SS_RSTB));
1561 	pmcs_wr_topunit(pwp, PMCS_SPC_RESET,
1562 	    spc | (PCS_IOP_SS_RSTB|PCS_AAP1_SS_RSTB));
1563 
1564 	/*
1565 	 * Step 14
1566 	 */
1567 	drv_usecwait(100);
1568 
1569 	/*
1570 	 * Step 15
1571 	 */
1572 	for (spc = 0, i = 0; i < 1000; i++) {
1573 		drv_usecwait(1000);
1574 		spc = pmcs_rd_msgunit(pwp, PMCS_MSGU_SCRATCH1);
1575 		if ((spc & PMCS_MSGU_AAP_SFR_PROGRESS) == sfrbits) {
1576 			break;
1577 		}
1578 	}
1579 
1580 	if ((spc & PMCS_MSGU_AAP_SFR_PROGRESS) != sfrbits) {
1581 		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
1582 		    "SFR didn't toggle (sfr 0x%x)", spc);
1583 		pwp->state = STATE_DEAD;
1584 		pwp->blocked = 0;
1585 		if (pwp->locks_initted) {
1586 			mutex_exit(&pwp->lock);
1587 		}
1588 		return (-1);
1589 	}
1590 
1591 	/*
1592 	 * Step 16
1593 	 */
1594 	pmcs_wr_msgunit(pwp, PMCS_MSGU_OBDB_MASK, 0xffffffff);
1595 	pmcs_wr_msgunit(pwp, PMCS_MSGU_OBDB_CLEAR, 0xffffffff);
1596 
1597 	/*
1598 	 * Wait for up to 5 seconds for AAP state to come either ready or error.
1599 	 */
1600 	for (i = 0; i < 50; i++) {
1601 		spc = pmcs_rd_msgunit(pwp, PMCS_MSGU_SCRATCH1) &
1602 		    PMCS_MSGU_AAP_STATE_MASK;
1603 		if (spc == PMCS_MSGU_AAP_STATE_ERROR ||
1604 		    spc == PMCS_MSGU_AAP_STATE_READY) {
1605 			break;
1606 		}
1607 		drv_usecwait(100000);
1608 	}
1609 	spc = pmcs_rd_msgunit(pwp, PMCS_MSGU_SCRATCH1);
1610 	if ((spc & PMCS_MSGU_AAP_STATE_MASK) != PMCS_MSGU_AAP_STATE_READY) {
1611 		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
1612 		    "soft reset failed (state 0x%x)", spc);
1613 		pwp->state = STATE_DEAD;
1614 		pwp->blocked = 0;
1615 		if (pwp->locks_initted) {
1616 			mutex_exit(&pwp->lock);
1617 		}
1618 		return (-1);
1619 	}
1620 
1621 
1622 	if (pwp->state == STATE_DEAD || pwp->state == STATE_UNPROBING ||
1623 	    pwp->state == STATE_PROBING || pwp->locks_initted == 0) {
1624 		pwp->blocked = 0;
1625 		if (pwp->locks_initted) {
1626 			mutex_exit(&pwp->lock);
1627 		}
1628 		return (0);
1629 	}
1630 
1631 	/*
1632 	 * Return at this point if we dont need to startup.
1633 	 */
1634 	if (no_restart) {
1635 		return (0);
1636 	}
1637 
1638 	ASSERT(pwp->locks_initted != 0);
1639 
1640 	/*
1641 	 * Clean up various soft state.
1642 	 */
1643 	bzero(pwp->ports, sizeof (pwp->ports));
1644 
1645 	pmcs_free_all_phys(pwp, pwp->root_phys);
1646 
1647 	for (pptr = pwp->root_phys; pptr; pptr = pptr->sibling) {
1648 		pmcs_lock_phy(pptr);
1649 		pmcs_clear_phy(pwp, pptr);
1650 		pmcs_unlock_phy(pptr);
1651 	}
1652 
1653 	if (pwp->targets) {
1654 		for (i = 0; i < pwp->max_dev; i++) {
1655 			pmcs_xscsi_t *xp = pwp->targets[i];
1656 
1657 			if (xp == NULL) {
1658 				continue;
1659 			}
1660 			mutex_enter(&xp->statlock);
1661 			pmcs_clear_xp(pwp, xp);
1662 			mutex_exit(&xp->statlock);
1663 		}
1664 	}
1665 
1666 	bzero(pwp->shadow_iqpi, sizeof (pwp->shadow_iqpi));
1667 	for (i = 0; i < PMCS_NIQ; i++) {
1668 		if (pwp->iqp[i]) {
1669 			bzero(pwp->iqp[i], PMCS_QENTRY_SIZE * pwp->ioq_depth);
1670 			pmcs_wr_iqpi(pwp, i, 0);
1671 			pmcs_wr_iqci(pwp, i, 0);
1672 		}
1673 	}
1674 	for (i = 0; i < PMCS_NOQ; i++) {
1675 		if (pwp->oqp[i]) {
1676 			bzero(pwp->oqp[i], PMCS_QENTRY_SIZE * pwp->ioq_depth);
1677 			pmcs_wr_oqpi(pwp, i, 0);
1678 			pmcs_wr_oqci(pwp, i, 0);
1679 		}
1680 
1681 	}
1682 	if (pwp->fwlogp) {
1683 		bzero(pwp->fwlogp, PMCS_FWLOG_SIZE);
1684 	}
1685 	STAILQ_INIT(&pwp->wf);
1686 	bzero(pwp->work, sizeof (pmcwork_t) * pwp->max_cmd);
1687 	for (i = 0; i < pwp->max_cmd - 1; i++) {
1688 		pmcwork_t *pwrk = &pwp->work[i];
1689 		STAILQ_INSERT_TAIL(&pwp->wf, pwrk, next);
1690 	}
1691 
1692 	/*
1693 	 * Clear out any leftover commands sitting in the work list
1694 	 */
1695 	for (i = 0; i < pwp->max_cmd; i++) {
1696 		pmcwork_t *pwrk = &pwp->work[i];
1697 		mutex_enter(&pwrk->lock);
1698 		if (pwrk->state == PMCS_WORK_STATE_ONCHIP) {
1699 			switch (PMCS_TAG_TYPE(pwrk->htag)) {
1700 			case PMCS_TAG_TYPE_WAIT:
1701 				mutex_exit(&pwrk->lock);
1702 				break;
1703 			case PMCS_TAG_TYPE_CBACK:
1704 			case PMCS_TAG_TYPE_NONE:
1705 				pmcs_pwork(pwp, pwrk);
1706 				break;
1707 			default:
1708 				break;
1709 			}
1710 		} else if (pwrk->state == PMCS_WORK_STATE_IOCOMPQ) {
1711 			pwrk->dead = 1;
1712 			mutex_exit(&pwrk->lock);
1713 		} else {
1714 			/*
1715 			 * The other states of NIL, READY and INTR
1716 			 * should not be visible outside of a lock being held.
1717 			 */
1718 			pmcs_pwork(pwp, pwrk);
1719 		}
1720 	}
1721 
1722 	/*
1723 	 * Restore Interrupt Mask
1724 	 */
1725 	pmcs_wr_msgunit(pwp, PMCS_MSGU_OBDB_MASK, pwp->intr_mask);
1726 	pmcs_wr_msgunit(pwp, PMCS_MSGU_OBDB_CLEAR, 0xffffffff);
1727 
1728 	pwp->blocked = 0;
1729 	pwp->mpi_table_setup = 0;
1730 	mutex_exit(&pwp->lock);
1731 
1732 	/*
1733 	 * Set up MPI again.
1734 	 */
1735 	if (pmcs_setup(pwp)) {
1736 		msg = "unable to setup MPI tables again";
1737 		goto fail_restart;
1738 	}
1739 	pmcs_report_fwversion(pwp);
1740 
1741 	/*
1742 	 * Restart MPI
1743 	 */
1744 	if (pmcs_start_mpi(pwp)) {
1745 		msg = "unable to restart MPI again";
1746 		goto fail_restart;
1747 	}
1748 
1749 	mutex_enter(&pwp->lock);
1750 	pwp->blocked = 0;
1751 	SCHEDULE_WORK(pwp, PMCS_WORK_RUN_QUEUES);
1752 	mutex_exit(&pwp->lock);
1753 
1754 	/*
1755 	 * Run any completions
1756 	 */
1757 	PMCS_CQ_RUN(pwp);
1758 
1759 	/*
1760 	 * Delay
1761 	 */
1762 	drv_usecwait(1000000);
1763 	return (0);
1764 
1765 fail_restart:
1766 	mutex_enter(&pwp->lock);
1767 	pwp->state = STATE_DEAD;
1768 	mutex_exit(&pwp->lock);
1769 	pmcs_prt(pwp, PMCS_PRT_ERR, NULL, NULL,
1770 	    "%s: Failed: %s", __func__, msg);
1771 	return (-1);
1772 }
1773 
1774 /*
1775  * Reset a device or a logical unit.
1776  */
1777 int
1778 pmcs_reset_dev(pmcs_hw_t *pwp, pmcs_phy_t *pptr, uint64_t lun)
1779 {
1780 	int rval = 0;
1781 
1782 	if (pptr == NULL) {
1783 		return (ENXIO);
1784 	}
1785 
1786 	pmcs_lock_phy(pptr);
1787 	if (pptr->dtype == SAS) {
1788 		/*
1789 		 * Some devices do not support SAS_I_T_NEXUS_RESET as
1790 		 * it is not a mandatory (in SAM4) task management
1791 		 * function, while LOGIC_UNIT_RESET is mandatory.
1792 		 *
1793 		 * The problem here is that we need to iterate over
1794 		 * all known LUNs to emulate the semantics of
1795 		 * "RESET_TARGET".
1796 		 *
1797 		 * XXX: FIX ME
1798 		 */
1799 		if (lun == (uint64_t)-1) {
1800 			lun = 0;
1801 		}
1802 		rval = pmcs_ssp_tmf(pwp, pptr, SAS_LOGICAL_UNIT_RESET, 0, lun,
1803 		    NULL);
1804 	} else if (pptr->dtype == SATA) {
1805 		if (lun != 0ull) {
1806 			pmcs_unlock_phy(pptr);
1807 			return (EINVAL);
1808 		}
1809 		rval = pmcs_reset_phy(pwp, pptr, PMCS_PHYOP_LINK_RESET);
1810 	} else {
1811 		pmcs_unlock_phy(pptr);
1812 		pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, NULL,
1813 		    "%s: cannot reset a SMP device yet (%s)",
1814 		    __func__, pptr->path);
1815 		return (EINVAL);
1816 	}
1817 
1818 	/*
1819 	 * Now harvest any commands killed by this action
1820 	 * by issuing an ABORT for all commands on this device.
1821 	 *
1822 	 * We do this even if the the tmf or reset fails (in case there
1823 	 * are any dead commands around to be harvested *anyway*).
1824 	 * We don't have to await for the abort to complete.
1825 	 */
1826 	if (pmcs_abort(pwp, pptr, 0, 1, 0)) {
1827 		pptr->abort_pending = 1;
1828 		SCHEDULE_WORK(pwp, PMCS_WORK_ABORT_HANDLE);
1829 	}
1830 
1831 	pmcs_unlock_phy(pptr);
1832 	return (rval);
1833 }
1834 
1835 /*
1836  * Called with PHY locked.
1837  */
1838 static int
1839 pmcs_get_device_handle(pmcs_hw_t *pwp, pmcs_phy_t *pptr)
1840 {
1841 	if (pptr->valid_device_id == 0) {
1842 		int result = pmcs_register_device(pwp, pptr);
1843 
1844 		/*
1845 		 * If we changed while registering, punt
1846 		 */
1847 		if (pptr->changed) {
1848 			RESTART_DISCOVERY(pwp);
1849 			return (-1);
1850 		}
1851 
1852 		/*
1853 		 * If we had a failure to register, check against errors.
1854 		 * An ENOMEM error means we just retry (temp resource shortage).
1855 		 */
1856 		if (result == ENOMEM) {
1857 			PHY_CHANGED(pwp, pptr);
1858 			RESTART_DISCOVERY(pwp);
1859 			return (-1);
1860 		}
1861 
1862 		/*
1863 		 * An ETIMEDOUT error means we retry (if our counter isn't
1864 		 * exhausted)
1865 		 */
1866 		if (result == ETIMEDOUT) {
1867 			if (ddi_get_lbolt() < pptr->config_stop) {
1868 				PHY_CHANGED(pwp, pptr);
1869 				RESTART_DISCOVERY(pwp);
1870 			} else {
1871 				pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, pptr, NULL,
1872 				    "%s: Retries exhausted for %s, killing",
1873 				    __func__, pptr->path);
1874 				pptr->config_stop = 0;
1875 				pmcs_kill_changed(pwp, pptr, 0);
1876 			}
1877 			return (-1);
1878 		}
1879 		/*
1880 		 * Other errors or no valid device id is fatal, but don't
1881 		 * preclude a future action.
1882 		 */
1883 		if (result || pptr->valid_device_id == 0) {
1884 			pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, pptr, NULL,
1885 			    "%s: %s could not be registered", __func__,
1886 			    pptr->path);
1887 			return (-1);
1888 		}
1889 	}
1890 	return (0);
1891 }
1892 
1893 int
1894 pmcs_iport_tgtmap_create(pmcs_iport_t *iport)
1895 {
1896 	ASSERT(iport);
1897 	if (iport == NULL)
1898 		return (B_FALSE);
1899 
1900 	pmcs_prt(iport->pwp, PMCS_PRT_DEBUG_MAP, NULL, NULL, "%s", __func__);
1901 
1902 	/* create target map */
1903 	if (scsi_hba_tgtmap_create(iport->dip, SCSI_TM_FULLSET, tgtmap_usec,
1904 	    NULL, NULL, NULL, &iport->iss_tgtmap) != DDI_SUCCESS) {
1905 		pmcs_prt(iport->pwp, PMCS_PRT_DEBUG, NULL, NULL,
1906 		    "%s: failed to create tgtmap", __func__);
1907 		return (B_FALSE);
1908 	}
1909 	return (B_TRUE);
1910 }
1911 
1912 int
1913 pmcs_iport_tgtmap_destroy(pmcs_iport_t *iport)
1914 {
1915 	ASSERT(iport && iport->iss_tgtmap);
1916 	if ((iport == NULL) || (iport->iss_tgtmap == NULL))
1917 		return (B_FALSE);
1918 
1919 	pmcs_prt(iport->pwp, PMCS_PRT_DEBUG_MAP, NULL, NULL, "%s", __func__);
1920 
1921 	/* destroy target map */
1922 	scsi_hba_tgtmap_destroy(iport->iss_tgtmap);
1923 	return (B_TRUE);
1924 }
1925 
1926 /*
1927  * Remove all phys from an iport's phymap and empty it's phylist.
1928  * Called when a port has been reset by the host (see pmcs_intr.c).
1929  */
1930 void
1931 pmcs_iport_teardown_phys(pmcs_iport_t *iport)
1932 {
1933 	pmcs_hw_t		*pwp;
1934 	sas_phymap_phys_t	*phys;
1935 	int			phynum;
1936 
1937 	ASSERT(iport);
1938 	ASSERT(mutex_owned(&iport->lock));
1939 	pwp = iport->pwp;
1940 	ASSERT(pwp);
1941 
1942 	/*
1943 	 * Remove all phys from the iport handle's phy list, unset its
1944 	 * primary phy and update its state.
1945 	 */
1946 	pmcs_remove_phy_from_iport(iport, NULL);
1947 	iport->pptr = NULL;
1948 	iport->ua_state = UA_PEND_DEACTIVATE;
1949 
1950 	/* Remove all phys from the phymap */
1951 	phys = sas_phymap_ua2phys(pwp->hss_phymap, iport->ua);
1952 	while ((phynum = sas_phymap_phys_next(phys)) != -1) {
1953 		sas_phymap_phy_rem(pwp->hss_phymap, phynum);
1954 	}
1955 	sas_phymap_phys_free(phys);
1956 }
1957 
1958 /*
1959  * Query the phymap and populate the iport handle passed in.
1960  * Called with iport lock held.
1961  */
1962 int
1963 pmcs_iport_configure_phys(pmcs_iport_t *iport)
1964 {
1965 	pmcs_hw_t		*pwp;
1966 	pmcs_phy_t		*pptr;
1967 	sas_phymap_phys_t	*phys;
1968 	int			phynum;
1969 	int			inst;
1970 
1971 	ASSERT(iport);
1972 	ASSERT(mutex_owned(&iport->lock));
1973 	pwp = iport->pwp;
1974 	ASSERT(pwp);
1975 	inst = ddi_get_instance(iport->dip);
1976 
1977 	mutex_enter(&pwp->lock);
1978 	ASSERT(pwp->root_phys != NULL);
1979 
1980 	/*
1981 	 * Query the phymap regarding the phys in this iport and populate
1982 	 * the iport's phys list. Hereafter this list is maintained via
1983 	 * port up and down events in pmcs_intr.c
1984 	 */
1985 	ASSERT(list_is_empty(&iport->phys));
1986 	phys = sas_phymap_ua2phys(pwp->hss_phymap, iport->ua);
1987 	while ((phynum = sas_phymap_phys_next(phys)) != -1) {
1988 		/* Grab the phy pointer from root_phys */
1989 		pptr = pwp->root_phys + phynum;
1990 		ASSERT(pptr);
1991 		pmcs_lock_phy(pptr);
1992 		ASSERT(pptr->phynum == phynum);
1993 
1994 		/*
1995 		 * Set a back pointer in the phy to this iport.
1996 		 */
1997 		pptr->iport = iport;
1998 
1999 		/*
2000 		 * If this phy is the primary, set a pointer to it on our
2001 		 * iport handle, and set our portid from it.
2002 		 */
2003 		if (!pptr->subsidiary) {
2004 			iport->pptr = pptr;
2005 			iport->portid = pptr->portid;
2006 		}
2007 
2008 		/*
2009 		 * Finally, insert the phy into our list
2010 		 */
2011 		pmcs_unlock_phy(pptr);
2012 		pmcs_add_phy_to_iport(iport, pptr);
2013 
2014 		pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, pptr, NULL, "%s: found "
2015 		    "phy %d [0x%p] on iport%d, refcnt(%d)", __func__, phynum,
2016 		    (void *)pptr, inst, iport->refcnt);
2017 	}
2018 	mutex_exit(&pwp->lock);
2019 	sas_phymap_phys_free(phys);
2020 	RESTART_DISCOVERY(pwp);
2021 	return (DDI_SUCCESS);
2022 }
2023 
2024 /*
2025  * Return the iport that ua is associated with, or NULL.  If an iport is
2026  * returned, it will be held and the caller must release the hold.
2027  */
2028 static pmcs_iport_t *
2029 pmcs_get_iport_by_ua(pmcs_hw_t *pwp, char *ua)
2030 {
2031 	pmcs_iport_t	*iport = NULL;
2032 
2033 	rw_enter(&pwp->iports_lock, RW_READER);
2034 	for (iport = list_head(&pwp->iports);
2035 	    iport != NULL;
2036 	    iport = list_next(&pwp->iports, iport)) {
2037 		mutex_enter(&iport->lock);
2038 		if (strcmp(iport->ua, ua) == 0) {
2039 			mutex_exit(&iport->lock);
2040 			mutex_enter(&iport->refcnt_lock);
2041 			iport->refcnt++;
2042 			mutex_exit(&iport->refcnt_lock);
2043 			break;
2044 		}
2045 		mutex_exit(&iport->lock);
2046 	}
2047 	rw_exit(&pwp->iports_lock);
2048 
2049 	return (iport);
2050 }
2051 
2052 /*
2053  * Return the iport that pptr is associated with, or NULL.
2054  * If an iport is returned, there is a hold that the caller must release.
2055  */
2056 pmcs_iport_t *
2057 pmcs_get_iport_by_phy(pmcs_hw_t *pwp, pmcs_phy_t *pptr)
2058 {
2059 	pmcs_iport_t	*iport = NULL;
2060 	char		*ua;
2061 
2062 	ua = sas_phymap_lookup_ua(pwp->hss_phymap, pwp->sas_wwns[0],
2063 	    pmcs_barray2wwn(pptr->sas_address));
2064 	if (ua) {
2065 		iport = pmcs_get_iport_by_ua(pwp, ua);
2066 		if (iport) {
2067 			mutex_enter(&iport->lock);
2068 			iport->ua_state = UA_ACTIVE;
2069 			pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, pptr, NULL, "%s: "
2070 			    "found iport [0x%p] on ua (%s) for phy [0x%p], "
2071 			    "refcnt (%d)", __func__, (void *)iport, ua,
2072 			    (void *)pptr, iport->refcnt);
2073 			mutex_exit(&iport->lock);
2074 		}
2075 	}
2076 
2077 	return (iport);
2078 }
2079 
2080 /*
2081  * Promote the next phy on this port to primary, and return it.
2082  * Called when the primary PHY on a port is going down, but the port
2083  * remains up (see pmcs_intr.c).
2084  */
2085 pmcs_phy_t *
2086 pmcs_promote_next_phy(pmcs_phy_t *prev_primary)
2087 {
2088 	pmcs_hw_t	*pwp;
2089 	pmcs_iport_t	*iport;
2090 	pmcs_phy_t	*pptr, *child;
2091 	int		portid;
2092 
2093 	pmcs_lock_phy(prev_primary);
2094 	portid = prev_primary->portid;
2095 	iport  = prev_primary->iport;
2096 	pwp    = prev_primary->pwp;
2097 
2098 	/* Use the first available phy in this port */
2099 	for (pptr = pwp->root_phys; pptr; pptr = pptr->sibling) {
2100 		if ((pptr->portid == portid) && (pptr != prev_primary)) {
2101 			mutex_enter(&pptr->phy_lock);
2102 			break;
2103 		}
2104 	}
2105 
2106 	if (pptr == NULL) {
2107 		pmcs_unlock_phy(prev_primary);
2108 		return (NULL);
2109 	}
2110 
2111 	if (iport) {
2112 		mutex_enter(&iport->lock);
2113 		iport->pptr = pptr;
2114 		mutex_exit(&iport->lock);
2115 	}
2116 
2117 	/* Update the phy handle with the data from the previous primary */
2118 	pptr->children		= prev_primary->children;
2119 	child = pptr->children;
2120 	while (child) {
2121 		child->parent = pptr;
2122 		child = child->sibling;
2123 	}
2124 	pptr->ncphy		= prev_primary->ncphy;
2125 	pptr->width		= prev_primary->width;
2126 	pptr->dtype		= prev_primary->dtype;
2127 	pptr->pend_dtype	= prev_primary->pend_dtype;
2128 	pptr->tolerates_sas2	= prev_primary->tolerates_sas2;
2129 	pptr->atdt		= prev_primary->atdt;
2130 	pptr->portid		= prev_primary->portid;
2131 	pptr->link_rate		= prev_primary->link_rate;
2132 	pptr->configured	= prev_primary->configured;
2133 	pptr->iport		= prev_primary->iport;
2134 	pptr->target		= prev_primary->target;
2135 	if (pptr->target) {
2136 		pptr->target->phy = pptr;
2137 	}
2138 	pptr->subsidiary = 0;
2139 
2140 	prev_primary->subsidiary = 1;
2141 	prev_primary->children = NULL;
2142 	prev_primary->target = NULL;
2143 	pmcs_unlock_phy(prev_primary);
2144 
2145 	/*
2146 	 * We call pmcs_unlock_phy() on pptr because it now contains the
2147 	 * list of children.
2148 	 */
2149 	pmcs_unlock_phy(pptr);
2150 
2151 	return (pptr);
2152 }
2153 
2154 void
2155 pmcs_rele_iport(pmcs_iport_t *iport)
2156 {
2157 	/*
2158 	 * Release a refcnt on this iport. If this is the last reference,
2159 	 * signal the potential waiter in pmcs_iport_unattach().
2160 	 */
2161 	ASSERT(iport->refcnt > 0);
2162 	mutex_enter(&iport->refcnt_lock);
2163 	iport->refcnt--;
2164 	mutex_exit(&iport->refcnt_lock);
2165 	if (iport->refcnt == 0) {
2166 		cv_signal(&iport->refcnt_cv);
2167 	}
2168 	pmcs_prt(iport->pwp, PMCS_PRT_DEBUG_CONFIG, NULL, NULL, "%s: iport "
2169 	    "[0x%p] refcnt (%d)", __func__, (void *)iport, iport->refcnt);
2170 }
2171 
2172 void
2173 pmcs_phymap_activate(void *arg, char *ua, void **privp)
2174 {
2175 	_NOTE(ARGUNUSED(privp));
2176 	pmcs_hw_t	*pwp = arg;
2177 	pmcs_iport_t	*iport = NULL;
2178 
2179 	mutex_enter(&pwp->lock);
2180 	if ((pwp->state == STATE_UNPROBING) || (pwp->state == STATE_DEAD)) {
2181 		mutex_exit(&pwp->lock);
2182 		return;
2183 	}
2184 	pwp->phymap_active++;
2185 	mutex_exit(&pwp->lock);
2186 
2187 	if (scsi_hba_iportmap_iport_add(pwp->hss_iportmap, ua, NULL) !=
2188 	    DDI_SUCCESS) {
2189 		pmcs_prt(pwp, PMCS_PRT_DEBUG_MAP, NULL, NULL, "%s: failed to "
2190 		    "add iport handle on unit address [%s]", __func__, ua);
2191 	} else {
2192 		pmcs_prt(pwp, PMCS_PRT_DEBUG_MAP, NULL, NULL, "%s: "
2193 		    "phymap_active count (%d), added iport handle on unit "
2194 		    "address [%s]", __func__, pwp->phymap_active, ua);
2195 	}
2196 
2197 	/* Set the HBA softstate as our private data for this unit address */
2198 	*privp = (void *)pwp;
2199 
2200 	/*
2201 	 * We are waiting on attach for this iport node, unless it is still
2202 	 * attached. This can happen if a consumer has an outstanding open
2203 	 * on our iport node, but the port is down.  If this is the case, we
2204 	 * need to configure our iport here for reuse.
2205 	 */
2206 	iport = pmcs_get_iport_by_ua(pwp, ua);
2207 	if (iport) {
2208 		mutex_enter(&iport->lock);
2209 		if (pmcs_iport_configure_phys(iport) != DDI_SUCCESS) {
2210 			pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, NULL, NULL, "%s: "
2211 			    "failed to configure phys on iport [0x%p] at "
2212 			    "unit address (%s)", __func__, (void *)iport, ua);
2213 		}
2214 		iport->ua_state = UA_ACTIVE;
2215 		pmcs_smhba_add_iport_prop(iport, DATA_TYPE_INT32, PMCS_NUM_PHYS,
2216 		    &iport->nphy);
2217 		mutex_exit(&iport->lock);
2218 		pmcs_rele_iport(iport);
2219 	}
2220 
2221 }
2222 
2223 void
2224 pmcs_phymap_deactivate(void *arg, char *ua, void *privp)
2225 {
2226 	_NOTE(ARGUNUSED(privp));
2227 	pmcs_hw_t	*pwp = arg;
2228 	pmcs_iport_t	*iport;
2229 
2230 	mutex_enter(&pwp->lock);
2231 	pwp->phymap_active--;
2232 	mutex_exit(&pwp->lock);
2233 
2234 	if (scsi_hba_iportmap_iport_remove(pwp->hss_iportmap, ua) !=
2235 	    DDI_SUCCESS) {
2236 		pmcs_prt(pwp, PMCS_PRT_DEBUG_MAP, NULL, NULL, "%s: failed to "
2237 		    "remove iport handle on unit address [%s]", __func__, ua);
2238 	} else {
2239 		pmcs_prt(pwp, PMCS_PRT_DEBUG_MAP, NULL, NULL, "%s: "
2240 		    "phymap_active count (%d), removed iport handle on unit "
2241 		    "address [%s]", __func__, pwp->phymap_active, ua);
2242 	}
2243 
2244 	iport = pmcs_get_iport_by_ua(pwp, ua);
2245 
2246 	if (iport == NULL) {
2247 		pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, NULL, NULL, "%s: failed "
2248 		    "lookup of iport handle on unit addr (%s)", __func__, ua);
2249 		return;
2250 	}
2251 
2252 	mutex_enter(&iport->lock);
2253 	iport->ua_state = UA_INACTIVE;
2254 	iport->portid = PMCS_IPORT_INVALID_PORT_ID;
2255 	pmcs_remove_phy_from_iport(iport, NULL);
2256 	mutex_exit(&iport->lock);
2257 	pmcs_rele_iport(iport);
2258 }
2259 
2260 /*
2261  * Top-level discovery function
2262  */
2263 void
2264 pmcs_discover(pmcs_hw_t *pwp)
2265 {
2266 	pmcs_phy_t		*pptr;
2267 	pmcs_phy_t		*root_phy;
2268 	boolean_t		config_changed;
2269 
2270 	DTRACE_PROBE2(pmcs__discover__entry, ulong_t, pwp->work_flags,
2271 	    boolean_t, pwp->config_changed);
2272 
2273 	mutex_enter(&pwp->lock);
2274 
2275 	if (pwp->state != STATE_RUNNING) {
2276 		mutex_exit(&pwp->lock);
2277 		return;
2278 	}
2279 
2280 	/* Ensure we have at least one phymap active */
2281 	if (pwp->phymap_active == 0) {
2282 		mutex_exit(&pwp->lock);
2283 		pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, NULL, NULL,
2284 		    "%s: phymap inactive, exiting", __func__);
2285 		return;
2286 	}
2287 
2288 	mutex_exit(&pwp->lock);
2289 
2290 	/*
2291 	 * If no iports have attached, but we have PHYs that are up, we
2292 	 * are waiting for iport attach to complete.  Restart discovery.
2293 	 */
2294 	rw_enter(&pwp->iports_lock, RW_READER);
2295 	if (!pwp->iports_attached) {
2296 		rw_exit(&pwp->iports_lock);
2297 		pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, NULL, NULL,
2298 		    "%s: no iports attached, retry discovery", __func__);
2299 		SCHEDULE_WORK(pwp, PMCS_WORK_DISCOVER);
2300 		return;
2301 	}
2302 	rw_exit(&pwp->iports_lock);
2303 
2304 	mutex_enter(&pwp->config_lock);
2305 	if (pwp->configuring) {
2306 		mutex_exit(&pwp->config_lock);
2307 		pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, NULL, NULL,
2308 		    "%s: configuration already in progress", __func__);
2309 		return;
2310 	}
2311 
2312 	if (pmcs_acquire_scratch(pwp, B_FALSE)) {
2313 		mutex_exit(&pwp->config_lock);
2314 		pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, NULL, NULL,
2315 		    "%s: cannot allocate scratch", __func__);
2316 		SCHEDULE_WORK(pwp, PMCS_WORK_DISCOVER);
2317 		return;
2318 	}
2319 
2320 	pwp->configuring = 1;
2321 	pwp->config_changed = B_FALSE;
2322 	mutex_exit(&pwp->config_lock);
2323 
2324 	pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, NULL, NULL, "Discovery begin");
2325 
2326 	/*
2327 	 * The order of the following traversals is important.
2328 	 *
2329 	 * The first one checks for changed expanders.
2330 	 *
2331 	 * The second one aborts commands for dead devices and deregisters them.
2332 	 *
2333 	 * The third one clears the contents of dead expanders from the tree
2334 	 *
2335 	 * The fourth one clears now dead devices in expanders that remain.
2336 	 */
2337 
2338 	/*
2339 	 * 1. Check expanders marked changed (but not dead) to see if they still
2340 	 * have the same number of phys and the same SAS address. Mark them,
2341 	 * their subsidiary phys (if wide) and their descendents dead if
2342 	 * anything has changed. Check the devices they contain to see if
2343 	 * *they* have changed. If they've changed from type NOTHING we leave
2344 	 * them marked changed to be configured later (picking up a new SAS
2345 	 * address and link rate if possible). Otherwise, any change in type,
2346 	 * SAS address or removal of target role will cause us to mark them
2347 	 * (and their descendents) as dead (and cause any pending commands
2348 	 * and associated devices to be removed).
2349 	 *
2350 	 * NOTE: We don't want to bail on discovery if the config has
2351 	 * changed until *after* we run pmcs_kill_devices.
2352 	 */
2353 	root_phy = pwp->root_phys;
2354 	config_changed = pmcs_check_expanders(pwp, root_phy);
2355 
2356 	/*
2357 	 * 2. Descend the tree looking for dead devices and kill them
2358 	 * by aborting all active commands and then deregistering them.
2359 	 */
2360 	if (pmcs_kill_devices(pwp, root_phy) || config_changed) {
2361 		goto out;
2362 	}
2363 
2364 	/*
2365 	 * 3. Check for dead expanders and remove their children from the tree.
2366 	 * By the time we get here, the devices and commands for them have
2367 	 * already been terminated and removed.
2368 	 *
2369 	 * We do this independent of the configuration count changing so we can
2370 	 * free any dead device PHYs that were discovered while checking
2371 	 * expanders. We ignore any subsidiary phys as pmcs_clear_expander
2372 	 * will take care of those.
2373 	 *
2374 	 * NOTE: pmcs_clear_expander requires softstate lock
2375 	 */
2376 	mutex_enter(&pwp->lock);
2377 	for (pptr = pwp->root_phys; pptr; pptr = pptr->sibling) {
2378 		/*
2379 		 * Call pmcs_clear_expander for every root PHY.  It will
2380 		 * recurse and determine which (if any) expanders actually
2381 		 * need to be cleared.
2382 		 */
2383 		pmcs_lock_phy(pptr);
2384 		pmcs_clear_expander(pwp, pptr, 0);
2385 		pmcs_unlock_phy(pptr);
2386 	}
2387 	mutex_exit(&pwp->lock);
2388 
2389 	/*
2390 	 * 4. Check for dead devices and nullify them. By the time we get here,
2391 	 * the devices and commands for them have already been terminated
2392 	 * and removed. This is different from step 2 in that this just nulls
2393 	 * phys that are part of expanders that are still here but used to
2394 	 * be something but are no longer something (e.g., after a pulled
2395 	 * disk drive). Note that dead expanders had their contained phys
2396 	 * removed from the tree- here, the expanders themselves are
2397 	 * nullified (unless they were removed by being contained in another
2398 	 * expander phy).
2399 	 */
2400 	pmcs_clear_phys(pwp, root_phy);
2401 
2402 	/*
2403 	 * 5. Now check for and configure new devices.
2404 	 */
2405 	if (pmcs_configure_new_devices(pwp, root_phy)) {
2406 		goto restart;
2407 	}
2408 
2409 out:
2410 	DTRACE_PROBE2(pmcs__discover__exit, ulong_t, pwp->work_flags,
2411 	    boolean_t, pwp->config_changed);
2412 	pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, NULL, NULL, "Discovery end");
2413 
2414 	mutex_enter(&pwp->config_lock);
2415 
2416 	if (pwp->config_changed == B_FALSE) {
2417 		/*
2418 		 * Observation is stable, report what we currently see to
2419 		 * the tgtmaps for delta processing. Start by setting
2420 		 * BEGIN on all tgtmaps.
2421 		 */
2422 		mutex_exit(&pwp->config_lock);
2423 		if (pmcs_report_observations(pwp) == B_FALSE) {
2424 			goto restart;
2425 		}
2426 		mutex_enter(&pwp->config_lock);
2427 	} else {
2428 		/*
2429 		 * If config_changed is TRUE, we need to reschedule
2430 		 * discovery now.
2431 		 */
2432 		pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, NULL, NULL,
2433 		    "%s: Config has changed, will re-run discovery", __func__);
2434 		SCHEDULE_WORK(pwp, PMCS_WORK_DISCOVER);
2435 	}
2436 
2437 	pmcs_release_scratch(pwp);
2438 	pwp->configuring = 0;
2439 	mutex_exit(&pwp->config_lock);
2440 
2441 #ifdef DEBUG
2442 	pptr = pmcs_find_phy_needing_work(pwp, pwp->root_phys);
2443 	if (pptr != NULL) {
2444 		if (!WORK_IS_SCHEDULED(pwp, PMCS_WORK_DISCOVER)) {
2445 			pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, NULL,
2446 			    "PHY %s dead=%d changed=%d configured=%d "
2447 			    "but no work scheduled", pptr->path, pptr->dead,
2448 			    pptr->changed, pptr->configured);
2449 		}
2450 		pmcs_unlock_phy(pptr);
2451 	}
2452 #endif
2453 
2454 	return;
2455 
2456 restart:
2457 	/* Clean up and restart discovery */
2458 	pmcs_release_scratch(pwp);
2459 	mutex_enter(&pwp->config_lock);
2460 	pwp->configuring = 0;
2461 	RESTART_DISCOVERY_LOCKED(pwp);
2462 	mutex_exit(&pwp->config_lock);
2463 }
2464 
2465 /*
2466  * Return any PHY that needs to have scheduled work done.  The PHY is returned
2467  * locked.
2468  */
2469 static pmcs_phy_t *
2470 pmcs_find_phy_needing_work(pmcs_hw_t *pwp, pmcs_phy_t *pptr)
2471 {
2472 	pmcs_phy_t *cphyp, *pnext;
2473 
2474 	while (pptr) {
2475 		pmcs_lock_phy(pptr);
2476 
2477 		if (pptr->changed || (pptr->dead && pptr->valid_device_id)) {
2478 			return (pptr);
2479 		}
2480 
2481 		pnext = pptr->sibling;
2482 
2483 		if (pptr->children) {
2484 			cphyp = pptr->children;
2485 			pmcs_unlock_phy(pptr);
2486 			cphyp = pmcs_find_phy_needing_work(pwp, cphyp);
2487 			if (cphyp) {
2488 				return (cphyp);
2489 			}
2490 		} else {
2491 			pmcs_unlock_phy(pptr);
2492 		}
2493 
2494 		pptr = pnext;
2495 	}
2496 
2497 	return (NULL);
2498 }
2499 
2500 /*
2501  * Report current observations to SCSA.
2502  */
2503 static boolean_t
2504 pmcs_report_observations(pmcs_hw_t *pwp)
2505 {
2506 	pmcs_iport_t		*iport;
2507 	scsi_hba_tgtmap_t	*tgtmap;
2508 	char			*ap;
2509 	pmcs_phy_t		*pptr;
2510 	uint64_t		wwn;
2511 
2512 	/*
2513 	 * Observation is stable, report what we currently see to the tgtmaps
2514 	 * for delta processing. Start by setting BEGIN on all tgtmaps.
2515 	 */
2516 	rw_enter(&pwp->iports_lock, RW_READER);
2517 	for (iport = list_head(&pwp->iports); iport != NULL;
2518 	    iport = list_next(&pwp->iports, iport)) {
2519 		/*
2520 		 * Unless we have at least one phy up, skip this iport.
2521 		 * Note we don't need to lock the iport for report_skip
2522 		 * since it is only used here.  We are doing the skip so that
2523 		 * the phymap and iportmap stabilization times are honored -
2524 		 * giving us the ability to recover port operation within the
2525 		 * stabilization time without unconfiguring targets using the
2526 		 * port.
2527 		 */
2528 		if (!sas_phymap_uahasphys(pwp->hss_phymap, iport->ua)) {
2529 			iport->report_skip = 1;
2530 			continue;		/* skip set_begin */
2531 		}
2532 		iport->report_skip = 0;
2533 
2534 		tgtmap = iport->iss_tgtmap;
2535 		ASSERT(tgtmap);
2536 		if (scsi_hba_tgtmap_set_begin(tgtmap) != DDI_SUCCESS) {
2537 			pmcs_prt(pwp, PMCS_PRT_DEBUG_MAP, NULL, NULL,
2538 			    "%s: cannot set_begin tgtmap ", __func__);
2539 			rw_exit(&pwp->iports_lock);
2540 			return (B_FALSE);
2541 		}
2542 		pmcs_prt(pwp, PMCS_PRT_DEBUG_MAP, NULL, NULL,
2543 		    "%s: set begin on tgtmap [0x%p]", __func__, (void *)tgtmap);
2544 	}
2545 	rw_exit(&pwp->iports_lock);
2546 
2547 	/*
2548 	 * Now, cycle through all levels of all phys and report
2549 	 * observations into their respective tgtmaps.
2550 	 */
2551 	pptr = pwp->root_phys;
2552 
2553 	while (pptr) {
2554 		pmcs_lock_phy(pptr);
2555 
2556 		/*
2557 		 * Skip PHYs that have nothing attached or are dead.
2558 		 */
2559 		if ((pptr->dtype == NOTHING) || pptr->dead) {
2560 			pmcs_unlock_phy(pptr);
2561 			pptr = pptr->sibling;
2562 			continue;
2563 		}
2564 
2565 		if (pptr->changed) {
2566 			pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, pptr, NULL,
2567 			    "%s: oops, PHY %s changed; restart discovery",
2568 			    __func__, pptr->path);
2569 			pmcs_unlock_phy(pptr);
2570 			return (B_FALSE);
2571 		}
2572 
2573 		/*
2574 		 * Get the iport for this root PHY, then call the helper
2575 		 * to report observations for this iport's targets
2576 		 */
2577 		iport = pmcs_get_iport_by_phy(pwp, pptr);
2578 		if (iport == NULL) {
2579 			/* No iport for this tgt */
2580 			pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, NULL, NULL,
2581 			    "%s: no iport for this target", __func__);
2582 			pmcs_unlock_phy(pptr);
2583 			pptr = pptr->sibling;
2584 			continue;
2585 		}
2586 
2587 		if (!iport->report_skip) {
2588 			if (pmcs_report_iport_observations(
2589 			    pwp, iport, pptr) == B_FALSE) {
2590 				pmcs_rele_iport(iport);
2591 				pmcs_unlock_phy(pptr);
2592 				return (B_FALSE);
2593 			}
2594 		}
2595 		pmcs_rele_iport(iport);
2596 		pmcs_unlock_phy(pptr);
2597 		pptr = pptr->sibling;
2598 	}
2599 
2600 	/*
2601 	 * The observation is complete, end sets. Note we will skip any
2602 	 * iports that are active, but have no PHYs in them (i.e. awaiting
2603 	 * unconfigure). Set to restart discovery if we find this.
2604 	 */
2605 	rw_enter(&pwp->iports_lock, RW_READER);
2606 	for (iport = list_head(&pwp->iports);
2607 	    iport != NULL;
2608 	    iport = list_next(&pwp->iports, iport)) {
2609 
2610 		if (iport->report_skip)
2611 			continue;		/* skip set_end */
2612 
2613 		tgtmap = iport->iss_tgtmap;
2614 		ASSERT(tgtmap);
2615 		if (scsi_hba_tgtmap_set_end(tgtmap, 0) != DDI_SUCCESS) {
2616 			pmcs_prt(pwp, PMCS_PRT_DEBUG_MAP, NULL, NULL,
2617 			    "%s: cannot set_end tgtmap ", __func__);
2618 			rw_exit(&pwp->iports_lock);
2619 			return (B_FALSE);
2620 		}
2621 		pmcs_prt(pwp, PMCS_PRT_DEBUG_MAP, NULL, NULL,
2622 		    "%s: set end on tgtmap [0x%p]", __func__, (void *)tgtmap);
2623 	}
2624 
2625 	/*
2626 	 * Now that discovery is complete, set up the necessary
2627 	 * DDI properties on each iport node.
2628 	 */
2629 	for (iport = list_head(&pwp->iports); iport != NULL;
2630 	    iport = list_next(&pwp->iports, iport)) {
2631 		/* Set up the 'attached-port' property on the iport */
2632 		ap = kmem_zalloc(PMCS_MAX_UA_SIZE, KM_SLEEP);
2633 		mutex_enter(&iport->lock);
2634 		pptr = iport->pptr;
2635 		mutex_exit(&iport->lock);
2636 		if (pptr == NULL) {
2637 			/*
2638 			 * This iport is down, but has not been
2639 			 * removed from our list (unconfigured).
2640 			 * Set our value to '0'.
2641 			 */
2642 			(void) snprintf(ap, 1, "%s", "0");
2643 		} else {
2644 			/* Otherwise, set it to remote phy's wwn */
2645 			pmcs_lock_phy(pptr);
2646 			wwn = pmcs_barray2wwn(pptr->sas_address);
2647 			(void) scsi_wwn_to_wwnstr(wwn, 1, ap);
2648 			pmcs_unlock_phy(pptr);
2649 		}
2650 		if (ndi_prop_update_string(DDI_DEV_T_NONE, iport->dip,
2651 		    SCSI_ADDR_PROP_ATTACHED_PORT,  ap) != DDI_SUCCESS) {
2652 			pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL, "%s: Failed "
2653 			    "to set prop ("SCSI_ADDR_PROP_ATTACHED_PORT")",
2654 			    __func__);
2655 		}
2656 		kmem_free(ap, PMCS_MAX_UA_SIZE);
2657 	}
2658 	rw_exit(&pwp->iports_lock);
2659 
2660 	return (B_TRUE);
2661 }
2662 
2663 /*
2664  * Report observations into a particular iport's target map
2665  *
2666  * Called with phyp (and all descendents) locked
2667  */
2668 static boolean_t
2669 pmcs_report_iport_observations(pmcs_hw_t *pwp, pmcs_iport_t *iport,
2670     pmcs_phy_t *phyp)
2671 {
2672 	pmcs_phy_t		*lphyp;
2673 	scsi_hba_tgtmap_t	*tgtmap;
2674 	scsi_tgtmap_tgt_type_t	tgt_type;
2675 	char			*ua;
2676 	uint64_t		wwn;
2677 
2678 	tgtmap = iport->iss_tgtmap;
2679 	ASSERT(tgtmap);
2680 
2681 	lphyp = phyp;
2682 	while (lphyp) {
2683 		switch (lphyp->dtype) {
2684 		default:		/* Skip unknown PHYs. */
2685 			/* for non-root phys, skip to sibling */
2686 			goto next_phy;
2687 
2688 		case SATA:
2689 		case SAS:
2690 			tgt_type = SCSI_TGT_SCSI_DEVICE;
2691 			break;
2692 
2693 		case EXPANDER:
2694 			tgt_type = SCSI_TGT_SMP_DEVICE;
2695 			break;
2696 		}
2697 
2698 		if (lphyp->dead) {
2699 			goto next_phy;
2700 		}
2701 
2702 		wwn = pmcs_barray2wwn(lphyp->sas_address);
2703 		ua = scsi_wwn_to_wwnstr(wwn, 1, NULL);
2704 
2705 		pmcs_prt(pwp, PMCS_PRT_DEBUG_MAP, lphyp, NULL,
2706 		    "iport_observation: adding %s on tgtmap [0x%p] phy [0x%p]",
2707 		    ua, (void *)tgtmap, (void*)lphyp);
2708 
2709 		if (scsi_hba_tgtmap_set_add(tgtmap, tgt_type, ua, NULL) !=
2710 		    DDI_SUCCESS) {
2711 			pmcs_prt(pwp, PMCS_PRT_DEBUG_MAP,  NULL, NULL,
2712 			    "%s: failed to add address %s", __func__, ua);
2713 			scsi_free_wwnstr(ua);
2714 			return (B_FALSE);
2715 		}
2716 		scsi_free_wwnstr(ua);
2717 
2718 		if (lphyp->children) {
2719 			if (pmcs_report_iport_observations(pwp, iport,
2720 			    lphyp->children) == B_FALSE) {
2721 				return (B_FALSE);
2722 			}
2723 		}
2724 
2725 		/* for non-root phys, report siblings too */
2726 next_phy:
2727 		if (IS_ROOT_PHY(lphyp)) {
2728 			lphyp = NULL;
2729 		} else {
2730 			lphyp = lphyp->sibling;
2731 		}
2732 	}
2733 
2734 	return (B_TRUE);
2735 }
2736 
2737 /*
2738  * Check for and configure new devices.
2739  *
2740  * If the changed device is a SATA device, add a SATA device.
2741  *
2742  * If the changed device is a SAS device, add a SAS device.
2743  *
2744  * If the changed device is an EXPANDER device, do a REPORT
2745  * GENERAL SMP command to find out the number of contained phys.
2746  *
2747  * For each number of contained phys, allocate a phy, do a
2748  * DISCOVERY SMP command to find out what kind of device it
2749  * is and add it to the linked list of phys on the *next* level.
2750  *
2751  * NOTE: pptr passed in by the caller will be a root PHY
2752  */
2753 static int
2754 pmcs_configure_new_devices(pmcs_hw_t *pwp, pmcs_phy_t *pptr)
2755 {
2756 	int rval = 0;
2757 	pmcs_iport_t *iport;
2758 	pmcs_phy_t *pnext, *orig_pptr = pptr, *root_phy, *pchild;
2759 
2760 	/*
2761 	 * First, walk through each PHY at this level
2762 	 */
2763 	while (pptr) {
2764 		pmcs_lock_phy(pptr);
2765 		pnext = pptr->sibling;
2766 
2767 		/*
2768 		 * Set the new dtype if it has changed
2769 		 */
2770 		if ((pptr->pend_dtype != NEW) &&
2771 		    (pptr->pend_dtype != pptr->dtype)) {
2772 			pptr->dtype = pptr->pend_dtype;
2773 		}
2774 
2775 		if (pptr->changed == 0 || pptr->dead || pptr->configured) {
2776 			goto next_phy;
2777 		}
2778 
2779 		/*
2780 		 * Confirm that this target's iport is configured
2781 		 */
2782 		root_phy = pmcs_get_root_phy(pptr);
2783 		iport = pmcs_get_iport_by_phy(pwp, root_phy);
2784 		if (iport == NULL) {
2785 			/* No iport for this tgt, restart */
2786 			pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, NULL, NULL,
2787 			    "%s: iport not yet configured, "
2788 			    "retry discovery", __func__);
2789 			pnext = NULL;
2790 			rval = -1;
2791 			goto next_phy;
2792 		}
2793 
2794 		switch (pptr->dtype) {
2795 		case NOTHING:
2796 			pptr->changed = 0;
2797 			break;
2798 		case SATA:
2799 		case SAS:
2800 			pptr->iport = iport;
2801 			pmcs_new_tport(pwp, pptr);
2802 			break;
2803 		case EXPANDER:
2804 			pmcs_configure_expander(pwp, pptr, iport);
2805 			break;
2806 		}
2807 		pmcs_rele_iport(iport);
2808 
2809 		mutex_enter(&pwp->config_lock);
2810 		if (pwp->config_changed) {
2811 			mutex_exit(&pwp->config_lock);
2812 			pnext = NULL;
2813 			goto next_phy;
2814 		}
2815 		mutex_exit(&pwp->config_lock);
2816 
2817 next_phy:
2818 		pmcs_unlock_phy(pptr);
2819 		pptr = pnext;
2820 	}
2821 
2822 	if (rval != 0) {
2823 		return (rval);
2824 	}
2825 
2826 	/*
2827 	 * Now walk through each PHY again, recalling ourselves if they
2828 	 * have children
2829 	 */
2830 	pptr = orig_pptr;
2831 	while (pptr) {
2832 		pmcs_lock_phy(pptr);
2833 		pnext = pptr->sibling;
2834 		pchild = pptr->children;
2835 		pmcs_unlock_phy(pptr);
2836 
2837 		if (pchild) {
2838 			rval = pmcs_configure_new_devices(pwp, pchild);
2839 			if (rval != 0) {
2840 				break;
2841 			}
2842 		}
2843 
2844 		pptr = pnext;
2845 	}
2846 
2847 	return (rval);
2848 }
2849 
2850 /*
2851  * Set all phys and descendent phys as changed if changed == B_TRUE, otherwise
2852  * mark them all as not changed.
2853  *
2854  * Called with parent PHY locked.
2855  */
2856 void
2857 pmcs_set_changed(pmcs_hw_t *pwp, pmcs_phy_t *parent, boolean_t changed,
2858     int level)
2859 {
2860 	pmcs_phy_t *pptr;
2861 
2862 	if (level == 0) {
2863 		if (changed) {
2864 			PHY_CHANGED(pwp, parent);
2865 		} else {
2866 			parent->changed = 0;
2867 		}
2868 		if (parent->dtype == EXPANDER && parent->level) {
2869 			parent->width = 1;
2870 		}
2871 		if (parent->children) {
2872 			pmcs_set_changed(pwp, parent->children, changed,
2873 			    level + 1);
2874 		}
2875 	} else {
2876 		pptr = parent;
2877 		while (pptr) {
2878 			if (changed) {
2879 				PHY_CHANGED(pwp, pptr);
2880 			} else {
2881 				pptr->changed = 0;
2882 			}
2883 			if (pptr->dtype == EXPANDER && pptr->level) {
2884 				pptr->width = 1;
2885 			}
2886 			if (pptr->children) {
2887 				pmcs_set_changed(pwp, pptr->children, changed,
2888 				    level + 1);
2889 			}
2890 			pptr = pptr->sibling;
2891 		}
2892 	}
2893 }
2894 
2895 /*
2896  * Take the passed phy mark it and its descendants as dead.
2897  * Fire up reconfiguration to abort commands and bury it.
2898  *
2899  * Called with the parent PHY locked.
2900  */
2901 void
2902 pmcs_kill_changed(pmcs_hw_t *pwp, pmcs_phy_t *parent, int level)
2903 {
2904 	pmcs_phy_t *pptr = parent;
2905 
2906 	while (pptr) {
2907 		pptr->link_rate = 0;
2908 		pptr->abort_sent = 0;
2909 		pptr->abort_pending = 1;
2910 		SCHEDULE_WORK(pwp, PMCS_WORK_ABORT_HANDLE);
2911 		pptr->need_rl_ext = 0;
2912 
2913 		if (pptr->dead == 0) {
2914 			PHY_CHANGED(pwp, pptr);
2915 			RESTART_DISCOVERY(pwp);
2916 		}
2917 
2918 		pptr->dead = 1;
2919 
2920 		if (pptr->children) {
2921 			pmcs_kill_changed(pwp, pptr->children, level + 1);
2922 		}
2923 
2924 		/*
2925 		 * Only kill siblings at level > 0
2926 		 */
2927 		if (level == 0) {
2928 			return;
2929 		}
2930 
2931 		pptr = pptr->sibling;
2932 	}
2933 }
2934 
2935 /*
2936  * Go through every PHY and clear any that are dead (unless they're expanders)
2937  */
2938 static void
2939 pmcs_clear_phys(pmcs_hw_t *pwp, pmcs_phy_t *pptr)
2940 {
2941 	pmcs_phy_t *pnext, *phyp;
2942 
2943 	phyp = pptr;
2944 	while (phyp) {
2945 		if (IS_ROOT_PHY(phyp)) {
2946 			pmcs_lock_phy(phyp);
2947 		}
2948 
2949 		if ((phyp->dtype != EXPANDER) && phyp->dead) {
2950 			pmcs_clear_phy(pwp, phyp);
2951 		}
2952 
2953 		if (phyp->children) {
2954 			pmcs_clear_phys(pwp, phyp->children);
2955 		}
2956 
2957 		pnext = phyp->sibling;
2958 
2959 		if (IS_ROOT_PHY(phyp)) {
2960 			pmcs_unlock_phy(phyp);
2961 		}
2962 
2963 		phyp = pnext;
2964 	}
2965 }
2966 
2967 /*
2968  * Clear volatile parts of a phy.  Called with PHY locked.
2969  */
2970 void
2971 pmcs_clear_phy(pmcs_hw_t *pwp, pmcs_phy_t *pptr)
2972 {
2973 	pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, pptr, NULL, "%s: %s",
2974 	    __func__, pptr->path);
2975 	ASSERT(mutex_owned(&pptr->phy_lock));
2976 	/* keep sibling */
2977 	/* keep children */
2978 	/* keep parent */
2979 	pptr->device_id = PMCS_INVALID_DEVICE_ID;
2980 	/* keep hw_event_ack */
2981 	pptr->ncphy = 0;
2982 	/* keep phynum */
2983 	pptr->width = 0;
2984 	pptr->ds_recovery_retries = 0;
2985 	pptr->ds_prev_good_recoveries = 0;
2986 	pptr->last_good_recovery = 0;
2987 	pptr->prev_recovery = 0;
2988 
2989 	/* keep dtype */
2990 	pptr->config_stop = 0;
2991 	pptr->spinup_hold = 0;
2992 	pptr->atdt = 0;
2993 	/* keep portid */
2994 	pptr->link_rate = 0;
2995 	pptr->valid_device_id = 0;
2996 	pptr->abort_sent = 0;
2997 	pptr->abort_pending = 0;
2998 	pptr->need_rl_ext = 0;
2999 	pptr->subsidiary = 0;
3000 	pptr->configured = 0;
3001 	/* Only mark dead if it's not a root PHY and its dtype isn't NOTHING */
3002 	/* XXX: What about directly attached disks? */
3003 	if (!IS_ROOT_PHY(pptr) && (pptr->dtype != NOTHING))
3004 		pptr->dead = 1;
3005 	pptr->changed = 0;
3006 	/* keep SAS address */
3007 	/* keep path */
3008 	/* keep ref_count */
3009 	/* Don't clear iport on root PHYs - they are handled in pmcs_intr.c */
3010 	if (!IS_ROOT_PHY(pptr)) {
3011 		pptr->iport = NULL;
3012 	}
3013 	/* keep target */
3014 }
3015 
3016 /*
3017  * Allocate softstate for this target if there isn't already one.  If there
3018  * is, just redo our internal configuration.  If it is actually "new", we'll
3019  * soon get a tran_tgt_init for it.
3020  *
3021  * Called with PHY locked.
3022  */
3023 static void
3024 pmcs_new_tport(pmcs_hw_t *pwp, pmcs_phy_t *pptr)
3025 {
3026 	pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, pptr, NULL, "%s: phy 0x%p @ %s",
3027 	    __func__, (void *)pptr, pptr->path);
3028 
3029 	if (pmcs_configure_phy(pwp, pptr) == B_FALSE) {
3030 		/*
3031 		 * If the config failed, mark the PHY as changed.
3032 		 */
3033 		PHY_CHANGED(pwp, pptr);
3034 		pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, pptr, NULL,
3035 		    "%s: pmcs_configure_phy failed for phy 0x%p", __func__,
3036 		    (void *)pptr);
3037 		return;
3038 	}
3039 
3040 	/* Mark PHY as no longer changed */
3041 	pptr->changed = 0;
3042 
3043 	/*
3044 	 * If the PHY has no target pointer, see if there's a dead PHY that
3045 	 * matches.
3046 	 */
3047 	if (pptr->target == NULL) {
3048 		pmcs_reap_dead_phy(pptr);
3049 	}
3050 
3051 	/*
3052 	 * Only assign the device if there is a target for this PHY with a
3053 	 * matching SAS address.  If an iport is disconnected from one piece
3054 	 * of storage and connected to another within the iport stabilization
3055 	 * time, we can get the PHY/target mismatch situation.
3056 	 *
3057 	 * Otherwise, it'll get done in tran_tgt_init.
3058 	 */
3059 	if (pptr->target) {
3060 		mutex_enter(&pptr->target->statlock);
3061 		if (pmcs_phy_target_match(pptr) == B_FALSE) {
3062 			mutex_exit(&pptr->target->statlock);
3063 			if (!IS_ROOT_PHY(pptr)) {
3064 				pmcs_dec_phy_ref_count(pptr);
3065 			}
3066 			pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, NULL,
3067 			    "%s: Not assigning existing tgt %p for PHY %p "
3068 			    "(WWN mismatch)", __func__, (void *)pptr->target,
3069 			    (void *)pptr);
3070 			pptr->target = NULL;
3071 			return;
3072 		}
3073 
3074 		if (!pmcs_assign_device(pwp, pptr->target)) {
3075 			pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, pptr, pptr->target,
3076 			    "%s: pmcs_assign_device failed for target 0x%p",
3077 			    __func__, (void *)pptr->target);
3078 		}
3079 		mutex_exit(&pptr->target->statlock);
3080 	}
3081 }
3082 
3083 /*
3084  * Called with PHY lock held.
3085  */
3086 static boolean_t
3087 pmcs_configure_phy(pmcs_hw_t *pwp, pmcs_phy_t *pptr)
3088 {
3089 	char *dtype;
3090 
3091 	ASSERT(mutex_owned(&pptr->phy_lock));
3092 
3093 	/*
3094 	 * Mark this device as no longer changed.
3095 	 */
3096 	pptr->changed = 0;
3097 
3098 	/*
3099 	 * If we don't have a device handle, get one.
3100 	 */
3101 	if (pmcs_get_device_handle(pwp, pptr)) {
3102 		return (B_FALSE);
3103 	}
3104 
3105 	pptr->configured = 1;
3106 
3107 	switch (pptr->dtype) {
3108 	case SAS:
3109 		dtype = "SAS";
3110 		break;
3111 	case SATA:
3112 		dtype = "SATA";
3113 		break;
3114 	case EXPANDER:
3115 		dtype = "SMP";
3116 		break;
3117 	default:
3118 		dtype = "???";
3119 	}
3120 
3121 	pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, pptr, NULL, "config_dev: %s "
3122 	    "dev %s " SAS_ADDR_FMT " dev id 0x%x lr 0x%x", dtype, pptr->path,
3123 	    SAS_ADDR_PRT(pptr->sas_address), pptr->device_id, pptr->link_rate);
3124 
3125 	return (B_TRUE);
3126 }
3127 
3128 /*
3129  * Called with PHY locked
3130  */
3131 static void
3132 pmcs_configure_expander(pmcs_hw_t *pwp, pmcs_phy_t *pptr, pmcs_iport_t *iport)
3133 {
3134 	pmcs_phy_t *ctmp, *clist = NULL, *cnext;
3135 	int result, i, nphy = 0;
3136 	boolean_t root_phy = B_FALSE;
3137 
3138 	ASSERT(iport);
3139 
3140 	/*
3141 	 * Step 1- clear our "changed" bit. If we need to retry/restart due
3142 	 * to resource shortages, we'll set it again. While we're doing
3143 	 * configuration, other events may set it again as well.  If the PHY
3144 	 * is a root PHY and is currently marked as having changed, reset the
3145 	 * config_stop timer as well.
3146 	 */
3147 	if (IS_ROOT_PHY(pptr) && pptr->changed) {
3148 		pptr->config_stop = ddi_get_lbolt() +
3149 		    drv_usectohz(PMCS_MAX_CONFIG_TIME);
3150 	}
3151 	pptr->changed = 0;
3152 
3153 	/*
3154 	 * Step 2- make sure we don't overflow
3155 	 */
3156 	if (pptr->level == PMCS_MAX_XPND-1) {
3157 		pmcs_prt(pwp, PMCS_PRT_WARN, pptr, NULL,
3158 		    "%s: SAS expansion tree too deep", __func__);
3159 		return;
3160 	}
3161 
3162 	/*
3163 	 * Step 3- Check if this expander is part of a wide phy that has
3164 	 * already been configured.
3165 	 *
3166 	 * This is known by checking this level for another EXPANDER device
3167 	 * with the same SAS address and isn't already marked as a subsidiary
3168 	 * phy and a parent whose SAS address is the same as our SAS address
3169 	 * (if there are parents).
3170 	 */
3171 	if (!IS_ROOT_PHY(pptr)) {
3172 		/*
3173 		 * No need to lock the parent here because we're in discovery
3174 		 * and the only time a PHY's children pointer can change is
3175 		 * in discovery; either in pmcs_clear_expander (which has
3176 		 * already been called) or here, down below.  Plus, trying to
3177 		 * grab the parent's lock here can cause deadlock.
3178 		 */
3179 		ctmp = pptr->parent->children;
3180 	} else {
3181 		ctmp = pwp->root_phys;
3182 		root_phy = B_TRUE;
3183 	}
3184 
3185 	while (ctmp) {
3186 		/*
3187 		 * If we've checked all PHYs up to pptr, we stop. Otherwise,
3188 		 * we'll be checking for a primary PHY with a higher PHY
3189 		 * number than pptr, which will never happen.  The primary
3190 		 * PHY on non-root expanders will ALWAYS be the lowest
3191 		 * numbered PHY.
3192 		 */
3193 		if (ctmp == pptr) {
3194 			break;
3195 		}
3196 
3197 		/*
3198 		 * If pptr and ctmp are root PHYs, just grab the mutex on
3199 		 * ctmp.  No need to lock the entire tree.  If they are not
3200 		 * root PHYs, there is no need to lock since a non-root PHY's
3201 		 * SAS address and other characteristics can only change in
3202 		 * discovery anyway.
3203 		 */
3204 		if (root_phy) {
3205 			mutex_enter(&ctmp->phy_lock);
3206 		}
3207 
3208 		if (ctmp->dtype == EXPANDER && ctmp->width &&
3209 		    memcmp(ctmp->sas_address, pptr->sas_address, 8) == 0) {
3210 			int widephy = 0;
3211 			/*
3212 			 * If these phys are not root PHYs, compare their SAS
3213 			 * addresses too.
3214 			 */
3215 			if (!root_phy) {
3216 				if (memcmp(ctmp->parent->sas_address,
3217 				    pptr->parent->sas_address, 8) == 0) {
3218 					widephy = 1;
3219 				}
3220 			} else {
3221 				widephy = 1;
3222 			}
3223 			if (widephy) {
3224 				ctmp->width++;
3225 				pptr->subsidiary = 1;
3226 				pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, pptr, NULL,
3227 				    "%s: PHY %s part of wide PHY %s "
3228 				    "(now %d wide)", __func__, pptr->path,
3229 				    ctmp->path, ctmp->width);
3230 				if (root_phy) {
3231 					mutex_exit(&ctmp->phy_lock);
3232 				}
3233 				return;
3234 			}
3235 		}
3236 
3237 		cnext = ctmp->sibling;
3238 		if (root_phy) {
3239 			mutex_exit(&ctmp->phy_lock);
3240 		}
3241 		ctmp = cnext;
3242 	}
3243 
3244 	/*
3245 	 * Step 4- If we don't have a device handle, get one.  Since this
3246 	 * is the primary PHY, make sure subsidiary is cleared.
3247 	 */
3248 	pptr->subsidiary = 0;
3249 	pptr->iport = iport;
3250 	if (pmcs_get_device_handle(pwp, pptr)) {
3251 		goto out;
3252 	}
3253 	pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, pptr, NULL, "Config expander %s "
3254 	    SAS_ADDR_FMT " dev id 0x%x lr 0x%x", pptr->path,
3255 	    SAS_ADDR_PRT(pptr->sas_address), pptr->device_id, pptr->link_rate);
3256 
3257 	/*
3258 	 * Step 5- figure out how many phys are in this expander.
3259 	 */
3260 	nphy = pmcs_expander_get_nphy(pwp, pptr);
3261 	if (nphy <= 0) {
3262 		if (nphy == 0 && ddi_get_lbolt() < pptr->config_stop) {
3263 			PHY_CHANGED(pwp, pptr);
3264 			RESTART_DISCOVERY(pwp);
3265 		} else {
3266 			pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, pptr, NULL,
3267 			    "%s: Retries exhausted for %s, killing", __func__,
3268 			    pptr->path);
3269 			pptr->config_stop = 0;
3270 			pmcs_kill_changed(pwp, pptr, 0);
3271 		}
3272 		goto out;
3273 	}
3274 
3275 	/*
3276 	 * Step 6- Allocate a list of phys for this expander and figure out
3277 	 * what each one is.
3278 	 */
3279 	for (i = 0; i < nphy; i++) {
3280 		ctmp = kmem_cache_alloc(pwp->phy_cache, KM_SLEEP);
3281 		bzero(ctmp, sizeof (pmcs_phy_t));
3282 		ctmp->device_id = PMCS_INVALID_DEVICE_ID;
3283 		ctmp->sibling = clist;
3284 		ctmp->pend_dtype = NEW;	/* Init pending dtype */
3285 		ctmp->config_stop = ddi_get_lbolt() +
3286 		    drv_usectohz(PMCS_MAX_CONFIG_TIME);
3287 		clist = ctmp;
3288 	}
3289 
3290 	mutex_enter(&pwp->config_lock);
3291 	if (pwp->config_changed) {
3292 		RESTART_DISCOVERY_LOCKED(pwp);
3293 		mutex_exit(&pwp->config_lock);
3294 		/*
3295 		 * Clean up the newly allocated PHYs and return
3296 		 */
3297 		while (clist) {
3298 			ctmp = clist->sibling;
3299 			kmem_cache_free(pwp->phy_cache, clist);
3300 			clist = ctmp;
3301 		}
3302 		return;
3303 	}
3304 	mutex_exit(&pwp->config_lock);
3305 
3306 	/*
3307 	 * Step 7- Now fill in the rest of the static portions of the phy.
3308 	 */
3309 	for (i = 0, ctmp = clist; ctmp; ctmp = ctmp->sibling, i++) {
3310 		ctmp->parent = pptr;
3311 		ctmp->pwp = pwp;
3312 		ctmp->level = pptr->level+1;
3313 		ctmp->portid = pptr->portid;
3314 		if (ctmp->tolerates_sas2) {
3315 			ASSERT(i < SAS2_PHYNUM_MAX);
3316 			ctmp->phynum = i & SAS2_PHYNUM_MASK;
3317 		} else {
3318 			ASSERT(i < SAS_PHYNUM_MAX);
3319 			ctmp->phynum = i & SAS_PHYNUM_MASK;
3320 		}
3321 		pmcs_phy_name(pwp, ctmp, ctmp->path, sizeof (ctmp->path));
3322 		pmcs_lock_phy(ctmp);
3323 	}
3324 
3325 	/*
3326 	 * Step 8- Discover things about each phy in the expander.
3327 	 */
3328 	for (i = 0, ctmp = clist; ctmp; ctmp = ctmp->sibling, i++) {
3329 		result = pmcs_expander_content_discover(pwp, pptr, ctmp);
3330 		if (result <= 0) {
3331 			if (ddi_get_lbolt() < pptr->config_stop) {
3332 				PHY_CHANGED(pwp, pptr);
3333 				RESTART_DISCOVERY(pwp);
3334 			} else {
3335 				pptr->config_stop = 0;
3336 				pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, pptr, NULL,
3337 				    "%s: Retries exhausted for %s, killing",
3338 				    __func__, pptr->path);
3339 				pmcs_kill_changed(pwp, pptr, 0);
3340 			}
3341 			goto out;
3342 		}
3343 
3344 		/* Set pend_dtype to dtype for 1st time initialization */
3345 		ctmp->pend_dtype = ctmp->dtype;
3346 	}
3347 
3348 	/*
3349 	 * Step 9- Install the new list on the next level. There should be
3350 	 * no children pointer on this PHY.  If there is, we'd need to know
3351 	 * how it happened (The expander suddenly got more PHYs?).
3352 	 */
3353 	ASSERT(pptr->children == NULL);
3354 	if (pptr->children != NULL) {
3355 		pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, NULL, "%s: Already child "
3356 		    "PHYs attached  to PHY %s: This should never happen",
3357 		    __func__, pptr->path);
3358 		goto out;
3359 	} else {
3360 		pptr->children = clist;
3361 	}
3362 
3363 	clist = NULL;
3364 	pptr->ncphy = nphy;
3365 	pptr->configured = 1;
3366 
3367 	/*
3368 	 * We only set width if we're greater than level 0.
3369 	 */
3370 	if (pptr->level) {
3371 		pptr->width = 1;
3372 	}
3373 
3374 	/*
3375 	 * Now tell the rest of the world about us, as an SMP node.
3376 	 */
3377 	pptr->iport = iport;
3378 	pmcs_new_tport(pwp, pptr);
3379 
3380 out:
3381 	while (clist) {
3382 		ctmp = clist->sibling;
3383 		pmcs_unlock_phy(clist);
3384 		kmem_cache_free(pwp->phy_cache, clist);
3385 		clist = ctmp;
3386 	}
3387 }
3388 
3389 /*
3390  * 2. Check expanders marked changed (but not dead) to see if they still have
3391  * the same number of phys and the same SAS address. Mark them, their subsidiary
3392  * phys (if wide) and their descendents dead if anything has changed. Check the
3393  * the devices they contain to see if *they* have changed. If they've changed
3394  * from type NOTHING we leave them marked changed to be configured later
3395  * (picking up a new SAS address and link rate if possible). Otherwise, any
3396  * change in type, SAS address or removal of target role will cause us to
3397  * mark them (and their descendents) as dead and cause any pending commands
3398  * and associated devices to be removed.
3399  *
3400  * Called with PHY (pptr) locked.
3401  */
3402 
3403 static void
3404 pmcs_check_expander(pmcs_hw_t *pwp, pmcs_phy_t *pptr)
3405 {
3406 	int nphy, result;
3407 	pmcs_phy_t *ctmp, *local, *local_list = NULL, *local_tail = NULL;
3408 	boolean_t kill_changed, changed;
3409 
3410 	pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, pptr, NULL,
3411 	    "%s: check %s", __func__, pptr->path);
3412 
3413 	/*
3414 	 * Step 1: Mark phy as not changed. We will mark it changed if we need
3415 	 * to retry.
3416 	 */
3417 	pptr->changed = 0;
3418 
3419 	/*
3420 	 * Reset the config_stop time. Although we're not actually configuring
3421 	 * anything here, we do want some indication of when to give up trying
3422 	 * if we can't communicate with the expander.
3423 	 */
3424 	pptr->config_stop = ddi_get_lbolt() +
3425 	    drv_usectohz(PMCS_MAX_CONFIG_TIME);
3426 
3427 	/*
3428 	 * Step 2: Figure out how many phys are in this expander. If
3429 	 * pmcs_expander_get_nphy returns 0 we ran out of resources,
3430 	 * so reschedule and try later. If it returns another error,
3431 	 * just return.
3432 	 */
3433 	nphy = pmcs_expander_get_nphy(pwp, pptr);
3434 	if (nphy <= 0) {
3435 		if ((nphy == 0) && (ddi_get_lbolt() < pptr->config_stop)) {
3436 			PHY_CHANGED(pwp, pptr);
3437 			RESTART_DISCOVERY(pwp);
3438 		} else {
3439 			pptr->config_stop = 0;
3440 			pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, pptr, NULL,
3441 			    "%s: Retries exhausted for %s, killing", __func__,
3442 			    pptr->path);
3443 			pmcs_kill_changed(pwp, pptr, 0);
3444 		}
3445 		return;
3446 	}
3447 
3448 	/*
3449 	 * Step 3: If the number of phys don't agree, kill the old sub-tree.
3450 	 */
3451 	if (nphy != pptr->ncphy) {
3452 		pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, pptr, NULL,
3453 		    "%s: number of contained phys for %s changed from %d to %d",
3454 		    __func__, pptr->path, pptr->ncphy, nphy);
3455 		/*
3456 		 * Force a rescan of this expander after dead contents
3457 		 * are cleared and removed.
3458 		 */
3459 		pmcs_kill_changed(pwp, pptr, 0);
3460 		return;
3461 	}
3462 
3463 	/*
3464 	 * Step 4: if we're at the bottom of the stack, we're done
3465 	 * (we can't have any levels below us)
3466 	 */
3467 	if (pptr->level == PMCS_MAX_XPND-1) {
3468 		return;
3469 	}
3470 
3471 	/*
3472 	 * Step 5: Discover things about each phy in this expander.  We do
3473 	 * this by walking the current list of contained phys and doing a
3474 	 * content discovery for it to a local phy.
3475 	 */
3476 	ctmp = pptr->children;
3477 	ASSERT(ctmp);
3478 	if (ctmp == NULL) {
3479 		pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, pptr, NULL,
3480 		    "%s: No children attached to expander @ %s?", __func__,
3481 		    pptr->path);
3482 		return;
3483 	}
3484 
3485 	while (ctmp) {
3486 		/*
3487 		 * Allocate a local PHY to contain the proposed new contents
3488 		 * and link it to the rest of the local PHYs so that they
3489 		 * can all be freed later.
3490 		 */
3491 		local = pmcs_clone_phy(ctmp);
3492 
3493 		if (local_list == NULL) {
3494 			local_list = local;
3495 			local_tail = local;
3496 		} else {
3497 			local_tail->sibling = local;
3498 			local_tail = local;
3499 		}
3500 
3501 		/*
3502 		 * Need to lock the local PHY since pmcs_expander_content_
3503 		 * discovery may call pmcs_clear_phy on it, which expects
3504 		 * the PHY to be locked.
3505 		 */
3506 		pmcs_lock_phy(local);
3507 		result = pmcs_expander_content_discover(pwp, pptr, local);
3508 		pmcs_unlock_phy(local);
3509 		if (result <= 0) {
3510 			if (ddi_get_lbolt() < pptr->config_stop) {
3511 				PHY_CHANGED(pwp, pptr);
3512 				RESTART_DISCOVERY(pwp);
3513 			} else {
3514 				pptr->config_stop = 0;
3515 				pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, pptr, NULL,
3516 				    "%s: Retries exhausted for %s, killing",
3517 				    __func__, pptr->path);
3518 				pmcs_kill_changed(pwp, pptr, 0);
3519 			}
3520 
3521 			/*
3522 			 * Release all the local PHYs that we allocated.
3523 			 */
3524 			pmcs_free_phys(pwp, local_list);
3525 			return;
3526 		}
3527 
3528 		ctmp = ctmp->sibling;
3529 	}
3530 
3531 	/*
3532 	 * Step 6: Compare the local PHY's contents to our current PHY.  If
3533 	 * there are changes, take the appropriate action.
3534 	 * This is done in two steps (step 5 above, and 6 here) so that if we
3535 	 * have to bail during this process (e.g. pmcs_expander_content_discover
3536 	 * fails), we haven't actually changed the state of any of the real
3537 	 * PHYs.  Next time we come through here, we'll be starting over from
3538 	 * scratch.  This keeps us from marking a changed PHY as no longer
3539 	 * changed, but then having to bail only to come back next time and
3540 	 * think that the PHY hadn't changed.  If this were to happen, we
3541 	 * would fail to properly configure the device behind this PHY.
3542 	 */
3543 	local = local_list;
3544 	ctmp = pptr->children;
3545 
3546 	while (ctmp) {
3547 		changed = B_FALSE;
3548 		kill_changed = B_FALSE;
3549 
3550 		/*
3551 		 * We set local to local_list prior to this loop so that we
3552 		 * can simply walk the local_list while we walk this list.  The
3553 		 * two lists should be completely in sync.
3554 		 *
3555 		 * Clear the changed flag here.
3556 		 */
3557 		ctmp->changed = 0;
3558 
3559 		if (ctmp->dtype != local->dtype) {
3560 			if (ctmp->dtype != NOTHING) {
3561 				pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, ctmp, NULL,
3562 				    "%s: %s type changed from %s to %s "
3563 				    "(killing)", __func__, ctmp->path,
3564 				    PHY_TYPE(ctmp), PHY_TYPE(local));
3565 				/*
3566 				 * Force a rescan of this expander after dead
3567 				 * contents are cleared and removed.
3568 				 */
3569 				changed = B_TRUE;
3570 				kill_changed = B_TRUE;
3571 			} else {
3572 				changed = B_TRUE;
3573 				pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, ctmp, NULL,
3574 				    "%s: %s type changed from NOTHING to %s",
3575 				    __func__, ctmp->path, PHY_TYPE(local));
3576 			}
3577 
3578 		} else if (ctmp->atdt != local->atdt) {
3579 			pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, ctmp, NULL, "%s: "
3580 			    "%s attached device type changed from %d to %d "
3581 			    "(killing)", __func__, ctmp->path, ctmp->atdt,
3582 			    local->atdt);
3583 			/*
3584 			 * Force a rescan of this expander after dead
3585 			 * contents are cleared and removed.
3586 			 */
3587 			changed = B_TRUE;
3588 
3589 			if (local->atdt == 0) {
3590 				kill_changed = B_TRUE;
3591 			}
3592 		} else if (ctmp->link_rate != local->link_rate) {
3593 			pmcs_prt(pwp, PMCS_PRT_INFO, ctmp, NULL, "%s: %s "
3594 			    "changed speed from %s to %s", __func__, ctmp->path,
3595 			    pmcs_get_rate(ctmp->link_rate),
3596 			    pmcs_get_rate(local->link_rate));
3597 			/* If the speed changed from invalid, force rescan */
3598 			if (!PMCS_VALID_LINK_RATE(ctmp->link_rate)) {
3599 				changed = B_TRUE;
3600 				RESTART_DISCOVERY(pwp);
3601 			} else {
3602 				/* Just update to the new link rate */
3603 				ctmp->link_rate = local->link_rate;
3604 			}
3605 
3606 			if (!PMCS_VALID_LINK_RATE(local->link_rate)) {
3607 				kill_changed = B_TRUE;
3608 			}
3609 		} else if (memcmp(ctmp->sas_address, local->sas_address,
3610 		    sizeof (ctmp->sas_address)) != 0) {
3611 			pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, ctmp, NULL,
3612 			    "%s: SAS Addr for %s changed from " SAS_ADDR_FMT
3613 			    "to " SAS_ADDR_FMT " (kill old tree)", __func__,
3614 			    ctmp->path, SAS_ADDR_PRT(ctmp->sas_address),
3615 			    SAS_ADDR_PRT(local->sas_address));
3616 			/*
3617 			 * Force a rescan of this expander after dead
3618 			 * contents are cleared and removed.
3619 			 */
3620 			changed = B_TRUE;
3621 		} else {
3622 			pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, ctmp, NULL,
3623 			    "%s: %s looks the same (type %s)",
3624 			    __func__, ctmp->path, PHY_TYPE(ctmp));
3625 			/*
3626 			 * If EXPANDER, still mark it changed so we
3627 			 * re-evaluate its contents.  If it's not an expander,
3628 			 * but it hasn't been configured, also mark it as
3629 			 * changed so that it will undergo configuration.
3630 			 */
3631 			if (ctmp->dtype == EXPANDER) {
3632 				changed = B_TRUE;
3633 			} else if ((ctmp->dtype != NOTHING) &&
3634 			    !ctmp->configured) {
3635 				ctmp->changed = 1;
3636 			} else {
3637 				/* It simply hasn't changed */
3638 				ctmp->changed = 0;
3639 			}
3640 		}
3641 
3642 		/*
3643 		 * If the PHY changed, call pmcs_kill_changed if indicated,
3644 		 * update its contents to reflect its current state and mark it
3645 		 * as changed.
3646 		 */
3647 		if (changed) {
3648 			/*
3649 			 * pmcs_kill_changed will mark the PHY as changed, so
3650 			 * only do PHY_CHANGED if we did not do kill_changed.
3651 			 */
3652 			if (kill_changed) {
3653 				pmcs_kill_changed(pwp, ctmp, 0);
3654 			} else {
3655 				/*
3656 				 * If we're not killing the device, it's not
3657 				 * dead.  Mark the PHY as changed.
3658 				 */
3659 				PHY_CHANGED(pwp, ctmp);
3660 
3661 				if (ctmp->dead) {
3662 					pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG,
3663 					    ctmp, NULL, "%s: Unmarking PHY %s "
3664 					    "dead, restarting discovery",
3665 					    __func__, ctmp->path);
3666 					ctmp->dead = 0;
3667 					RESTART_DISCOVERY(pwp);
3668 				}
3669 			}
3670 
3671 			/*
3672 			 * If the dtype of this PHY is now NOTHING, mark it as
3673 			 * unconfigured.  Set pend_dtype to what the new dtype
3674 			 * is.  It'll get updated at the end of the discovery
3675 			 * process.
3676 			 */
3677 			if (local->dtype == NOTHING) {
3678 				bzero(ctmp->sas_address,
3679 				    sizeof (local->sas_address));
3680 				ctmp->atdt = 0;
3681 				ctmp->link_rate = 0;
3682 				ctmp->pend_dtype = NOTHING;
3683 				ctmp->configured = 0;
3684 			} else {
3685 				(void) memcpy(ctmp->sas_address,
3686 				    local->sas_address,
3687 				    sizeof (local->sas_address));
3688 				ctmp->atdt = local->atdt;
3689 				ctmp->link_rate = local->link_rate;
3690 				ctmp->pend_dtype = local->dtype;
3691 			}
3692 		}
3693 
3694 		local = local->sibling;
3695 		ctmp = ctmp->sibling;
3696 	}
3697 
3698 	/*
3699 	 * If we got to here, that means we were able to see all the PHYs
3700 	 * and we can now update all of the real PHYs with the information
3701 	 * we got on the local PHYs.  Once that's done, free all the local
3702 	 * PHYs.
3703 	 */
3704 
3705 	pmcs_free_phys(pwp, local_list);
3706 }
3707 
3708 /*
3709  * Top level routine to check expanders.  We call pmcs_check_expander for
3710  * each expander.  Since we're not doing any configuration right now, it
3711  * doesn't matter if this is breadth-first.
3712  */
3713 static boolean_t
3714 pmcs_check_expanders(pmcs_hw_t *pwp, pmcs_phy_t *pptr)
3715 {
3716 	pmcs_phy_t *phyp, *pnext, *pchild;
3717 	boolean_t config_changed = B_FALSE;
3718 
3719 	pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, pptr, NULL,
3720 	    "%s: %s", __func__, pptr->path);
3721 
3722 	/*
3723 	 * Check each expander at this level
3724 	 */
3725 	phyp = pptr;
3726 	while (phyp && !config_changed) {
3727 		pmcs_lock_phy(phyp);
3728 
3729 		if ((phyp->dtype == EXPANDER) && phyp->changed &&
3730 		    !phyp->dead && !phyp->subsidiary &&
3731 		    phyp->configured) {
3732 			pmcs_check_expander(pwp, phyp);
3733 		}
3734 
3735 		pnext = phyp->sibling;
3736 		pmcs_unlock_phy(phyp);
3737 
3738 		mutex_enter(&pwp->config_lock);
3739 		config_changed = pwp->config_changed;
3740 		mutex_exit(&pwp->config_lock);
3741 
3742 		phyp = pnext;
3743 	}
3744 
3745 	if (config_changed) {
3746 		return (config_changed);
3747 	}
3748 
3749 	/*
3750 	 * Now check the children
3751 	 */
3752 	phyp = pptr;
3753 	while (phyp && !config_changed) {
3754 		pmcs_lock_phy(phyp);
3755 		pnext = phyp->sibling;
3756 		pchild = phyp->children;
3757 		pmcs_unlock_phy(phyp);
3758 
3759 		if (pchild) {
3760 			(void) pmcs_check_expanders(pwp, pchild);
3761 		}
3762 
3763 		mutex_enter(&pwp->config_lock);
3764 		config_changed = pwp->config_changed;
3765 		mutex_exit(&pwp->config_lock);
3766 
3767 		phyp = pnext;
3768 	}
3769 
3770 	/*
3771 	 * We're done
3772 	 */
3773 	return (config_changed);
3774 }
3775 
3776 /*
3777  * Called with softstate and PHY locked
3778  */
3779 static void
3780 pmcs_clear_expander(pmcs_hw_t *pwp, pmcs_phy_t *pptr, int level)
3781 {
3782 	pmcs_phy_t *ctmp;
3783 
3784 	ASSERT(mutex_owned(&pwp->lock));
3785 	ASSERT(mutex_owned(&pptr->phy_lock));
3786 	ASSERT(pptr->level < PMCS_MAX_XPND - 1);
3787 
3788 	pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, pptr, NULL,
3789 	    "%s: checking %s", __func__, pptr->path);
3790 
3791 	ctmp = pptr->children;
3792 	while (ctmp) {
3793 		/*
3794 		 * If the expander is dead, mark its children dead
3795 		 */
3796 		if (pptr->dead) {
3797 			ctmp->dead = 1;
3798 		}
3799 		if (ctmp->dtype == EXPANDER) {
3800 			pmcs_clear_expander(pwp, ctmp, level + 1);
3801 		}
3802 		ctmp = ctmp->sibling;
3803 	}
3804 
3805 	/*
3806 	 * If this expander is not dead, we're done here.
3807 	 */
3808 	if (!pptr->dead) {
3809 		return;
3810 	}
3811 
3812 	/*
3813 	 * Now snip out the list of children below us and release them
3814 	 */
3815 	ctmp = pptr->children;
3816 	while (ctmp) {
3817 		pmcs_phy_t *nxt = ctmp->sibling;
3818 		pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, ctmp, NULL,
3819 		    "%s: dead PHY 0x%p (%s) (ref_count %d)", __func__,
3820 		    (void *)ctmp, ctmp->path, ctmp->ref_count);
3821 		/*
3822 		 * Put this PHY on the dead PHY list for the watchdog to
3823 		 * clean up after any outstanding work has completed.
3824 		 */
3825 		mutex_enter(&pwp->dead_phylist_lock);
3826 		ctmp->dead_next = pwp->dead_phys;
3827 		pwp->dead_phys = ctmp;
3828 		mutex_exit(&pwp->dead_phylist_lock);
3829 		pmcs_unlock_phy(ctmp);
3830 		ctmp = nxt;
3831 	}
3832 
3833 	pptr->children = NULL;
3834 
3835 	/*
3836 	 * Clear subsidiary phys as well.  Getting the parent's PHY lock
3837 	 * is only necessary if level == 0 since otherwise the parent is
3838 	 * already locked.
3839 	 */
3840 	if (!IS_ROOT_PHY(pptr)) {
3841 		if (level == 0) {
3842 			mutex_enter(&pptr->parent->phy_lock);
3843 		}
3844 		ctmp = pptr->parent->children;
3845 		if (level == 0) {
3846 			mutex_exit(&pptr->parent->phy_lock);
3847 		}
3848 	} else {
3849 		ctmp = pwp->root_phys;
3850 	}
3851 
3852 	while (ctmp) {
3853 		if (ctmp == pptr) {
3854 			ctmp = ctmp->sibling;
3855 			continue;
3856 		}
3857 		/*
3858 		 * We only need to lock subsidiary PHYs on the level 0
3859 		 * expander.  Any children of that expander, subsidiaries or
3860 		 * not, will already be locked.
3861 		 */
3862 		if (level == 0) {
3863 			pmcs_lock_phy(ctmp);
3864 		}
3865 		if (ctmp->dtype != EXPANDER || ctmp->subsidiary == 0 ||
3866 		    memcmp(ctmp->sas_address, pptr->sas_address,
3867 		    sizeof (ctmp->sas_address)) != 0) {
3868 			if (level == 0) {
3869 				pmcs_unlock_phy(ctmp);
3870 			}
3871 			ctmp = ctmp->sibling;
3872 			continue;
3873 		}
3874 		pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, ctmp, NULL,
3875 		    "%s: subsidiary %s", __func__, ctmp->path);
3876 		pmcs_clear_phy(pwp, ctmp);
3877 		if (level == 0) {
3878 			pmcs_unlock_phy(ctmp);
3879 		}
3880 		ctmp = ctmp->sibling;
3881 	}
3882 
3883 	pmcs_clear_phy(pwp, pptr);
3884 }
3885 
3886 /*
3887  * Called with PHY locked and with scratch acquired. We return 0 if
3888  * we fail to allocate resources or notice that the configuration
3889  * count changed while we were running the command. We return
3890  * less than zero if we had an I/O error or received an unsupported
3891  * configuration. Otherwise we return the number of phys in the
3892  * expander.
3893  */
3894 #define	DFM(m, y) if (m == NULL) m = y
3895 static int
3896 pmcs_expander_get_nphy(pmcs_hw_t *pwp, pmcs_phy_t *pptr)
3897 {
3898 	struct pmcwork *pwrk;
3899 	char buf[64];
3900 	const uint_t rdoff = 0x100;	/* returned data offset */
3901 	smp_response_frame_t *srf;
3902 	smp_report_general_resp_t *srgr;
3903 	uint32_t msg[PMCS_MSG_SIZE], *ptr, htag, status, ival;
3904 	int result;
3905 
3906 	ival = 0x40001100;
3907 again:
3908 	pwrk = pmcs_gwork(pwp, PMCS_TAG_TYPE_WAIT, pptr);
3909 	if (pwrk == NULL) {
3910 		result = 0;
3911 		goto out;
3912 	}
3913 	(void) memset(pwp->scratch, 0x77, PMCS_SCRATCH_SIZE);
3914 	pwrk->arg = pwp->scratch;
3915 	pwrk->dtype = pptr->dtype;
3916 	mutex_enter(&pwp->iqp_lock[PMCS_IQ_OTHER]);
3917 	ptr = GET_IQ_ENTRY(pwp, PMCS_IQ_OTHER);
3918 	if (ptr == NULL) {
3919 		mutex_exit(&pwp->iqp_lock[PMCS_IQ_OTHER]);
3920 		pmcs_prt(pwp, PMCS_PRT_DEBUG2, pptr, NULL,
3921 		    "%s: GET_IQ_ENTRY failed", __func__);
3922 		pmcs_pwork(pwp, pwrk);
3923 		result = 0;
3924 		goto out;
3925 	}
3926 
3927 	msg[0] = LE_32(PMCS_HIPRI(pwp, PMCS_OQ_GENERAL, PMCIN_SMP_REQUEST));
3928 	msg[1] = LE_32(pwrk->htag);
3929 	msg[2] = LE_32(pptr->device_id);
3930 	msg[3] = LE_32((4 << SMP_REQUEST_LENGTH_SHIFT) | SMP_INDIRECT_RESPONSE);
3931 	/*
3932 	 * Send SMP REPORT GENERAL (of either SAS1.1 or SAS2 flavors).
3933 	 */
3934 	msg[4] = BE_32(ival);
3935 	msg[5] = 0;
3936 	msg[6] = 0;
3937 	msg[7] = 0;
3938 	msg[8] = 0;
3939 	msg[9] = 0;
3940 	msg[10] = 0;
3941 	msg[11] = 0;
3942 	msg[12] = LE_32(DWORD0(pwp->scratch_dma+rdoff));
3943 	msg[13] = LE_32(DWORD1(pwp->scratch_dma+rdoff));
3944 	msg[14] = LE_32(PMCS_SCRATCH_SIZE - rdoff);
3945 	msg[15] = 0;
3946 
3947 	COPY_MESSAGE(ptr, msg, PMCS_MSG_SIZE);
3948 
3949 	/* SMP serialization */
3950 	pmcs_smp_acquire(pptr->iport);
3951 
3952 	pwrk->state = PMCS_WORK_STATE_ONCHIP;
3953 	htag = pwrk->htag;
3954 	INC_IQ_ENTRY(pwp, PMCS_IQ_OTHER);
3955 
3956 	pmcs_unlock_phy(pptr);
3957 	WAIT_FOR(pwrk, 1000, result);
3958 	pmcs_lock_phy(pptr);
3959 
3960 	/* SMP serialization */
3961 	pmcs_smp_release(pptr->iport);
3962 
3963 	pmcs_pwork(pwp, pwrk);
3964 
3965 	mutex_enter(&pwp->config_lock);
3966 	if (pwp->config_changed) {
3967 		RESTART_DISCOVERY_LOCKED(pwp);
3968 		mutex_exit(&pwp->config_lock);
3969 		result = 0;
3970 		goto out;
3971 	}
3972 	mutex_exit(&pwp->config_lock);
3973 
3974 	if (result) {
3975 		pmcs_timed_out(pwp, htag, __func__);
3976 		pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, pptr, NULL,
3977 		    "%s: Issuing SMP ABORT for htag 0x%08x", __func__, htag);
3978 		if (pmcs_abort(pwp, pptr, htag, 0, 0)) {
3979 			pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, pptr, NULL,
3980 			    "%s: Unable to issue SMP ABORT for htag 0x%08x",
3981 			    __func__, htag);
3982 		} else {
3983 			pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, pptr, NULL,
3984 			    "%s: Issuing SMP ABORT for htag 0x%08x",
3985 			    __func__, htag);
3986 		}
3987 		result = 0;
3988 		goto out;
3989 	}
3990 	ptr = (void *)pwp->scratch;
3991 	status = LE_32(ptr[2]);
3992 	if (status == PMCOUT_STATUS_UNDERFLOW ||
3993 	    status == PMCOUT_STATUS_OVERFLOW) {
3994 		pmcs_prt(pwp, PMCS_PRT_DEBUG_UNDERFLOW, pptr, NULL,
3995 		    "%s: over/underflow", __func__);
3996 		status = PMCOUT_STATUS_OK;
3997 	}
3998 	srf = (smp_response_frame_t *)&((uint32_t *)pwp->scratch)[rdoff >> 2];
3999 	srgr = (smp_report_general_resp_t *)
4000 	    &((uint32_t *)pwp->scratch)[(rdoff >> 2)+1];
4001 
4002 	if (status != PMCOUT_STATUS_OK) {
4003 		char *nag = NULL;
4004 		(void) snprintf(buf, sizeof (buf),
4005 		    "%s: SMP op failed (0x%x)", __func__, status);
4006 		switch (status) {
4007 		case PMCOUT_STATUS_IO_PORT_IN_RESET:
4008 			DFM(nag, "I/O Port In Reset");
4009 			/* FALLTHROUGH */
4010 		case PMCOUT_STATUS_ERROR_HW_TIMEOUT:
4011 			DFM(nag, "Hardware Timeout");
4012 			/* FALLTHROUGH */
4013 		case PMCOUT_STATUS_ERROR_INTERNAL_SMP_RESOURCE:
4014 			DFM(nag, "Internal SMP Resource Failure");
4015 			/* FALLTHROUGH */
4016 		case PMCOUT_STATUS_XFER_ERR_PHY_NOT_READY:
4017 			DFM(nag, "PHY Not Ready");
4018 			/* FALLTHROUGH */
4019 		case PMCOUT_STATUS_OPEN_CNX_ERROR_CONNECTION_RATE_NOT_SUPPORTED:
4020 			DFM(nag, "Connection Rate Not Supported");
4021 			/* FALLTHROUGH */
4022 		case PMCOUT_STATUS_IO_XFER_OPEN_RETRY_TIMEOUT:
4023 			DFM(nag, "Open Retry Timeout");
4024 			/* FALLTHROUGH */
4025 		case PMCOUT_STATUS_SMP_RESP_CONNECTION_ERROR:
4026 			DFM(nag, "Response Connection Error");
4027 			pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, NULL,
4028 			    "%s: expander %s SMP operation failed (%s)",
4029 			    __func__, pptr->path, nag);
4030 			break;
4031 
4032 		/*
4033 		 * For the IO_DS_NON_OPERATIONAL case, we need to kick off
4034 		 * device state recovery and return 0 so that the caller
4035 		 * doesn't assume this expander is dead for good.
4036 		 */
4037 		case PMCOUT_STATUS_IO_DS_NON_OPERATIONAL: {
4038 			pmcs_xscsi_t *xp = pptr->target;
4039 
4040 			pmcs_prt(pwp, PMCS_PRT_DEBUG_DEV_STATE, pptr, xp,
4041 			    "%s: expander %s device state non-operational",
4042 			    __func__, pptr->path);
4043 
4044 			if (xp == NULL) {
4045 				/*
4046 				 * Kick off recovery right now.
4047 				 */
4048 				SCHEDULE_WORK(pwp, PMCS_WORK_DS_ERR_RECOVERY);
4049 				(void) ddi_taskq_dispatch(pwp->tq, pmcs_worker,
4050 				    pwp, DDI_NOSLEEP);
4051 			} else {
4052 				mutex_enter(&xp->statlock);
4053 				pmcs_start_dev_state_recovery(xp, pptr);
4054 				mutex_exit(&xp->statlock);
4055 			}
4056 
4057 			break;
4058 		}
4059 
4060 		default:
4061 			pmcs_print_entry(pwp, PMCS_PRT_DEBUG, buf, ptr);
4062 			result = -EIO;
4063 			break;
4064 		}
4065 	} else if (srf->srf_frame_type != SMP_FRAME_TYPE_RESPONSE) {
4066 		pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, NULL,
4067 		    "%s: bad response frame type 0x%x",
4068 		    __func__, srf->srf_frame_type);
4069 		result = -EINVAL;
4070 	} else if (srf->srf_function != SMP_FUNC_REPORT_GENERAL) {
4071 		pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, NULL,
4072 		    "%s: bad response function 0x%x",
4073 		    __func__, srf->srf_function);
4074 		result = -EINVAL;
4075 	} else if (srf->srf_result != 0) {
4076 		/*
4077 		 * Check to see if we have a value of 3 for failure and
4078 		 * whether we were using a SAS2.0 allocation length value
4079 		 * and retry without it.
4080 		 */
4081 		if (srf->srf_result == 3 && (ival & 0xff00)) {
4082 			ival &= ~0xff00;
4083 			pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, NULL,
4084 			    "%s: err 0x%x with SAS2 request- retry with SAS1",
4085 			    __func__, srf->srf_result);
4086 			goto again;
4087 		}
4088 		pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, NULL,
4089 		    "%s: bad response 0x%x", __func__, srf->srf_result);
4090 		result = -EINVAL;
4091 	} else if (srgr->srgr_configuring) {
4092 		pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, NULL,
4093 		    "%s: expander at phy %s is still configuring",
4094 		    __func__, pptr->path);
4095 		result = 0;
4096 	} else {
4097 		result = srgr->srgr_number_of_phys;
4098 		if (ival & 0xff00) {
4099 			pptr->tolerates_sas2 = 1;
4100 		}
4101 		pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, pptr, NULL,
4102 		    "%s has %d phys and %s SAS2", pptr->path, result,
4103 		    pptr->tolerates_sas2? "tolerates" : "does not tolerate");
4104 	}
4105 out:
4106 	return (result);
4107 }
4108 
4109 /*
4110  * Called with expander locked (and thus, pptr) as well as all PHYs up to
4111  * the root, and scratch acquired. Return 0 if we fail to allocate resources
4112  * or notice that the configuration changed while we were running the command.
4113  *
4114  * We return less than zero if we had an I/O error or received an
4115  * unsupported configuration.
4116  */
4117 static int
4118 pmcs_expander_content_discover(pmcs_hw_t *pwp, pmcs_phy_t *expander,
4119     pmcs_phy_t *pptr)
4120 {
4121 	struct pmcwork *pwrk;
4122 	char buf[64];
4123 	uint8_t sas_address[8];
4124 	uint8_t att_sas_address[8];
4125 	smp_response_frame_t *srf;
4126 	smp_discover_resp_t *sdr;
4127 	const uint_t rdoff = 0x100;	/* returned data offset */
4128 	uint8_t *roff;
4129 	uint32_t status, *ptr, msg[PMCS_MSG_SIZE], htag;
4130 	int result;
4131 	uint8_t	ini_support;
4132 	uint8_t	tgt_support;
4133 
4134 	pwrk = pmcs_gwork(pwp, PMCS_TAG_TYPE_WAIT, expander);
4135 	if (pwrk == NULL) {
4136 		result = 0;
4137 		goto out;
4138 	}
4139 	(void) memset(pwp->scratch, 0x77, PMCS_SCRATCH_SIZE);
4140 	pwrk->arg = pwp->scratch;
4141 	pwrk->dtype = expander->dtype;
4142 	msg[0] = LE_32(PMCS_HIPRI(pwp, PMCS_OQ_GENERAL, PMCIN_SMP_REQUEST));
4143 	msg[1] = LE_32(pwrk->htag);
4144 	msg[2] = LE_32(expander->device_id);
4145 	msg[3] = LE_32((12 << SMP_REQUEST_LENGTH_SHIFT) |
4146 	    SMP_INDIRECT_RESPONSE);
4147 	/*
4148 	 * Send SMP DISCOVER (of either SAS1.1 or SAS2 flavors).
4149 	 */
4150 	if (expander->tolerates_sas2) {
4151 		msg[4] = BE_32(0x40101B00);
4152 	} else {
4153 		msg[4] = BE_32(0x40100000);
4154 	}
4155 	msg[5] = 0;
4156 	msg[6] = BE_32((pptr->phynum << 16));
4157 	msg[7] = 0;
4158 	msg[8] = 0;
4159 	msg[9] = 0;
4160 	msg[10] = 0;
4161 	msg[11] = 0;
4162 	msg[12] = LE_32(DWORD0(pwp->scratch_dma+rdoff));
4163 	msg[13] = LE_32(DWORD1(pwp->scratch_dma+rdoff));
4164 	msg[14] = LE_32(PMCS_SCRATCH_SIZE - rdoff);
4165 	msg[15] = 0;
4166 	mutex_enter(&pwp->iqp_lock[PMCS_IQ_OTHER]);
4167 	ptr = GET_IQ_ENTRY(pwp, PMCS_IQ_OTHER);
4168 	if (ptr == NULL) {
4169 		mutex_exit(&pwp->iqp_lock[PMCS_IQ_OTHER]);
4170 		result = 0;
4171 		goto out;
4172 	}
4173 
4174 	COPY_MESSAGE(ptr, msg, PMCS_MSG_SIZE);
4175 
4176 	/* SMP serialization */
4177 	pmcs_smp_acquire(expander->iport);
4178 
4179 	pwrk->state = PMCS_WORK_STATE_ONCHIP;
4180 	htag = pwrk->htag;
4181 	INC_IQ_ENTRY(pwp, PMCS_IQ_OTHER);
4182 
4183 	/*
4184 	 * Drop PHY lock while waiting so other completions aren't potentially
4185 	 * blocked.
4186 	 */
4187 	pmcs_unlock_phy(expander);
4188 	WAIT_FOR(pwrk, 1000, result);
4189 	pmcs_lock_phy(expander);
4190 
4191 	/* SMP serialization */
4192 	pmcs_smp_release(expander->iport);
4193 
4194 	pmcs_pwork(pwp, pwrk);
4195 
4196 	mutex_enter(&pwp->config_lock);
4197 	if (pwp->config_changed) {
4198 		RESTART_DISCOVERY_LOCKED(pwp);
4199 		mutex_exit(&pwp->config_lock);
4200 		result = 0;
4201 		goto out;
4202 	}
4203 	mutex_exit(&pwp->config_lock);
4204 
4205 	if (result) {
4206 		pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, NULL, pmcs_timeo, __func__);
4207 		if (pmcs_abort(pwp, expander, htag, 0, 0)) {
4208 			pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, pptr, NULL,
4209 			    "%s: Unable to issue SMP ABORT for htag 0x%08x",
4210 			    __func__, htag);
4211 		} else {
4212 			pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, pptr, NULL,
4213 			    "%s: Issuing SMP ABORT for htag 0x%08x",
4214 			    __func__, htag);
4215 		}
4216 		result = -ETIMEDOUT;
4217 		goto out;
4218 	}
4219 	ptr = (void *)pwp->scratch;
4220 	/*
4221 	 * Point roff to the DMA offset for returned data
4222 	 */
4223 	roff = pwp->scratch;
4224 	roff += rdoff;
4225 	srf = (smp_response_frame_t *)roff;
4226 	sdr = (smp_discover_resp_t *)(roff+4);
4227 	status = LE_32(ptr[2]);
4228 	if (status == PMCOUT_STATUS_UNDERFLOW ||
4229 	    status == PMCOUT_STATUS_OVERFLOW) {
4230 		pmcs_prt(pwp, PMCS_PRT_DEBUG_UNDERFLOW, pptr, NULL,
4231 		    "%s: over/underflow", __func__);
4232 		status = PMCOUT_STATUS_OK;
4233 	}
4234 	if (status != PMCOUT_STATUS_OK) {
4235 		char *nag = NULL;
4236 		(void) snprintf(buf, sizeof (buf),
4237 		    "%s: SMP op failed (0x%x)", __func__, status);
4238 		switch (status) {
4239 		case PMCOUT_STATUS_ERROR_HW_TIMEOUT:
4240 			DFM(nag, "Hardware Timeout");
4241 			/* FALLTHROUGH */
4242 		case PMCOUT_STATUS_ERROR_INTERNAL_SMP_RESOURCE:
4243 			DFM(nag, "Internal SMP Resource Failure");
4244 			/* FALLTHROUGH */
4245 		case PMCOUT_STATUS_XFER_ERR_PHY_NOT_READY:
4246 			DFM(nag, "PHY Not Ready");
4247 			/* FALLTHROUGH */
4248 		case PMCOUT_STATUS_OPEN_CNX_ERROR_CONNECTION_RATE_NOT_SUPPORTED:
4249 			DFM(nag, "Connection Rate Not Supported");
4250 			/* FALLTHROUGH */
4251 		case PMCOUT_STATUS_IO_XFER_OPEN_RETRY_TIMEOUT:
4252 			DFM(nag, "Open Retry Timeout");
4253 			/* FALLTHROUGH */
4254 		case PMCOUT_STATUS_SMP_RESP_CONNECTION_ERROR:
4255 			DFM(nag, "Response Connection Error");
4256 			pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, NULL,
4257 			    "%s: expander %s SMP operation failed (%s)",
4258 			    __func__, pptr->path, nag);
4259 			break;
4260 		default:
4261 			pmcs_print_entry(pwp, PMCS_PRT_DEBUG, buf, ptr);
4262 			result = -EIO;
4263 			break;
4264 		}
4265 		goto out;
4266 	} else if (srf->srf_frame_type != SMP_FRAME_TYPE_RESPONSE) {
4267 		pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, NULL,
4268 		    "%s: bad response frame type 0x%x",
4269 		    __func__, srf->srf_frame_type);
4270 		result = -EINVAL;
4271 		goto out;
4272 	} else if (srf->srf_function != SMP_FUNC_DISCOVER) {
4273 		pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, NULL,
4274 		    "%s: bad response function 0x%x",
4275 		    __func__, srf->srf_function);
4276 		result = -EINVAL;
4277 		goto out;
4278 	} else if (srf->srf_result != SMP_RES_FUNCTION_ACCEPTED) {
4279 		result = pmcs_smp_function_result(pwp, srf);
4280 		/* Need not fail if PHY is Vacant */
4281 		if (result != SMP_RES_PHY_VACANT) {
4282 			result = -EINVAL;
4283 			goto out;
4284 		}
4285 	}
4286 
4287 	ini_support = (sdr->sdr_attached_sata_host |
4288 	    (sdr->sdr_attached_smp_initiator << 1) |
4289 	    (sdr->sdr_attached_stp_initiator << 2) |
4290 	    (sdr->sdr_attached_ssp_initiator << 3));
4291 
4292 	tgt_support = (sdr->sdr_attached_sata_device |
4293 	    (sdr->sdr_attached_smp_target << 1) |
4294 	    (sdr->sdr_attached_stp_target << 2) |
4295 	    (sdr->sdr_attached_ssp_target << 3));
4296 
4297 	pmcs_wwn2barray(BE_64(sdr->sdr_sas_addr), sas_address);
4298 	pmcs_wwn2barray(BE_64(sdr->sdr_attached_sas_addr), att_sas_address);
4299 
4300 	switch (sdr->sdr_attached_device_type) {
4301 	case SAS_IF_DTYPE_ENDPOINT:
4302 		pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, pptr, NULL,
4303 		    "exp_content: %s atdt=0x%x lr=%x is=%x ts=%x SAS="
4304 		    SAS_ADDR_FMT " attSAS=" SAS_ADDR_FMT " atPHY=%x",
4305 		    pptr->path,
4306 		    sdr->sdr_attached_device_type,
4307 		    sdr->sdr_negotiated_logical_link_rate,
4308 		    ini_support,
4309 		    tgt_support,
4310 		    SAS_ADDR_PRT(sas_address),
4311 		    SAS_ADDR_PRT(att_sas_address),
4312 		    sdr->sdr_attached_phy_identifier);
4313 
4314 		if (sdr->sdr_attached_sata_device ||
4315 		    sdr->sdr_attached_stp_target) {
4316 			pptr->dtype = SATA;
4317 		} else if (sdr->sdr_attached_ssp_target) {
4318 			pptr->dtype = SAS;
4319 		} else if (tgt_support || ini_support) {
4320 			pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, pptr, NULL,
4321 			    "%s: %s has tgt support=%x init support=(%x)",
4322 			    __func__, pptr->path, tgt_support, ini_support);
4323 		}
4324 		break;
4325 	case SAS_IF_DTYPE_EDGE:
4326 	case SAS_IF_DTYPE_FANOUT:
4327 		pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, pptr, NULL,
4328 		    "exp_content: %s atdt=0x%x lr=%x is=%x ts=%x SAS="
4329 		    SAS_ADDR_FMT " attSAS=" SAS_ADDR_FMT " atPHY=%x",
4330 		    pptr->path,
4331 		    sdr->sdr_attached_device_type,
4332 		    sdr->sdr_negotiated_logical_link_rate,
4333 		    ini_support,
4334 		    tgt_support,
4335 		    SAS_ADDR_PRT(sas_address),
4336 		    SAS_ADDR_PRT(att_sas_address),
4337 		    sdr->sdr_attached_phy_identifier);
4338 		if (sdr->sdr_attached_smp_target) {
4339 			/*
4340 			 * Avoid configuring phys that just point back
4341 			 * at a parent phy
4342 			 */
4343 			if (expander->parent &&
4344 			    memcmp(expander->parent->sas_address,
4345 			    att_sas_address,
4346 			    sizeof (expander->parent->sas_address)) == 0) {
4347 				pmcs_prt(pwp, PMCS_PRT_DEBUG3, pptr, NULL,
4348 				    "%s: skipping port back to parent "
4349 				    "expander (%s)", __func__, pptr->path);
4350 				pptr->dtype = NOTHING;
4351 				break;
4352 			}
4353 			pptr->dtype = EXPANDER;
4354 
4355 		} else if (tgt_support || ini_support) {
4356 			pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, pptr, NULL,
4357 			    "%s has tgt support=%x init support=(%x)",
4358 			    pptr->path, tgt_support, ini_support);
4359 			pptr->dtype = EXPANDER;
4360 		}
4361 		break;
4362 	default:
4363 		pptr->dtype = NOTHING;
4364 		break;
4365 	}
4366 	if (pptr->dtype != NOTHING) {
4367 		pmcs_phy_t *ctmp;
4368 
4369 		/*
4370 		 * If the attached device is a SATA device and the expander
4371 		 * is (possibly) a SAS2 compliant expander, check for whether
4372 		 * there is a NAA=5 WWN field starting at this offset and
4373 		 * use that for the SAS Address for this device.
4374 		 */
4375 		if (expander->tolerates_sas2 && pptr->dtype == SATA &&
4376 		    (roff[SAS_ATTACHED_NAME_OFFSET] >> 8) == 0x5) {
4377 			(void) memcpy(pptr->sas_address,
4378 			    &roff[SAS_ATTACHED_NAME_OFFSET], 8);
4379 		} else {
4380 			(void) memcpy(pptr->sas_address, att_sas_address, 8);
4381 		}
4382 		pptr->atdt = (sdr->sdr_attached_device_type);
4383 		/*
4384 		 * Now run up from the expander's parent up to the top to
4385 		 * make sure we only use the least common link_rate.
4386 		 */
4387 		for (ctmp = expander->parent; ctmp; ctmp = ctmp->parent) {
4388 			if (ctmp->link_rate <
4389 			    sdr->sdr_negotiated_logical_link_rate) {
4390 				pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, pptr, NULL,
4391 				    "%s: derating link rate from %x to %x due "
4392 				    "to %s being slower", pptr->path,
4393 				    sdr->sdr_negotiated_logical_link_rate,
4394 				    ctmp->link_rate,
4395 				    ctmp->path);
4396 				sdr->sdr_negotiated_logical_link_rate =
4397 				    ctmp->link_rate;
4398 			}
4399 		}
4400 		pptr->link_rate = sdr->sdr_negotiated_logical_link_rate;
4401 		pptr->state.prog_min_rate = sdr->sdr_prog_min_phys_link_rate;
4402 		pptr->state.hw_min_rate = sdr->sdr_hw_min_phys_link_rate;
4403 		pptr->state.prog_max_rate = sdr->sdr_prog_max_phys_link_rate;
4404 		pptr->state.hw_max_rate = sdr->sdr_hw_max_phys_link_rate;
4405 		PHY_CHANGED(pwp, pptr);
4406 	} else {
4407 		pmcs_clear_phy(pwp, pptr);
4408 	}
4409 	result = 1;
4410 out:
4411 	return (result);
4412 }
4413 
4414 /*
4415  * Get a work structure and assign it a tag with type and serial number
4416  * If a structure is returned, it is returned locked.
4417  */
4418 pmcwork_t *
4419 pmcs_gwork(pmcs_hw_t *pwp, uint32_t tag_type, pmcs_phy_t *phyp)
4420 {
4421 	pmcwork_t *p;
4422 	uint16_t snum;
4423 	uint32_t off;
4424 
4425 	mutex_enter(&pwp->wfree_lock);
4426 	p = STAILQ_FIRST(&pwp->wf);
4427 	if (p == NULL) {
4428 		/*
4429 		 * If we couldn't get a work structure, it's time to bite
4430 		 * the bullet, grab the pfree_lock and copy over all the
4431 		 * work structures from the pending free list to the actual
4432 		 * free list.  This shouldn't happen all that often.
4433 		 */
4434 		mutex_enter(&pwp->pfree_lock);
4435 		pwp->wf.stqh_first = pwp->pf.stqh_first;
4436 		pwp->wf.stqh_last = pwp->pf.stqh_last;
4437 		STAILQ_INIT(&pwp->pf);
4438 		mutex_exit(&pwp->pfree_lock);
4439 
4440 		p = STAILQ_FIRST(&pwp->wf);
4441 		if (p == NULL) {
4442 			mutex_exit(&pwp->wfree_lock);
4443 			return (NULL);
4444 		}
4445 	}
4446 	STAILQ_REMOVE(&pwp->wf, p, pmcwork, next);
4447 	snum = pwp->wserno++;
4448 	mutex_exit(&pwp->wfree_lock);
4449 
4450 	off = p - pwp->work;
4451 
4452 	mutex_enter(&p->lock);
4453 	ASSERT(p->state == PMCS_WORK_STATE_NIL);
4454 	ASSERT(p->htag == PMCS_TAG_FREE);
4455 	p->htag = (tag_type << PMCS_TAG_TYPE_SHIFT) & PMCS_TAG_TYPE_MASK;
4456 	p->htag |= ((snum << PMCS_TAG_SERNO_SHIFT) & PMCS_TAG_SERNO_MASK);
4457 	p->htag |= ((off << PMCS_TAG_INDEX_SHIFT) & PMCS_TAG_INDEX_MASK);
4458 	p->start = gethrtime();
4459 	p->state = PMCS_WORK_STATE_READY;
4460 	p->ssp_event = 0;
4461 	p->dead = 0;
4462 
4463 	if (phyp) {
4464 		p->phy = phyp;
4465 		pmcs_inc_phy_ref_count(phyp);
4466 	}
4467 
4468 	return (p);
4469 }
4470 
4471 /*
4472  * Called with pwrk lock held.  Returned with lock released.
4473  */
4474 void
4475 pmcs_pwork(pmcs_hw_t *pwp, pmcwork_t *p)
4476 {
4477 	ASSERT(p != NULL);
4478 	ASSERT(mutex_owned(&p->lock));
4479 
4480 	p->last_ptr = p->ptr;
4481 	p->last_arg = p->arg;
4482 	p->last_phy = p->phy;
4483 	p->last_xp = p->xp;
4484 	p->last_htag = p->htag;
4485 	p->last_state = p->state;
4486 	p->finish = gethrtime();
4487 
4488 	if (p->phy) {
4489 		pmcs_dec_phy_ref_count(p->phy);
4490 	}
4491 
4492 	p->state = PMCS_WORK_STATE_NIL;
4493 	p->htag = PMCS_TAG_FREE;
4494 	p->xp = NULL;
4495 	p->ptr = NULL;
4496 	p->arg = NULL;
4497 	p->phy = NULL;
4498 	p->abt_htag = 0;
4499 	p->timer = 0;
4500 	mutex_exit(&p->lock);
4501 
4502 	if (mutex_tryenter(&pwp->wfree_lock) == 0) {
4503 		mutex_enter(&pwp->pfree_lock);
4504 		STAILQ_INSERT_TAIL(&pwp->pf, p, next);
4505 		mutex_exit(&pwp->pfree_lock);
4506 	} else {
4507 		STAILQ_INSERT_TAIL(&pwp->wf, p, next);
4508 		mutex_exit(&pwp->wfree_lock);
4509 	}
4510 }
4511 
4512 /*
4513  * Find a work structure based upon a tag and make sure that the tag
4514  * serial number matches the work structure we've found.
4515  * If a structure is found, its lock is held upon return.
4516  */
4517 pmcwork_t *
4518 pmcs_tag2wp(pmcs_hw_t *pwp, uint32_t htag)
4519 {
4520 	pmcwork_t *p;
4521 	uint32_t idx = PMCS_TAG_INDEX(htag);
4522 
4523 	p = &pwp->work[idx];
4524 
4525 	mutex_enter(&p->lock);
4526 	if (p->htag == htag) {
4527 		return (p);
4528 	}
4529 	mutex_exit(&p->lock);
4530 	pmcs_prt(pwp, PMCS_PRT_DEBUG2, NULL, NULL,
4531 	    "INDEX 0x%x HTAG 0x%x got p->htag 0x%x", idx, htag, p->htag);
4532 	return (NULL);
4533 }
4534 
4535 /*
4536  * Issue an abort for a command or for all commands.
4537  *
4538  * Since this can be called from interrupt context,
4539  * we don't wait for completion if wait is not set.
4540  *
4541  * Called with PHY lock held.
4542  */
4543 int
4544 pmcs_abort(pmcs_hw_t *pwp, pmcs_phy_t *pptr, uint32_t tag, int all_cmds,
4545     int wait)
4546 {
4547 	pmcwork_t *pwrk;
4548 	pmcs_xscsi_t *tgt;
4549 	uint32_t msg[PMCS_MSG_SIZE], *ptr;
4550 	int result, abt_type;
4551 	uint32_t abt_htag, status;
4552 
4553 	if (pptr->abort_all_start) {
4554 		pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, NULL, "%s: ABORT_ALL for "
4555 		    "(%s) already in progress.", __func__, pptr->path);
4556 		return (EBUSY);
4557 	}
4558 
4559 	switch (pptr->dtype) {
4560 	case SAS:
4561 		abt_type = PMCIN_SSP_ABORT;
4562 		break;
4563 	case SATA:
4564 		abt_type = PMCIN_SATA_ABORT;
4565 		break;
4566 	case EXPANDER:
4567 		abt_type = PMCIN_SMP_ABORT;
4568 		break;
4569 	default:
4570 		return (0);
4571 	}
4572 
4573 	pwrk = pmcs_gwork(pwp, wait ? PMCS_TAG_TYPE_WAIT : PMCS_TAG_TYPE_NONE,
4574 	    pptr);
4575 
4576 	if (pwrk == NULL) {
4577 		pmcs_prt(pwp, PMCS_PRT_ERR, pptr, NULL, pmcs_nowrk, __func__);
4578 		return (ENOMEM);
4579 	}
4580 
4581 	pwrk->dtype = pptr->dtype;
4582 	if (wait) {
4583 		pwrk->arg = msg;
4584 	}
4585 	if (pptr->valid_device_id == 0) {
4586 		pmcs_pwork(pwp, pwrk);
4587 		pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, NULL,
4588 		    "%s: Invalid DeviceID", __func__);
4589 		return (ENODEV);
4590 	}
4591 	msg[0] = LE_32(PMCS_HIPRI(pwp, PMCS_OQ_GENERAL, abt_type));
4592 	msg[1] = LE_32(pwrk->htag);
4593 	msg[2] = LE_32(pptr->device_id);
4594 	if (all_cmds) {
4595 		msg[3] = 0;
4596 		msg[4] = LE_32(1);
4597 		pwrk->ptr = NULL;
4598 		pptr->abort_all_start = gethrtime();
4599 	} else {
4600 		msg[3] = LE_32(tag);
4601 		msg[4] = 0;
4602 		pwrk->abt_htag = tag;
4603 	}
4604 	mutex_enter(&pwp->iqp_lock[PMCS_IQ_OTHER]);
4605 	ptr = GET_IQ_ENTRY(pwp, PMCS_IQ_OTHER);
4606 	if (ptr == NULL) {
4607 		mutex_exit(&pwp->iqp_lock[PMCS_IQ_OTHER]);
4608 		pmcs_pwork(pwp, pwrk);
4609 		pmcs_prt(pwp, PMCS_PRT_ERR, pptr, NULL, pmcs_nomsg, __func__);
4610 		return (ENOMEM);
4611 	}
4612 
4613 	COPY_MESSAGE(ptr, msg, 5);
4614 	if (all_cmds) {
4615 		pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, NULL,
4616 		    "%s: aborting all commands for %s device %s. (htag=0x%x)",
4617 		    __func__, pmcs_get_typename(pptr->dtype), pptr->path,
4618 		    msg[1]);
4619 	} else {
4620 		pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, NULL,
4621 		    "%s: aborting tag 0x%x for %s device %s. (htag=0x%x)",
4622 		    __func__, tag, pmcs_get_typename(pptr->dtype), pptr->path,
4623 		    msg[1]);
4624 	}
4625 	pwrk->state = PMCS_WORK_STATE_ONCHIP;
4626 
4627 	INC_IQ_ENTRY(pwp, PMCS_IQ_OTHER);
4628 	if (!wait) {
4629 		mutex_exit(&pwrk->lock);
4630 		return (0);
4631 	}
4632 
4633 	abt_htag = pwrk->htag;
4634 	pmcs_unlock_phy(pwrk->phy);
4635 	WAIT_FOR(pwrk, 1000, result);
4636 	pmcs_lock_phy(pwrk->phy);
4637 
4638 	tgt = pwrk->xp;
4639 	pmcs_pwork(pwp, pwrk);
4640 
4641 	if (tgt != NULL) {
4642 		mutex_enter(&tgt->aqlock);
4643 		if (!STAILQ_EMPTY(&tgt->aq)) {
4644 			pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, tgt,
4645 			    "%s: Abort complete (result=0x%x), but "
4646 			    "aq not empty (tgt 0x%p), waiting",
4647 			    __func__, result, (void *)tgt);
4648 			cv_wait(&tgt->abort_cv, &tgt->aqlock);
4649 		}
4650 		mutex_exit(&tgt->aqlock);
4651 	}
4652 
4653 	if (all_cmds) {
4654 		pptr->abort_all_start = 0;
4655 		cv_signal(&pptr->abort_all_cv);
4656 	}
4657 
4658 	if (result) {
4659 		pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, tgt,
4660 		    "%s: Abort (htag 0x%08x) request timed out",
4661 		    __func__, abt_htag);
4662 		if (tgt != NULL) {
4663 			mutex_enter(&tgt->statlock);
4664 			if ((tgt->dev_state != PMCS_DEVICE_STATE_IN_RECOVERY) &&
4665 			    (tgt->dev_state !=
4666 			    PMCS_DEVICE_STATE_NON_OPERATIONAL)) {
4667 				pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, tgt,
4668 				    "%s: Trying DS error recovery for tgt 0x%p",
4669 				    __func__, (void *)tgt);
4670 				(void) pmcs_send_err_recovery_cmd(pwp,
4671 				    PMCS_DEVICE_STATE_IN_RECOVERY, pptr, tgt);
4672 			}
4673 			mutex_exit(&tgt->statlock);
4674 		}
4675 		return (ETIMEDOUT);
4676 	}
4677 
4678 	status = LE_32(msg[2]);
4679 	if (status != PMCOUT_STATUS_OK) {
4680 		/*
4681 		 * The only non-success status are IO_NOT_VALID &
4682 		 * IO_ABORT_IN_PROGRESS.
4683 		 * In case of IO_ABORT_IN_PROGRESS, the other ABORT cmd's
4684 		 * status is of concern and this duplicate cmd status can
4685 		 * be ignored.
4686 		 * If IO_NOT_VALID, that's not an error per-se.
4687 		 * For abort of single I/O complete the command anyway.
4688 		 * If, however, we were aborting all, that is a problem
4689 		 * as IO_NOT_VALID really means that the IO or device is
4690 		 * not there. So, discovery process will take of the cleanup.
4691 		 */
4692 		pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, tgt,
4693 		    "%s: abort result 0x%x", __func__, LE_32(msg[2]));
4694 		if (all_cmds) {
4695 			PHY_CHANGED(pwp, pptr);
4696 			RESTART_DISCOVERY(pwp);
4697 		} else {
4698 			return (EINVAL);
4699 		}
4700 
4701 		return (0);
4702 	}
4703 
4704 	if (tgt != NULL) {
4705 		mutex_enter(&tgt->statlock);
4706 		if (tgt->dev_state == PMCS_DEVICE_STATE_IN_RECOVERY) {
4707 			pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, tgt,
4708 			    "%s: Restoring OPERATIONAL dev_state for tgt 0x%p",
4709 			    __func__, (void *)tgt);
4710 			(void) pmcs_send_err_recovery_cmd(pwp,
4711 			    PMCS_DEVICE_STATE_OPERATIONAL, pptr, tgt);
4712 		}
4713 		mutex_exit(&tgt->statlock);
4714 	}
4715 
4716 	return (0);
4717 }
4718 
4719 /*
4720  * Issue a task management function to an SSP device.
4721  *
4722  * Called with PHY lock held.
4723  * statlock CANNOT be held upon entry.
4724  */
4725 int
4726 pmcs_ssp_tmf(pmcs_hw_t *pwp, pmcs_phy_t *pptr, uint8_t tmf, uint32_t tag,
4727     uint64_t lun, uint32_t *response)
4728 {
4729 	int result, ds;
4730 	uint8_t local[PMCS_QENTRY_SIZE << 1], *xd;
4731 	sas_ssp_rsp_iu_t *rptr = (void *)local;
4732 	static const uint8_t ssp_rsp_evec[] = {
4733 		0x58, 0x61, 0x56, 0x72, 0x00
4734 	};
4735 	uint32_t msg[PMCS_MSG_SIZE], *ptr, status;
4736 	struct pmcwork *pwrk;
4737 	pmcs_xscsi_t *xp;
4738 
4739 	pwrk = pmcs_gwork(pwp, PMCS_TAG_TYPE_WAIT, pptr);
4740 	if (pwrk == NULL) {
4741 		pmcs_prt(pwp, PMCS_PRT_ERR, pptr, NULL, pmcs_nowrk, __func__);
4742 		return (ENOMEM);
4743 	}
4744 	/*
4745 	 * NB: We use the PMCS_OQ_GENERAL outbound queue
4746 	 * NB: so as to not get entangled in normal I/O
4747 	 * NB: processing.
4748 	 */
4749 	msg[0] = LE_32(PMCS_HIPRI(pwp, PMCS_OQ_GENERAL,
4750 	    PMCIN_SSP_INI_TM_START));
4751 	msg[1] = LE_32(pwrk->htag);
4752 	msg[2] = LE_32(pptr->device_id);
4753 	if (tmf == SAS_ABORT_TASK || tmf == SAS_QUERY_TASK) {
4754 		msg[3] = LE_32(tag);
4755 	} else {
4756 		msg[3] = 0;
4757 	}
4758 	msg[4] = LE_32(tmf);
4759 	msg[5] = BE_32((uint32_t)lun);
4760 	msg[6] = BE_32((uint32_t)(lun >> 32));
4761 	msg[7] = LE_32(PMCIN_MESSAGE_REPORT);
4762 
4763 	mutex_enter(&pwp->iqp_lock[PMCS_IQ_OTHER]);
4764 	ptr = GET_IQ_ENTRY(pwp, PMCS_IQ_OTHER);
4765 	if (ptr == NULL) {
4766 		mutex_exit(&pwp->iqp_lock[PMCS_IQ_OTHER]);
4767 		pmcs_pwork(pwp, pwrk);
4768 		pmcs_prt(pwp, PMCS_PRT_ERR, pptr, NULL, pmcs_nomsg, __func__);
4769 		return (ENOMEM);
4770 	}
4771 	COPY_MESSAGE(ptr, msg, 7);
4772 	pwrk->arg = msg;
4773 	pwrk->dtype = pptr->dtype;
4774 
4775 	xp = pptr->target;
4776 	if (xp != NULL) {
4777 		mutex_enter(&xp->statlock);
4778 		if (xp->dev_state == PMCS_DEVICE_STATE_NON_OPERATIONAL) {
4779 			mutex_exit(&xp->statlock);
4780 			mutex_exit(&pwp->iqp_lock[PMCS_IQ_OTHER]);
4781 			pmcs_pwork(pwp, pwrk);
4782 			pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, xp, "%s: Not "
4783 			    "sending '%s' because DS is '%s'", __func__,
4784 			    pmcs_tmf2str(tmf), pmcs_status_str
4785 			    (PMCOUT_STATUS_IO_DS_NON_OPERATIONAL));
4786 			return (EIO);
4787 		}
4788 		mutex_exit(&xp->statlock);
4789 	}
4790 
4791 	pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, xp,
4792 	    "%s: sending '%s' to %s (lun %llu) tag 0x%x", __func__,
4793 	    pmcs_tmf2str(tmf), pptr->path, (unsigned long long) lun, tag);
4794 	pwrk->state = PMCS_WORK_STATE_ONCHIP;
4795 	INC_IQ_ENTRY(pwp, PMCS_IQ_OTHER);
4796 
4797 	pmcs_unlock_phy(pptr);
4798 	/*
4799 	 * This is a command sent to the target device, so it can take
4800 	 * significant amount of time to complete when path & device is busy.
4801 	 * Set a timeout to 20 seconds
4802 	 */
4803 	WAIT_FOR(pwrk, 20000, result);
4804 	pmcs_lock_phy(pptr);
4805 	pmcs_pwork(pwp, pwrk);
4806 
4807 	if (result) {
4808 		if (xp == NULL) {
4809 			return (ETIMEDOUT);
4810 		}
4811 
4812 		mutex_enter(&xp->statlock);
4813 		pmcs_start_dev_state_recovery(xp, pptr);
4814 		mutex_exit(&xp->statlock);
4815 		return (ETIMEDOUT);
4816 	}
4817 
4818 	status = LE_32(msg[2]);
4819 	if (status != PMCOUT_STATUS_OK) {
4820 		pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, xp,
4821 		    "%s: status %s for TMF %s action to %s, lun %llu",
4822 		    __func__, pmcs_status_str(status),  pmcs_tmf2str(tmf),
4823 		    pptr->path, (unsigned long long) lun);
4824 		if ((status == PMCOUT_STATUS_IO_DS_NON_OPERATIONAL) ||
4825 		    (status == PMCOUT_STATUS_OPEN_CNX_ERROR_BREAK) ||
4826 		    (status == PMCOUT_STATUS_OPEN_CNX_ERROR_IT_NEXUS_LOSS)) {
4827 			ds = PMCS_DEVICE_STATE_NON_OPERATIONAL;
4828 		} else if (status == PMCOUT_STATUS_IO_DS_IN_RECOVERY) {
4829 			/*
4830 			 * If the status is IN_RECOVERY, it's an indication
4831 			 * that it's now time for us to request to have the
4832 			 * device state set to OPERATIONAL since we're the ones
4833 			 * that requested recovery to begin with.
4834 			 */
4835 			ds = PMCS_DEVICE_STATE_OPERATIONAL;
4836 		} else {
4837 			ds = PMCS_DEVICE_STATE_IN_RECOVERY;
4838 		}
4839 		if (xp != NULL) {
4840 			mutex_enter(&xp->statlock);
4841 			if (xp->dev_state != ds) {
4842 				pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, xp,
4843 				    "%s: Sending err recovery cmd"
4844 				    " for tgt 0x%p (status = %s)",
4845 				    __func__, (void *)xp,
4846 				    pmcs_status_str(status));
4847 				(void) pmcs_send_err_recovery_cmd(pwp, ds,
4848 				    pptr, xp);
4849 			}
4850 			mutex_exit(&xp->statlock);
4851 		}
4852 		return (EIO);
4853 	} else {
4854 		ds = PMCS_DEVICE_STATE_OPERATIONAL;
4855 		if (xp != NULL) {
4856 			mutex_enter(&xp->statlock);
4857 			if (xp->dev_state != ds) {
4858 				pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, xp,
4859 				    "%s: Sending err recovery cmd"
4860 				    " for tgt 0x%p (status = %s)",
4861 				    __func__, (void *)xp,
4862 				    pmcs_status_str(status));
4863 				(void) pmcs_send_err_recovery_cmd(pwp, ds,
4864 				    pptr, xp);
4865 			}
4866 			mutex_exit(&xp->statlock);
4867 		}
4868 	}
4869 	if (LE_32(msg[3]) == 0) {
4870 		pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, xp,
4871 		    "TMF completed with no response");
4872 		return (EIO);
4873 	}
4874 	pmcs_endian_transform(pwp, local, &msg[5], ssp_rsp_evec);
4875 	xd = (uint8_t *)(&msg[5]);
4876 	xd += SAS_RSP_HDR_SIZE;
4877 	if (rptr->datapres != SAS_RSP_DATAPRES_RESPONSE_DATA) {
4878 		pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, xp,
4879 		    "%s: TMF response not RESPONSE DATA (0x%x)",
4880 		    __func__, rptr->datapres);
4881 		return (EIO);
4882 	}
4883 	if (rptr->response_data_length != 4) {
4884 		pmcs_print_entry(pwp, PMCS_PRT_DEBUG,
4885 		    "Bad SAS RESPONSE DATA LENGTH", msg);
4886 		return (EIO);
4887 	}
4888 	(void) memcpy(&status, xd, sizeof (uint32_t));
4889 	status = BE_32(status);
4890 	if (response != NULL)
4891 		*response = status;
4892 	/*
4893 	 * The status is actually in the low-order byte.  The upper three
4894 	 * bytes contain additional information for the TMFs that support them.
4895 	 * However, at this time we do not issue any of those.  In the other
4896 	 * cases, the upper three bytes are supposed to be 0, but it appears
4897 	 * they aren't always.  Just mask them off.
4898 	 */
4899 	switch (status & 0xff) {
4900 	case SAS_RSP_TMF_COMPLETE:
4901 		pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, xp,
4902 		    "%s: TMF complete", __func__);
4903 		result = 0;
4904 		break;
4905 	case SAS_RSP_TMF_SUCCEEDED:
4906 		pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, xp,
4907 		    "%s: TMF succeeded", __func__);
4908 		result = 0;
4909 		break;
4910 	case SAS_RSP_INVALID_FRAME:
4911 		pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, xp,
4912 		    "%s: TMF returned INVALID FRAME", __func__);
4913 		result = EIO;
4914 		break;
4915 	case SAS_RSP_TMF_NOT_SUPPORTED:
4916 		pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, xp,
4917 		    "%s: TMF returned TMF NOT SUPPORTED", __func__);
4918 		result = EIO;
4919 		break;
4920 	case SAS_RSP_TMF_FAILED:
4921 		pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, xp,
4922 		    "%s: TMF returned TMF FAILED", __func__);
4923 		result = EIO;
4924 		break;
4925 	case SAS_RSP_TMF_INCORRECT_LUN:
4926 		pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, xp,
4927 		    "%s: TMF returned INCORRECT LUN", __func__);
4928 		result = EIO;
4929 		break;
4930 	case SAS_RSP_OVERLAPPED_OIPTTA:
4931 		pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, xp,
4932 		    "%s: TMF returned OVERLAPPED INITIATOR PORT TRANSFER TAG "
4933 		    "ATTEMPTED", __func__);
4934 		result = EIO;
4935 		break;
4936 	default:
4937 		pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, xp,
4938 		    "%s: TMF returned unknown code 0x%x", __func__, status);
4939 		result = EIO;
4940 		break;
4941 	}
4942 	return (result);
4943 }
4944 
4945 /*
4946  * Called with PHY lock held and scratch acquired
4947  */
4948 int
4949 pmcs_sata_abort_ncq(pmcs_hw_t *pwp, pmcs_phy_t *pptr)
4950 {
4951 	const char *utag_fail_fmt = "%s: untagged NCQ command failure";
4952 	const char *tag_fail_fmt = "%s: NCQ command failure (tag 0x%x)";
4953 	uint32_t msg[PMCS_QENTRY_SIZE], *ptr, result, status;
4954 	uint8_t *fp = pwp->scratch, ds;
4955 	fis_t fis;
4956 	pmcwork_t *pwrk;
4957 	pmcs_xscsi_t *tgt;
4958 
4959 	pwrk = pmcs_gwork(pwp, PMCS_TAG_TYPE_WAIT, pptr);
4960 	if (pwrk == NULL) {
4961 		return (ENOMEM);
4962 	}
4963 	msg[0] = LE_32(PMCS_IOMB_IN_SAS(PMCS_OQ_IODONE,
4964 	    PMCIN_SATA_HOST_IO_START));
4965 	msg[1] = LE_32(pwrk->htag);
4966 	msg[2] = LE_32(pptr->device_id);
4967 	msg[3] = LE_32(512);
4968 	msg[4] = LE_32(SATA_PROTOCOL_PIO | PMCIN_DATADIR_2_INI);
4969 	msg[5] = LE_32((READ_LOG_EXT << 16) | (C_BIT << 8) | FIS_REG_H2DEV);
4970 	msg[6] = LE_32(0x10);
4971 	msg[8] = LE_32(1);
4972 	msg[9] = 0;
4973 	msg[10] = 0;
4974 	msg[11] = 0;
4975 	msg[12] = LE_32(DWORD0(pwp->scratch_dma));
4976 	msg[13] = LE_32(DWORD1(pwp->scratch_dma));
4977 	msg[14] = LE_32(512);
4978 	msg[15] = 0;
4979 
4980 	pwrk->arg = msg;
4981 	pwrk->dtype = pptr->dtype;
4982 
4983 	mutex_enter(&pwp->iqp_lock[PMCS_IQ_OTHER]);
4984 	ptr = GET_IQ_ENTRY(pwp, PMCS_IQ_OTHER);
4985 	if (ptr == NULL) {
4986 		mutex_exit(&pwp->iqp_lock[PMCS_IQ_OTHER]);
4987 		pmcs_pwork(pwp, pwrk);
4988 		return (ENOMEM);
4989 	}
4990 	COPY_MESSAGE(ptr, msg, PMCS_QENTRY_SIZE);
4991 	pwrk->state = PMCS_WORK_STATE_ONCHIP;
4992 	INC_IQ_ENTRY(pwp, PMCS_IQ_OTHER);
4993 
4994 	pmcs_unlock_phy(pptr);
4995 	WAIT_FOR(pwrk, 250, result);
4996 	pmcs_lock_phy(pptr);
4997 	pmcs_pwork(pwp, pwrk);
4998 
4999 	tgt = pptr->target;
5000 	if (result) {
5001 		pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, tgt, pmcs_timeo, __func__);
5002 		return (EIO);
5003 	}
5004 	status = LE_32(msg[2]);
5005 	if (status != PMCOUT_STATUS_OK || LE_32(msg[3])) {
5006 		if (tgt == NULL) {
5007 			pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, tgt,
5008 			    "%s: cannot find target for phy 0x%p for "
5009 			    "dev state recovery", __func__, (void *)pptr);
5010 			return (EIO);
5011 		}
5012 
5013 		mutex_enter(&tgt->statlock);
5014 
5015 		pmcs_print_entry(pwp, PMCS_PRT_DEBUG, "READ LOG EXT", msg);
5016 		if ((status == PMCOUT_STATUS_IO_DS_NON_OPERATIONAL) ||
5017 		    (status == PMCOUT_STATUS_OPEN_CNX_ERROR_BREAK) ||
5018 		    (status == PMCOUT_STATUS_OPEN_CNX_ERROR_IT_NEXUS_LOSS)) {
5019 			ds = PMCS_DEVICE_STATE_NON_OPERATIONAL;
5020 		} else {
5021 			ds = PMCS_DEVICE_STATE_IN_RECOVERY;
5022 		}
5023 		if (tgt->dev_state != ds) {
5024 			pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, tgt, "%s: Trying "
5025 			    "SATA DS Recovery for tgt(0x%p) for status(%s)",
5026 			    __func__, (void *)tgt, pmcs_status_str(status));
5027 			(void) pmcs_send_err_recovery_cmd(pwp, ds, pptr, tgt);
5028 		}
5029 
5030 		mutex_exit(&tgt->statlock);
5031 		return (EIO);
5032 	}
5033 	fis[0] = (fp[4] << 24) | (fp[3] << 16) | (fp[2] << 8) | FIS_REG_D2H;
5034 	fis[1] = (fp[8] << 24) | (fp[7] << 16) | (fp[6] << 8) | fp[5];
5035 	fis[2] = (fp[12] << 24) | (fp[11] << 16) | (fp[10] << 8) | fp[9];
5036 	fis[3] = (fp[16] << 24) | (fp[15] << 16) | (fp[14] << 8) | fp[13];
5037 	fis[4] = 0;
5038 	if (fp[0] & 0x80) {
5039 		pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, tgt,
5040 		    utag_fail_fmt, __func__);
5041 	} else {
5042 		pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, tgt,
5043 		    tag_fail_fmt, __func__, fp[0] & 0x1f);
5044 	}
5045 	pmcs_fis_dump(pwp, fis);
5046 	pptr->need_rl_ext = 0;
5047 	return (0);
5048 }
5049 
5050 /*
5051  * Transform a structure from CPU to Device endian format, or
5052  * vice versa, based upon a transformation vector.
5053  *
5054  * A transformation vector is an array of bytes, each byte
5055  * of which is defined thusly:
5056  *
5057  *  bit 7: from CPU to desired endian, otherwise from desired endian
5058  *	   to CPU format
5059  *  bit 6: Big Endian, else Little Endian
5060  *  bits 5-4:
5061  *       00 Undefined
5062  *       01 One Byte quantities
5063  *       02 Two Byte quantities
5064  *       03 Four Byte quantities
5065  *
5066  *  bits 3-0:
5067  *       00 Undefined
5068  *       Number of quantities to transform
5069  *
5070  * The vector is terminated by a 0 value.
5071  */
5072 
5073 void
5074 pmcs_endian_transform(pmcs_hw_t *pwp, void *orig_out, void *orig_in,
5075     const uint8_t *xfvec)
5076 {
5077 	uint8_t c, *out = orig_out, *in = orig_in;
5078 
5079 	if (xfvec == NULL) {
5080 		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
5081 		    "%s: null xfvec", __func__);
5082 		return;
5083 	}
5084 	if (out == NULL) {
5085 		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
5086 		    "%s: null out", __func__);
5087 		return;
5088 	}
5089 	if (in == NULL) {
5090 		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
5091 		    "%s: null in", __func__);
5092 		return;
5093 	}
5094 	while ((c = *xfvec++) != 0) {
5095 		int nbyt = (c & 0xf);
5096 		int size = (c >> 4) & 0x3;
5097 		int bige = (c >> 4) & 0x4;
5098 
5099 		switch (size) {
5100 		case 1:
5101 		{
5102 			while (nbyt-- > 0) {
5103 				*out++ = *in++;
5104 			}
5105 			break;
5106 		}
5107 		case 2:
5108 		{
5109 			uint16_t tmp;
5110 			while (nbyt-- > 0) {
5111 				(void) memcpy(&tmp, in, sizeof (uint16_t));
5112 				if (bige) {
5113 					tmp = BE_16(tmp);
5114 				} else {
5115 					tmp = LE_16(tmp);
5116 				}
5117 				(void) memcpy(out, &tmp, sizeof (uint16_t));
5118 				out += sizeof (uint16_t);
5119 				in += sizeof (uint16_t);
5120 			}
5121 			break;
5122 		}
5123 		case 3:
5124 		{
5125 			uint32_t tmp;
5126 			while (nbyt-- > 0) {
5127 				(void) memcpy(&tmp, in, sizeof (uint32_t));
5128 				if (bige) {
5129 					tmp = BE_32(tmp);
5130 				} else {
5131 					tmp = LE_32(tmp);
5132 				}
5133 				(void) memcpy(out, &tmp, sizeof (uint32_t));
5134 				out += sizeof (uint32_t);
5135 				in += sizeof (uint32_t);
5136 			}
5137 			break;
5138 		}
5139 		default:
5140 			pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
5141 			    "%s: bad size", __func__);
5142 			return;
5143 		}
5144 	}
5145 }
5146 
5147 const char *
5148 pmcs_get_rate(unsigned int linkrt)
5149 {
5150 	const char *rate;
5151 	switch (linkrt) {
5152 	case SAS_LINK_RATE_1_5GBIT:
5153 		rate = "1.5";
5154 		break;
5155 	case SAS_LINK_RATE_3GBIT:
5156 		rate = "3.0";
5157 		break;
5158 	case SAS_LINK_RATE_6GBIT:
5159 		rate = "6.0";
5160 		break;
5161 	default:
5162 		rate = "???";
5163 		break;
5164 	}
5165 	return (rate);
5166 }
5167 
5168 const char *
5169 pmcs_get_typename(pmcs_dtype_t type)
5170 {
5171 	switch (type) {
5172 	case NOTHING:
5173 		return ("NIL");
5174 	case SATA:
5175 		return ("SATA");
5176 	case SAS:
5177 		return ("SSP");
5178 	case EXPANDER:
5179 		return ("EXPANDER");
5180 	}
5181 	return ("????");
5182 }
5183 
5184 const char *
5185 pmcs_tmf2str(int tmf)
5186 {
5187 	switch (tmf) {
5188 	case SAS_ABORT_TASK:
5189 		return ("Abort Task");
5190 	case SAS_ABORT_TASK_SET:
5191 		return ("Abort Task Set");
5192 	case SAS_CLEAR_TASK_SET:
5193 		return ("Clear Task Set");
5194 	case SAS_LOGICAL_UNIT_RESET:
5195 		return ("Logical Unit Reset");
5196 	case SAS_I_T_NEXUS_RESET:
5197 		return ("I_T Nexus Reset");
5198 	case SAS_CLEAR_ACA:
5199 		return ("Clear ACA");
5200 	case SAS_QUERY_TASK:
5201 		return ("Query Task");
5202 	case SAS_QUERY_TASK_SET:
5203 		return ("Query Task Set");
5204 	case SAS_QUERY_UNIT_ATTENTION:
5205 		return ("Query Unit Attention");
5206 	default:
5207 		return ("Unknown");
5208 	}
5209 }
5210 
5211 const char *
5212 pmcs_status_str(uint32_t status)
5213 {
5214 	switch (status) {
5215 	case PMCOUT_STATUS_OK:
5216 		return ("OK");
5217 	case PMCOUT_STATUS_ABORTED:
5218 		return ("ABORTED");
5219 	case PMCOUT_STATUS_OVERFLOW:
5220 		return ("OVERFLOW");
5221 	case PMCOUT_STATUS_UNDERFLOW:
5222 		return ("UNDERFLOW");
5223 	case PMCOUT_STATUS_FAILED:
5224 		return ("FAILED");
5225 	case PMCOUT_STATUS_ABORT_RESET:
5226 		return ("ABORT_RESET");
5227 	case PMCOUT_STATUS_IO_NOT_VALID:
5228 		return ("IO_NOT_VALID");
5229 	case PMCOUT_STATUS_NO_DEVICE:
5230 		return ("NO_DEVICE");
5231 	case PMCOUT_STATUS_ILLEGAL_PARAMETER:
5232 		return ("ILLEGAL_PARAMETER");
5233 	case PMCOUT_STATUS_LINK_FAILURE:
5234 		return ("LINK_FAILURE");
5235 	case PMCOUT_STATUS_PROG_ERROR:
5236 		return ("PROG_ERROR");
5237 	case PMCOUT_STATUS_EDC_IN_ERROR:
5238 		return ("EDC_IN_ERROR");
5239 	case PMCOUT_STATUS_EDC_OUT_ERROR:
5240 		return ("EDC_OUT_ERROR");
5241 	case PMCOUT_STATUS_ERROR_HW_TIMEOUT:
5242 		return ("ERROR_HW_TIMEOUT");
5243 	case PMCOUT_STATUS_XFER_ERR_BREAK:
5244 		return ("XFER_ERR_BREAK");
5245 	case PMCOUT_STATUS_XFER_ERR_PHY_NOT_READY:
5246 		return ("XFER_ERR_PHY_NOT_READY");
5247 	case PMCOUT_STATUS_OPEN_CNX_PROTOCOL_NOT_SUPPORTED:
5248 		return ("OPEN_CNX_PROTOCOL_NOT_SUPPORTED");
5249 	case PMCOUT_STATUS_OPEN_CNX_ERROR_ZONE_VIOLATION:
5250 		return ("OPEN_CNX_ERROR_ZONE_VIOLATION");
5251 	case PMCOUT_STATUS_OPEN_CNX_ERROR_BREAK:
5252 		return ("OPEN_CNX_ERROR_BREAK");
5253 	case PMCOUT_STATUS_OPEN_CNX_ERROR_IT_NEXUS_LOSS:
5254 		return ("OPEN_CNX_ERROR_IT_NEXUS_LOSS");
5255 	case PMCOUT_STATUS_OPENCNX_ERROR_BAD_DESTINATION:
5256 		return ("OPENCNX_ERROR_BAD_DESTINATION");
5257 	case PMCOUT_STATUS_OPEN_CNX_ERROR_CONNECTION_RATE_NOT_SUPPORTED:
5258 		return ("OPEN_CNX_ERROR_CONNECTION_RATE_NOT_SUPPORTED");
5259 	case PMCOUT_STATUS_OPEN_CNX_ERROR_STP_RESOURCES_BUSY:
5260 		return ("OPEN_CNX_ERROR_STP_RESOURCES_BUSY");
5261 	case PMCOUT_STATUS_OPEN_CNX_ERROR_WRONG_DESTINATION:
5262 		return ("OPEN_CNX_ERROR_WRONG_DESTINATION");
5263 	case PMCOUT_STATUS_OPEN_CNX_ERROR_UNKNOWN_EROOR:
5264 		return ("OPEN_CNX_ERROR_UNKNOWN_EROOR");
5265 	case PMCOUT_STATUS_IO_XFER_ERROR_NAK_RECEIVED:
5266 		return ("IO_XFER_ERROR_NAK_RECEIVED");
5267 	case PMCOUT_STATUS_XFER_ERROR_ACK_NAK_TIMEOUT:
5268 		return ("XFER_ERROR_ACK_NAK_TIMEOUT");
5269 	case PMCOUT_STATUS_XFER_ERROR_PEER_ABORTED:
5270 		return ("XFER_ERROR_PEER_ABORTED");
5271 	case PMCOUT_STATUS_XFER_ERROR_RX_FRAME:
5272 		return ("XFER_ERROR_RX_FRAME");
5273 	case PMCOUT_STATUS_IO_XFER_ERROR_DMA:
5274 		return ("IO_XFER_ERROR_DMA");
5275 	case PMCOUT_STATUS_XFER_ERROR_CREDIT_TIMEOUT:
5276 		return ("XFER_ERROR_CREDIT_TIMEOUT");
5277 	case PMCOUT_STATUS_XFER_ERROR_SATA_LINK_TIMEOUT:
5278 		return ("XFER_ERROR_SATA_LINK_TIMEOUT");
5279 	case PMCOUT_STATUS_XFER_ERROR_SATA:
5280 		return ("XFER_ERROR_SATA");
5281 	case PMCOUT_STATUS_XFER_ERROR_REJECTED_NCQ_MODE:
5282 		return ("XFER_ERROR_REJECTED_NCQ_MODE");
5283 	case PMCOUT_STATUS_XFER_ERROR_ABORTED_DUE_TO_SRST:
5284 		return ("XFER_ERROR_ABORTED_DUE_TO_SRST");
5285 	case PMCOUT_STATUS_XFER_ERROR_ABORTED_NCQ_MODE:
5286 		return ("XFER_ERROR_ABORTED_NCQ_MODE");
5287 	case PMCOUT_STATUS_IO_XFER_OPEN_RETRY_TIMEOUT:
5288 		return ("IO_XFER_OPEN_RETRY_TIMEOUT");
5289 	case PMCOUT_STATUS_SMP_RESP_CONNECTION_ERROR:
5290 		return ("SMP_RESP_CONNECTION_ERROR");
5291 	case PMCOUT_STATUS_XFER_ERROR_UNEXPECTED_PHASE:
5292 		return ("XFER_ERROR_UNEXPECTED_PHASE");
5293 	case PMCOUT_STATUS_XFER_ERROR_RDY_OVERRUN:
5294 		return ("XFER_ERROR_RDY_OVERRUN");
5295 	case PMCOUT_STATUS_XFER_ERROR_RDY_NOT_EXPECTED:
5296 		return ("XFER_ERROR_RDY_NOT_EXPECTED");
5297 	case PMCOUT_STATUS_XFER_ERROR_CMD_ISSUE_ACK_NAK_TIMEOUT:
5298 		return ("XFER_ERROR_CMD_ISSUE_ACK_NAK_TIMEOUT");
5299 	case PMCOUT_STATUS_XFER_ERROR_CMD_ISSUE_BREAK_BEFORE_ACK_NACK:
5300 		return ("XFER_ERROR_CMD_ISSUE_BREAK_BEFORE_ACK_NACK");
5301 	case PMCOUT_STATUS_XFER_ERROR_CMD_ISSUE_PHY_DOWN_BEFORE_ACK_NAK:
5302 		return ("XFER_ERROR_CMD_ISSUE_PHY_DOWN_BEFORE_ACK_NAK");
5303 	case PMCOUT_STATUS_XFER_ERROR_OFFSET_MISMATCH:
5304 		return ("XFER_ERROR_OFFSET_MISMATCH");
5305 	case PMCOUT_STATUS_XFER_ERROR_ZERO_DATA_LEN:
5306 		return ("XFER_ERROR_ZERO_DATA_LEN");
5307 	case PMCOUT_STATUS_XFER_CMD_FRAME_ISSUED:
5308 		return ("XFER_CMD_FRAME_ISSUED");
5309 	case PMCOUT_STATUS_ERROR_INTERNAL_SMP_RESOURCE:
5310 		return ("ERROR_INTERNAL_SMP_RESOURCE");
5311 	case PMCOUT_STATUS_IO_PORT_IN_RESET:
5312 		return ("IO_PORT_IN_RESET");
5313 	case PMCOUT_STATUS_IO_DS_NON_OPERATIONAL:
5314 		return ("DEVICE STATE NON-OPERATIONAL");
5315 	case PMCOUT_STATUS_IO_DS_IN_RECOVERY:
5316 		return ("DEVICE STATE IN RECOVERY");
5317 	default:
5318 		return (NULL);
5319 	}
5320 }
5321 
5322 uint64_t
5323 pmcs_barray2wwn(uint8_t ba[8])
5324 {
5325 	uint64_t result = 0;
5326 	int i;
5327 
5328 	for (i = 0; i < 8; i++) {
5329 		result <<= 8;
5330 		result |= ba[i];
5331 	}
5332 	return (result);
5333 }
5334 
5335 void
5336 pmcs_wwn2barray(uint64_t wwn, uint8_t ba[8])
5337 {
5338 	int i;
5339 	for (i = 0; i < 8; i++) {
5340 		ba[7 - i] = wwn & 0xff;
5341 		wwn >>= 8;
5342 	}
5343 }
5344 
5345 void
5346 pmcs_report_fwversion(pmcs_hw_t *pwp)
5347 {
5348 	const char *fwsupport;
5349 	switch (PMCS_FW_TYPE(pwp)) {
5350 	case PMCS_FW_TYPE_RELEASED:
5351 		fwsupport = "Released";
5352 		break;
5353 	case PMCS_FW_TYPE_DEVELOPMENT:
5354 		fwsupport = "Development";
5355 		break;
5356 	case PMCS_FW_TYPE_ALPHA:
5357 		fwsupport = "Alpha";
5358 		break;
5359 	case PMCS_FW_TYPE_BETA:
5360 		fwsupport = "Beta";
5361 		break;
5362 	default:
5363 		fwsupport = "Special";
5364 		break;
5365 	}
5366 	pmcs_prt(pwp, PMCS_PRT_INFO, NULL, NULL,
5367 	    "Chip Revision: %c; F/W Revision %x.%x.%x %s", 'A' + pwp->chiprev,
5368 	    PMCS_FW_MAJOR(pwp), PMCS_FW_MINOR(pwp), PMCS_FW_MICRO(pwp),
5369 	    fwsupport);
5370 }
5371 
5372 void
5373 pmcs_phy_name(pmcs_hw_t *pwp, pmcs_phy_t *pptr, char *obuf, size_t olen)
5374 {
5375 	if (pptr->parent) {
5376 		pmcs_phy_name(pwp, pptr->parent, obuf, olen);
5377 		(void) snprintf(obuf, olen, "%s.%02x", obuf, pptr->phynum);
5378 	} else {
5379 		(void) snprintf(obuf, olen, "pp%02x", pptr->phynum);
5380 	}
5381 }
5382 
5383 /*
5384  * Implementation for pmcs_find_phy_by_devid.
5385  * If the PHY is found, it is returned locked.
5386  */
5387 static pmcs_phy_t *
5388 pmcs_find_phy_by_devid_impl(pmcs_phy_t *phyp, uint32_t device_id)
5389 {
5390 	pmcs_phy_t *match, *cphyp, *nphyp;
5391 
5392 	ASSERT(!mutex_owned(&phyp->phy_lock));
5393 
5394 	while (phyp) {
5395 		pmcs_lock_phy(phyp);
5396 
5397 		if ((phyp->valid_device_id) && (phyp->device_id == device_id)) {
5398 			return (phyp);
5399 		}
5400 		if (phyp->children) {
5401 			cphyp = phyp->children;
5402 			pmcs_unlock_phy(phyp);
5403 			match = pmcs_find_phy_by_devid_impl(cphyp, device_id);
5404 			if (match) {
5405 				ASSERT(mutex_owned(&match->phy_lock));
5406 				return (match);
5407 			}
5408 			pmcs_lock_phy(phyp);
5409 		}
5410 
5411 		if (IS_ROOT_PHY(phyp)) {
5412 			pmcs_unlock_phy(phyp);
5413 			phyp = NULL;
5414 		} else {
5415 			nphyp = phyp->sibling;
5416 			pmcs_unlock_phy(phyp);
5417 			phyp = nphyp;
5418 		}
5419 	}
5420 
5421 	return (NULL);
5422 }
5423 
5424 /*
5425  * If the PHY is found, it is returned locked
5426  */
5427 pmcs_phy_t *
5428 pmcs_find_phy_by_devid(pmcs_hw_t *pwp, uint32_t device_id)
5429 {
5430 	pmcs_phy_t *phyp, *match = NULL;
5431 
5432 	phyp = pwp->root_phys;
5433 
5434 	while (phyp) {
5435 		match = pmcs_find_phy_by_devid_impl(phyp, device_id);
5436 		if (match) {
5437 			ASSERT(mutex_owned(&match->phy_lock));
5438 			return (match);
5439 		}
5440 		phyp = phyp->sibling;
5441 	}
5442 
5443 	return (NULL);
5444 }
5445 
5446 /*
5447  * This function is called as a sanity check to ensure that a newly registered
5448  * PHY doesn't have a device_id that exists with another registered PHY.
5449  */
5450 static boolean_t
5451 pmcs_validate_devid(pmcs_phy_t *parent, pmcs_phy_t *phyp, uint32_t device_id)
5452 {
5453 	pmcs_phy_t *pptr;
5454 	boolean_t rval;
5455 
5456 	pptr = parent;
5457 
5458 	while (pptr) {
5459 		if (pptr->valid_device_id && (pptr != phyp) &&
5460 		    (pptr->device_id == device_id)) {
5461 			pmcs_prt(pptr->pwp, PMCS_PRT_DEBUG, pptr, NULL,
5462 			    "%s: phy %s already exists as %s with "
5463 			    "device id 0x%x", __func__, phyp->path,
5464 			    pptr->path, device_id);
5465 			return (B_FALSE);
5466 		}
5467 
5468 		if (pptr->children) {
5469 			rval = pmcs_validate_devid(pptr->children, phyp,
5470 			    device_id);
5471 			if (rval == B_FALSE) {
5472 				return (rval);
5473 			}
5474 		}
5475 
5476 		pptr = pptr->sibling;
5477 	}
5478 
5479 	/* This PHY and device_id are valid */
5480 	return (B_TRUE);
5481 }
5482 
5483 /*
5484  * If the PHY is found, it is returned locked
5485  */
5486 static pmcs_phy_t *
5487 pmcs_find_phy_by_wwn_impl(pmcs_phy_t *phyp, uint8_t *wwn)
5488 {
5489 	pmcs_phy_t *matched_phy, *cphyp, *nphyp;
5490 
5491 	ASSERT(!mutex_owned(&phyp->phy_lock));
5492 
5493 	while (phyp) {
5494 		pmcs_lock_phy(phyp);
5495 
5496 		if (phyp->valid_device_id) {
5497 			if (memcmp(phyp->sas_address, wwn, 8) == 0) {
5498 				return (phyp);
5499 			}
5500 		}
5501 
5502 		if (phyp->children) {
5503 			cphyp = phyp->children;
5504 			pmcs_unlock_phy(phyp);
5505 			matched_phy = pmcs_find_phy_by_wwn_impl(cphyp, wwn);
5506 			if (matched_phy) {
5507 				ASSERT(mutex_owned(&matched_phy->phy_lock));
5508 				return (matched_phy);
5509 			}
5510 			pmcs_lock_phy(phyp);
5511 		}
5512 
5513 		/*
5514 		 * Only iterate through non-root PHYs
5515 		 */
5516 		if (IS_ROOT_PHY(phyp)) {
5517 			pmcs_unlock_phy(phyp);
5518 			phyp = NULL;
5519 		} else {
5520 			nphyp = phyp->sibling;
5521 			pmcs_unlock_phy(phyp);
5522 			phyp = nphyp;
5523 		}
5524 	}
5525 
5526 	return (NULL);
5527 }
5528 
5529 pmcs_phy_t *
5530 pmcs_find_phy_by_wwn(pmcs_hw_t *pwp, uint64_t wwn)
5531 {
5532 	uint8_t ebstr[8];
5533 	pmcs_phy_t *pptr, *matched_phy;
5534 
5535 	pmcs_wwn2barray(wwn, ebstr);
5536 
5537 	pptr = pwp->root_phys;
5538 	while (pptr) {
5539 		matched_phy = pmcs_find_phy_by_wwn_impl(pptr, ebstr);
5540 		if (matched_phy) {
5541 			ASSERT(mutex_owned(&matched_phy->phy_lock));
5542 			return (matched_phy);
5543 		}
5544 
5545 		pptr = pptr->sibling;
5546 	}
5547 
5548 	return (NULL);
5549 }
5550 
5551 
5552 /*
5553  * pmcs_find_phy_by_sas_address
5554  *
5555  * Find a PHY that both matches "sas_addr" and is on "iport".
5556  * If a matching PHY is found, it is returned locked.
5557  */
5558 pmcs_phy_t *
5559 pmcs_find_phy_by_sas_address(pmcs_hw_t *pwp, pmcs_iport_t *iport,
5560     pmcs_phy_t *root, char *sas_addr)
5561 {
5562 	int ua_form = 1;
5563 	uint64_t wwn;
5564 	char addr[PMCS_MAX_UA_SIZE];
5565 	pmcs_phy_t *pptr, *pnext, *pchild;
5566 
5567 	if (root == NULL) {
5568 		pptr = pwp->root_phys;
5569 	} else {
5570 		pptr = root;
5571 	}
5572 
5573 	while (pptr) {
5574 		pmcs_lock_phy(pptr);
5575 		/*
5576 		 * If the PHY is dead or does not have a valid device ID,
5577 		 * skip it.
5578 		 */
5579 		if ((pptr->dead) || (!pptr->valid_device_id)) {
5580 			goto next_phy;
5581 		}
5582 
5583 		if (pptr->iport != iport) {
5584 			goto next_phy;
5585 		}
5586 
5587 		wwn = pmcs_barray2wwn(pptr->sas_address);
5588 		(void *) scsi_wwn_to_wwnstr(wwn, ua_form, addr);
5589 		if (strncmp(addr, sas_addr, strlen(addr)) == 0) {
5590 			return (pptr);
5591 		}
5592 
5593 		if (pptr->children) {
5594 			pchild = pptr->children;
5595 			pmcs_unlock_phy(pptr);
5596 			pnext = pmcs_find_phy_by_sas_address(pwp, iport, pchild,
5597 			    sas_addr);
5598 			if (pnext) {
5599 				return (pnext);
5600 			}
5601 			pmcs_lock_phy(pptr);
5602 		}
5603 
5604 next_phy:
5605 		pnext = pptr->sibling;
5606 		pmcs_unlock_phy(pptr);
5607 		pptr = pnext;
5608 	}
5609 
5610 	return (NULL);
5611 }
5612 
5613 void
5614 pmcs_fis_dump(pmcs_hw_t *pwp, fis_t fis)
5615 {
5616 	switch (fis[0] & 0xff) {
5617 	case FIS_REG_H2DEV:
5618 		pmcs_prt(pwp, PMCS_PRT_INFO, NULL, NULL,
5619 		    "FIS REGISTER HOST TO DEVICE: "
5620 		    "OP=0x%02x Feature=0x%04x Count=0x%04x Device=0x%02x "
5621 		    "LBA=%llu", BYTE2(fis[0]), BYTE3(fis[2]) << 8 |
5622 		    BYTE3(fis[0]), WORD0(fis[3]), BYTE3(fis[1]),
5623 		    (unsigned long long)
5624 		    (((uint64_t)fis[2] & 0x00ffffff) << 24 |
5625 		    ((uint64_t)fis[1] & 0x00ffffff)));
5626 		break;
5627 	case FIS_REG_D2H:
5628 		pmcs_prt(pwp, PMCS_PRT_INFO, NULL, NULL,
5629 		    "FIS REGISTER DEVICE TO HOST: Status=0x%02x "
5630 		    "Error=0x%02x Dev=0x%02x Count=0x%04x LBA=%llu",
5631 		    BYTE2(fis[0]), BYTE3(fis[0]), BYTE3(fis[1]), WORD0(fis[3]),
5632 		    (unsigned long long)(((uint64_t)fis[2] & 0x00ffffff) << 24 |
5633 		    ((uint64_t)fis[1] & 0x00ffffff)));
5634 		break;
5635 	default:
5636 		pmcs_prt(pwp, PMCS_PRT_INFO, NULL, NULL,
5637 		    "FIS: 0x%08x 0x%08x 0x%08x 0x%08x 0x%08x 0x%08x 0x%08x",
5638 		    fis[0], fis[1], fis[2], fis[3], fis[4], fis[5], fis[6]);
5639 		break;
5640 	}
5641 }
5642 
5643 void
5644 pmcs_print_entry(pmcs_hw_t *pwp, int level, char *msg, void *arg)
5645 {
5646 	uint32_t *mb = arg;
5647 	size_t i;
5648 
5649 	pmcs_prt(pwp, level, NULL, NULL, msg);
5650 	for (i = 0; i < (PMCS_QENTRY_SIZE / sizeof (uint32_t)); i += 4) {
5651 		pmcs_prt(pwp, level, NULL, NULL,
5652 		    "Offset %2lu: 0x%08x 0x%08x 0x%08x 0x%08x",
5653 		    i * sizeof (uint32_t), LE_32(mb[i]),
5654 		    LE_32(mb[i+1]), LE_32(mb[i+2]), LE_32(mb[i+3]));
5655 	}
5656 }
5657 
5658 /*
5659  * If phyp == NULL we're being called from the worker thread, in which
5660  * case we need to check all the PHYs.  In this case, the softstate lock
5661  * will be held.
5662  * If phyp is non-NULL, just issue the spinup release for the specified PHY
5663  * (which will already be locked).
5664  */
5665 void
5666 pmcs_spinup_release(pmcs_hw_t *pwp, pmcs_phy_t *phyp)
5667 {
5668 	uint32_t *msg;
5669 	struct pmcwork *pwrk;
5670 	pmcs_phy_t *tphyp;
5671 
5672 	if (phyp != NULL) {
5673 		ASSERT(mutex_owned(&phyp->phy_lock));
5674 		pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, phyp, NULL,
5675 		    "%s: Issuing spinup release only for PHY %s", __func__,
5676 		    phyp->path);
5677 		mutex_enter(&pwp->iqp_lock[PMCS_IQ_OTHER]);
5678 		msg = GET_IQ_ENTRY(pwp, PMCS_IQ_OTHER);
5679 		if (msg == NULL || (pwrk =
5680 		    pmcs_gwork(pwp, PMCS_TAG_TYPE_NONE, NULL)) == NULL) {
5681 			mutex_exit(&pwp->iqp_lock[PMCS_IQ_OTHER]);
5682 			SCHEDULE_WORK(pwp, PMCS_WORK_SPINUP_RELEASE);
5683 			return;
5684 		}
5685 
5686 		phyp->spinup_hold = 0;
5687 		bzero(msg, PMCS_QENTRY_SIZE);
5688 		msg[0] = LE_32(PMCS_HIPRI(pwp, PMCS_OQ_GENERAL,
5689 		    PMCIN_LOCAL_PHY_CONTROL));
5690 		msg[1] = LE_32(pwrk->htag);
5691 		msg[2] = LE_32((0x10 << 8) | phyp->phynum);
5692 
5693 		pwrk->dtype = phyp->dtype;
5694 		pwrk->state = PMCS_WORK_STATE_ONCHIP;
5695 		mutex_exit(&pwrk->lock);
5696 		INC_IQ_ENTRY(pwp, PMCS_IQ_OTHER);
5697 		return;
5698 	}
5699 
5700 	ASSERT(mutex_owned(&pwp->lock));
5701 
5702 	tphyp = pwp->root_phys;
5703 	while (tphyp) {
5704 		pmcs_lock_phy(tphyp);
5705 		if (tphyp->spinup_hold == 0) {
5706 			pmcs_unlock_phy(tphyp);
5707 			tphyp = tphyp->sibling;
5708 			continue;
5709 		}
5710 
5711 		pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, phyp, NULL,
5712 		    "%s: Issuing spinup release for PHY %s", __func__,
5713 		    phyp->path);
5714 
5715 		mutex_enter(&pwp->iqp_lock[PMCS_IQ_OTHER]);
5716 		msg = GET_IQ_ENTRY(pwp, PMCS_IQ_OTHER);
5717 		if (msg == NULL || (pwrk =
5718 		    pmcs_gwork(pwp, PMCS_TAG_TYPE_NONE, NULL)) == NULL) {
5719 			pmcs_unlock_phy(tphyp);
5720 			mutex_exit(&pwp->iqp_lock[PMCS_IQ_OTHER]);
5721 			SCHEDULE_WORK(pwp, PMCS_WORK_SPINUP_RELEASE);
5722 			break;
5723 		}
5724 
5725 		tphyp->spinup_hold = 0;
5726 		bzero(msg, PMCS_QENTRY_SIZE);
5727 		msg[0] = LE_32(PMCS_HIPRI(pwp, PMCS_OQ_GENERAL,
5728 		    PMCIN_LOCAL_PHY_CONTROL));
5729 		msg[1] = LE_32(pwrk->htag);
5730 		msg[2] = LE_32((0x10 << 8) | tphyp->phynum);
5731 
5732 		pwrk->dtype = phyp->dtype;
5733 		pwrk->state = PMCS_WORK_STATE_ONCHIP;
5734 		mutex_exit(&pwrk->lock);
5735 		INC_IQ_ENTRY(pwp, PMCS_IQ_OTHER);
5736 		pmcs_unlock_phy(tphyp);
5737 
5738 		tphyp = tphyp->sibling;
5739 	}
5740 }
5741 
5742 /*
5743  * Abort commands on dead PHYs and deregister them as well as removing
5744  * the associated targets.
5745  */
5746 static int
5747 pmcs_kill_devices(pmcs_hw_t *pwp, pmcs_phy_t *phyp)
5748 {
5749 	pmcs_phy_t *pnext, *pchild;
5750 	boolean_t remove_device;
5751 	int rval = 0;
5752 
5753 	while (phyp) {
5754 		pmcs_lock_phy(phyp);
5755 		pchild = phyp->children;
5756 		pnext = phyp->sibling;
5757 		pmcs_unlock_phy(phyp);
5758 
5759 		if (pchild) {
5760 			rval = pmcs_kill_devices(pwp, pchild);
5761 			if (rval) {
5762 				return (rval);
5763 			}
5764 		}
5765 
5766 		/*
5767 		 * pmcs_remove_device requires the softstate lock.
5768 		 */
5769 		mutex_enter(&pwp->lock);
5770 		pmcs_lock_phy(phyp);
5771 		if (phyp->dead && phyp->valid_device_id) {
5772 			remove_device = B_TRUE;
5773 		} else {
5774 			remove_device = B_FALSE;
5775 		}
5776 
5777 		if (remove_device) {
5778 			pmcs_remove_device(pwp, phyp);
5779 			mutex_exit(&pwp->lock);
5780 
5781 			rval = pmcs_kill_device(pwp, phyp);
5782 
5783 			if (rval) {
5784 				pmcs_unlock_phy(phyp);
5785 				return (rval);
5786 			}
5787 		} else {
5788 			mutex_exit(&pwp->lock);
5789 		}
5790 
5791 		pmcs_unlock_phy(phyp);
5792 		phyp = pnext;
5793 	}
5794 
5795 	return (rval);
5796 }
5797 
5798 /*
5799  * Called with PHY locked
5800  */
5801 int
5802 pmcs_kill_device(pmcs_hw_t *pwp, pmcs_phy_t *pptr)
5803 {
5804 	int r, result;
5805 	uint32_t msg[PMCS_MSG_SIZE], *ptr, status;
5806 	struct pmcwork *pwrk;
5807 
5808 	pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, NULL, "kill %s device @ %s",
5809 	    pmcs_get_typename(pptr->dtype), pptr->path);
5810 
5811 	/*
5812 	 * There may be an outstanding ABORT_ALL running, which we wouldn't
5813 	 * know just by checking abort_pending.  We can, however, check
5814 	 * abort_all_start.  If it's non-zero, there is one, and we'll just
5815 	 * sit here and wait for it to complete.  If we don't, we'll remove
5816 	 * the device while there are still commands pending.
5817 	 */
5818 	if (pptr->abort_all_start) {
5819 		while (pptr->abort_all_start) {
5820 			pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, NULL,
5821 			    "%s: Waiting for outstanding ABORT_ALL on PHY 0x%p",
5822 			    __func__, (void *)pptr);
5823 			cv_wait(&pptr->abort_all_cv, &pptr->phy_lock);
5824 		}
5825 	} else if (pptr->abort_pending) {
5826 		r = pmcs_abort(pwp, pptr, pptr->device_id, 1, 1);
5827 
5828 		if (r) {
5829 			pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, NULL,
5830 			    "%s: ABORT_ALL returned non-zero status (%d) for "
5831 			    "PHY 0x%p", __func__, r, (void *)pptr);
5832 			return (r);
5833 		}
5834 		pptr->abort_pending = 0;
5835 	}
5836 
5837 	if (pptr->valid_device_id == 0) {
5838 		return (0);
5839 	}
5840 
5841 	if ((pwrk = pmcs_gwork(pwp, PMCS_TAG_TYPE_WAIT, pptr)) == NULL) {
5842 		pmcs_prt(pwp, PMCS_PRT_ERR, pptr, NULL, pmcs_nowrk, __func__);
5843 		return (ENOMEM);
5844 	}
5845 	pwrk->arg = msg;
5846 	pwrk->dtype = pptr->dtype;
5847 	msg[0] = LE_32(PMCS_HIPRI(pwp, PMCS_OQ_GENERAL,
5848 	    PMCIN_DEREGISTER_DEVICE_HANDLE));
5849 	msg[1] = LE_32(pwrk->htag);
5850 	msg[2] = LE_32(pptr->device_id);
5851 
5852 	mutex_enter(&pwp->iqp_lock[PMCS_IQ_OTHER]);
5853 	ptr = GET_IQ_ENTRY(pwp, PMCS_IQ_OTHER);
5854 	if (ptr == NULL) {
5855 		mutex_exit(&pwp->iqp_lock[PMCS_IQ_OTHER]);
5856 		mutex_exit(&pwrk->lock);
5857 		pmcs_prt(pwp, PMCS_PRT_ERR, pptr, NULL, pmcs_nomsg, __func__);
5858 		return (ENOMEM);
5859 	}
5860 
5861 	COPY_MESSAGE(ptr, msg, 3);
5862 	pwrk->state = PMCS_WORK_STATE_ONCHIP;
5863 	INC_IQ_ENTRY(pwp, PMCS_IQ_OTHER);
5864 
5865 	pmcs_unlock_phy(pptr);
5866 	WAIT_FOR(pwrk, 250, result);
5867 	pmcs_lock_phy(pptr);
5868 	pmcs_pwork(pwp, pwrk);
5869 
5870 	if (result) {
5871 		return (ETIMEDOUT);
5872 	}
5873 	status = LE_32(msg[2]);
5874 	if (status != PMCOUT_STATUS_OK) {
5875 		pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, NULL,
5876 		    "%s: status 0x%x when trying to deregister device %s",
5877 		    __func__, status, pptr->path);
5878 	}
5879 
5880 	pptr->device_id = PMCS_INVALID_DEVICE_ID;
5881 	PHY_CHANGED(pwp, pptr);
5882 	RESTART_DISCOVERY(pwp);
5883 	pptr->valid_device_id = 0;
5884 	return (0);
5885 }
5886 
5887 /*
5888  * Acknowledge the SAS h/w events that need acknowledgement.
5889  * This is only needed for first level PHYs.
5890  */
5891 void
5892 pmcs_ack_events(pmcs_hw_t *pwp)
5893 {
5894 	uint32_t msg[PMCS_MSG_SIZE], *ptr;
5895 	struct pmcwork *pwrk;
5896 	pmcs_phy_t *pptr;
5897 
5898 	for (pptr = pwp->root_phys; pptr; pptr = pptr->sibling) {
5899 		pmcs_lock_phy(pptr);
5900 		if (pptr->hw_event_ack == 0) {
5901 			pmcs_unlock_phy(pptr);
5902 			continue;
5903 		}
5904 		mutex_enter(&pwp->iqp_lock[PMCS_IQ_OTHER]);
5905 		ptr = GET_IQ_ENTRY(pwp, PMCS_IQ_OTHER);
5906 
5907 		if ((ptr == NULL) || (pwrk =
5908 		    pmcs_gwork(pwp, PMCS_TAG_TYPE_NONE, NULL)) == NULL) {
5909 			mutex_exit(&pwp->iqp_lock[PMCS_IQ_OTHER]);
5910 			pmcs_unlock_phy(pptr);
5911 			SCHEDULE_WORK(pwp, PMCS_WORK_SAS_HW_ACK);
5912 			break;
5913 		}
5914 
5915 		msg[0] = LE_32(PMCS_HIPRI(pwp, PMCS_OQ_GENERAL,
5916 		    PMCIN_SAW_HW_EVENT_ACK));
5917 		msg[1] = LE_32(pwrk->htag);
5918 		msg[2] = LE_32(pptr->hw_event_ack);
5919 
5920 		mutex_exit(&pwrk->lock);
5921 		pwrk->dtype = pptr->dtype;
5922 		pptr->hw_event_ack = 0;
5923 		COPY_MESSAGE(ptr, msg, 3);
5924 		INC_IQ_ENTRY(pwp, PMCS_IQ_OTHER);
5925 		pmcs_unlock_phy(pptr);
5926 	}
5927 }
5928 
5929 /*
5930  * Load DMA
5931  */
5932 int
5933 pmcs_dma_load(pmcs_hw_t *pwp, pmcs_cmd_t *sp, uint32_t *msg)
5934 {
5935 	ddi_dma_cookie_t *sg;
5936 	pmcs_dmachunk_t *tc;
5937 	pmcs_dmasgl_t *sgl, *prior;
5938 	int seg, tsc;
5939 	uint64_t sgl_addr;
5940 
5941 	/*
5942 	 * If we have no data segments, we're done.
5943 	 */
5944 	if (CMD2PKT(sp)->pkt_numcookies == 0) {
5945 		return (0);
5946 	}
5947 
5948 	/*
5949 	 * Get the S/G list pointer.
5950 	 */
5951 	sg = CMD2PKT(sp)->pkt_cookies;
5952 
5953 	/*
5954 	 * If we only have one dma segment, we can directly address that
5955 	 * data within the Inbound message itself.
5956 	 */
5957 	if (CMD2PKT(sp)->pkt_numcookies == 1) {
5958 		msg[12] = LE_32(DWORD0(sg->dmac_laddress));
5959 		msg[13] = LE_32(DWORD1(sg->dmac_laddress));
5960 		msg[14] = LE_32(sg->dmac_size);
5961 		msg[15] = 0;
5962 		return (0);
5963 	}
5964 
5965 	/*
5966 	 * Otherwise, we'll need one or more external S/G list chunks.
5967 	 * Get the first one and its dma address into the Inbound message.
5968 	 */
5969 	mutex_enter(&pwp->dma_lock);
5970 	tc = pwp->dma_freelist;
5971 	if (tc == NULL) {
5972 		SCHEDULE_WORK(pwp, PMCS_WORK_ADD_DMA_CHUNKS);
5973 		mutex_exit(&pwp->dma_lock);
5974 		pmcs_prt(pwp, PMCS_PRT_DEBUG2, NULL, NULL,
5975 		    "%s: out of SG lists", __func__);
5976 		return (-1);
5977 	}
5978 	pwp->dma_freelist = tc->nxt;
5979 	mutex_exit(&pwp->dma_lock);
5980 
5981 	tc->nxt = NULL;
5982 	sp->cmd_clist = tc;
5983 	sgl = tc->chunks;
5984 	(void) memset(tc->chunks, 0, PMCS_SGL_CHUNKSZ);
5985 	sgl_addr = tc->addr;
5986 	msg[12] = LE_32(DWORD0(sgl_addr));
5987 	msg[13] = LE_32(DWORD1(sgl_addr));
5988 	msg[14] = 0;
5989 	msg[15] = LE_32(PMCS_DMASGL_EXTENSION);
5990 
5991 	prior = sgl;
5992 	tsc = 0;
5993 
5994 	for (seg = 0; seg < CMD2PKT(sp)->pkt_numcookies; seg++) {
5995 		/*
5996 		 * If the current segment count for this chunk is one less than
5997 		 * the number s/g lists per chunk and we have more than one seg
5998 		 * to go, we need another chunk. Get it, and make sure that the
5999 		 * tail end of the the previous chunk points the new chunk
6000 		 * (if remembering an offset can be called 'pointing to').
6001 		 *
6002 		 * Note that we can store the offset into our command area that
6003 		 * represents the new chunk in the length field of the part
6004 		 * that points the PMC chip at the next chunk- the PMC chip
6005 		 * ignores this field when the EXTENSION bit is set.
6006 		 *
6007 		 * This is required for dma unloads later.
6008 		 */
6009 		if (tsc == (PMCS_SGL_NCHUNKS - 1) &&
6010 		    seg < (CMD2PKT(sp)->pkt_numcookies - 1)) {
6011 			mutex_enter(&pwp->dma_lock);
6012 			tc = pwp->dma_freelist;
6013 			if (tc == NULL) {
6014 				SCHEDULE_WORK(pwp, PMCS_WORK_ADD_DMA_CHUNKS);
6015 				mutex_exit(&pwp->dma_lock);
6016 				pmcs_dma_unload(pwp, sp);
6017 				pmcs_prt(pwp, PMCS_PRT_DEBUG2, NULL, NULL,
6018 				    "%s: out of SG lists", __func__);
6019 				return (-1);
6020 			}
6021 			pwp->dma_freelist = tc->nxt;
6022 			tc->nxt = sp->cmd_clist;
6023 			mutex_exit(&pwp->dma_lock);
6024 
6025 			sp->cmd_clist = tc;
6026 			(void) memset(tc->chunks, 0, PMCS_SGL_CHUNKSZ);
6027 			sgl = tc->chunks;
6028 			sgl_addr = tc->addr;
6029 			prior[PMCS_SGL_NCHUNKS-1].sglal =
6030 			    LE_32(DWORD0(sgl_addr));
6031 			prior[PMCS_SGL_NCHUNKS-1].sglah =
6032 			    LE_32(DWORD1(sgl_addr));
6033 			prior[PMCS_SGL_NCHUNKS-1].sglen = 0;
6034 			prior[PMCS_SGL_NCHUNKS-1].flags =
6035 			    LE_32(PMCS_DMASGL_EXTENSION);
6036 			prior = sgl;
6037 			tsc = 0;
6038 		}
6039 		sgl[tsc].sglal = LE_32(DWORD0(sg->dmac_laddress));
6040 		sgl[tsc].sglah = LE_32(DWORD1(sg->dmac_laddress));
6041 		sgl[tsc].sglen = LE_32(sg->dmac_size);
6042 		sgl[tsc++].flags = 0;
6043 		sg++;
6044 	}
6045 	return (0);
6046 }
6047 
6048 /*
6049  * Unload DMA
6050  */
6051 void
6052 pmcs_dma_unload(pmcs_hw_t *pwp, pmcs_cmd_t *sp)
6053 {
6054 	pmcs_dmachunk_t *cp;
6055 
6056 	mutex_enter(&pwp->dma_lock);
6057 	while ((cp = sp->cmd_clist) != NULL) {
6058 		sp->cmd_clist = cp->nxt;
6059 		cp->nxt = pwp->dma_freelist;
6060 		pwp->dma_freelist = cp;
6061 	}
6062 	mutex_exit(&pwp->dma_lock);
6063 }
6064 
6065 /*
6066  * Take a chunk of consistent memory that has just been allocated and inserted
6067  * into the cip indices and prepare it for DMA chunk usage and add it to the
6068  * freelist.
6069  *
6070  * Called with dma_lock locked (except during attach when it's unnecessary)
6071  */
6072 void
6073 pmcs_idma_chunks(pmcs_hw_t *pwp, pmcs_dmachunk_t *dcp,
6074     pmcs_chunk_t *pchunk, unsigned long lim)
6075 {
6076 	unsigned long off, n;
6077 	pmcs_dmachunk_t *np = dcp;
6078 	pmcs_chunk_t *tmp_chunk;
6079 
6080 	if (pwp->dma_chunklist == NULL) {
6081 		pwp->dma_chunklist = pchunk;
6082 	} else {
6083 		tmp_chunk = pwp->dma_chunklist;
6084 		while (tmp_chunk->next) {
6085 			tmp_chunk = tmp_chunk->next;
6086 		}
6087 		tmp_chunk->next = pchunk;
6088 	}
6089 
6090 	/*
6091 	 * Install offsets into chunk lists.
6092 	 */
6093 	for (n = 0, off = 0; off < lim; off += PMCS_SGL_CHUNKSZ, n++) {
6094 		np->chunks = (void *)&pchunk->addrp[off];
6095 		np->addr = pchunk->dma_addr + off;
6096 		np->acc_handle = pchunk->acc_handle;
6097 		np->dma_handle = pchunk->dma_handle;
6098 		if ((off + PMCS_SGL_CHUNKSZ) < lim) {
6099 			np = np->nxt;
6100 		}
6101 	}
6102 	np->nxt = pwp->dma_freelist;
6103 	pwp->dma_freelist = dcp;
6104 	pmcs_prt(pwp, PMCS_PRT_DEBUG2, NULL, NULL,
6105 	    "added %lu DMA chunks ", n);
6106 }
6107 
6108 /*
6109  * Change the value of the interrupt coalescing timer.  This is done currently
6110  * only for I/O completions.  If we're using the "auto clear" feature, it can
6111  * be turned back on when interrupt coalescing is turned off and must be
6112  * turned off when the coalescing timer is on.
6113  * NOTE: PMCS_MSIX_GENERAL and PMCS_OQ_IODONE are the same value.  As long
6114  * as that's true, we don't need to distinguish between them.
6115  */
6116 
6117 void
6118 pmcs_set_intr_coal_timer(pmcs_hw_t *pwp, pmcs_coal_timer_adj_t adj)
6119 {
6120 	if (adj == DECREASE_TIMER) {
6121 		/* If the timer is already off, nothing to do. */
6122 		if (pwp->io_intr_coal.timer_on == B_FALSE) {
6123 			return;
6124 		}
6125 
6126 		pwp->io_intr_coal.intr_coal_timer -= PMCS_COAL_TIMER_GRAN;
6127 
6128 		if (pwp->io_intr_coal.intr_coal_timer == 0) {
6129 			/* Disable the timer */
6130 			pmcs_wr_topunit(pwp, PMCS_INT_COALESCING_CONTROL, 0);
6131 
6132 			if (pwp->odb_auto_clear & (1 << PMCS_MSIX_IODONE)) {
6133 				pmcs_wr_topunit(pwp, PMCS_OBDB_AUTO_CLR,
6134 				    pwp->odb_auto_clear);
6135 			}
6136 
6137 			pwp->io_intr_coal.timer_on = B_FALSE;
6138 			pwp->io_intr_coal.max_io_completions = B_FALSE;
6139 			pwp->io_intr_coal.num_intrs = 0;
6140 			pwp->io_intr_coal.int_cleared = B_FALSE;
6141 			pwp->io_intr_coal.num_io_completions = 0;
6142 
6143 			DTRACE_PROBE1(pmcs__intr__coalesce__timer__off,
6144 			    pmcs_io_intr_coal_t *, &pwp->io_intr_coal);
6145 		} else {
6146 			pmcs_wr_topunit(pwp, PMCS_INT_COALESCING_TIMER,
6147 			    pwp->io_intr_coal.intr_coal_timer);
6148 		}
6149 	} else {
6150 		/*
6151 		 * If the timer isn't on yet, do the setup for it now.
6152 		 */
6153 		if (pwp->io_intr_coal.timer_on == B_FALSE) {
6154 			/* If auto clear is being used, turn it off. */
6155 			if (pwp->odb_auto_clear & (1 << PMCS_MSIX_IODONE)) {
6156 				pmcs_wr_topunit(pwp, PMCS_OBDB_AUTO_CLR,
6157 				    (pwp->odb_auto_clear &
6158 				    ~(1 << PMCS_MSIX_IODONE)));
6159 			}
6160 
6161 			pmcs_wr_topunit(pwp, PMCS_INT_COALESCING_CONTROL,
6162 			    (1 << PMCS_MSIX_IODONE));
6163 			pwp->io_intr_coal.timer_on = B_TRUE;
6164 			pwp->io_intr_coal.intr_coal_timer =
6165 			    PMCS_COAL_TIMER_GRAN;
6166 
6167 			DTRACE_PROBE1(pmcs__intr__coalesce__timer__on,
6168 			    pmcs_io_intr_coal_t *, &pwp->io_intr_coal);
6169 		} else {
6170 			pwp->io_intr_coal.intr_coal_timer +=
6171 			    PMCS_COAL_TIMER_GRAN;
6172 		}
6173 
6174 		if (pwp->io_intr_coal.intr_coal_timer > PMCS_MAX_COAL_TIMER) {
6175 			pwp->io_intr_coal.intr_coal_timer = PMCS_MAX_COAL_TIMER;
6176 		}
6177 
6178 		pmcs_wr_topunit(pwp, PMCS_INT_COALESCING_TIMER,
6179 		    pwp->io_intr_coal.intr_coal_timer);
6180 	}
6181 
6182 	/*
6183 	 * Adjust the interrupt threshold based on the current timer value
6184 	 */
6185 	pwp->io_intr_coal.intr_threshold =
6186 	    PMCS_INTR_THRESHOLD(PMCS_QUANTUM_TIME_USECS * 1000 /
6187 	    (pwp->io_intr_coal.intr_latency +
6188 	    (pwp->io_intr_coal.intr_coal_timer * 1000)));
6189 }
6190 
6191 /*
6192  * Register Access functions
6193  */
6194 uint32_t
6195 pmcs_rd_iqci(pmcs_hw_t *pwp, uint32_t qnum)
6196 {
6197 	uint32_t iqci;
6198 
6199 	if (ddi_dma_sync(pwp->cip_handles, 0, 0, DDI_DMA_SYNC_FORKERNEL) !=
6200 	    DDI_SUCCESS) {
6201 		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
6202 		    "%s: ddi_dma_sync failed?", __func__);
6203 	}
6204 
6205 	iqci = LE_32(
6206 	    ((uint32_t *)((void *)pwp->cip))[IQ_OFFSET(qnum) >> 2]);
6207 
6208 	return (iqci);
6209 }
6210 
6211 uint32_t
6212 pmcs_rd_oqpi(pmcs_hw_t *pwp, uint32_t qnum)
6213 {
6214 	uint32_t oqpi;
6215 
6216 	if (ddi_dma_sync(pwp->cip_handles, 0, 0, DDI_DMA_SYNC_FORKERNEL) !=
6217 	    DDI_SUCCESS) {
6218 		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
6219 		    "%s: ddi_dma_sync failed?", __func__);
6220 	}
6221 
6222 	oqpi = LE_32(
6223 	    ((uint32_t *)((void *)pwp->cip))[OQ_OFFSET(qnum) >> 2]);
6224 
6225 	return (oqpi);
6226 }
6227 
6228 uint32_t
6229 pmcs_rd_gsm_reg(pmcs_hw_t *pwp, uint32_t off)
6230 {
6231 	uint32_t rv, newaxil, oldaxil;
6232 
6233 	newaxil = off & ~GSM_BASE_MASK;
6234 	off &= GSM_BASE_MASK;
6235 	mutex_enter(&pwp->axil_lock);
6236 	oldaxil = ddi_get32(pwp->top_acc_handle,
6237 	    &pwp->top_regs[PMCS_AXI_TRANS >> 2]);
6238 	ddi_put32(pwp->top_acc_handle,
6239 	    &pwp->top_regs[PMCS_AXI_TRANS >> 2], newaxil);
6240 	drv_usecwait(10);
6241 	if (ddi_get32(pwp->top_acc_handle,
6242 	    &pwp->top_regs[PMCS_AXI_TRANS >> 2]) != newaxil) {
6243 		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
6244 		    "AXIL register update failed");
6245 	}
6246 	rv = ddi_get32(pwp->gsm_acc_handle, &pwp->gsm_regs[off >> 2]);
6247 	ddi_put32(pwp->top_acc_handle,
6248 	    &pwp->top_regs[PMCS_AXI_TRANS >> 2], oldaxil);
6249 	drv_usecwait(10);
6250 	if (ddi_get32(pwp->top_acc_handle,
6251 	    &pwp->top_regs[PMCS_AXI_TRANS >> 2]) != oldaxil) {
6252 		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
6253 		    "AXIL register restore failed");
6254 	}
6255 	mutex_exit(&pwp->axil_lock);
6256 	return (rv);
6257 }
6258 
6259 void
6260 pmcs_wr_gsm_reg(pmcs_hw_t *pwp, uint32_t off, uint32_t val)
6261 {
6262 	uint32_t newaxil, oldaxil;
6263 
6264 	newaxil = off & ~GSM_BASE_MASK;
6265 	off &= GSM_BASE_MASK;
6266 	mutex_enter(&pwp->axil_lock);
6267 	oldaxil = ddi_get32(pwp->top_acc_handle,
6268 	    &pwp->top_regs[PMCS_AXI_TRANS >> 2]);
6269 	ddi_put32(pwp->top_acc_handle,
6270 	    &pwp->top_regs[PMCS_AXI_TRANS >> 2], newaxil);
6271 	drv_usecwait(10);
6272 	if (ddi_get32(pwp->top_acc_handle,
6273 	    &pwp->top_regs[PMCS_AXI_TRANS >> 2]) != newaxil) {
6274 		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
6275 		    "AXIL register update failed");
6276 	}
6277 	ddi_put32(pwp->gsm_acc_handle, &pwp->gsm_regs[off >> 2], val);
6278 	ddi_put32(pwp->top_acc_handle,
6279 	    &pwp->top_regs[PMCS_AXI_TRANS >> 2], oldaxil);
6280 	drv_usecwait(10);
6281 	if (ddi_get32(pwp->top_acc_handle,
6282 	    &pwp->top_regs[PMCS_AXI_TRANS >> 2]) != oldaxil) {
6283 		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
6284 		    "AXIL register restore failed");
6285 	}
6286 	mutex_exit(&pwp->axil_lock);
6287 }
6288 
6289 uint32_t
6290 pmcs_rd_topunit(pmcs_hw_t *pwp, uint32_t off)
6291 {
6292 	switch (off) {
6293 	case PMCS_SPC_RESET:
6294 	case PMCS_SPC_BOOT_STRAP:
6295 	case PMCS_SPC_DEVICE_ID:
6296 	case PMCS_DEVICE_REVISION:
6297 		off = pmcs_rd_gsm_reg(pwp, off);
6298 		break;
6299 	default:
6300 		off = ddi_get32(pwp->top_acc_handle,
6301 		    &pwp->top_regs[off >> 2]);
6302 		break;
6303 	}
6304 	return (off);
6305 }
6306 
6307 void
6308 pmcs_wr_topunit(pmcs_hw_t *pwp, uint32_t off, uint32_t val)
6309 {
6310 	switch (off) {
6311 	case PMCS_SPC_RESET:
6312 	case PMCS_DEVICE_REVISION:
6313 		pmcs_wr_gsm_reg(pwp, off, val);
6314 		break;
6315 	default:
6316 		ddi_put32(pwp->top_acc_handle, &pwp->top_regs[off >> 2], val);
6317 		break;
6318 	}
6319 }
6320 
6321 uint32_t
6322 pmcs_rd_msgunit(pmcs_hw_t *pwp, uint32_t off)
6323 {
6324 	return (ddi_get32(pwp->msg_acc_handle, &pwp->msg_regs[off >> 2]));
6325 }
6326 
6327 uint32_t
6328 pmcs_rd_mpi_tbl(pmcs_hw_t *pwp, uint32_t off)
6329 {
6330 	return (ddi_get32(pwp->mpi_acc_handle,
6331 	    &pwp->mpi_regs[(pwp->mpi_offset + off) >> 2]));
6332 }
6333 
6334 uint32_t
6335 pmcs_rd_gst_tbl(pmcs_hw_t *pwp, uint32_t off)
6336 {
6337 	return (ddi_get32(pwp->mpi_acc_handle,
6338 	    &pwp->mpi_regs[(pwp->mpi_gst_offset + off) >> 2]));
6339 }
6340 
6341 uint32_t
6342 pmcs_rd_iqc_tbl(pmcs_hw_t *pwp, uint32_t off)
6343 {
6344 	return (ddi_get32(pwp->mpi_acc_handle,
6345 	    &pwp->mpi_regs[(pwp->mpi_iqc_offset + off) >> 2]));
6346 }
6347 
6348 uint32_t
6349 pmcs_rd_oqc_tbl(pmcs_hw_t *pwp, uint32_t off)
6350 {
6351 	return (ddi_get32(pwp->mpi_acc_handle,
6352 	    &pwp->mpi_regs[(pwp->mpi_oqc_offset + off) >> 2]));
6353 }
6354 
6355 uint32_t
6356 pmcs_rd_iqpi(pmcs_hw_t *pwp, uint32_t qnum)
6357 {
6358 	return (ddi_get32(pwp->mpi_acc_handle,
6359 	    &pwp->mpi_regs[pwp->iqpi_offset[qnum] >> 2]));
6360 }
6361 
6362 uint32_t
6363 pmcs_rd_oqci(pmcs_hw_t *pwp, uint32_t qnum)
6364 {
6365 	return (ddi_get32(pwp->mpi_acc_handle,
6366 	    &pwp->mpi_regs[pwp->oqci_offset[qnum] >> 2]));
6367 }
6368 
6369 void
6370 pmcs_wr_msgunit(pmcs_hw_t *pwp, uint32_t off, uint32_t val)
6371 {
6372 	ddi_put32(pwp->msg_acc_handle, &pwp->msg_regs[off >> 2], val);
6373 }
6374 
6375 void
6376 pmcs_wr_mpi_tbl(pmcs_hw_t *pwp, uint32_t off, uint32_t val)
6377 {
6378 	ddi_put32(pwp->mpi_acc_handle,
6379 	    &pwp->mpi_regs[(pwp->mpi_offset + off) >> 2], (val));
6380 }
6381 
6382 void
6383 pmcs_wr_gst_tbl(pmcs_hw_t *pwp, uint32_t off, uint32_t val)
6384 {
6385 	ddi_put32(pwp->mpi_acc_handle,
6386 	    &pwp->mpi_regs[(pwp->mpi_gst_offset + off) >> 2], val);
6387 }
6388 
6389 void
6390 pmcs_wr_iqc_tbl(pmcs_hw_t *pwp, uint32_t off, uint32_t val)
6391 {
6392 	ddi_put32(pwp->mpi_acc_handle,
6393 	    &pwp->mpi_regs[(pwp->mpi_iqc_offset + off) >> 2], val);
6394 }
6395 
6396 void
6397 pmcs_wr_oqc_tbl(pmcs_hw_t *pwp, uint32_t off, uint32_t val)
6398 {
6399 	ddi_put32(pwp->mpi_acc_handle,
6400 	    &pwp->mpi_regs[(pwp->mpi_oqc_offset + off) >> 2], val);
6401 }
6402 
6403 void
6404 pmcs_wr_iqci(pmcs_hw_t *pwp, uint32_t qnum, uint32_t val)
6405 {
6406 	((uint32_t *)((void *)pwp->cip))[IQ_OFFSET(qnum) >> 2] = val;
6407 	if (ddi_dma_sync(pwp->cip_handles, 0, 0, DDI_DMA_SYNC_FORDEV) !=
6408 	    DDI_SUCCESS) {
6409 		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
6410 		    "%s: ddi_dma_sync failed?", __func__);
6411 	}
6412 }
6413 
6414 void
6415 pmcs_wr_iqpi(pmcs_hw_t *pwp, uint32_t qnum, uint32_t val)
6416 {
6417 	ddi_put32(pwp->mpi_acc_handle,
6418 	    &pwp->mpi_regs[pwp->iqpi_offset[qnum] >> 2], val);
6419 }
6420 
6421 void
6422 pmcs_wr_oqci(pmcs_hw_t *pwp, uint32_t qnum, uint32_t val)
6423 {
6424 	ddi_put32(pwp->mpi_acc_handle,
6425 	    &pwp->mpi_regs[pwp->oqci_offset[qnum] >> 2], val);
6426 }
6427 
6428 void
6429 pmcs_wr_oqpi(pmcs_hw_t *pwp, uint32_t qnum, uint32_t val)
6430 {
6431 	((uint32_t *)((void *)pwp->cip))[OQ_OFFSET(qnum) >> 2] = val;
6432 	if (ddi_dma_sync(pwp->cip_handles, 0, 0, DDI_DMA_SYNC_FORDEV) !=
6433 	    DDI_SUCCESS) {
6434 		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
6435 		    "%s: ddi_dma_sync failed?", __func__);
6436 	}
6437 }
6438 
6439 /*
6440  * Check the status value of an outbound IOMB and report anything bad
6441  */
6442 
6443 void
6444 pmcs_check_iomb_status(pmcs_hw_t *pwp, uint32_t *iomb)
6445 {
6446 	uint16_t 	opcode;
6447 	int		offset;
6448 
6449 	if (iomb == NULL) {
6450 		return;
6451 	}
6452 
6453 	opcode = LE_32(iomb[0]) & 0xfff;
6454 
6455 	switch (opcode) {
6456 		/*
6457 		 * The following have no status field, so ignore them
6458 		 */
6459 	case PMCOUT_ECHO:
6460 	case PMCOUT_SAS_HW_EVENT:
6461 	case PMCOUT_GET_DEVICE_HANDLE:
6462 	case PMCOUT_SATA_EVENT:
6463 	case PMCOUT_SSP_EVENT:
6464 	case PMCOUT_DEVICE_HANDLE_ARRIVED:
6465 	case PMCOUT_SMP_REQUEST_RECEIVED:
6466 	case PMCOUT_GPIO:
6467 	case PMCOUT_GPIO_EVENT:
6468 	case PMCOUT_GET_TIME_STAMP:
6469 	case PMCOUT_SKIP_ENTRIES:
6470 	case PMCOUT_GET_NVMD_DATA:	/* Actually lower 16 bits of word 3 */
6471 	case PMCOUT_SET_NVMD_DATA:	/* but ignore - we don't use these */
6472 	case PMCOUT_DEVICE_HANDLE_REMOVED:
6473 	case PMCOUT_SSP_REQUEST_RECEIVED:
6474 		return;
6475 
6476 	case PMCOUT_GENERAL_EVENT:
6477 		offset = 1;
6478 		break;
6479 
6480 	case PMCOUT_SSP_COMPLETION:
6481 	case PMCOUT_SMP_COMPLETION:
6482 	case PMCOUT_DEVICE_REGISTRATION:
6483 	case PMCOUT_DEREGISTER_DEVICE_HANDLE:
6484 	case PMCOUT_SATA_COMPLETION:
6485 	case PMCOUT_DEVICE_INFO:
6486 	case PMCOUT_FW_FLASH_UPDATE:
6487 	case PMCOUT_SSP_ABORT:
6488 	case PMCOUT_SATA_ABORT:
6489 	case PMCOUT_SAS_DIAG_MODE_START_END:
6490 	case PMCOUT_SAS_HW_EVENT_ACK_ACK:
6491 	case PMCOUT_SMP_ABORT:
6492 	case PMCOUT_SET_DEVICE_STATE:
6493 	case PMCOUT_GET_DEVICE_STATE:
6494 	case PMCOUT_SET_DEVICE_INFO:
6495 		offset = 2;
6496 		break;
6497 
6498 	case PMCOUT_LOCAL_PHY_CONTROL:
6499 	case PMCOUT_SAS_DIAG_EXECUTE:
6500 	case PMCOUT_PORT_CONTROL:
6501 		offset = 3;
6502 		break;
6503 
6504 	case PMCOUT_GET_INFO:
6505 	case PMCOUT_GET_VPD:
6506 	case PMCOUT_SAS_ASSISTED_DISCOVERY_EVENT:
6507 	case PMCOUT_SATA_ASSISTED_DISCOVERY_EVENT:
6508 	case PMCOUT_SET_VPD:
6509 	case PMCOUT_TWI:
6510 		pmcs_print_entry(pwp, PMCS_PRT_DEBUG,
6511 		    "Got response for deprecated opcode", iomb);
6512 		return;
6513 
6514 	default:
6515 		pmcs_print_entry(pwp, PMCS_PRT_DEBUG,
6516 		    "Got response for unknown opcode", iomb);
6517 		return;
6518 	}
6519 
6520 	if (LE_32(iomb[offset]) != PMCOUT_STATUS_OK) {
6521 		pmcs_print_entry(pwp, PMCS_PRT_DEBUG,
6522 		    "bad status on TAG_TYPE_NONE command", iomb);
6523 	}
6524 }
6525 
6526 /*
6527  * Called with statlock held
6528  */
6529 void
6530 pmcs_clear_xp(pmcs_hw_t *pwp, pmcs_xscsi_t *xp)
6531 {
6532 	_NOTE(ARGUNUSED(pwp));
6533 
6534 	ASSERT(mutex_owned(&xp->statlock));
6535 
6536 	pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, xp, "%s: Device 0x%p is gone.",
6537 	    __func__, (void *)xp);
6538 
6539 	/*
6540 	 * Clear the dip now.  This keeps pmcs_remove_device from attempting
6541 	 * to call us on the same device while we're still flushing queues.
6542 	 * The only side effect is we can no longer update SM-HBA properties,
6543 	 * but this device is going away anyway, so no matter.
6544 	 */
6545 	xp->dip = NULL;
6546 
6547 	xp->special_running = 0;
6548 	xp->recovering = 0;
6549 	xp->recover_wait = 0;
6550 	xp->draining = 0;
6551 	xp->new = 0;
6552 	xp->assigned = 0;
6553 	xp->dev_state = 0;
6554 	xp->tagmap = 0;
6555 	xp->dev_gone = 1;
6556 	xp->event_recovery = 0;
6557 	xp->dtype = NOTHING;
6558 	xp->wq_recovery_tail = NULL;
6559 	/* Don't clear xp->phy */
6560 	/* Don't clear xp->actv_cnt */
6561 
6562 	/*
6563 	 * Flush all target queues
6564 	 */
6565 	pmcs_flush_target_queues(pwp, xp, PMCS_TGT_ALL_QUEUES);
6566 }
6567 
6568 static int
6569 pmcs_smp_function_result(pmcs_hw_t *pwp, smp_response_frame_t *srf)
6570 {
6571 	int result = srf->srf_result;
6572 
6573 	switch (result) {
6574 	case SMP_RES_UNKNOWN_FUNCTION:
6575 		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
6576 		    "%s: SMP DISCOVER Response "
6577 		    "Function Result: Unknown SMP Function(0x%x)",
6578 		    __func__, result);
6579 		break;
6580 	case SMP_RES_FUNCTION_FAILED:
6581 		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
6582 		    "%s: SMP DISCOVER Response "
6583 		    "Function Result: SMP Function Failed(0x%x)",
6584 		    __func__, result);
6585 		break;
6586 	case SMP_RES_INVALID_REQUEST_FRAME_LENGTH:
6587 		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
6588 		    "%s: SMP DISCOVER Response "
6589 		    "Function Result: Invalid Request Frame Length(0x%x)",
6590 		    __func__, result);
6591 		break;
6592 	case SMP_RES_INCOMPLETE_DESCRIPTOR_LIST:
6593 		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
6594 		    "%s: SMP DISCOVER Response "
6595 		    "Function Result: Incomplete Descriptor List(0x%x)",
6596 		    __func__, result);
6597 		break;
6598 	case SMP_RES_PHY_DOES_NOT_EXIST:
6599 		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
6600 		    "%s: SMP DISCOVER Response "
6601 		    "Function Result: PHY does not exist(0x%x)",
6602 		    __func__, result);
6603 		break;
6604 	case SMP_RES_PHY_VACANT:
6605 		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
6606 		    "%s: SMP DISCOVER Response "
6607 		    "Function Result: PHY Vacant(0x%x)",
6608 		    __func__, result);
6609 		break;
6610 	default:
6611 		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
6612 		    "%s: SMP DISCOVER Response "
6613 		    "Function Result: (0x%x)",
6614 		    __func__, result);
6615 		break;
6616 	}
6617 
6618 	return (result);
6619 }
6620 
6621 /*
6622  * Do all the repetitive stuff necessary to setup for DMA
6623  *
6624  * pwp: Used for dip
6625  * dma_attr: ddi_dma_attr_t to use for the mapping
6626  * acch: ddi_acc_handle_t to use for the mapping
6627  * dmah: ddi_dma_handle_t to use
6628  * length: Amount of memory for mapping
6629  * kvap: Pointer filled in with kernel virtual address on successful return
6630  * dma_addr: Pointer filled in with DMA address on successful return
6631  */
6632 boolean_t
6633 pmcs_dma_setup(pmcs_hw_t *pwp, ddi_dma_attr_t *dma_attr, ddi_acc_handle_t *acch,
6634     ddi_dma_handle_t *dmah, size_t length, caddr_t *kvap, uint64_t *dma_addr)
6635 {
6636 	dev_info_t		*dip = pwp->dip;
6637 	ddi_dma_cookie_t	cookie;
6638 	size_t			real_length;
6639 	uint_t			ddma_flag = DDI_DMA_CONSISTENT;
6640 	uint_t			ddabh_flag = DDI_DMA_CONSISTENT | DDI_DMA_RDWR;
6641 	uint_t			cookie_cnt;
6642 	ddi_device_acc_attr_t	mattr = {
6643 		DDI_DEVICE_ATTR_V0,
6644 		DDI_NEVERSWAP_ACC,
6645 		DDI_STRICTORDER_ACC,
6646 		DDI_DEFAULT_ACC
6647 	};
6648 
6649 	*acch = NULL;
6650 	*dmah = NULL;
6651 
6652 	if (ddi_dma_alloc_handle(dip, dma_attr, DDI_DMA_SLEEP, NULL, dmah) !=
6653 	    DDI_SUCCESS) {
6654 		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
6655 		    "Failed to allocate DMA handle");
6656 		return (B_FALSE);
6657 	}
6658 
6659 	if (ddi_dma_mem_alloc(*dmah, length, &mattr, ddma_flag, DDI_DMA_SLEEP,
6660 	    NULL, kvap, &real_length, acch) != DDI_SUCCESS) {
6661 		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
6662 		    "Failed to allocate DMA mem");
6663 		ddi_dma_free_handle(dmah);
6664 		*dmah = NULL;
6665 		return (B_FALSE);
6666 	}
6667 
6668 	if (ddi_dma_addr_bind_handle(*dmah, NULL, *kvap, real_length,
6669 	    ddabh_flag, DDI_DMA_SLEEP, NULL, &cookie, &cookie_cnt)
6670 	    != DDI_DMA_MAPPED) {
6671 		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL, "Failed to bind DMA");
6672 		ddi_dma_free_handle(dmah);
6673 		ddi_dma_mem_free(acch);
6674 		*dmah = NULL;
6675 		*acch = NULL;
6676 		return (B_FALSE);
6677 	}
6678 
6679 	if (cookie_cnt != 1) {
6680 		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL, "Multiple cookies");
6681 		if (ddi_dma_unbind_handle(*dmah) != DDI_SUCCESS) {
6682 			pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL, "Condition "
6683 			    "failed at %s():%d", __func__, __LINE__);
6684 		}
6685 		ddi_dma_free_handle(dmah);
6686 		ddi_dma_mem_free(acch);
6687 		*dmah = NULL;
6688 		*acch = NULL;
6689 		return (B_FALSE);
6690 	}
6691 
6692 	*dma_addr = cookie.dmac_laddress;
6693 
6694 	return (B_TRUE);
6695 }
6696 
6697 /*
6698  * Flush requested queues for a particular target.  Called with statlock held
6699  */
6700 void
6701 pmcs_flush_target_queues(pmcs_hw_t *pwp, pmcs_xscsi_t *tgt, uint8_t queues)
6702 {
6703 	pmcs_cmd_t	*sp;
6704 	pmcwork_t	*pwrk;
6705 
6706 	ASSERT(pwp != NULL);
6707 	ASSERT(tgt != NULL);
6708 
6709 	pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, tgt,
6710 	    "%s: Flushing queues (%d) for target 0x%p", __func__,
6711 	    queues, (void *)tgt);
6712 
6713 	/*
6714 	 * Commands on the wait queue (or the special queue below) don't have
6715 	 * work structures associated with them.
6716 	 */
6717 	if (queues & PMCS_TGT_WAIT_QUEUE) {
6718 		mutex_enter(&tgt->wqlock);
6719 		while ((sp = STAILQ_FIRST(&tgt->wq)) != NULL) {
6720 			STAILQ_REMOVE(&tgt->wq, sp, pmcs_cmd, cmd_next);
6721 			pmcs_prt(pwp, PMCS_PRT_DEBUG1, NULL, tgt,
6722 			    "%s: Removing cmd 0x%p from wq for target 0x%p",
6723 			    __func__, (void *)sp, (void *)tgt);
6724 			CMD2PKT(sp)->pkt_reason = CMD_DEV_GONE;
6725 			CMD2PKT(sp)->pkt_state = STATE_GOT_BUS;
6726 			mutex_exit(&tgt->wqlock);
6727 			pmcs_dma_unload(pwp, sp);
6728 			mutex_enter(&pwp->cq_lock);
6729 			STAILQ_INSERT_TAIL(&pwp->cq, sp, cmd_next);
6730 			mutex_exit(&pwp->cq_lock);
6731 			mutex_enter(&tgt->wqlock);
6732 		}
6733 		mutex_exit(&tgt->wqlock);
6734 	}
6735 
6736 	/*
6737 	 * Commands on the active queue will have work structures associated
6738 	 * with them.
6739 	 */
6740 	if (queues & PMCS_TGT_ACTIVE_QUEUE) {
6741 		mutex_enter(&tgt->aqlock);
6742 		while ((sp = STAILQ_FIRST(&tgt->aq)) != NULL) {
6743 			STAILQ_REMOVE(&tgt->aq, sp, pmcs_cmd, cmd_next);
6744 			pwrk = pmcs_tag2wp(pwp, sp->cmd_tag);
6745 			mutex_exit(&tgt->aqlock);
6746 			mutex_exit(&tgt->statlock);
6747 			/*
6748 			 * If we found a work structure, mark it as dead
6749 			 * and complete it
6750 			 */
6751 			if (pwrk != NULL) {
6752 				pwrk->dead = 1;
6753 				CMD2PKT(sp)->pkt_reason = CMD_DEV_GONE;
6754 				CMD2PKT(sp)->pkt_state = STATE_GOT_BUS;
6755 				pmcs_complete_work_impl(pwp, pwrk, NULL, 0);
6756 			}
6757 			pmcs_prt(pwp, PMCS_PRT_DEBUG1, NULL, tgt,
6758 			    "%s: Removing cmd 0x%p from aq for target 0x%p",
6759 			    __func__, (void *)sp, (void *)tgt);
6760 			pmcs_dma_unload(pwp, sp);
6761 			mutex_enter(&pwp->cq_lock);
6762 			STAILQ_INSERT_TAIL(&pwp->cq, sp, cmd_next);
6763 			mutex_exit(&pwp->cq_lock);
6764 			mutex_enter(&tgt->aqlock);
6765 			mutex_enter(&tgt->statlock);
6766 		}
6767 		mutex_exit(&tgt->aqlock);
6768 	}
6769 
6770 	if (queues & PMCS_TGT_SPECIAL_QUEUE) {
6771 		while ((sp = STAILQ_FIRST(&tgt->sq)) != NULL) {
6772 			STAILQ_REMOVE(&tgt->sq, sp, pmcs_cmd, cmd_next);
6773 			pmcs_prt(pwp, PMCS_PRT_DEBUG1, NULL, tgt,
6774 			    "%s: Removing cmd 0x%p from sq for target 0x%p",
6775 			    __func__, (void *)sp, (void *)tgt);
6776 			CMD2PKT(sp)->pkt_reason = CMD_DEV_GONE;
6777 			CMD2PKT(sp)->pkt_state = STATE_GOT_BUS;
6778 			pmcs_dma_unload(pwp, sp);
6779 			mutex_enter(&pwp->cq_lock);
6780 			STAILQ_INSERT_TAIL(&pwp->cq, sp, cmd_next);
6781 			mutex_exit(&pwp->cq_lock);
6782 		}
6783 	}
6784 }
6785 
6786 void
6787 pmcs_complete_work_impl(pmcs_hw_t *pwp, pmcwork_t *pwrk, uint32_t *iomb,
6788     size_t amt)
6789 {
6790 	switch (PMCS_TAG_TYPE(pwrk->htag)) {
6791 	case PMCS_TAG_TYPE_CBACK:
6792 	{
6793 		pmcs_cb_t callback = (pmcs_cb_t)pwrk->ptr;
6794 		(*callback)(pwp, pwrk, iomb);
6795 		break;
6796 	}
6797 	case PMCS_TAG_TYPE_WAIT:
6798 		if (pwrk->arg && iomb && amt) {
6799 			(void) memcpy(pwrk->arg, iomb, amt);
6800 		}
6801 		cv_signal(&pwrk->sleep_cv);
6802 		mutex_exit(&pwrk->lock);
6803 		break;
6804 	case PMCS_TAG_TYPE_NONE:
6805 #ifdef DEBUG
6806 		pmcs_check_iomb_status(pwp, iomb);
6807 #endif
6808 		pmcs_pwork(pwp, pwrk);
6809 		break;
6810 	default:
6811 		/*
6812 		 * We will leak a structure here if we don't know
6813 		 * what happened
6814 		 */
6815 		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
6816 		    "%s: Unknown PMCS_TAG_TYPE (%x)",
6817 		    __func__, PMCS_TAG_TYPE(pwrk->htag));
6818 		break;
6819 	}
6820 }
6821 
6822 /*
6823  * Determine if iport still has targets. During detach(9E), if SCSA is
6824  * successfull in its guarantee of tran_tgt_free(9E) before detach(9E),
6825  * this should always return B_FALSE.
6826  */
6827 boolean_t
6828 pmcs_iport_has_targets(pmcs_hw_t *pwp, pmcs_iport_t *iport)
6829 {
6830 	pmcs_xscsi_t *xp;
6831 	int i;
6832 
6833 	mutex_enter(&pwp->lock);
6834 
6835 	if (!pwp->targets || !pwp->max_dev) {
6836 		mutex_exit(&pwp->lock);
6837 		return (B_FALSE);
6838 	}
6839 
6840 	for (i = 0; i < pwp->max_dev; i++) {
6841 		xp = pwp->targets[i];
6842 		if ((xp == NULL) || (xp->phy == NULL) ||
6843 		    (xp->phy->iport != iport)) {
6844 			continue;
6845 		}
6846 
6847 		mutex_exit(&pwp->lock);
6848 		return (B_TRUE);
6849 	}
6850 
6851 	mutex_exit(&pwp->lock);
6852 	return (B_FALSE);
6853 }
6854 
6855 /*
6856  * Called with softstate lock held
6857  */
6858 void
6859 pmcs_destroy_target(pmcs_xscsi_t *target)
6860 {
6861 	pmcs_hw_t *pwp = target->pwp;
6862 	pmcs_iport_t *iport;
6863 
6864 	ASSERT(pwp);
6865 	ASSERT(mutex_owned(&pwp->lock));
6866 
6867 	if (!target->ua) {
6868 		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, target,
6869 		    "%s: target %p iport address is null",
6870 		    __func__, (void *)target);
6871 	}
6872 
6873 	iport = pmcs_get_iport_by_ua(pwp, target->ua);
6874 	if (iport == NULL) {
6875 		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, target,
6876 		    "%s: no iport associated with tgt(0x%p)",
6877 		    __func__, (void *)target);
6878 		return;
6879 	}
6880 
6881 	pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, NULL, target,
6882 	    "%s: free target %p", __func__, (void *)target);
6883 	if (target->ua) {
6884 		strfree(target->ua);
6885 	}
6886 
6887 	mutex_destroy(&target->wqlock);
6888 	mutex_destroy(&target->aqlock);
6889 	mutex_destroy(&target->statlock);
6890 	cv_destroy(&target->reset_cv);
6891 	cv_destroy(&target->abort_cv);
6892 	ddi_soft_state_bystr_fini(&target->lun_sstate);
6893 	ddi_soft_state_bystr_free(iport->tgt_sstate, target->unit_address);
6894 	pmcs_rele_iport(iport);
6895 }
6896 
6897 /*
6898  * pmcs_lock_phy_impl
6899  *
6900  * This function is what does the actual work for pmcs_lock_phy.  It will
6901  * lock all PHYs from phyp down in a top-down fashion.
6902  *
6903  * Locking notes:
6904  * 1. level starts from 0 for the PHY ("parent") that's passed in.  It is
6905  * not a reflection of the actual level of the PHY in the SAS topology.
6906  * 2. If parent is an expander, then parent is locked along with all its
6907  * descendents.
6908  * 3. Expander subsidiary PHYs at level 0 are not locked.  It is the
6909  * responsibility of the caller to individually lock expander subsidiary PHYs
6910  * at level 0 if necessary.
6911  * 4. Siblings at level 0 are not traversed due to the possibility that we're
6912  * locking a PHY on the dead list.  The siblings could be pointing to invalid
6913  * PHYs.  We don't lock siblings at level 0 anyway.
6914  */
6915 static void
6916 pmcs_lock_phy_impl(pmcs_phy_t *phyp, int level)
6917 {
6918 	pmcs_phy_t *tphyp;
6919 
6920 	ASSERT((phyp->dtype == SAS) || (phyp->dtype == SATA) ||
6921 	    (phyp->dtype == EXPANDER) || (phyp->dtype == NOTHING));
6922 
6923 	/*
6924 	 * Start walking the PHYs.
6925 	 */
6926 	tphyp = phyp;
6927 	while (tphyp) {
6928 		/*
6929 		 * If we're at the top level, only lock ourselves.  For anything
6930 		 * at level > 0, traverse children while locking everything.
6931 		 */
6932 		if ((level > 0) || (tphyp == phyp)) {
6933 			pmcs_prt(tphyp->pwp, PMCS_PRT_DEBUG_PHY_LOCKING, tphyp,
6934 			    NULL, "%s: PHY 0x%p parent 0x%p path %s lvl %d",
6935 			    __func__, (void *)tphyp, (void *)tphyp->parent,
6936 			    tphyp->path, level);
6937 			mutex_enter(&tphyp->phy_lock);
6938 
6939 			if (tphyp->children) {
6940 				pmcs_lock_phy_impl(tphyp->children, level + 1);
6941 			}
6942 		}
6943 
6944 		if (level == 0) {
6945 			return;
6946 		}
6947 
6948 		tphyp = tphyp->sibling;
6949 	}
6950 }
6951 
6952 /*
6953  * pmcs_lock_phy
6954  *
6955  * This function is responsible for locking a PHY and all its descendents
6956  */
6957 void
6958 pmcs_lock_phy(pmcs_phy_t *phyp)
6959 {
6960 #ifdef DEBUG
6961 	char *callername = NULL;
6962 	ulong_t off;
6963 
6964 	ASSERT(phyp != NULL);
6965 
6966 	callername = modgetsymname((uintptr_t)caller(), &off);
6967 
6968 	if (callername == NULL) {
6969 		pmcs_prt(phyp->pwp, PMCS_PRT_DEBUG_PHY_LOCKING, phyp, NULL,
6970 		    "%s: PHY 0x%p path %s caller: unknown", __func__,
6971 		    (void *)phyp, phyp->path);
6972 	} else {
6973 		pmcs_prt(phyp->pwp, PMCS_PRT_DEBUG_PHY_LOCKING, phyp, NULL,
6974 		    "%s: PHY 0x%p path %s caller: %s+%lx", __func__,
6975 		    (void *)phyp, phyp->path, callername, off);
6976 	}
6977 #else
6978 	pmcs_prt(phyp->pwp, PMCS_PRT_DEBUG_PHY_LOCKING, phyp, NULL,
6979 	    "%s: PHY 0x%p path %s", __func__, (void *)phyp, phyp->path);
6980 #endif
6981 	pmcs_lock_phy_impl(phyp, 0);
6982 }
6983 
6984 /*
6985  * pmcs_unlock_phy_impl
6986  *
6987  * Unlock all PHYs from phyp down in a bottom-up fashion.
6988  */
6989 static void
6990 pmcs_unlock_phy_impl(pmcs_phy_t *phyp, int level)
6991 {
6992 	pmcs_phy_t *phy_next;
6993 
6994 	ASSERT((phyp->dtype == SAS) || (phyp->dtype == SATA) ||
6995 	    (phyp->dtype == EXPANDER) || (phyp->dtype == NOTHING));
6996 
6997 	/*
6998 	 * Recurse down to the bottom PHYs
6999 	 */
7000 	if (level == 0) {
7001 		if (phyp->children) {
7002 			pmcs_unlock_phy_impl(phyp->children, level + 1);
7003 		}
7004 	} else {
7005 		phy_next = phyp;
7006 		while (phy_next) {
7007 			if (phy_next->children) {
7008 				pmcs_unlock_phy_impl(phy_next->children,
7009 				    level + 1);
7010 			}
7011 			phy_next = phy_next->sibling;
7012 		}
7013 	}
7014 
7015 	/*
7016 	 * Iterate through PHYs unlocking all at level > 0 as well the top PHY
7017 	 */
7018 	phy_next = phyp;
7019 	while (phy_next) {
7020 		if ((level > 0) || (phy_next == phyp)) {
7021 			pmcs_prt(phy_next->pwp, PMCS_PRT_DEBUG_PHY_LOCKING,
7022 			    phy_next, NULL,
7023 			    "%s: PHY 0x%p parent 0x%p path %s lvl %d",
7024 			    __func__, (void *)phy_next,
7025 			    (void *)phy_next->parent, phy_next->path, level);
7026 			mutex_exit(&phy_next->phy_lock);
7027 		}
7028 
7029 		if (level == 0) {
7030 			return;
7031 		}
7032 
7033 		phy_next = phy_next->sibling;
7034 	}
7035 }
7036 
7037 /*
7038  * pmcs_unlock_phy
7039  *
7040  * Unlock a PHY and all its descendents
7041  */
7042 void
7043 pmcs_unlock_phy(pmcs_phy_t *phyp)
7044 {
7045 #ifdef DEBUG
7046 	char *callername = NULL;
7047 	ulong_t off;
7048 
7049 	ASSERT(phyp != NULL);
7050 
7051 	callername = modgetsymname((uintptr_t)caller(), &off);
7052 
7053 	if (callername == NULL) {
7054 		pmcs_prt(phyp->pwp, PMCS_PRT_DEBUG_PHY_LOCKING, phyp, NULL,
7055 		    "%s: PHY 0x%p path %s caller: unknown", __func__,
7056 		    (void *)phyp, phyp->path);
7057 	} else {
7058 		pmcs_prt(phyp->pwp, PMCS_PRT_DEBUG_PHY_LOCKING, phyp, NULL,
7059 		    "%s: PHY 0x%p path %s caller: %s+%lx", __func__,
7060 		    (void *)phyp, phyp->path, callername, off);
7061 	}
7062 #else
7063 	pmcs_prt(phyp->pwp, PMCS_PRT_DEBUG_PHY_LOCKING, phyp, NULL,
7064 	    "%s: PHY 0x%p path %s", __func__, (void *)phyp, phyp->path);
7065 #endif
7066 	pmcs_unlock_phy_impl(phyp, 0);
7067 }
7068 
7069 /*
7070  * pmcs_get_root_phy
7071  *
7072  * For a given phy pointer return its root phy.
7073  * The caller must be holding the lock on every PHY from phyp up to the root.
7074  */
7075 pmcs_phy_t *
7076 pmcs_get_root_phy(pmcs_phy_t *phyp)
7077 {
7078 	ASSERT(phyp);
7079 
7080 	while (phyp) {
7081 		if (IS_ROOT_PHY(phyp)) {
7082 			break;
7083 		}
7084 		phyp = phyp->parent;
7085 	}
7086 
7087 	return (phyp);
7088 }
7089 
7090 /*
7091  * pmcs_free_dma_chunklist
7092  *
7093  * Free DMA S/G chunk list
7094  */
7095 void
7096 pmcs_free_dma_chunklist(pmcs_hw_t *pwp)
7097 {
7098 	pmcs_chunk_t	*pchunk;
7099 
7100 	while (pwp->dma_chunklist) {
7101 		pchunk = pwp->dma_chunklist;
7102 		pwp->dma_chunklist = pwp->dma_chunklist->next;
7103 		if (pchunk->dma_handle) {
7104 			if (ddi_dma_unbind_handle(pchunk->dma_handle) !=
7105 			    DDI_SUCCESS) {
7106 				pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
7107 				    "Condition failed at %s():%d",
7108 				    __func__, __LINE__);
7109 			}
7110 			ddi_dma_free_handle(&pchunk->dma_handle);
7111 			ddi_dma_mem_free(&pchunk->acc_handle);
7112 		}
7113 		kmem_free(pchunk, sizeof (pmcs_chunk_t));
7114 	}
7115 }
7116 
7117 /*ARGSUSED2*/
7118 int
7119 pmcs_phy_constructor(void *buf, void *arg, int kmflags)
7120 {
7121 	pmcs_hw_t *pwp = (pmcs_hw_t *)arg;
7122 	pmcs_phy_t *phyp = (pmcs_phy_t *)buf;
7123 
7124 	mutex_init(&phyp->phy_lock, NULL, MUTEX_DRIVER,
7125 	    DDI_INTR_PRI(pwp->intr_pri));
7126 	cv_init(&phyp->abort_all_cv, NULL, CV_DRIVER, NULL);
7127 	return (0);
7128 }
7129 
7130 /*ARGSUSED1*/
7131 void
7132 pmcs_phy_destructor(void *buf, void *arg)
7133 {
7134 	pmcs_phy_t *phyp = (pmcs_phy_t *)buf;
7135 
7136 	cv_destroy(&phyp->abort_all_cv);
7137 	mutex_destroy(&phyp->phy_lock);
7138 }
7139 
7140 /*
7141  * Free all PHYs from the kmem_cache starting at phyp as well as everything
7142  * on the dead_phys list.
7143  *
7144  * NOTE: This function does not free root PHYs as they are not allocated
7145  * from the kmem_cache.
7146  *
7147  * No PHY locks are acquired as this should only be called during DDI_DETACH
7148  * or soft reset (while pmcs interrupts are disabled).
7149  */
7150 void
7151 pmcs_free_all_phys(pmcs_hw_t *pwp, pmcs_phy_t *phyp)
7152 {
7153 	pmcs_phy_t *tphyp, *nphyp;
7154 
7155 	if (phyp == NULL) {
7156 		return;
7157 	}
7158 
7159 	tphyp = phyp;
7160 	while (tphyp) {
7161 		nphyp = tphyp->sibling;
7162 
7163 		if (tphyp->children) {
7164 			pmcs_free_all_phys(pwp, tphyp->children);
7165 			tphyp->children = NULL;
7166 		}
7167 		if (!IS_ROOT_PHY(tphyp)) {
7168 			kmem_cache_free(pwp->phy_cache, tphyp);
7169 		}
7170 
7171 		tphyp = nphyp;
7172 	}
7173 
7174 	tphyp = pwp->dead_phys;
7175 	while (tphyp) {
7176 		nphyp = tphyp->sibling;
7177 		kmem_cache_free(pwp->phy_cache, tphyp);
7178 		tphyp = nphyp;
7179 	}
7180 	pwp->dead_phys = NULL;
7181 }
7182 
7183 /*
7184  * Free a list of PHYs linked together by the sibling pointer back to the
7185  * kmem cache from whence they came.  This function does not recurse, so the
7186  * caller must ensure there are no children.
7187  */
7188 void
7189 pmcs_free_phys(pmcs_hw_t *pwp, pmcs_phy_t *phyp)
7190 {
7191 	pmcs_phy_t *next_phy;
7192 
7193 	while (phyp) {
7194 		next_phy = phyp->sibling;
7195 		ASSERT(!mutex_owned(&phyp->phy_lock));
7196 		kmem_cache_free(pwp->phy_cache, phyp);
7197 		phyp = next_phy;
7198 	}
7199 }
7200 
7201 /*
7202  * Make a copy of an existing PHY structure.  This is used primarily in
7203  * discovery to compare the contents of an existing PHY with what gets
7204  * reported back by an expander.
7205  *
7206  * This function must not be called from any context where sleeping is
7207  * not possible.
7208  *
7209  * The new PHY is returned unlocked.
7210  */
7211 static pmcs_phy_t *
7212 pmcs_clone_phy(pmcs_phy_t *orig_phy)
7213 {
7214 	pmcs_phy_t *local;
7215 
7216 	local = kmem_cache_alloc(orig_phy->pwp->phy_cache, KM_SLEEP);
7217 
7218 	/*
7219 	 * Go ahead and just copy everything...
7220 	 */
7221 	*local = *orig_phy;
7222 
7223 	/*
7224 	 * But the following must be set appropriately for this copy
7225 	 */
7226 	local->sibling = NULL;
7227 	local->children = NULL;
7228 	mutex_init(&local->phy_lock, NULL, MUTEX_DRIVER,
7229 	    DDI_INTR_PRI(orig_phy->pwp->intr_pri));
7230 
7231 	return (local);
7232 }
7233 
7234 int
7235 pmcs_check_acc_handle(ddi_acc_handle_t handle)
7236 {
7237 	ddi_fm_error_t de;
7238 
7239 	if (handle == NULL) {
7240 		return (DDI_FAILURE);
7241 	}
7242 	ddi_fm_acc_err_get(handle, &de, DDI_FME_VER0);
7243 	return (de.fme_status);
7244 }
7245 
7246 int
7247 pmcs_check_dma_handle(ddi_dma_handle_t handle)
7248 {
7249 	ddi_fm_error_t de;
7250 
7251 	if (handle == NULL) {
7252 		return (DDI_FAILURE);
7253 	}
7254 	ddi_fm_dma_err_get(handle, &de, DDI_FME_VER0);
7255 	return (de.fme_status);
7256 }
7257 
7258 
7259 void
7260 pmcs_fm_ereport(pmcs_hw_t *pwp, char *detail)
7261 {
7262 	uint64_t ena;
7263 	char buf[FM_MAX_CLASS];
7264 
7265 	(void) snprintf(buf, FM_MAX_CLASS, "%s.%s", DDI_FM_DEVICE, detail);
7266 	ena = fm_ena_generate(0, FM_ENA_FMT1);
7267 	if (DDI_FM_EREPORT_CAP(pwp->fm_capabilities)) {
7268 		ddi_fm_ereport_post(pwp->dip, buf, ena, DDI_NOSLEEP,
7269 		    FM_VERSION, DATA_TYPE_UINT8, FM_EREPORT_VERS0, NULL);
7270 	}
7271 }
7272 
7273 int
7274 pmcs_check_acc_dma_handle(pmcs_hw_t *pwp)
7275 {
7276 	pmcs_chunk_t *pchunk;
7277 	int i;
7278 
7279 	/* check all acc & dma handles allocated in attach */
7280 	if ((pmcs_check_acc_handle(pwp->pci_acc_handle) != DDI_SUCCESS) ||
7281 	    (pmcs_check_acc_handle(pwp->msg_acc_handle) != DDI_SUCCESS) ||
7282 	    (pmcs_check_acc_handle(pwp->top_acc_handle) != DDI_SUCCESS) ||
7283 	    (pmcs_check_acc_handle(pwp->mpi_acc_handle) != DDI_SUCCESS) ||
7284 	    (pmcs_check_acc_handle(pwp->gsm_acc_handle) != DDI_SUCCESS)) {
7285 		goto check_failed;
7286 	}
7287 
7288 	for (i = 0; i < PMCS_NIQ; i++) {
7289 		if ((pmcs_check_dma_handle(
7290 		    pwp->iqp_handles[i]) != DDI_SUCCESS) ||
7291 		    (pmcs_check_acc_handle(
7292 		    pwp->iqp_acchdls[i]) != DDI_SUCCESS)) {
7293 			goto check_failed;
7294 		}
7295 	}
7296 
7297 	for (i = 0; i < PMCS_NOQ; i++) {
7298 		if ((pmcs_check_dma_handle(
7299 		    pwp->oqp_handles[i]) != DDI_SUCCESS) ||
7300 		    (pmcs_check_acc_handle(
7301 		    pwp->oqp_acchdls[i]) != DDI_SUCCESS)) {
7302 			goto check_failed;
7303 		}
7304 	}
7305 
7306 	if ((pmcs_check_dma_handle(pwp->cip_handles) != DDI_SUCCESS) ||
7307 	    (pmcs_check_acc_handle(pwp->cip_acchdls) != DDI_SUCCESS)) {
7308 		goto check_failed;
7309 	}
7310 
7311 	if (pwp->fwlog &&
7312 	    ((pmcs_check_dma_handle(pwp->fwlog_hndl) != DDI_SUCCESS) ||
7313 	    (pmcs_check_acc_handle(pwp->fwlog_acchdl) != DDI_SUCCESS))) {
7314 		goto check_failed;
7315 	}
7316 
7317 	if (pwp->regdump_hndl && pwp->regdump_acchdl &&
7318 	    ((pmcs_check_dma_handle(pwp->regdump_hndl) != DDI_SUCCESS) ||
7319 	    (pmcs_check_acc_handle(pwp->regdump_acchdl)
7320 	    != DDI_SUCCESS))) {
7321 		goto check_failed;
7322 	}
7323 
7324 
7325 	pchunk = pwp->dma_chunklist;
7326 	while (pchunk) {
7327 		if ((pmcs_check_acc_handle(pchunk->acc_handle)
7328 		    != DDI_SUCCESS) ||
7329 		    (pmcs_check_dma_handle(pchunk->dma_handle)
7330 		    != DDI_SUCCESS)) {
7331 			goto check_failed;
7332 		}
7333 		pchunk = pchunk->next;
7334 	}
7335 
7336 	return (0);
7337 
7338 check_failed:
7339 
7340 	return (1);
7341 }
7342 
7343 /*
7344  * pmcs_handle_dead_phys
7345  *
7346  * If the PHY has no outstanding work associated with it, remove it from
7347  * the dead PHY list and free it.
7348  *
7349  * If pwp->ds_err_recovering or pwp->configuring is set, don't run.
7350  * This keeps routines that need to submit work to the chip from having to
7351  * hold PHY locks to ensure that PHYs don't disappear while they do their work.
7352  */
7353 void
7354 pmcs_handle_dead_phys(pmcs_hw_t *pwp)
7355 {
7356 	pmcs_phy_t *phyp, *nphyp, *pphyp;
7357 
7358 	mutex_enter(&pwp->lock);
7359 	mutex_enter(&pwp->config_lock);
7360 
7361 	if (pwp->configuring | pwp->ds_err_recovering) {
7362 		mutex_exit(&pwp->config_lock);
7363 		mutex_exit(&pwp->lock);
7364 		return;
7365 	}
7366 
7367 	/*
7368 	 * Check every PHY in the dead PHY list
7369 	 */
7370 	mutex_enter(&pwp->dead_phylist_lock);
7371 	phyp = pwp->dead_phys;
7372 	pphyp = NULL;	/* Set previous PHY to NULL */
7373 
7374 	while (phyp != NULL) {
7375 		pmcs_lock_phy(phyp);
7376 		ASSERT(phyp->dead);
7377 
7378 		nphyp = phyp->dead_next;
7379 
7380 		/*
7381 		 * Check for outstanding work
7382 		 */
7383 		if (phyp->ref_count > 0) {
7384 			pmcs_unlock_phy(phyp);
7385 			pphyp = phyp;	/* This PHY becomes "previous" */
7386 		} else if (phyp->target) {
7387 			pmcs_unlock_phy(phyp);
7388 			pmcs_prt(pwp, PMCS_PRT_DEBUG1, phyp, phyp->target,
7389 			    "%s: Not freeing PHY 0x%p: target 0x%p is not free",
7390 			    __func__, (void *)phyp, (void *)phyp->target);
7391 			pphyp = phyp;
7392 		} else {
7393 			/*
7394 			 * No outstanding work or target references. Remove it
7395 			 * from the list and free it
7396 			 */
7397 			pmcs_prt(pwp, PMCS_PRT_DEBUG, phyp, phyp->target,
7398 			    "%s: Freeing inactive dead PHY 0x%p @ %s "
7399 			    "target = 0x%p", __func__, (void *)phyp,
7400 			    phyp->path, (void *)phyp->target);
7401 			/*
7402 			 * If pphyp is NULL, then phyp was the head of the list,
7403 			 * so just reset the head to nphyp. Otherwise, the
7404 			 * previous PHY will now point to nphyp (the next PHY)
7405 			 */
7406 			if (pphyp == NULL) {
7407 				pwp->dead_phys = nphyp;
7408 			} else {
7409 				pphyp->dead_next = nphyp;
7410 			}
7411 			/*
7412 			 * If the target still points to this PHY, remove
7413 			 * that linkage now.
7414 			 */
7415 			if (phyp->target) {
7416 				mutex_enter(&phyp->target->statlock);
7417 				if (phyp->target->phy == phyp) {
7418 					phyp->target->phy = NULL;
7419 				}
7420 				mutex_exit(&phyp->target->statlock);
7421 			}
7422 			pmcs_unlock_phy(phyp);
7423 			kmem_cache_free(pwp->phy_cache, phyp);
7424 		}
7425 
7426 		phyp = nphyp;
7427 	}
7428 
7429 	mutex_exit(&pwp->dead_phylist_lock);
7430 	mutex_exit(&pwp->config_lock);
7431 	mutex_exit(&pwp->lock);
7432 }
7433 
7434 void
7435 pmcs_inc_phy_ref_count(pmcs_phy_t *phyp)
7436 {
7437 	atomic_inc_32(&phyp->ref_count);
7438 }
7439 
7440 void
7441 pmcs_dec_phy_ref_count(pmcs_phy_t *phyp)
7442 {
7443 	ASSERT(phyp->ref_count != 0);
7444 	atomic_dec_32(&phyp->ref_count);
7445 }
7446 
7447 /*
7448  * pmcs_reap_dead_phy
7449  *
7450  * This function is called from pmcs_new_tport when we have a PHY
7451  * without a target pointer.  It's possible in that case that this PHY
7452  * may have a "brother" on the dead_phys list.  That is, it may be the same as
7453  * this one but with a different root PHY number (e.g. pp05 vs. pp04).  If
7454  * that's the case, update the dead PHY and this new PHY.  If that's not the
7455  * case, we should get a tran_tgt_init on this after it's reported to SCSA.
7456  *
7457  * Called with PHY locked.
7458  */
7459 static void
7460 pmcs_reap_dead_phy(pmcs_phy_t *phyp)
7461 {
7462 	pmcs_hw_t *pwp = phyp->pwp;
7463 	pmcs_phy_t *ctmp;
7464 
7465 	ASSERT(mutex_owned(&phyp->phy_lock));
7466 
7467 	/*
7468 	 * Check the dead PHYs list
7469 	 */
7470 	mutex_enter(&pwp->dead_phylist_lock);
7471 	ctmp = pwp->dead_phys;
7472 	while (ctmp) {
7473 		if ((ctmp->iport != phyp->iport) ||
7474 		    (memcmp((void *)&ctmp->sas_address[0],
7475 		    (void *)&phyp->sas_address[0], 8))) {
7476 			ctmp = ctmp->dead_next;
7477 			continue;
7478 		}
7479 
7480 		/*
7481 		 * Same SAS address on same iport.  Now check to see if
7482 		 * the PHY path is the same with the possible exception
7483 		 * of the root PHY number.
7484 		 * The "5" is the string length of "pp00."
7485 		 */
7486 		if ((strnlen(phyp->path, 5) >= 5) &&
7487 		    (strnlen(ctmp->path, 5) >= 5)) {
7488 			if (memcmp((void *)&phyp->path[5],
7489 			    (void *)&ctmp->path[5],
7490 			    strnlen(phyp->path, 32) - 5) == 0) {
7491 				break;
7492 			}
7493 		}
7494 
7495 		ctmp = ctmp->dead_next;
7496 	}
7497 	mutex_exit(&pwp->dead_phylist_lock);
7498 
7499 	/*
7500 	 * Found a match.  Remove the target linkage and drop the
7501 	 * ref count on the old PHY.  Then, increment the ref count
7502 	 * on the new PHY to compensate.
7503 	 */
7504 	if (ctmp) {
7505 		pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, ctmp, NULL,
7506 		    "%s: Found match in dead PHY list for new PHY %s",
7507 		    __func__, phyp->path);
7508 		if (ctmp->target) {
7509 			/*
7510 			 * If there is a pointer to the target in the dead
7511 			 * PHY, and that PHY's ref_count drops to 0, we can
7512 			 * clear the target linkage now.  If the PHY's
7513 			 * ref_count is > 1, then there may be multiple
7514 			 * LUNs still remaining, so leave the linkage.
7515 			 */
7516 			pmcs_inc_phy_ref_count(phyp);
7517 			pmcs_dec_phy_ref_count(ctmp);
7518 			phyp->target = ctmp->target;
7519 			/*
7520 			 * Update the target's linkage as well
7521 			 */
7522 			mutex_enter(&phyp->target->statlock);
7523 			phyp->target->phy = phyp;
7524 			phyp->target->dtype = phyp->dtype;
7525 			mutex_exit(&phyp->target->statlock);
7526 
7527 			if (ctmp->ref_count == 0) {
7528 				ctmp->target = NULL;
7529 			}
7530 		}
7531 	}
7532 }
7533 
7534 /*
7535  * Called with iport lock held
7536  */
7537 void
7538 pmcs_add_phy_to_iport(pmcs_iport_t *iport, pmcs_phy_t *phyp)
7539 {
7540 	ASSERT(mutex_owned(&iport->lock));
7541 	ASSERT(phyp);
7542 	ASSERT(!list_link_active(&phyp->list_node));
7543 	iport->nphy++;
7544 	list_insert_tail(&iport->phys, phyp);
7545 	pmcs_smhba_add_iport_prop(iport, DATA_TYPE_INT32, PMCS_NUM_PHYS,
7546 	    &iport->nphy);
7547 	mutex_enter(&iport->refcnt_lock);
7548 	iport->refcnt++;
7549 	mutex_exit(&iport->refcnt_lock);
7550 }
7551 
7552 /*
7553  * Called with the iport lock held
7554  */
7555 void
7556 pmcs_remove_phy_from_iport(pmcs_iport_t *iport, pmcs_phy_t *phyp)
7557 {
7558 	pmcs_phy_t *pptr, *next_pptr;
7559 
7560 	ASSERT(mutex_owned(&iport->lock));
7561 
7562 	/*
7563 	 * If phyp is NULL, remove all PHYs from the iport
7564 	 */
7565 	if (phyp == NULL) {
7566 		for (pptr = list_head(&iport->phys); pptr != NULL;
7567 		    pptr = next_pptr) {
7568 			next_pptr = list_next(&iport->phys, pptr);
7569 			mutex_enter(&pptr->phy_lock);
7570 			pptr->iport = NULL;
7571 			mutex_exit(&pptr->phy_lock);
7572 			pmcs_rele_iport(iport);
7573 			list_remove(&iport->phys, pptr);
7574 		}
7575 		iport->nphy = 0;
7576 		return;
7577 	}
7578 
7579 	ASSERT(phyp);
7580 	ASSERT(iport->nphy > 0);
7581 	ASSERT(list_link_active(&phyp->list_node));
7582 	iport->nphy--;
7583 	list_remove(&iport->phys, phyp);
7584 	pmcs_smhba_add_iport_prop(iport, DATA_TYPE_INT32, PMCS_NUM_PHYS,
7585 	    &iport->nphy);
7586 	pmcs_rele_iport(iport);
7587 }
7588 
7589 /*
7590  * This function checks to see if the target pointed to by phyp is still
7591  * correct.  This is done by comparing the target's unit address with the
7592  * SAS address in phyp.
7593  *
7594  * Called with PHY locked and target statlock held
7595  */
7596 static boolean_t
7597 pmcs_phy_target_match(pmcs_phy_t *phyp)
7598 {
7599 	uint64_t wwn;
7600 	char unit_address[PMCS_MAX_UA_SIZE];
7601 	boolean_t rval = B_FALSE;
7602 
7603 	ASSERT(phyp);
7604 	ASSERT(phyp->target);
7605 	ASSERT(mutex_owned(&phyp->phy_lock));
7606 	ASSERT(mutex_owned(&phyp->target->statlock));
7607 
7608 	wwn = pmcs_barray2wwn(phyp->sas_address);
7609 	(void) scsi_wwn_to_wwnstr(wwn, 1, unit_address);
7610 
7611 	if (memcmp((void *)unit_address, (void *)phyp->target->unit_address,
7612 	    strnlen(phyp->target->unit_address, PMCS_MAX_UA_SIZE)) == 0) {
7613 		rval = B_TRUE;
7614 	}
7615 
7616 	return (rval);
7617 }
7618 /*
7619  * Commands used to serialize SMP requests.
7620  *
7621  * The SPC only allows 2 SMP commands per SMP target: 1 cmd pending and 1 cmd
7622  * queued for the same SMP target. If a third SMP cmd is sent to the SPC for an
7623  * SMP target that already has a SMP cmd pending and one queued, then the
7624  * SPC responds with the ERROR_INTERNAL_SMP_RESOURCE response.
7625  *
7626  * Additionally, the SPC has an 8 entry deep cmd queue and the number of SMP
7627  * cmds that can be queued is controlled by the PORT_CONTROL IOMB. The
7628  * SPC default is 1 SMP command/port (iport).  These 2 queued SMP cmds would
7629  * have to be for different SMP targets.  The INTERNAL_SMP_RESOURCE error will
7630  * also be returned if a 2nd SMP cmd is sent to the controller when there is
7631  * already 1 SMP cmd queued for that port or if a 3rd SMP cmd is sent to the
7632  * queue if there are already 2 queued SMP cmds.
7633  */
7634 void
7635 pmcs_smp_acquire(pmcs_iport_t *iport)
7636 {
7637 	ASSERT(iport);
7638 
7639 	if (iport == NULL) {
7640 		pmcs_prt(iport->pwp, PMCS_PRT_DEBUG_IPORT, NULL, NULL,
7641 		    "%s: iport is NULL...", __func__);
7642 		return;
7643 	}
7644 
7645 	mutex_enter(&iport->smp_lock);
7646 	while (iport->smp_active) {
7647 		pmcs_prt(iport->pwp, PMCS_PRT_DEBUG_IPORT, NULL, NULL,
7648 		    "%s: SMP is active on thread 0x%p, waiting", __func__,
7649 		    (void *)iport->smp_active_thread);
7650 		cv_wait(&iport->smp_cv, &iport->smp_lock);
7651 	}
7652 	iport->smp_active = B_TRUE;
7653 	iport->smp_active_thread = curthread;
7654 	pmcs_prt(iport->pwp, PMCS_PRT_DEBUG_IPORT, NULL, NULL,
7655 	    "%s: SMP acquired by thread 0x%p", __func__,
7656 	    (void *)iport->smp_active_thread);
7657 	mutex_exit(&iport->smp_lock);
7658 }
7659 
7660 void
7661 pmcs_smp_release(pmcs_iport_t *iport)
7662 {
7663 	ASSERT(iport);
7664 
7665 	if (iport == NULL) {
7666 		pmcs_prt(iport->pwp, PMCS_PRT_DEBUG_IPORT, NULL, NULL,
7667 		    "%s: iport is NULL...", __func__);
7668 		return;
7669 	}
7670 
7671 	mutex_enter(&iport->smp_lock);
7672 	pmcs_prt(iport->pwp, PMCS_PRT_DEBUG_IPORT, NULL, NULL,
7673 	    "%s: SMP released by thread 0x%p", __func__, (void *)curthread);
7674 	iport->smp_active = B_FALSE;
7675 	iport->smp_active_thread = NULL;
7676 	cv_signal(&iport->smp_cv);
7677 	mutex_exit(&iport->smp_lock);
7678 }
7679