xref: /freebsd/sys/dev/mpr/mpr_sas.c (revision ca420b4ef2ceac00f6c6905252d553a86100ab6a)
1991554f2SKenneth D. Merry /*-
2991554f2SKenneth D. Merry  * Copyright (c) 2009 Yahoo! Inc.
3a2c14879SStephen McConnell  * Copyright (c) 2011-2015 LSI Corp.
47a2a6a1aSStephen McConnell  * Copyright (c) 2013-2016 Avago Technologies
546b23587SKashyap D Desai  * Copyright 2000-2020 Broadcom Inc.
6991554f2SKenneth D. Merry  * All rights reserved.
7991554f2SKenneth D. Merry  *
8991554f2SKenneth D. Merry  * Redistribution and use in source and binary forms, with or without
9991554f2SKenneth D. Merry  * modification, are permitted provided that the following conditions
10991554f2SKenneth D. Merry  * are met:
11991554f2SKenneth D. Merry  * 1. Redistributions of source code must retain the above copyright
12991554f2SKenneth D. Merry  *    notice, this list of conditions and the following disclaimer.
13991554f2SKenneth D. Merry  * 2. Redistributions in binary form must reproduce the above copyright
14991554f2SKenneth D. Merry  *    notice, this list of conditions and the following disclaimer in the
15991554f2SKenneth D. Merry  *    documentation and/or other materials provided with the distribution.
16991554f2SKenneth D. Merry  *
17991554f2SKenneth D. Merry  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18991554f2SKenneth D. Merry  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19991554f2SKenneth D. Merry  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20991554f2SKenneth D. Merry  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
21991554f2SKenneth D. Merry  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22991554f2SKenneth D. Merry  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23991554f2SKenneth D. Merry  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24991554f2SKenneth D. Merry  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25991554f2SKenneth D. Merry  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26991554f2SKenneth D. Merry  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27991554f2SKenneth D. Merry  * SUCH DAMAGE.
28a2c14879SStephen McConnell  *
2946b23587SKashyap D Desai  * Broadcom Inc. (LSI) MPT-Fusion Host Adapter FreeBSD
30a2c14879SStephen McConnell  *
31991554f2SKenneth D. Merry  */
32991554f2SKenneth D. Merry 
33991554f2SKenneth D. Merry #include <sys/cdefs.h>
34991554f2SKenneth D. Merry __FBSDID("$FreeBSD$");
35991554f2SKenneth D. Merry 
36a2c14879SStephen McConnell /* Communications core for Avago Technologies (LSI) MPT3 */
37991554f2SKenneth D. Merry 
38991554f2SKenneth D. Merry /* TODO Move headers to mprvar */
39991554f2SKenneth D. Merry #include <sys/types.h>
40991554f2SKenneth D. Merry #include <sys/param.h>
41991554f2SKenneth D. Merry #include <sys/systm.h>
42991554f2SKenneth D. Merry #include <sys/kernel.h>
43991554f2SKenneth D. Merry #include <sys/selinfo.h>
44991554f2SKenneth D. Merry #include <sys/module.h>
45991554f2SKenneth D. Merry #include <sys/bus.h>
46991554f2SKenneth D. Merry #include <sys/conf.h>
47991554f2SKenneth D. Merry #include <sys/bio.h>
48991554f2SKenneth D. Merry #include <sys/malloc.h>
49991554f2SKenneth D. Merry #include <sys/uio.h>
50991554f2SKenneth D. Merry #include <sys/sysctl.h>
51991554f2SKenneth D. Merry #include <sys/endian.h>
52991554f2SKenneth D. Merry #include <sys/queue.h>
53991554f2SKenneth D. Merry #include <sys/kthread.h>
54991554f2SKenneth D. Merry #include <sys/taskqueue.h>
55991554f2SKenneth D. Merry #include <sys/sbuf.h>
56991554f2SKenneth D. Merry 
57991554f2SKenneth D. Merry #include <machine/bus.h>
58991554f2SKenneth D. Merry #include <machine/resource.h>
59991554f2SKenneth D. Merry #include <sys/rman.h>
60991554f2SKenneth D. Merry 
61991554f2SKenneth D. Merry #include <machine/stdarg.h>
62991554f2SKenneth D. Merry 
63991554f2SKenneth D. Merry #include <cam/cam.h>
64991554f2SKenneth D. Merry #include <cam/cam_ccb.h>
65991554f2SKenneth D. Merry #include <cam/cam_debug.h>
66991554f2SKenneth D. Merry #include <cam/cam_sim.h>
67991554f2SKenneth D. Merry #include <cam/cam_xpt_sim.h>
68991554f2SKenneth D. Merry #include <cam/cam_xpt_periph.h>
69991554f2SKenneth D. Merry #include <cam/cam_periph.h>
70991554f2SKenneth D. Merry #include <cam/scsi/scsi_all.h>
71991554f2SKenneth D. Merry #include <cam/scsi/scsi_message.h>
72991554f2SKenneth D. Merry #include <cam/scsi/smp_all.h>
73991554f2SKenneth D. Merry 
7467feec50SStephen McConnell #include <dev/nvme/nvme.h>
7567feec50SStephen McConnell 
76991554f2SKenneth D. Merry #include <dev/mpr/mpi/mpi2_type.h>
77991554f2SKenneth D. Merry #include <dev/mpr/mpi/mpi2.h>
78991554f2SKenneth D. Merry #include <dev/mpr/mpi/mpi2_ioc.h>
79991554f2SKenneth D. Merry #include <dev/mpr/mpi/mpi2_sas.h>
8067feec50SStephen McConnell #include <dev/mpr/mpi/mpi2_pci.h>
81991554f2SKenneth D. Merry #include <dev/mpr/mpi/mpi2_cnfg.h>
82991554f2SKenneth D. Merry #include <dev/mpr/mpi/mpi2_init.h>
83991554f2SKenneth D. Merry #include <dev/mpr/mpi/mpi2_tool.h>
84991554f2SKenneth D. Merry #include <dev/mpr/mpr_ioctl.h>
85991554f2SKenneth D. Merry #include <dev/mpr/mprvar.h>
86991554f2SKenneth D. Merry #include <dev/mpr/mpr_table.h>
87991554f2SKenneth D. Merry #include <dev/mpr/mpr_sas.h>
88991554f2SKenneth D. Merry 
89991554f2SKenneth D. Merry #define MPRSAS_DISCOVERY_TIMEOUT	20
90991554f2SKenneth D. Merry #define MPRSAS_MAX_DISCOVERY_TIMEOUTS	10 /* 200 seconds */
91991554f2SKenneth D. Merry 
92991554f2SKenneth D. Merry /*
93991554f2SKenneth D. Merry  * static array to check SCSI OpCode for EEDP protection bits
94991554f2SKenneth D. Merry  */
95991554f2SKenneth D. Merry #define	PRO_R MPI2_SCSIIO_EEDPFLAGS_CHECK_REMOVE_OP
96991554f2SKenneth D. Merry #define	PRO_W MPI2_SCSIIO_EEDPFLAGS_INSERT_OP
97991554f2SKenneth D. Merry #define	PRO_V MPI2_SCSIIO_EEDPFLAGS_INSERT_OP
98991554f2SKenneth D. Merry static uint8_t op_code_prot[256] = {
99991554f2SKenneth D. Merry 	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
100991554f2SKenneth D. Merry 	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
101991554f2SKenneth D. Merry 	0, 0, 0, 0, 0, 0, 0, 0, PRO_R, 0, PRO_W, 0, 0, 0, PRO_W, PRO_V,
102991554f2SKenneth D. Merry 	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
103991554f2SKenneth D. Merry 	0, PRO_W, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
104991554f2SKenneth D. Merry 	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
105991554f2SKenneth D. Merry 	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
106991554f2SKenneth D. Merry 	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
107991554f2SKenneth D. Merry 	0, 0, 0, 0, 0, 0, 0, 0, PRO_R, 0, PRO_W, 0, 0, 0, PRO_W, PRO_V,
108991554f2SKenneth D. Merry 	0, 0, 0, PRO_W, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
109991554f2SKenneth D. Merry 	0, 0, 0, 0, 0, 0, 0, 0, PRO_R, 0, PRO_W, 0, 0, 0, PRO_W, PRO_V,
110991554f2SKenneth D. Merry 	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
111991554f2SKenneth D. Merry 	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
112991554f2SKenneth D. Merry 	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
113991554f2SKenneth D. Merry 	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
114991554f2SKenneth D. Merry 	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
115991554f2SKenneth D. Merry };
116991554f2SKenneth D. Merry 
117991554f2SKenneth D. Merry MALLOC_DEFINE(M_MPRSAS, "MPRSAS", "MPR SAS memory");
118991554f2SKenneth D. Merry 
119991554f2SKenneth D. Merry static void mprsas_remove_device(struct mpr_softc *, struct mpr_command *);
120991554f2SKenneth D. Merry static void mprsas_remove_complete(struct mpr_softc *, struct mpr_command *);
121991554f2SKenneth D. Merry static void mprsas_action(struct cam_sim *sim, union ccb *ccb);
122991554f2SKenneth D. Merry static void mprsas_poll(struct cam_sim *sim);
123991554f2SKenneth D. Merry static void mprsas_scsiio_timeout(void *data);
1247a2a6a1aSStephen McConnell static void mprsas_abort_complete(struct mpr_softc *sc, struct mpr_command *cm);
125991554f2SKenneth D. Merry static void mprsas_action_scsiio(struct mprsas_softc *, union ccb *);
126991554f2SKenneth D. Merry static void mprsas_scsiio_complete(struct mpr_softc *, struct mpr_command *);
127991554f2SKenneth D. Merry static void mprsas_action_resetdev(struct mprsas_softc *, union ccb *);
1287a2a6a1aSStephen McConnell static void mprsas_resetdev_complete(struct mpr_softc *, struct mpr_command *);
129991554f2SKenneth D. Merry static int mprsas_send_abort(struct mpr_softc *sc, struct mpr_command *tm,
130991554f2SKenneth D. Merry     struct mpr_command *cm);
131991554f2SKenneth D. Merry static void mprsas_async(void *callback_arg, uint32_t code,
132991554f2SKenneth D. Merry     struct cam_path *path, void *arg);
133991554f2SKenneth D. Merry static int mprsas_send_portenable(struct mpr_softc *sc);
134991554f2SKenneth D. Merry static void mprsas_portenable_complete(struct mpr_softc *sc,
135991554f2SKenneth D. Merry     struct mpr_command *cm);
136991554f2SKenneth D. Merry 
1377a2a6a1aSStephen McConnell static void mprsas_smpio_complete(struct mpr_softc *sc, struct mpr_command *cm);
1387a2a6a1aSStephen McConnell static void mprsas_send_smpcmd(struct mprsas_softc *sassc, union ccb *ccb,
1397a2a6a1aSStephen McConnell     uint64_t sasaddr);
140a2c14879SStephen McConnell static void mprsas_action_smpio(struct mprsas_softc *sassc, union ccb *ccb);
141991554f2SKenneth D. Merry 
142991554f2SKenneth D. Merry struct mprsas_target *
143991554f2SKenneth D. Merry mprsas_find_target_by_handle(struct mprsas_softc *sassc, int start,
144991554f2SKenneth D. Merry     uint16_t handle)
145991554f2SKenneth D. Merry {
146991554f2SKenneth D. Merry 	struct mprsas_target *target;
147991554f2SKenneth D. Merry 	int i;
148991554f2SKenneth D. Merry 
149991554f2SKenneth D. Merry 	for (i = start; i < sassc->maxtargets; i++) {
150991554f2SKenneth D. Merry 		target = &sassc->targets[i];
151991554f2SKenneth D. Merry 		if (target->handle == handle)
152991554f2SKenneth D. Merry 			return (target);
153991554f2SKenneth D. Merry 	}
154991554f2SKenneth D. Merry 
155991554f2SKenneth D. Merry 	return (NULL);
156991554f2SKenneth D. Merry }
157991554f2SKenneth D. Merry 
158991554f2SKenneth D. Merry /* we need to freeze the simq during attach and diag reset, to avoid failing
159991554f2SKenneth D. Merry  * commands before device handles have been found by discovery.  Since
160991554f2SKenneth D. Merry  * discovery involves reading config pages and possibly sending commands,
161991554f2SKenneth D. Merry  * discovery actions may continue even after we receive the end of discovery
162991554f2SKenneth D. Merry  * event, so refcount discovery actions instead of assuming we can unfreeze
163991554f2SKenneth D. Merry  * the simq when we get the event.
164991554f2SKenneth D. Merry  */
165991554f2SKenneth D. Merry void
166991554f2SKenneth D. Merry mprsas_startup_increment(struct mprsas_softc *sassc)
167991554f2SKenneth D. Merry {
168991554f2SKenneth D. Merry 	MPR_FUNCTRACE(sassc->sc);
169991554f2SKenneth D. Merry 
170991554f2SKenneth D. Merry 	if ((sassc->flags & MPRSAS_IN_STARTUP) != 0) {
171991554f2SKenneth D. Merry 		if (sassc->startup_refcount++ == 0) {
172991554f2SKenneth D. Merry 			/* just starting, freeze the simq */
173991554f2SKenneth D. Merry 			mpr_dprint(sassc->sc, MPR_INIT,
174991554f2SKenneth D. Merry 			    "%s freezing simq\n", __func__);
175991554f2SKenneth D. Merry 			xpt_hold_boot();
176991554f2SKenneth D. Merry 			xpt_freeze_simq(sassc->sim, 1);
177991554f2SKenneth D. Merry 		}
178991554f2SKenneth D. Merry 		mpr_dprint(sassc->sc, MPR_INIT, "%s refcount %u\n", __func__,
179991554f2SKenneth D. Merry 		    sassc->startup_refcount);
180991554f2SKenneth D. Merry 	}
181991554f2SKenneth D. Merry }
182991554f2SKenneth D. Merry 
183991554f2SKenneth D. Merry void
184991554f2SKenneth D. Merry mprsas_release_simq_reinit(struct mprsas_softc *sassc)
185991554f2SKenneth D. Merry {
186991554f2SKenneth D. Merry 	if (sassc->flags & MPRSAS_QUEUE_FROZEN) {
187991554f2SKenneth D. Merry 		sassc->flags &= ~MPRSAS_QUEUE_FROZEN;
188991554f2SKenneth D. Merry 		xpt_release_simq(sassc->sim, 1);
189991554f2SKenneth D. Merry 		mpr_dprint(sassc->sc, MPR_INFO, "Unfreezing SIM queue\n");
190991554f2SKenneth D. Merry 	}
191991554f2SKenneth D. Merry }
192991554f2SKenneth D. Merry 
193991554f2SKenneth D. Merry void
194991554f2SKenneth D. Merry mprsas_startup_decrement(struct mprsas_softc *sassc)
195991554f2SKenneth D. Merry {
196991554f2SKenneth D. Merry 	MPR_FUNCTRACE(sassc->sc);
197991554f2SKenneth D. Merry 
198991554f2SKenneth D. Merry 	if ((sassc->flags & MPRSAS_IN_STARTUP) != 0) {
199991554f2SKenneth D. Merry 		if (--sassc->startup_refcount == 0) {
200991554f2SKenneth D. Merry 			/* finished all discovery-related actions, release
201991554f2SKenneth D. Merry 			 * the simq and rescan for the latest topology.
202991554f2SKenneth D. Merry 			 */
203991554f2SKenneth D. Merry 			mpr_dprint(sassc->sc, MPR_INIT,
204991554f2SKenneth D. Merry 			    "%s releasing simq\n", __func__);
205991554f2SKenneth D. Merry 			sassc->flags &= ~MPRSAS_IN_STARTUP;
206991554f2SKenneth D. Merry 			xpt_release_simq(sassc->sim, 1);
207991554f2SKenneth D. Merry 			xpt_release_boot();
208991554f2SKenneth D. Merry 		}
209991554f2SKenneth D. Merry 		mpr_dprint(sassc->sc, MPR_INIT, "%s refcount %u\n", __func__,
210991554f2SKenneth D. Merry 		    sassc->startup_refcount);
211991554f2SKenneth D. Merry 	}
212991554f2SKenneth D. Merry }
213991554f2SKenneth D. Merry 
214b7f1ee79SScott Long /*
215b7f1ee79SScott Long  * The firmware requires us to stop sending commands when we're doing task
216b7f1ee79SScott Long  * management.
217991554f2SKenneth D. Merry  * use.
218b7f1ee79SScott Long  * XXX The logic for serializing the device has been made lazy and moved to
219b7f1ee79SScott Long  * mprsas_prepare_for_tm().
220991554f2SKenneth D. Merry  */
221991554f2SKenneth D. Merry struct mpr_command *
222991554f2SKenneth D. Merry mprsas_alloc_tm(struct mpr_softc *sc)
223991554f2SKenneth D. Merry {
22446b9415fSScott Long 	MPI2_SCSI_TASK_MANAGE_REQUEST *req;
225991554f2SKenneth D. Merry 	struct mpr_command *tm;
226991554f2SKenneth D. Merry 
227991554f2SKenneth D. Merry 	MPR_FUNCTRACE(sc);
228991554f2SKenneth D. Merry 	tm = mpr_alloc_high_priority_command(sc);
22946b9415fSScott Long 	if (tm == NULL)
23046b9415fSScott Long 		return (NULL);
23146b9415fSScott Long 
23246b9415fSScott Long 	req = (MPI2_SCSI_TASK_MANAGE_REQUEST *)tm->cm_req;
23346b9415fSScott Long 	req->Function = MPI2_FUNCTION_SCSI_TASK_MGMT;
234991554f2SKenneth D. Merry 	return tm;
235991554f2SKenneth D. Merry }
236991554f2SKenneth D. Merry 
237991554f2SKenneth D. Merry void
238991554f2SKenneth D. Merry mprsas_free_tm(struct mpr_softc *sc, struct mpr_command *tm)
239991554f2SKenneth D. Merry {
24088619392SStephen McConnell 
241a2c14879SStephen McConnell 	MPR_FUNCTRACE(sc);
242991554f2SKenneth D. Merry 	if (tm == NULL)
243991554f2SKenneth D. Merry 		return;
244991554f2SKenneth D. Merry 
245a2c14879SStephen McConnell 	/*
246a2c14879SStephen McConnell 	 * For TM's the devq is frozen for the device.  Unfreeze it here and
247a2c14879SStephen McConnell 	 * free the resources used for freezing the devq.  Must clear the
248a2c14879SStephen McConnell 	 * INRESET flag as well or scsi I/O will not work.
249991554f2SKenneth D. Merry 	 */
250a2c14879SStephen McConnell 	if (tm->cm_ccb) {
251a8837c77SWarner Losh 		mpr_dprint(sc, MPR_XINFO | MPR_RECOVERY,
252a8837c77SWarner Losh 		    "Unfreezing devq for target ID %d\n",
2539781c28cSAlexander Motin 		    tm->cm_targ->tid);
2549781c28cSAlexander Motin 		tm->cm_targ->flags &= ~MPRSAS_TARGET_INRESET;
255a2c14879SStephen McConnell 		xpt_release_devq(tm->cm_ccb->ccb_h.path, 1, TRUE);
256a2c14879SStephen McConnell 		xpt_free_path(tm->cm_ccb->ccb_h.path);
257a2c14879SStephen McConnell 		xpt_free_ccb(tm->cm_ccb);
258a2c14879SStephen McConnell 	}
259991554f2SKenneth D. Merry 
260991554f2SKenneth D. Merry 	mpr_free_high_priority_command(sc, tm);
261991554f2SKenneth D. Merry }
262991554f2SKenneth D. Merry 
263991554f2SKenneth D. Merry void
264991554f2SKenneth D. Merry mprsas_rescan_target(struct mpr_softc *sc, struct mprsas_target *targ)
265991554f2SKenneth D. Merry {
266991554f2SKenneth D. Merry 	struct mprsas_softc *sassc = sc->sassc;
267991554f2SKenneth D. Merry 	path_id_t pathid;
268991554f2SKenneth D. Merry 	target_id_t targetid;
269991554f2SKenneth D. Merry 	union ccb *ccb;
270991554f2SKenneth D. Merry 
271991554f2SKenneth D. Merry 	MPR_FUNCTRACE(sc);
272991554f2SKenneth D. Merry 	pathid = cam_sim_path(sassc->sim);
273991554f2SKenneth D. Merry 	if (targ == NULL)
274991554f2SKenneth D. Merry 		targetid = CAM_TARGET_WILDCARD;
275991554f2SKenneth D. Merry 	else
276991554f2SKenneth D. Merry 		targetid = targ - sassc->targets;
277991554f2SKenneth D. Merry 
278991554f2SKenneth D. Merry 	/*
279991554f2SKenneth D. Merry 	 * Allocate a CCB and schedule a rescan.
280991554f2SKenneth D. Merry 	 */
281991554f2SKenneth D. Merry 	ccb = xpt_alloc_ccb_nowait();
282991554f2SKenneth D. Merry 	if (ccb == NULL) {
283991554f2SKenneth D. Merry 		mpr_dprint(sc, MPR_ERROR, "unable to alloc CCB for rescan\n");
284991554f2SKenneth D. Merry 		return;
285991554f2SKenneth D. Merry 	}
286991554f2SKenneth D. Merry 
287a2c14879SStephen McConnell 	if (xpt_create_path(&ccb->ccb_h.path, NULL, pathid, targetid,
288a2c14879SStephen McConnell 	    CAM_LUN_WILDCARD) != CAM_REQ_CMP) {
289991554f2SKenneth D. Merry 		mpr_dprint(sc, MPR_ERROR, "unable to create path for rescan\n");
290991554f2SKenneth D. Merry 		xpt_free_ccb(ccb);
291991554f2SKenneth D. Merry 		return;
292991554f2SKenneth D. Merry 	}
293991554f2SKenneth D. Merry 
294991554f2SKenneth D. Merry 	if (targetid == CAM_TARGET_WILDCARD)
295991554f2SKenneth D. Merry 		ccb->ccb_h.func_code = XPT_SCAN_BUS;
296991554f2SKenneth D. Merry 	else
297991554f2SKenneth D. Merry 		ccb->ccb_h.func_code = XPT_SCAN_TGT;
298991554f2SKenneth D. Merry 
299991554f2SKenneth D. Merry 	mpr_dprint(sc, MPR_TRACE, "%s targetid %u\n", __func__, targetid);
300991554f2SKenneth D. Merry 	xpt_rescan(ccb);
301991554f2SKenneth D. Merry }
302991554f2SKenneth D. Merry 
303991554f2SKenneth D. Merry static void
304991554f2SKenneth D. Merry mprsas_log_command(struct mpr_command *cm, u_int level, const char *fmt, ...)
305991554f2SKenneth D. Merry {
306991554f2SKenneth D. Merry 	struct sbuf sb;
307991554f2SKenneth D. Merry 	va_list ap;
308a2386b6fSAlexander Motin 	char str[224];
309991554f2SKenneth D. Merry 	char path_str[64];
310991554f2SKenneth D. Merry 
311991554f2SKenneth D. Merry 	if (cm == NULL)
312991554f2SKenneth D. Merry 		return;
313991554f2SKenneth D. Merry 
314991554f2SKenneth D. Merry 	/* No need to be in here if debugging isn't enabled */
315991554f2SKenneth D. Merry 	if ((cm->cm_sc->mpr_debug & level) == 0)
316991554f2SKenneth D. Merry 		return;
317991554f2SKenneth D. Merry 
318991554f2SKenneth D. Merry 	sbuf_new(&sb, str, sizeof(str), 0);
319991554f2SKenneth D. Merry 
320991554f2SKenneth D. Merry 	va_start(ap, fmt);
321991554f2SKenneth D. Merry 
322991554f2SKenneth D. Merry 	if (cm->cm_ccb != NULL) {
323991554f2SKenneth D. Merry 		xpt_path_string(cm->cm_ccb->csio.ccb_h.path, path_str,
324991554f2SKenneth D. Merry 		    sizeof(path_str));
325991554f2SKenneth D. Merry 		sbuf_cat(&sb, path_str);
326991554f2SKenneth D. Merry 		if (cm->cm_ccb->ccb_h.func_code == XPT_SCSI_IO) {
327991554f2SKenneth D. Merry 			scsi_command_string(&cm->cm_ccb->csio, &sb);
328991554f2SKenneth D. Merry 			sbuf_printf(&sb, "length %d ",
329991554f2SKenneth D. Merry 			    cm->cm_ccb->csio.dxfer_len);
330991554f2SKenneth D. Merry 		}
331991554f2SKenneth D. Merry 	} else {
332991554f2SKenneth D. Merry 		sbuf_printf(&sb, "(noperiph:%s%d:%u:%u:%u): ",
333991554f2SKenneth D. Merry 		    cam_sim_name(cm->cm_sc->sassc->sim),
334991554f2SKenneth D. Merry 		    cam_sim_unit(cm->cm_sc->sassc->sim),
335991554f2SKenneth D. Merry 		    cam_sim_bus(cm->cm_sc->sassc->sim),
336991554f2SKenneth D. Merry 		    cm->cm_targ ? cm->cm_targ->tid : 0xFFFFFFFF,
337991554f2SKenneth D. Merry 		    cm->cm_lun);
338991554f2SKenneth D. Merry 	}
339991554f2SKenneth D. Merry 
340991554f2SKenneth D. Merry 	sbuf_printf(&sb, "SMID %u ", cm->cm_desc.Default.SMID);
341991554f2SKenneth D. Merry 	sbuf_vprintf(&sb, fmt, ap);
342991554f2SKenneth D. Merry 	sbuf_finish(&sb);
343c11c484fSScott Long 	mpr_print_field(cm->cm_sc, "%s", sbuf_data(&sb));
344991554f2SKenneth D. Merry 
345991554f2SKenneth D. Merry 	va_end(ap);
346991554f2SKenneth D. Merry }
347991554f2SKenneth D. Merry 
348991554f2SKenneth D. Merry static void
349991554f2SKenneth D. Merry mprsas_remove_volume(struct mpr_softc *sc, struct mpr_command *tm)
350991554f2SKenneth D. Merry {
351991554f2SKenneth D. Merry 	MPI2_SCSI_TASK_MANAGE_REPLY *reply;
352991554f2SKenneth D. Merry 	struct mprsas_target *targ;
353991554f2SKenneth D. Merry 	uint16_t handle;
354991554f2SKenneth D. Merry 
355991554f2SKenneth D. Merry 	MPR_FUNCTRACE(sc);
356991554f2SKenneth D. Merry 
357991554f2SKenneth D. Merry 	reply = (MPI2_SCSI_TASK_MANAGE_REPLY *)tm->cm_reply;
358991554f2SKenneth D. Merry 	handle = (uint16_t)(uintptr_t)tm->cm_complete_data;
359991554f2SKenneth D. Merry 	targ = tm->cm_targ;
360991554f2SKenneth D. Merry 
361991554f2SKenneth D. Merry 	if (reply == NULL) {
362991554f2SKenneth D. Merry 		/* XXX retry the remove after the diag reset completes? */
363991554f2SKenneth D. Merry 		mpr_dprint(sc, MPR_FAULT, "%s NULL reply resetting device "
364991554f2SKenneth D. Merry 		    "0x%04x\n", __func__, handle);
365991554f2SKenneth D. Merry 		mprsas_free_tm(sc, tm);
366991554f2SKenneth D. Merry 		return;
367991554f2SKenneth D. Merry 	}
368991554f2SKenneth D. Merry 
369d3f6eabfSStephen McConnell 	if ((le16toh(reply->IOCStatus) & MPI2_IOCSTATUS_MASK) !=
370d3f6eabfSStephen McConnell 	    MPI2_IOCSTATUS_SUCCESS) {
37158581c13SStephen McConnell 		mpr_dprint(sc, MPR_ERROR, "IOCStatus = 0x%x while resetting "
372d3f6eabfSStephen McConnell 		    "device 0x%x\n", le16toh(reply->IOCStatus), handle);
373991554f2SKenneth D. Merry 	}
374991554f2SKenneth D. Merry 
375991554f2SKenneth D. Merry 	mpr_dprint(sc, MPR_XINFO, "Reset aborted %u commands\n",
376d3f6eabfSStephen McConnell 	    le32toh(reply->TerminationCount));
377991554f2SKenneth D. Merry 	mpr_free_reply(sc, tm->cm_reply_data);
378991554f2SKenneth D. Merry 	tm->cm_reply = NULL;	/* Ensures the reply won't get re-freed */
379991554f2SKenneth D. Merry 
380991554f2SKenneth D. Merry 	mpr_dprint(sc, MPR_XINFO, "clearing target %u handle 0x%04x\n",
381991554f2SKenneth D. Merry 	    targ->tid, handle);
382991554f2SKenneth D. Merry 
383991554f2SKenneth D. Merry 	/*
384991554f2SKenneth D. Merry 	 * Don't clear target if remove fails because things will get confusing.
385991554f2SKenneth D. Merry 	 * Leave the devname and sasaddr intact so that we know to avoid reusing
386991554f2SKenneth D. Merry 	 * this target id if possible, and so we can assign the same target id
387991554f2SKenneth D. Merry 	 * to this device if it comes back in the future.
388991554f2SKenneth D. Merry 	 */
389d3f6eabfSStephen McConnell 	if ((le16toh(reply->IOCStatus) & MPI2_IOCSTATUS_MASK) ==
390d3f6eabfSStephen McConnell 	    MPI2_IOCSTATUS_SUCCESS) {
391991554f2SKenneth D. Merry 		targ = tm->cm_targ;
392991554f2SKenneth D. Merry 		targ->handle = 0x0;
393991554f2SKenneth D. Merry 		targ->encl_handle = 0x0;
394991554f2SKenneth D. Merry 		targ->encl_level_valid = 0x0;
395991554f2SKenneth D. Merry 		targ->encl_level = 0x0;
396991554f2SKenneth D. Merry 		targ->connector_name[0] = ' ';
397991554f2SKenneth D. Merry 		targ->connector_name[1] = ' ';
398991554f2SKenneth D. Merry 		targ->connector_name[2] = ' ';
399991554f2SKenneth D. Merry 		targ->connector_name[3] = ' ';
400991554f2SKenneth D. Merry 		targ->encl_slot = 0x0;
401991554f2SKenneth D. Merry 		targ->exp_dev_handle = 0x0;
402991554f2SKenneth D. Merry 		targ->phy_num = 0x0;
403991554f2SKenneth D. Merry 		targ->linkrate = 0x0;
404991554f2SKenneth D. Merry 		targ->devinfo = 0x0;
405991554f2SKenneth D. Merry 		targ->flags = 0x0;
406991554f2SKenneth D. Merry 		targ->scsi_req_desc_type = 0;
407991554f2SKenneth D. Merry 	}
408991554f2SKenneth D. Merry 
409991554f2SKenneth D. Merry 	mprsas_free_tm(sc, tm);
410991554f2SKenneth D. Merry }
411991554f2SKenneth D. Merry 
412991554f2SKenneth D. Merry /*
413e3c5965cSAlexander Motin  * Retry mprsas_prepare_remove() if some previous attempt failed to allocate
414e3c5965cSAlexander Motin  * high priority command due to limit reached.
415e3c5965cSAlexander Motin  */
416e3c5965cSAlexander Motin void
417e3c5965cSAlexander Motin mprsas_prepare_remove_retry(struct mprsas_softc *sassc)
418e3c5965cSAlexander Motin {
419e3c5965cSAlexander Motin 	struct mprsas_target *target;
420e3c5965cSAlexander Motin 	int i;
421e3c5965cSAlexander Motin 
422e3c5965cSAlexander Motin 	if ((sassc->flags & MPRSAS_TOREMOVE) == 0)
423e3c5965cSAlexander Motin 		return;
424e3c5965cSAlexander Motin 
425e3c5965cSAlexander Motin 	for (i = 0; i < sassc->maxtargets; i++) {
426e3c5965cSAlexander Motin 		target = &sassc->targets[i];
427e3c5965cSAlexander Motin 		if ((target->flags & MPRSAS_TARGET_TOREMOVE) == 0)
428e3c5965cSAlexander Motin 			continue;
429e3c5965cSAlexander Motin 		if (TAILQ_EMPTY(&sassc->sc->high_priority_req_list))
430e3c5965cSAlexander Motin 			return;
431e3c5965cSAlexander Motin 		target->flags &= ~MPRSAS_TARGET_TOREMOVE;
432e3c5965cSAlexander Motin 		if (target->flags & MPR_TARGET_FLAGS_VOLUME)
433e3c5965cSAlexander Motin 			mprsas_prepare_volume_remove(sassc, target->handle);
434e3c5965cSAlexander Motin 		else
435e3c5965cSAlexander Motin 			mprsas_prepare_remove(sassc, target->handle);
436e3c5965cSAlexander Motin 	}
437e3c5965cSAlexander Motin 	sassc->flags &= ~MPRSAS_TOREMOVE;
438e3c5965cSAlexander Motin }
439e3c5965cSAlexander Motin 
440e3c5965cSAlexander Motin /*
441991554f2SKenneth D. Merry  * No Need to call "MPI2_SAS_OP_REMOVE_DEVICE" For Volume removal.
442991554f2SKenneth D. Merry  * Otherwise Volume Delete is same as Bare Drive Removal.
443991554f2SKenneth D. Merry  */
444991554f2SKenneth D. Merry void
445991554f2SKenneth D. Merry mprsas_prepare_volume_remove(struct mprsas_softc *sassc, uint16_t handle)
446991554f2SKenneth D. Merry {
447991554f2SKenneth D. Merry 	MPI2_SCSI_TASK_MANAGE_REQUEST *req;
448991554f2SKenneth D. Merry 	struct mpr_softc *sc;
449991554f2SKenneth D. Merry 	struct mpr_command *cm;
450991554f2SKenneth D. Merry 	struct mprsas_target *targ = NULL;
451991554f2SKenneth D. Merry 
452991554f2SKenneth D. Merry 	MPR_FUNCTRACE(sassc->sc);
453991554f2SKenneth D. Merry 	sc = sassc->sc;
454991554f2SKenneth D. Merry 
455991554f2SKenneth D. Merry 	targ = mprsas_find_target_by_handle(sassc, 0, handle);
456991554f2SKenneth D. Merry 	if (targ == NULL) {
457991554f2SKenneth D. Merry 		/* FIXME: what is the action? */
458991554f2SKenneth D. Merry 		/* We don't know about this device? */
459991554f2SKenneth D. Merry 		mpr_dprint(sc, MPR_ERROR,
460991554f2SKenneth D. Merry 		   "%s %d : invalid handle 0x%x \n", __func__,__LINE__, handle);
461991554f2SKenneth D. Merry 		return;
462991554f2SKenneth D. Merry 	}
463991554f2SKenneth D. Merry 
464991554f2SKenneth D. Merry 	targ->flags |= MPRSAS_TARGET_INREMOVAL;
465991554f2SKenneth D. Merry 
466991554f2SKenneth D. Merry 	cm = mprsas_alloc_tm(sc);
467991554f2SKenneth D. Merry 	if (cm == NULL) {
468e3c5965cSAlexander Motin 		targ->flags |= MPRSAS_TARGET_TOREMOVE;
469e3c5965cSAlexander Motin 		sassc->flags |= MPRSAS_TOREMOVE;
470991554f2SKenneth D. Merry 		return;
471991554f2SKenneth D. Merry 	}
472991554f2SKenneth D. Merry 
473991554f2SKenneth D. Merry 	mprsas_rescan_target(sc, targ);
474991554f2SKenneth D. Merry 
475991554f2SKenneth D. Merry 	req = (MPI2_SCSI_TASK_MANAGE_REQUEST *)cm->cm_req;
476991554f2SKenneth D. Merry 	req->DevHandle = targ->handle;
477991554f2SKenneth D. Merry 	req->TaskType = MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET;
478991554f2SKenneth D. Merry 
47989d1c21fSKashyap D Desai 	if (!targ->is_nvme || sc->custom_nvme_tm_handling) {
480991554f2SKenneth D. Merry 		/* SAS Hard Link Reset / SATA Link Reset */
481991554f2SKenneth D. Merry 		req->MsgFlags = MPI2_SCSITASKMGMT_MSGFLAGS_LINK_RESET;
48289d1c21fSKashyap D Desai 	} else {
48389d1c21fSKashyap D Desai 		/* PCIe Protocol Level Reset*/
48489d1c21fSKashyap D Desai 		req->MsgFlags =
48589d1c21fSKashyap D Desai 		    MPI26_SCSITASKMGMT_MSGFLAGS_PROTOCOL_LVL_RST_PCIE;
48689d1c21fSKashyap D Desai 	}
487991554f2SKenneth D. Merry 
488991554f2SKenneth D. Merry 	cm->cm_targ = targ;
489991554f2SKenneth D. Merry 	cm->cm_data = NULL;
490991554f2SKenneth D. Merry 	cm->cm_complete = mprsas_remove_volume;
491991554f2SKenneth D. Merry 	cm->cm_complete_data = (void *)(uintptr_t)handle;
492a2c14879SStephen McConnell 
493a2c14879SStephen McConnell 	mpr_dprint(sc, MPR_INFO, "%s: Sending reset for target ID %d\n",
494a2c14879SStephen McConnell 	    __func__, targ->tid);
495a2c14879SStephen McConnell 	mprsas_prepare_for_tm(sc, cm, targ, CAM_LUN_WILDCARD);
496a2c14879SStephen McConnell 
497991554f2SKenneth D. Merry 	mpr_map_command(sc, cm);
498991554f2SKenneth D. Merry }
499991554f2SKenneth D. Merry 
500991554f2SKenneth D. Merry /*
50167feec50SStephen McConnell  * The firmware performs debounce on the link to avoid transient link errors
50267feec50SStephen McConnell  * and false removals.  When it does decide that link has been lost and a
50367feec50SStephen McConnell  * device needs to go away, it expects that the host will perform a target reset
50467feec50SStephen McConnell  * and then an op remove.  The reset has the side-effect of aborting any
50567feec50SStephen McConnell  * outstanding requests for the device, which is required for the op-remove to
50667feec50SStephen McConnell  * succeed.  It's not clear if the host should check for the device coming back
50767feec50SStephen McConnell  * alive after the reset.
508991554f2SKenneth D. Merry  */
509991554f2SKenneth D. Merry void
510991554f2SKenneth D. Merry mprsas_prepare_remove(struct mprsas_softc *sassc, uint16_t handle)
511991554f2SKenneth D. Merry {
512991554f2SKenneth D. Merry 	MPI2_SCSI_TASK_MANAGE_REQUEST *req;
513991554f2SKenneth D. Merry 	struct mpr_softc *sc;
51446b9415fSScott Long 	struct mpr_command *tm;
515991554f2SKenneth D. Merry 	struct mprsas_target *targ = NULL;
516991554f2SKenneth D. Merry 
517991554f2SKenneth D. Merry 	MPR_FUNCTRACE(sassc->sc);
518991554f2SKenneth D. Merry 
519991554f2SKenneth D. Merry 	sc = sassc->sc;
520991554f2SKenneth D. Merry 
521991554f2SKenneth D. Merry 	targ = mprsas_find_target_by_handle(sassc, 0, handle);
522991554f2SKenneth D. Merry 	if (targ == NULL) {
523991554f2SKenneth D. Merry 		/* FIXME: what is the action? */
524991554f2SKenneth D. Merry 		/* We don't know about this device? */
525991554f2SKenneth D. Merry 		mpr_dprint(sc, MPR_ERROR, "%s : invalid handle 0x%x \n",
526991554f2SKenneth D. Merry 		    __func__, handle);
527991554f2SKenneth D. Merry 		return;
528991554f2SKenneth D. Merry 	}
529991554f2SKenneth D. Merry 
530991554f2SKenneth D. Merry 	targ->flags |= MPRSAS_TARGET_INREMOVAL;
531991554f2SKenneth D. Merry 
53246b9415fSScott Long 	tm = mprsas_alloc_tm(sc);
53346b9415fSScott Long 	if (tm == NULL) {
534e3c5965cSAlexander Motin 		targ->flags |= MPRSAS_TARGET_TOREMOVE;
535e3c5965cSAlexander Motin 		sassc->flags |= MPRSAS_TOREMOVE;
536991554f2SKenneth D. Merry 		return;
537991554f2SKenneth D. Merry 	}
538991554f2SKenneth D. Merry 
539991554f2SKenneth D. Merry 	mprsas_rescan_target(sc, targ);
540991554f2SKenneth D. Merry 
54146b9415fSScott Long 	req = (MPI2_SCSI_TASK_MANAGE_REQUEST *)tm->cm_req;
542991554f2SKenneth D. Merry 	req->DevHandle = htole16(targ->handle);
543991554f2SKenneth D. Merry 	req->TaskType = MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET;
544991554f2SKenneth D. Merry 
545991554f2SKenneth D. Merry 	/* SAS Hard Link Reset / SATA Link Reset */
546991554f2SKenneth D. Merry 	req->MsgFlags = MPI2_SCSITASKMGMT_MSGFLAGS_LINK_RESET;
547991554f2SKenneth D. Merry 
54846b9415fSScott Long 	tm->cm_targ = targ;
54946b9415fSScott Long 	tm->cm_data = NULL;
55046b9415fSScott Long 	tm->cm_complete = mprsas_remove_device;
55146b9415fSScott Long 	tm->cm_complete_data = (void *)(uintptr_t)handle;
552a2c14879SStephen McConnell 
553a2c14879SStephen McConnell 	mpr_dprint(sc, MPR_INFO, "%s: Sending reset for target ID %d\n",
554a2c14879SStephen McConnell 	    __func__, targ->tid);
55546b9415fSScott Long 	mprsas_prepare_for_tm(sc, tm, targ, CAM_LUN_WILDCARD);
556a2c14879SStephen McConnell 
55746b9415fSScott Long 	mpr_map_command(sc, tm);
558991554f2SKenneth D. Merry }
559991554f2SKenneth D. Merry 
560991554f2SKenneth D. Merry static void
561991554f2SKenneth D. Merry mprsas_remove_device(struct mpr_softc *sc, struct mpr_command *tm)
562991554f2SKenneth D. Merry {
563991554f2SKenneth D. Merry 	MPI2_SCSI_TASK_MANAGE_REPLY *reply;
564991554f2SKenneth D. Merry 	MPI2_SAS_IOUNIT_CONTROL_REQUEST *req;
565991554f2SKenneth D. Merry 	struct mprsas_target *targ;
566991554f2SKenneth D. Merry 	uint16_t handle;
567991554f2SKenneth D. Merry 
568991554f2SKenneth D. Merry 	MPR_FUNCTRACE(sc);
569991554f2SKenneth D. Merry 
570991554f2SKenneth D. Merry 	reply = (MPI2_SCSI_TASK_MANAGE_REPLY *)tm->cm_reply;
571991554f2SKenneth D. Merry 	handle = (uint16_t)(uintptr_t)tm->cm_complete_data;
572991554f2SKenneth D. Merry 	targ = tm->cm_targ;
573991554f2SKenneth D. Merry 
574991554f2SKenneth D. Merry 	/*
575991554f2SKenneth D. Merry 	 * Currently there should be no way we can hit this case.  It only
576991554f2SKenneth D. Merry 	 * happens when we have a failure to allocate chain frames, and
577991554f2SKenneth D. Merry 	 * task management commands don't have S/G lists.
578991554f2SKenneth D. Merry 	 */
579991554f2SKenneth D. Merry 	if ((tm->cm_flags & MPR_CM_FLAGS_ERROR_MASK) != 0) {
580991554f2SKenneth D. Merry 		mpr_dprint(sc, MPR_ERROR, "%s: cm_flags = %#x for remove of "
581991554f2SKenneth D. Merry 		    "handle %#04x! This should not happen!\n", __func__,
582991554f2SKenneth D. Merry 		    tm->cm_flags, handle);
583991554f2SKenneth D. Merry 	}
584991554f2SKenneth D. Merry 
585991554f2SKenneth D. Merry 	if (reply == NULL) {
586991554f2SKenneth D. Merry 		/* XXX retry the remove after the diag reset completes? */
587991554f2SKenneth D. Merry 		mpr_dprint(sc, MPR_FAULT, "%s NULL reply resetting device "
588991554f2SKenneth D. Merry 		    "0x%04x\n", __func__, handle);
589991554f2SKenneth D. Merry 		mprsas_free_tm(sc, tm);
590991554f2SKenneth D. Merry 		return;
591991554f2SKenneth D. Merry 	}
592991554f2SKenneth D. Merry 
593d3f6eabfSStephen McConnell 	if ((le16toh(reply->IOCStatus) & MPI2_IOCSTATUS_MASK) !=
594d3f6eabfSStephen McConnell 	    MPI2_IOCSTATUS_SUCCESS) {
59558581c13SStephen McConnell 		mpr_dprint(sc, MPR_ERROR, "IOCStatus = 0x%x while resetting "
596991554f2SKenneth D. Merry 		    "device 0x%x\n", le16toh(reply->IOCStatus), handle);
597991554f2SKenneth D. Merry 	}
598991554f2SKenneth D. Merry 
599991554f2SKenneth D. Merry 	mpr_dprint(sc, MPR_XINFO, "Reset aborted %u commands\n",
600991554f2SKenneth D. Merry 	    le32toh(reply->TerminationCount));
601991554f2SKenneth D. Merry 	mpr_free_reply(sc, tm->cm_reply_data);
602991554f2SKenneth D. Merry 	tm->cm_reply = NULL;	/* Ensures the reply won't get re-freed */
603991554f2SKenneth D. Merry 
604991554f2SKenneth D. Merry 	/* Reuse the existing command */
605991554f2SKenneth D. Merry 	req = (MPI2_SAS_IOUNIT_CONTROL_REQUEST *)tm->cm_req;
606991554f2SKenneth D. Merry 	memset(req, 0, sizeof(*req));
607991554f2SKenneth D. Merry 	req->Function = MPI2_FUNCTION_SAS_IO_UNIT_CONTROL;
608991554f2SKenneth D. Merry 	req->Operation = MPI2_SAS_OP_REMOVE_DEVICE;
609991554f2SKenneth D. Merry 	req->DevHandle = htole16(handle);
610991554f2SKenneth D. Merry 	tm->cm_data = NULL;
611991554f2SKenneth D. Merry 	tm->cm_desc.Default.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_DEFAULT_TYPE;
612991554f2SKenneth D. Merry 	tm->cm_complete = mprsas_remove_complete;
613991554f2SKenneth D. Merry 	tm->cm_complete_data = (void *)(uintptr_t)handle;
614991554f2SKenneth D. Merry 
6154c1cdd4aSWarner Losh 	/*
6164c1cdd4aSWarner Losh 	 * Wait to send the REMOVE_DEVICE until all the commands have cleared.
6174c1cdd4aSWarner Losh 	 * They should be aborted or time out and we'll kick thus off there
6184c1cdd4aSWarner Losh 	 * if so.
6194c1cdd4aSWarner Losh 	 */
6204c1cdd4aSWarner Losh 	if (TAILQ_FIRST(&targ->commands) == NULL) {
621*ca420b4eSWarner Losh 		mpr_dprint(sc, MPR_INFO,
622*ca420b4eSWarner Losh 		    "No pending commands: starting remove_device for target %u handle 0x%04x\n",
623*ca420b4eSWarner Losh 		    targ->tid, handle);
624991554f2SKenneth D. Merry 		mpr_map_command(sc, tm);
6254c1cdd4aSWarner Losh 		targ->pending_remove_tm = NULL;
6264c1cdd4aSWarner Losh 	} else {
6274c1cdd4aSWarner Losh 		targ->pending_remove_tm = tm;
6284c1cdd4aSWarner Losh 	}
629991554f2SKenneth D. Merry 
630a2c14879SStephen McConnell 	mpr_dprint(sc, MPR_INFO, "clearing target %u handle 0x%04x\n",
631991554f2SKenneth D. Merry 	    targ->tid, handle);
632991554f2SKenneth D. Merry 	if (targ->encl_level_valid) {
633a2c14879SStephen McConnell 		mpr_dprint(sc, MPR_INFO, "At enclosure level %d, slot %d, "
634991554f2SKenneth D. Merry 		    "connector name (%4s)\n", targ->encl_level, targ->encl_slot,
635991554f2SKenneth D. Merry 		    targ->connector_name);
636991554f2SKenneth D. Merry 	}
637991554f2SKenneth D. Merry }
638991554f2SKenneth D. Merry 
639991554f2SKenneth D. Merry static void
640991554f2SKenneth D. Merry mprsas_remove_complete(struct mpr_softc *sc, struct mpr_command *tm)
641991554f2SKenneth D. Merry {
642991554f2SKenneth D. Merry 	MPI2_SAS_IOUNIT_CONTROL_REPLY *reply;
643991554f2SKenneth D. Merry 	uint16_t handle;
644991554f2SKenneth D. Merry 	struct mprsas_target *targ;
645991554f2SKenneth D. Merry 	struct mprsas_lun *lun;
646991554f2SKenneth D. Merry 
647991554f2SKenneth D. Merry 	MPR_FUNCTRACE(sc);
648991554f2SKenneth D. Merry 
649991554f2SKenneth D. Merry 	reply = (MPI2_SAS_IOUNIT_CONTROL_REPLY *)tm->cm_reply;
650991554f2SKenneth D. Merry 	handle = (uint16_t)(uintptr_t)tm->cm_complete_data;
651991554f2SKenneth D. Merry 
6524c1cdd4aSWarner Losh 	targ = tm->cm_targ;
6534c1cdd4aSWarner Losh 
6544c1cdd4aSWarner Losh 	/*
6554c1cdd4aSWarner Losh 	 * At this point, we should have no pending commands for the target.
6564c1cdd4aSWarner Losh 	 * The remove target has just completed.
6574c1cdd4aSWarner Losh 	 */
6584c1cdd4aSWarner Losh 	KASSERT(TAILQ_FIRST(&targ->commands) == NULL,
6594c1cdd4aSWarner Losh 	    ("%s: no commands should be pending\n", __func__));
6604c1cdd4aSWarner Losh 
661991554f2SKenneth D. Merry 	/*
662991554f2SKenneth D. Merry 	 * Currently there should be no way we can hit this case.  It only
663991554f2SKenneth D. Merry 	 * happens when we have a failure to allocate chain frames, and
664991554f2SKenneth D. Merry 	 * task management commands don't have S/G lists.
665991554f2SKenneth D. Merry 	 */
666991554f2SKenneth D. Merry 	if ((tm->cm_flags & MPR_CM_FLAGS_ERROR_MASK) != 0) {
667991554f2SKenneth D. Merry 		mpr_dprint(sc, MPR_XINFO, "%s: cm_flags = %#x for remove of "
668991554f2SKenneth D. Merry 		    "handle %#04x! This should not happen!\n", __func__,
669991554f2SKenneth D. Merry 		    tm->cm_flags, handle);
670991554f2SKenneth D. Merry 		mprsas_free_tm(sc, tm);
671991554f2SKenneth D. Merry 		return;
672991554f2SKenneth D. Merry 	}
673991554f2SKenneth D. Merry 
674991554f2SKenneth D. Merry 	if (reply == NULL) {
675991554f2SKenneth D. Merry 		/* most likely a chip reset */
676991554f2SKenneth D. Merry 		mpr_dprint(sc, MPR_FAULT, "%s NULL reply removing device "
677991554f2SKenneth D. Merry 		    "0x%04x\n", __func__, handle);
678991554f2SKenneth D. Merry 		mprsas_free_tm(sc, tm);
679991554f2SKenneth D. Merry 		return;
680991554f2SKenneth D. Merry 	}
681991554f2SKenneth D. Merry 
682991554f2SKenneth D. Merry 	mpr_dprint(sc, MPR_XINFO, "%s on handle 0x%04x, IOCStatus= 0x%x\n",
683991554f2SKenneth D. Merry 	    __func__, handle, le16toh(reply->IOCStatus));
684991554f2SKenneth D. Merry 
685991554f2SKenneth D. Merry 	/*
686991554f2SKenneth D. Merry 	 * Don't clear target if remove fails because things will get confusing.
687991554f2SKenneth D. Merry 	 * Leave the devname and sasaddr intact so that we know to avoid reusing
688991554f2SKenneth D. Merry 	 * this target id if possible, and so we can assign the same target id
689991554f2SKenneth D. Merry 	 * to this device if it comes back in the future.
690991554f2SKenneth D. Merry 	 */
691d3f6eabfSStephen McConnell 	if ((le16toh(reply->IOCStatus) & MPI2_IOCSTATUS_MASK) ==
692d3f6eabfSStephen McConnell 	    MPI2_IOCSTATUS_SUCCESS) {
693991554f2SKenneth D. Merry 		targ->handle = 0x0;
694991554f2SKenneth D. Merry 		targ->encl_handle = 0x0;
695991554f2SKenneth D. Merry 		targ->encl_level_valid = 0x0;
696991554f2SKenneth D. Merry 		targ->encl_level = 0x0;
697991554f2SKenneth D. Merry 		targ->connector_name[0] = ' ';
698991554f2SKenneth D. Merry 		targ->connector_name[1] = ' ';
699991554f2SKenneth D. Merry 		targ->connector_name[2] = ' ';
700991554f2SKenneth D. Merry 		targ->connector_name[3] = ' ';
701991554f2SKenneth D. Merry 		targ->encl_slot = 0x0;
702991554f2SKenneth D. Merry 		targ->exp_dev_handle = 0x0;
703991554f2SKenneth D. Merry 		targ->phy_num = 0x0;
704991554f2SKenneth D. Merry 		targ->linkrate = 0x0;
705991554f2SKenneth D. Merry 		targ->devinfo = 0x0;
706991554f2SKenneth D. Merry 		targ->flags = 0x0;
707991554f2SKenneth D. Merry 		targ->scsi_req_desc_type = 0;
708991554f2SKenneth D. Merry 
709991554f2SKenneth D. Merry 		while (!SLIST_EMPTY(&targ->luns)) {
710991554f2SKenneth D. Merry 			lun = SLIST_FIRST(&targ->luns);
711991554f2SKenneth D. Merry 			SLIST_REMOVE_HEAD(&targ->luns, lun_link);
712991554f2SKenneth D. Merry 			free(lun, M_MPR);
713991554f2SKenneth D. Merry 		}
714991554f2SKenneth D. Merry 	}
715991554f2SKenneth D. Merry 
716991554f2SKenneth D. Merry 	mprsas_free_tm(sc, tm);
717991554f2SKenneth D. Merry }
718991554f2SKenneth D. Merry 
719991554f2SKenneth D. Merry static int
720991554f2SKenneth D. Merry mprsas_register_events(struct mpr_softc *sc)
721991554f2SKenneth D. Merry {
722991554f2SKenneth D. Merry 	uint8_t events[16];
723991554f2SKenneth D. Merry 
724991554f2SKenneth D. Merry 	bzero(events, 16);
725991554f2SKenneth D. Merry 	setbit(events, MPI2_EVENT_SAS_DEVICE_STATUS_CHANGE);
726991554f2SKenneth D. Merry 	setbit(events, MPI2_EVENT_SAS_DISCOVERY);
727991554f2SKenneth D. Merry 	setbit(events, MPI2_EVENT_SAS_BROADCAST_PRIMITIVE);
728991554f2SKenneth D. Merry 	setbit(events, MPI2_EVENT_SAS_INIT_DEVICE_STATUS_CHANGE);
729991554f2SKenneth D. Merry 	setbit(events, MPI2_EVENT_SAS_INIT_TABLE_OVERFLOW);
730991554f2SKenneth D. Merry 	setbit(events, MPI2_EVENT_SAS_TOPOLOGY_CHANGE_LIST);
731991554f2SKenneth D. Merry 	setbit(events, MPI2_EVENT_SAS_ENCL_DEVICE_STATUS_CHANGE);
732991554f2SKenneth D. Merry 	setbit(events, MPI2_EVENT_IR_CONFIGURATION_CHANGE_LIST);
733991554f2SKenneth D. Merry 	setbit(events, MPI2_EVENT_IR_VOLUME);
734991554f2SKenneth D. Merry 	setbit(events, MPI2_EVENT_IR_PHYSICAL_DISK);
735991554f2SKenneth D. Merry 	setbit(events, MPI2_EVENT_IR_OPERATION_STATUS);
736991554f2SKenneth D. Merry 	setbit(events, MPI2_EVENT_TEMP_THRESHOLD);
7375f5baf0eSAlexander Motin 	setbit(events, MPI2_EVENT_SAS_DEVICE_DISCOVERY_ERROR);
73867feec50SStephen McConnell 	if (sc->facts->MsgVersion >= MPI2_VERSION_02_06) {
7392bbc5fcbSStephen McConnell 		setbit(events, MPI2_EVENT_ACTIVE_CABLE_EXCEPTION);
74067feec50SStephen McConnell 		if (sc->mpr_flags & MPR_FLAGS_GEN35_IOC) {
74167feec50SStephen McConnell 			setbit(events, MPI2_EVENT_PCIE_DEVICE_STATUS_CHANGE);
74267feec50SStephen McConnell 			setbit(events, MPI2_EVENT_PCIE_ENUMERATION);
74367feec50SStephen McConnell 			setbit(events, MPI2_EVENT_PCIE_TOPOLOGY_CHANGE_LIST);
74467feec50SStephen McConnell 		}
74567feec50SStephen McConnell 	}
746991554f2SKenneth D. Merry 
747991554f2SKenneth D. Merry 	mpr_register_events(sc, events, mprsas_evt_handler, NULL,
748991554f2SKenneth D. Merry 	    &sc->sassc->mprsas_eh);
749991554f2SKenneth D. Merry 
750991554f2SKenneth D. Merry 	return (0);
751991554f2SKenneth D. Merry }
752991554f2SKenneth D. Merry 
753991554f2SKenneth D. Merry int
754991554f2SKenneth D. Merry mpr_attach_sas(struct mpr_softc *sc)
755991554f2SKenneth D. Merry {
756991554f2SKenneth D. Merry 	struct mprsas_softc *sassc;
757991554f2SKenneth D. Merry 	cam_status status;
75862a09ee9SAlexander Motin 	int unit, error = 0, reqs;
759991554f2SKenneth D. Merry 
760991554f2SKenneth D. Merry 	MPR_FUNCTRACE(sc);
761757ff642SScott Long 	mpr_dprint(sc, MPR_INIT, "%s entered\n", __func__);
762991554f2SKenneth D. Merry 
763991554f2SKenneth D. Merry 	sassc = malloc(sizeof(struct mprsas_softc), M_MPR, M_WAITOK|M_ZERO);
764991554f2SKenneth D. Merry 
765991554f2SKenneth D. Merry 	/*
766a2c14879SStephen McConnell 	 * XXX MaxTargets could change during a reinit.  Since we don't
767991554f2SKenneth D. Merry 	 * resize the targets[] array during such an event, cache the value
768991554f2SKenneth D. Merry 	 * of MaxTargets here so that we don't get into trouble later.  This
769991554f2SKenneth D. Merry 	 * should move into the reinit logic.
770991554f2SKenneth D. Merry 	 */
771327f2e6cSStephen McConnell 	sassc->maxtargets = sc->facts->MaxTargets + sc->facts->MaxVolumes;
772991554f2SKenneth D. Merry 	sassc->targets = malloc(sizeof(struct mprsas_target) *
773991554f2SKenneth D. Merry 	    sassc->maxtargets, M_MPR, M_WAITOK|M_ZERO);
774991554f2SKenneth D. Merry 	sc->sassc = sassc;
775991554f2SKenneth D. Merry 	sassc->sc = sc;
776991554f2SKenneth D. Merry 
77762a09ee9SAlexander Motin 	reqs = sc->num_reqs - sc->num_prireqs - 1;
77862a09ee9SAlexander Motin 	if ((sassc->devq = cam_simq_alloc(reqs)) == NULL) {
779757ff642SScott Long 		mpr_dprint(sc, MPR_INIT|MPR_ERROR, "Cannot allocate SIMQ\n");
780991554f2SKenneth D. Merry 		error = ENOMEM;
781991554f2SKenneth D. Merry 		goto out;
782991554f2SKenneth D. Merry 	}
783991554f2SKenneth D. Merry 
784991554f2SKenneth D. Merry 	unit = device_get_unit(sc->mpr_dev);
785991554f2SKenneth D. Merry 	sassc->sim = cam_sim_alloc(mprsas_action, mprsas_poll, "mpr", sassc,
78662a09ee9SAlexander Motin 	    unit, &sc->mpr_mtx, reqs, reqs, sassc->devq);
787991554f2SKenneth D. Merry 	if (sassc->sim == NULL) {
788757ff642SScott Long 		mpr_dprint(sc, MPR_INIT|MPR_ERROR, "Cannot allocate SIM\n");
789991554f2SKenneth D. Merry 		error = EINVAL;
790991554f2SKenneth D. Merry 		goto out;
791991554f2SKenneth D. Merry 	}
792991554f2SKenneth D. Merry 
793991554f2SKenneth D. Merry 	TAILQ_INIT(&sassc->ev_queue);
794991554f2SKenneth D. Merry 
795991554f2SKenneth D. Merry 	/* Initialize taskqueue for Event Handling */
796991554f2SKenneth D. Merry 	TASK_INIT(&sassc->ev_task, 0, mprsas_firmware_event_work, sc);
797991554f2SKenneth D. Merry 	sassc->ev_tq = taskqueue_create("mpr_taskq", M_NOWAIT | M_ZERO,
798991554f2SKenneth D. Merry 	    taskqueue_thread_enqueue, &sassc->ev_tq);
799c2a0f07aSAlexander Motin 	taskqueue_start_threads(&sassc->ev_tq, 1, PRIBIO, "%s taskq",
800991554f2SKenneth D. Merry 	    device_get_nameunit(sc->mpr_dev));
801991554f2SKenneth D. Merry 
802991554f2SKenneth D. Merry 	mpr_lock(sc);
803991554f2SKenneth D. Merry 
804991554f2SKenneth D. Merry 	/*
805991554f2SKenneth D. Merry 	 * XXX There should be a bus for every port on the adapter, but since
806991554f2SKenneth D. Merry 	 * we're just going to fake the topology for now, we'll pretend that
807991554f2SKenneth D. Merry 	 * everything is just a target on a single bus.
808991554f2SKenneth D. Merry 	 */
809991554f2SKenneth D. Merry 	if ((error = xpt_bus_register(sassc->sim, sc->mpr_dev, 0)) != 0) {
810757ff642SScott Long 		mpr_dprint(sc, MPR_INIT|MPR_ERROR,
811757ff642SScott Long 		    "Error %d registering SCSI bus\n", error);
812991554f2SKenneth D. Merry 		mpr_unlock(sc);
813991554f2SKenneth D. Merry 		goto out;
814991554f2SKenneth D. Merry 	}
815991554f2SKenneth D. Merry 
816991554f2SKenneth D. Merry 	/*
817a2c14879SStephen McConnell 	 * Assume that discovery events will start right away.
818991554f2SKenneth D. Merry 	 *
819991554f2SKenneth D. Merry 	 * Hold off boot until discovery is complete.
820991554f2SKenneth D. Merry 	 */
821991554f2SKenneth D. Merry 	sassc->flags |= MPRSAS_IN_STARTUP | MPRSAS_IN_DISCOVERY;
822991554f2SKenneth D. Merry 	sc->sassc->startup_refcount = 0;
823991554f2SKenneth D. Merry 	mprsas_startup_increment(sassc);
824991554f2SKenneth D. Merry 
82502d81940SAlexander Motin 	mpr_unlock(sc);
82602d81940SAlexander Motin 
827991554f2SKenneth D. Merry 	/*
828991554f2SKenneth D. Merry 	 * Register for async events so we can determine the EEDP
829991554f2SKenneth D. Merry 	 * capabilities of devices.
830991554f2SKenneth D. Merry 	 */
831991554f2SKenneth D. Merry 	status = xpt_create_path(&sassc->path, /*periph*/NULL,
832991554f2SKenneth D. Merry 	    cam_sim_path(sc->sassc->sim), CAM_TARGET_WILDCARD,
833991554f2SKenneth D. Merry 	    CAM_LUN_WILDCARD);
834991554f2SKenneth D. Merry 	if (status != CAM_REQ_CMP) {
835757ff642SScott Long 		mpr_dprint(sc, MPR_INIT|MPR_ERROR,
836757ff642SScott Long 		    "Error %#x creating sim path\n", status);
837991554f2SKenneth D. Merry 		sassc->path = NULL;
838991554f2SKenneth D. Merry 	} else {
839991554f2SKenneth D. Merry 		int event;
840991554f2SKenneth D. Merry 
84102d81940SAlexander Motin 		event = AC_ADVINFO_CHANGED;
842991554f2SKenneth D. Merry 		status = xpt_register_async(event, mprsas_async, sc,
843991554f2SKenneth D. Merry 					    sassc->path);
84407aa4de1SKenneth D. Merry 
845991554f2SKenneth D. Merry 		if (status != CAM_REQ_CMP) {
846991554f2SKenneth D. Merry 			mpr_dprint(sc, MPR_ERROR,
847991554f2SKenneth D. Merry 			    "Error %#x registering async handler for "
848991554f2SKenneth D. Merry 			    "AC_ADVINFO_CHANGED events\n", status);
849991554f2SKenneth D. Merry 			xpt_free_path(sassc->path);
850991554f2SKenneth D. Merry 			sassc->path = NULL;
851991554f2SKenneth D. Merry 		}
852991554f2SKenneth D. Merry 	}
853991554f2SKenneth D. Merry 	if (status != CAM_REQ_CMP) {
854991554f2SKenneth D. Merry 		/*
855991554f2SKenneth D. Merry 		 * EEDP use is the exception, not the rule.
856991554f2SKenneth D. Merry 		 * Warn the user, but do not fail to attach.
857991554f2SKenneth D. Merry 		 */
858991554f2SKenneth D. Merry 		mpr_printf(sc, "EEDP capabilities disabled.\n");
859991554f2SKenneth D. Merry 	}
860991554f2SKenneth D. Merry 
861991554f2SKenneth D. Merry 	mprsas_register_events(sc);
862991554f2SKenneth D. Merry out:
863991554f2SKenneth D. Merry 	if (error)
864991554f2SKenneth D. Merry 		mpr_detach_sas(sc);
865757ff642SScott Long 
866757ff642SScott Long 	mpr_dprint(sc, MPR_INIT, "%s exit, error= %d\n", __func__, error);
867991554f2SKenneth D. Merry 	return (error);
868991554f2SKenneth D. Merry }
869991554f2SKenneth D. Merry 
870991554f2SKenneth D. Merry int
871991554f2SKenneth D. Merry mpr_detach_sas(struct mpr_softc *sc)
872991554f2SKenneth D. Merry {
873991554f2SKenneth D. Merry 	struct mprsas_softc *sassc;
874991554f2SKenneth D. Merry 	struct mprsas_lun *lun, *lun_tmp;
875991554f2SKenneth D. Merry 	struct mprsas_target *targ;
876991554f2SKenneth D. Merry 	int i;
877991554f2SKenneth D. Merry 
878991554f2SKenneth D. Merry 	MPR_FUNCTRACE(sc);
879991554f2SKenneth D. Merry 
880991554f2SKenneth D. Merry 	if (sc->sassc == NULL)
881991554f2SKenneth D. Merry 		return (0);
882991554f2SKenneth D. Merry 
883991554f2SKenneth D. Merry 	sassc = sc->sassc;
884991554f2SKenneth D. Merry 	mpr_deregister_events(sc, sassc->mprsas_eh);
885991554f2SKenneth D. Merry 
886991554f2SKenneth D. Merry 	/*
887991554f2SKenneth D. Merry 	 * Drain and free the event handling taskqueue with the lock
888991554f2SKenneth D. Merry 	 * unheld so that any parallel processing tasks drain properly
889991554f2SKenneth D. Merry 	 * without deadlocking.
890991554f2SKenneth D. Merry 	 */
891991554f2SKenneth D. Merry 	if (sassc->ev_tq != NULL)
892991554f2SKenneth D. Merry 		taskqueue_free(sassc->ev_tq);
893991554f2SKenneth D. Merry 
894991554f2SKenneth D. Merry 	/* Deregister our async handler */
895991554f2SKenneth D. Merry 	if (sassc->path != NULL) {
896991554f2SKenneth D. Merry 		xpt_register_async(0, mprsas_async, sc, sassc->path);
897991554f2SKenneth D. Merry 		xpt_free_path(sassc->path);
898991554f2SKenneth D. Merry 		sassc->path = NULL;
899991554f2SKenneth D. Merry 	}
900991554f2SKenneth D. Merry 
90102d81940SAlexander Motin 	/* Make sure CAM doesn't wedge if we had to bail out early. */
90202d81940SAlexander Motin 	mpr_lock(sc);
90302d81940SAlexander Motin 
90402d81940SAlexander Motin 	while (sassc->startup_refcount != 0)
90502d81940SAlexander Motin 		mprsas_startup_decrement(sassc);
90602d81940SAlexander Motin 
907991554f2SKenneth D. Merry 	if (sassc->flags & MPRSAS_IN_STARTUP)
908991554f2SKenneth D. Merry 		xpt_release_simq(sassc->sim, 1);
909991554f2SKenneth D. Merry 
910991554f2SKenneth D. Merry 	if (sassc->sim != NULL) {
911991554f2SKenneth D. Merry 		xpt_bus_deregister(cam_sim_path(sassc->sim));
912991554f2SKenneth D. Merry 		cam_sim_free(sassc->sim, FALSE);
913991554f2SKenneth D. Merry 	}
914991554f2SKenneth D. Merry 
915991554f2SKenneth D. Merry 	mpr_unlock(sc);
916991554f2SKenneth D. Merry 
917991554f2SKenneth D. Merry 	if (sassc->devq != NULL)
918991554f2SKenneth D. Merry 		cam_simq_free(sassc->devq);
919991554f2SKenneth D. Merry 
920991554f2SKenneth D. Merry 	for (i = 0; i < sassc->maxtargets; i++) {
921991554f2SKenneth D. Merry 		targ = &sassc->targets[i];
922991554f2SKenneth D. Merry 		SLIST_FOREACH_SAFE(lun, &targ->luns, lun_link, lun_tmp) {
923991554f2SKenneth D. Merry 			free(lun, M_MPR);
924991554f2SKenneth D. Merry 		}
925991554f2SKenneth D. Merry 	}
926991554f2SKenneth D. Merry 	free(sassc->targets, M_MPR);
927991554f2SKenneth D. Merry 	free(sassc, M_MPR);
928991554f2SKenneth D. Merry 	sc->sassc = NULL;
929991554f2SKenneth D. Merry 
930991554f2SKenneth D. Merry 	return (0);
931991554f2SKenneth D. Merry }
932991554f2SKenneth D. Merry 
933991554f2SKenneth D. Merry void
934991554f2SKenneth D. Merry mprsas_discovery_end(struct mprsas_softc *sassc)
935991554f2SKenneth D. Merry {
936991554f2SKenneth D. Merry 	struct mpr_softc *sc = sassc->sc;
937991554f2SKenneth D. Merry 
938991554f2SKenneth D. Merry 	MPR_FUNCTRACE(sc);
939991554f2SKenneth D. Merry 
940327f2e6cSStephen McConnell 	/*
941327f2e6cSStephen McConnell 	 * After discovery has completed, check the mapping table for any
942327f2e6cSStephen McConnell 	 * missing devices and update their missing counts. Only do this once
943327f2e6cSStephen McConnell 	 * whenever the driver is initialized so that missing counts aren't
944327f2e6cSStephen McConnell 	 * updated unnecessarily. Note that just because discovery has
945327f2e6cSStephen McConnell 	 * completed doesn't mean that events have been processed yet. The
946327f2e6cSStephen McConnell 	 * check_devices function is a callout timer that checks if ALL devices
947327f2e6cSStephen McConnell 	 * are missing. If so, it will wait a little longer for events to
948327f2e6cSStephen McConnell 	 * complete and keep resetting itself until some device in the mapping
949327f2e6cSStephen McConnell 	 * table is not missing, meaning that event processing has started.
950327f2e6cSStephen McConnell 	 */
951327f2e6cSStephen McConnell 	if (sc->track_mapping_events) {
952327f2e6cSStephen McConnell 		mpr_dprint(sc, MPR_XINFO | MPR_MAPPING, "Discovery has "
953327f2e6cSStephen McConnell 		    "completed. Check for missing devices in the mapping "
954327f2e6cSStephen McConnell 		    "table.\n");
955327f2e6cSStephen McConnell 		callout_reset(&sc->device_check_callout,
956327f2e6cSStephen McConnell 		    MPR_MISSING_CHECK_DELAY * hz, mpr_mapping_check_devices,
957327f2e6cSStephen McConnell 		    sc);
958327f2e6cSStephen McConnell 	}
959991554f2SKenneth D. Merry }
960991554f2SKenneth D. Merry 
961991554f2SKenneth D. Merry static void
962991554f2SKenneth D. Merry mprsas_action(struct cam_sim *sim, union ccb *ccb)
963991554f2SKenneth D. Merry {
964991554f2SKenneth D. Merry 	struct mprsas_softc *sassc;
965991554f2SKenneth D. Merry 
966991554f2SKenneth D. Merry 	sassc = cam_sim_softc(sim);
967991554f2SKenneth D. Merry 
968991554f2SKenneth D. Merry 	MPR_FUNCTRACE(sassc->sc);
969a2c14879SStephen McConnell 	mpr_dprint(sassc->sc, MPR_TRACE, "ccb func_code 0x%x\n",
970991554f2SKenneth D. Merry 	    ccb->ccb_h.func_code);
971991554f2SKenneth D. Merry 	mtx_assert(&sassc->sc->mpr_mtx, MA_OWNED);
972991554f2SKenneth D. Merry 
973991554f2SKenneth D. Merry 	switch (ccb->ccb_h.func_code) {
974991554f2SKenneth D. Merry 	case XPT_PATH_INQ:
975991554f2SKenneth D. Merry 	{
976991554f2SKenneth D. Merry 		struct ccb_pathinq *cpi = &ccb->cpi;
97732b0a21eSStephen McConnell 		struct mpr_softc *sc = sassc->sc;
978991554f2SKenneth D. Merry 
979991554f2SKenneth D. Merry 		cpi->version_num = 1;
980991554f2SKenneth D. Merry 		cpi->hba_inquiry = PI_SDTR_ABLE|PI_TAG_ABLE|PI_WIDE_16;
981991554f2SKenneth D. Merry 		cpi->target_sprt = 0;
982991554f2SKenneth D. Merry 		cpi->hba_misc = PIM_NOBUSRESET | PIM_UNMAPPED | PIM_NOSCAN;
983991554f2SKenneth D. Merry 		cpi->hba_eng_cnt = 0;
984991554f2SKenneth D. Merry 		cpi->max_target = sassc->maxtargets - 1;
985991554f2SKenneth D. Merry 		cpi->max_lun = 255;
986327f2e6cSStephen McConnell 
987327f2e6cSStephen McConnell 		/*
988327f2e6cSStephen McConnell 		 * initiator_id is set here to an ID outside the set of valid
989327f2e6cSStephen McConnell 		 * target IDs (including volumes).
990327f2e6cSStephen McConnell 		 */
991327f2e6cSStephen McConnell 		cpi->initiator_id = sassc->maxtargets;
9924195c7deSAlan Somers 		strlcpy(cpi->sim_vid, "FreeBSD", SIM_IDLEN);
9934195c7deSAlan Somers 		strlcpy(cpi->hba_vid, "Avago Tech", HBA_IDLEN);
9944195c7deSAlan Somers 		strlcpy(cpi->dev_name, cam_sim_name(sim), DEV_IDLEN);
995991554f2SKenneth D. Merry 		cpi->unit_number = cam_sim_unit(sim);
996991554f2SKenneth D. Merry 		cpi->bus_id = cam_sim_bus(sim);
997991554f2SKenneth D. Merry 		/*
998991554f2SKenneth D. Merry 		 * XXXSLM-I think this needs to change based on config page or
999991554f2SKenneth D. Merry 		 * something instead of hardcoded to 150000.
1000991554f2SKenneth D. Merry 		 */
1001991554f2SKenneth D. Merry 		cpi->base_transfer_speed = 150000;
1002991554f2SKenneth D. Merry 		cpi->transport = XPORT_SAS;
1003991554f2SKenneth D. Merry 		cpi->transport_version = 0;
1004991554f2SKenneth D. Merry 		cpi->protocol = PROTO_SCSI;
1005991554f2SKenneth D. Merry 		cpi->protocol_version = SCSI_REV_SPC;
10064f5d6573SAlexander Motin 		cpi->maxio = sc->maxio;
1007a2c14879SStephen McConnell 		mprsas_set_ccbstatus(ccb, CAM_REQ_CMP);
1008991554f2SKenneth D. Merry 		break;
1009991554f2SKenneth D. Merry 	}
1010991554f2SKenneth D. Merry 	case XPT_GET_TRAN_SETTINGS:
1011991554f2SKenneth D. Merry 	{
1012991554f2SKenneth D. Merry 		struct ccb_trans_settings	*cts;
1013991554f2SKenneth D. Merry 		struct ccb_trans_settings_sas	*sas;
1014991554f2SKenneth D. Merry 		struct ccb_trans_settings_scsi	*scsi;
1015991554f2SKenneth D. Merry 		struct mprsas_target *targ;
1016991554f2SKenneth D. Merry 
1017991554f2SKenneth D. Merry 		cts = &ccb->cts;
1018991554f2SKenneth D. Merry 		sas = &cts->xport_specific.sas;
1019991554f2SKenneth D. Merry 		scsi = &cts->proto_specific.scsi;
1020991554f2SKenneth D. Merry 
1021991554f2SKenneth D. Merry 		KASSERT(cts->ccb_h.target_id < sassc->maxtargets,
1022991554f2SKenneth D. Merry 		    ("Target %d out of bounds in XPT_GET_TRAN_SETTINGS\n",
1023991554f2SKenneth D. Merry 		    cts->ccb_h.target_id));
1024991554f2SKenneth D. Merry 		targ = &sassc->targets[cts->ccb_h.target_id];
1025991554f2SKenneth D. Merry 		if (targ->handle == 0x0) {
1026a2c14879SStephen McConnell 			mprsas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE);
1027991554f2SKenneth D. Merry 			break;
1028991554f2SKenneth D. Merry 		}
1029991554f2SKenneth D. Merry 
1030991554f2SKenneth D. Merry 		cts->protocol_version = SCSI_REV_SPC2;
1031991554f2SKenneth D. Merry 		cts->transport = XPORT_SAS;
1032991554f2SKenneth D. Merry 		cts->transport_version = 0;
1033991554f2SKenneth D. Merry 
1034991554f2SKenneth D. Merry 		sas->valid = CTS_SAS_VALID_SPEED;
1035991554f2SKenneth D. Merry 		switch (targ->linkrate) {
1036991554f2SKenneth D. Merry 		case 0x08:
1037991554f2SKenneth D. Merry 			sas->bitrate = 150000;
1038991554f2SKenneth D. Merry 			break;
1039991554f2SKenneth D. Merry 		case 0x09:
1040991554f2SKenneth D. Merry 			sas->bitrate = 300000;
1041991554f2SKenneth D. Merry 			break;
1042991554f2SKenneth D. Merry 		case 0x0a:
1043991554f2SKenneth D. Merry 			sas->bitrate = 600000;
1044991554f2SKenneth D. Merry 			break;
104582315915SAlexander Motin 		case 0x0b:
104682315915SAlexander Motin 			sas->bitrate = 1200000;
104782315915SAlexander Motin 			break;
1048991554f2SKenneth D. Merry 		default:
1049991554f2SKenneth D. Merry 			sas->valid = 0;
1050991554f2SKenneth D. Merry 		}
1051991554f2SKenneth D. Merry 
1052991554f2SKenneth D. Merry 		cts->protocol = PROTO_SCSI;
1053991554f2SKenneth D. Merry 		scsi->valid = CTS_SCSI_VALID_TQ;
1054991554f2SKenneth D. Merry 		scsi->flags = CTS_SCSI_FLAGS_TAG_ENB;
1055991554f2SKenneth D. Merry 
1056a2c14879SStephen McConnell 		mprsas_set_ccbstatus(ccb, CAM_REQ_CMP);
1057991554f2SKenneth D. Merry 		break;
1058991554f2SKenneth D. Merry 	}
1059991554f2SKenneth D. Merry 	case XPT_CALC_GEOMETRY:
1060991554f2SKenneth D. Merry 		cam_calc_geometry(&ccb->ccg, /*extended*/1);
1061a2c14879SStephen McConnell 		mprsas_set_ccbstatus(ccb, CAM_REQ_CMP);
1062991554f2SKenneth D. Merry 		break;
1063991554f2SKenneth D. Merry 	case XPT_RESET_DEV:
10647a2a6a1aSStephen McConnell 		mpr_dprint(sassc->sc, MPR_XINFO, "mprsas_action "
10657a2a6a1aSStephen McConnell 		    "XPT_RESET_DEV\n");
1066991554f2SKenneth D. Merry 		mprsas_action_resetdev(sassc, ccb);
1067991554f2SKenneth D. Merry 		return;
1068991554f2SKenneth D. Merry 	case XPT_RESET_BUS:
1069991554f2SKenneth D. Merry 	case XPT_ABORT:
1070991554f2SKenneth D. Merry 	case XPT_TERM_IO:
10717a2a6a1aSStephen McConnell 		mpr_dprint(sassc->sc, MPR_XINFO, "mprsas_action faking success "
10727a2a6a1aSStephen McConnell 		    "for abort or reset\n");
1073a2c14879SStephen McConnell 		mprsas_set_ccbstatus(ccb, CAM_REQ_CMP);
1074991554f2SKenneth D. Merry 		break;
1075991554f2SKenneth D. Merry 	case XPT_SCSI_IO:
1076991554f2SKenneth D. Merry 		mprsas_action_scsiio(sassc, ccb);
1077991554f2SKenneth D. Merry 		return;
1078991554f2SKenneth D. Merry 	case XPT_SMP_IO:
1079991554f2SKenneth D. Merry 		mprsas_action_smpio(sassc, ccb);
1080991554f2SKenneth D. Merry 		return;
1081991554f2SKenneth D. Merry 	default:
1082a2c14879SStephen McConnell 		mprsas_set_ccbstatus(ccb, CAM_FUNC_NOTAVAIL);
1083991554f2SKenneth D. Merry 		break;
1084991554f2SKenneth D. Merry 	}
1085991554f2SKenneth D. Merry 	xpt_done(ccb);
1086991554f2SKenneth D. Merry 
1087991554f2SKenneth D. Merry }
1088991554f2SKenneth D. Merry 
1089991554f2SKenneth D. Merry static void
1090991554f2SKenneth D. Merry mprsas_announce_reset(struct mpr_softc *sc, uint32_t ac_code,
1091991554f2SKenneth D. Merry     target_id_t target_id, lun_id_t lun_id)
1092991554f2SKenneth D. Merry {
1093991554f2SKenneth D. Merry 	path_id_t path_id = cam_sim_path(sc->sassc->sim);
1094991554f2SKenneth D. Merry 	struct cam_path *path;
1095991554f2SKenneth D. Merry 
1096991554f2SKenneth D. Merry 	mpr_dprint(sc, MPR_XINFO, "%s code %x target %d lun %jx\n", __func__,
1097991554f2SKenneth D. Merry 	    ac_code, target_id, (uintmax_t)lun_id);
1098991554f2SKenneth D. Merry 
1099991554f2SKenneth D. Merry 	if (xpt_create_path(&path, NULL,
1100991554f2SKenneth D. Merry 		path_id, target_id, lun_id) != CAM_REQ_CMP) {
1101991554f2SKenneth D. Merry 		mpr_dprint(sc, MPR_ERROR, "unable to create path for reset "
1102991554f2SKenneth D. Merry 		    "notification\n");
1103991554f2SKenneth D. Merry 		return;
1104991554f2SKenneth D. Merry 	}
1105991554f2SKenneth D. Merry 
1106991554f2SKenneth D. Merry 	xpt_async(ac_code, path, NULL);
1107991554f2SKenneth D. Merry 	xpt_free_path(path);
1108991554f2SKenneth D. Merry }
1109991554f2SKenneth D. Merry 
1110991554f2SKenneth D. Merry static void
1111991554f2SKenneth D. Merry mprsas_complete_all_commands(struct mpr_softc *sc)
1112991554f2SKenneth D. Merry {
1113991554f2SKenneth D. Merry 	struct mpr_command *cm;
1114991554f2SKenneth D. Merry 	int i;
1115991554f2SKenneth D. Merry 	int completed;
1116991554f2SKenneth D. Merry 
1117991554f2SKenneth D. Merry 	MPR_FUNCTRACE(sc);
1118991554f2SKenneth D. Merry 	mtx_assert(&sc->mpr_mtx, MA_OWNED);
1119991554f2SKenneth D. Merry 
1120991554f2SKenneth D. Merry 	/* complete all commands with a NULL reply */
1121991554f2SKenneth D. Merry 	for (i = 1; i < sc->num_reqs; i++) {
1122991554f2SKenneth D. Merry 		cm = &sc->commands[i];
1123f0779b04SScott Long 		if (cm->cm_state == MPR_CM_STATE_FREE)
1124f0779b04SScott Long 			continue;
1125f0779b04SScott Long 
1126f0779b04SScott Long 		cm->cm_state = MPR_CM_STATE_BUSY;
1127991554f2SKenneth D. Merry 		cm->cm_reply = NULL;
1128991554f2SKenneth D. Merry 		completed = 0;
1129991554f2SKenneth D. Merry 
11308277ce2bSConrad Meyer 		if (cm->cm_flags & MPR_CM_FLAGS_SATA_ID_TIMEOUT) {
11318277ce2bSConrad Meyer 			MPASS(cm->cm_data);
11328277ce2bSConrad Meyer 			free(cm->cm_data, M_MPR);
11338277ce2bSConrad Meyer 			cm->cm_data = NULL;
11348277ce2bSConrad Meyer 		}
11358277ce2bSConrad Meyer 
1136991554f2SKenneth D. Merry 		if (cm->cm_flags & MPR_CM_FLAGS_POLLED)
1137991554f2SKenneth D. Merry 			cm->cm_flags |= MPR_CM_FLAGS_COMPLETE;
1138991554f2SKenneth D. Merry 
1139991554f2SKenneth D. Merry 		if (cm->cm_complete != NULL) {
1140991554f2SKenneth D. Merry 			mprsas_log_command(cm, MPR_RECOVERY,
11417a2a6a1aSStephen McConnell 			    "completing cm %p state %x ccb %p for diag reset\n",
11427a2a6a1aSStephen McConnell 			    cm, cm->cm_state, cm->cm_ccb);
1143991554f2SKenneth D. Merry 			cm->cm_complete(sc, cm);
1144991554f2SKenneth D. Merry 			completed = 1;
1145f0779b04SScott Long 		} else if (cm->cm_flags & MPR_CM_FLAGS_WAKEUP) {
1146991554f2SKenneth D. Merry 			mprsas_log_command(cm, MPR_RECOVERY,
1147991554f2SKenneth D. Merry 			    "waking up cm %p state %x ccb %p for diag reset\n",
1148991554f2SKenneth D. Merry 			    cm, cm->cm_state, cm->cm_ccb);
1149991554f2SKenneth D. Merry 			wakeup(cm);
1150991554f2SKenneth D. Merry 			completed = 1;
1151991554f2SKenneth D. Merry 		}
1152991554f2SKenneth D. Merry 
1153991554f2SKenneth D. Merry 		if ((completed == 0) && (cm->cm_state != MPR_CM_STATE_FREE)) {
1154991554f2SKenneth D. Merry 			/* this should never happen, but if it does, log */
1155991554f2SKenneth D. Merry 			mprsas_log_command(cm, MPR_RECOVERY,
1156991554f2SKenneth D. Merry 			    "cm %p state %x flags 0x%x ccb %p during diag "
1157991554f2SKenneth D. Merry 			    "reset\n", cm, cm->cm_state, cm->cm_flags,
1158991554f2SKenneth D. Merry 			    cm->cm_ccb);
1159991554f2SKenneth D. Merry 		}
1160991554f2SKenneth D. Merry 	}
1161f0779b04SScott Long 
1162f0779b04SScott Long 	sc->io_cmds_active = 0;
1163991554f2SKenneth D. Merry }
1164991554f2SKenneth D. Merry 
1165991554f2SKenneth D. Merry void
1166991554f2SKenneth D. Merry mprsas_handle_reinit(struct mpr_softc *sc)
1167991554f2SKenneth D. Merry {
1168991554f2SKenneth D. Merry 	int i;
1169991554f2SKenneth D. Merry 
1170991554f2SKenneth D. Merry 	/* Go back into startup mode and freeze the simq, so that CAM
1171991554f2SKenneth D. Merry 	 * doesn't send any commands until after we've rediscovered all
1172991554f2SKenneth D. Merry 	 * targets and found the proper device handles for them.
1173991554f2SKenneth D. Merry 	 *
1174991554f2SKenneth D. Merry 	 * After the reset, portenable will trigger discovery, and after all
1175991554f2SKenneth D. Merry 	 * discovery-related activities have finished, the simq will be
1176991554f2SKenneth D. Merry 	 * released.
1177991554f2SKenneth D. Merry 	 */
1178991554f2SKenneth D. Merry 	mpr_dprint(sc, MPR_INIT, "%s startup\n", __func__);
1179991554f2SKenneth D. Merry 	sc->sassc->flags |= MPRSAS_IN_STARTUP;
1180991554f2SKenneth D. Merry 	sc->sassc->flags |= MPRSAS_IN_DISCOVERY;
1181991554f2SKenneth D. Merry 	mprsas_startup_increment(sc->sassc);
1182991554f2SKenneth D. Merry 
1183991554f2SKenneth D. Merry 	/* notify CAM of a bus reset */
1184991554f2SKenneth D. Merry 	mprsas_announce_reset(sc, AC_BUS_RESET, CAM_TARGET_WILDCARD,
1185991554f2SKenneth D. Merry 	    CAM_LUN_WILDCARD);
1186991554f2SKenneth D. Merry 
1187991554f2SKenneth D. Merry 	/* complete and cleanup after all outstanding commands */
1188991554f2SKenneth D. Merry 	mprsas_complete_all_commands(sc);
1189991554f2SKenneth D. Merry 
1190a2c14879SStephen McConnell 	mpr_dprint(sc, MPR_INIT, "%s startup %u after command completion\n",
1191a2c14879SStephen McConnell 	    __func__, sc->sassc->startup_refcount);
1192991554f2SKenneth D. Merry 
1193991554f2SKenneth D. Merry 	/* zero all the target handles, since they may change after the
1194991554f2SKenneth D. Merry 	 * reset, and we have to rediscover all the targets and use the new
1195991554f2SKenneth D. Merry 	 * handles.
1196991554f2SKenneth D. Merry 	 */
1197991554f2SKenneth D. Merry 	for (i = 0; i < sc->sassc->maxtargets; i++) {
1198991554f2SKenneth D. Merry 		if (sc->sassc->targets[i].outstanding != 0)
1199991554f2SKenneth D. Merry 			mpr_dprint(sc, MPR_INIT, "target %u outstanding %u\n",
1200991554f2SKenneth D. Merry 			    i, sc->sassc->targets[i].outstanding);
1201991554f2SKenneth D. Merry 		sc->sassc->targets[i].handle = 0x0;
1202991554f2SKenneth D. Merry 		sc->sassc->targets[i].exp_dev_handle = 0x0;
1203991554f2SKenneth D. Merry 		sc->sassc->targets[i].outstanding = 0;
1204991554f2SKenneth D. Merry 		sc->sassc->targets[i].flags = MPRSAS_TARGET_INDIAGRESET;
1205991554f2SKenneth D. Merry 	}
1206991554f2SKenneth D. Merry }
1207991554f2SKenneth D. Merry static void
1208991554f2SKenneth D. Merry mprsas_tm_timeout(void *data)
1209991554f2SKenneth D. Merry {
1210991554f2SKenneth D. Merry 	struct mpr_command *tm = data;
1211991554f2SKenneth D. Merry 	struct mpr_softc *sc = tm->cm_sc;
1212991554f2SKenneth D. Merry 
1213991554f2SKenneth D. Merry 	mtx_assert(&sc->mpr_mtx, MA_OWNED);
1214991554f2SKenneth D. Merry 
12157a2a6a1aSStephen McConnell 	mprsas_log_command(tm, MPR_INFO|MPR_RECOVERY, "task mgmt %p timed "
12167a2a6a1aSStephen McConnell 	    "out\n", tm);
1217f0779b04SScott Long 
1218f0779b04SScott Long 	KASSERT(tm->cm_state == MPR_CM_STATE_INQUEUE,
1219175ad3d0SKenneth D. Merry 	    ("command not inqueue, state = %u\n", tm->cm_state));
1220f0779b04SScott Long 
1221f0779b04SScott Long 	tm->cm_state = MPR_CM_STATE_BUSY;
1222991554f2SKenneth D. Merry 	mpr_reinit(sc);
1223991554f2SKenneth D. Merry }
1224991554f2SKenneth D. Merry 
1225991554f2SKenneth D. Merry static void
12267a2a6a1aSStephen McConnell mprsas_logical_unit_reset_complete(struct mpr_softc *sc, struct mpr_command *tm)
1227991554f2SKenneth D. Merry {
1228991554f2SKenneth D. Merry 	MPI2_SCSI_TASK_MANAGE_REPLY *reply;
1229991554f2SKenneth D. Merry 	unsigned int cm_count = 0;
1230991554f2SKenneth D. Merry 	struct mpr_command *cm;
1231991554f2SKenneth D. Merry 	struct mprsas_target *targ;
1232991554f2SKenneth D. Merry 
1233991554f2SKenneth D. Merry 	callout_stop(&tm->cm_callout);
1234991554f2SKenneth D. Merry 
1235991554f2SKenneth D. Merry 	reply = (MPI2_SCSI_TASK_MANAGE_REPLY *)tm->cm_reply;
1236991554f2SKenneth D. Merry 	targ = tm->cm_targ;
1237991554f2SKenneth D. Merry 
1238991554f2SKenneth D. Merry 	/*
1239991554f2SKenneth D. Merry 	 * Currently there should be no way we can hit this case.  It only
1240991554f2SKenneth D. Merry 	 * happens when we have a failure to allocate chain frames, and
1241991554f2SKenneth D. Merry 	 * task management commands don't have S/G lists.
1242991554f2SKenneth D. Merry 	 */
1243991554f2SKenneth D. Merry 	if ((tm->cm_flags & MPR_CM_FLAGS_ERROR_MASK) != 0) {
12446eea4f46SScott Long 		mpr_dprint(sc, MPR_RECOVERY|MPR_ERROR,
12456eea4f46SScott Long 		    "%s: cm_flags = %#x for LUN reset! "
1246991554f2SKenneth D. Merry 		    "This should not happen!\n", __func__, tm->cm_flags);
1247991554f2SKenneth D. Merry 		mprsas_free_tm(sc, tm);
1248991554f2SKenneth D. Merry 		return;
1249991554f2SKenneth D. Merry 	}
1250991554f2SKenneth D. Merry 
1251991554f2SKenneth D. Merry 	if (reply == NULL) {
12526eea4f46SScott Long 		mpr_dprint(sc, MPR_RECOVERY, "NULL reset reply for tm %p\n",
12536eea4f46SScott Long 		    tm);
1254991554f2SKenneth D. Merry 		if ((sc->mpr_flags & MPR_FLAGS_DIAGRESET) != 0) {
1255991554f2SKenneth D. Merry 			/* this completion was due to a reset, just cleanup */
12566eea4f46SScott Long 			mpr_dprint(sc, MPR_RECOVERY, "Hardware undergoing "
12576eea4f46SScott Long 			    "reset, ignoring NULL LUN reset reply\n");
1258991554f2SKenneth D. Merry 			targ->tm = NULL;
1259991554f2SKenneth D. Merry 			mprsas_free_tm(sc, tm);
1260991554f2SKenneth D. Merry 		}
1261991554f2SKenneth D. Merry 		else {
1262991554f2SKenneth D. Merry 			/* we should have gotten a reply. */
12636eea4f46SScott Long 			mpr_dprint(sc, MPR_INFO|MPR_RECOVERY, "NULL reply on "
12646eea4f46SScott Long 			    "LUN reset attempt, resetting controller\n");
1265991554f2SKenneth D. Merry 			mpr_reinit(sc);
1266991554f2SKenneth D. Merry 		}
1267991554f2SKenneth D. Merry 		return;
1268991554f2SKenneth D. Merry 	}
1269991554f2SKenneth D. Merry 
12706eea4f46SScott Long 	mpr_dprint(sc, MPR_RECOVERY,
1271991554f2SKenneth D. Merry 	    "logical unit reset status 0x%x code 0x%x count %u\n",
1272991554f2SKenneth D. Merry 	    le16toh(reply->IOCStatus), le32toh(reply->ResponseCode),
1273991554f2SKenneth D. Merry 	    le32toh(reply->TerminationCount));
1274991554f2SKenneth D. Merry 
12756eea4f46SScott Long 	/*
12766eea4f46SScott Long 	 * See if there are any outstanding commands for this LUN.
1277991554f2SKenneth D. Merry 	 * This could be made more efficient by using a per-LU data
1278991554f2SKenneth D. Merry 	 * structure of some sort.
1279991554f2SKenneth D. Merry 	 */
1280991554f2SKenneth D. Merry 	TAILQ_FOREACH(cm, &targ->commands, cm_link) {
1281991554f2SKenneth D. Merry 		if (cm->cm_lun == tm->cm_lun)
1282991554f2SKenneth D. Merry 			cm_count++;
1283991554f2SKenneth D. Merry 	}
1284991554f2SKenneth D. Merry 
1285991554f2SKenneth D. Merry 	if (cm_count == 0) {
12866eea4f46SScott Long 		mpr_dprint(sc, MPR_RECOVERY|MPR_INFO,
12876eea4f46SScott Long 		    "Finished recovery after LUN reset for target %u\n",
12886eea4f46SScott Long 		    targ->tid);
1289991554f2SKenneth D. Merry 
12906eea4f46SScott Long 		mprsas_announce_reset(sc, AC_SENT_BDR, targ->tid,
1291991554f2SKenneth D. Merry 		    tm->cm_lun);
1292991554f2SKenneth D. Merry 
12936eea4f46SScott Long 		/*
12946eea4f46SScott Long 		 * We've finished recovery for this logical unit.  check and
1295991554f2SKenneth D. Merry 		 * see if some other logical unit has a timedout command
1296991554f2SKenneth D. Merry 		 * that needs to be processed.
1297991554f2SKenneth D. Merry 		 */
1298991554f2SKenneth D. Merry 		cm = TAILQ_FIRST(&targ->timedout_commands);
1299991554f2SKenneth D. Merry 		if (cm) {
13006eea4f46SScott Long 			mpr_dprint(sc, MPR_INFO|MPR_RECOVERY,
13016eea4f46SScott Long 			   "More commands to abort for target %u\n", targ->tid);
1302991554f2SKenneth D. Merry 			mprsas_send_abort(sc, tm, cm);
13036eea4f46SScott Long 		} else {
1304991554f2SKenneth D. Merry 			targ->tm = NULL;
1305991554f2SKenneth D. Merry 			mprsas_free_tm(sc, tm);
1306991554f2SKenneth D. Merry 		}
13076eea4f46SScott Long 	} else {
1308991554f2SKenneth D. Merry 		/* if we still have commands for this LUN, the reset
1309991554f2SKenneth D. Merry 		 * effectively failed, regardless of the status reported.
1310991554f2SKenneth D. Merry 		 * Escalate to a target reset.
1311991554f2SKenneth D. Merry 		 */
13126eea4f46SScott Long 		mpr_dprint(sc, MPR_INFO|MPR_RECOVERY,
13136eea4f46SScott Long 		    "logical unit reset complete for target %u, but still "
13146eea4f46SScott Long 		    "have %u command(s), sending target reset\n", targ->tid,
13156eea4f46SScott Long 		    cm_count);
131689d1c21fSKashyap D Desai 		if (!targ->is_nvme || sc->custom_nvme_tm_handling)
1317991554f2SKenneth D. Merry 			mprsas_send_reset(sc, tm,
1318991554f2SKenneth D. Merry 			    MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET);
131989d1c21fSKashyap D Desai 		else
132089d1c21fSKashyap D Desai 			mpr_reinit(sc);
1321991554f2SKenneth D. Merry 	}
1322991554f2SKenneth D. Merry }
1323991554f2SKenneth D. Merry 
1324991554f2SKenneth D. Merry static void
1325991554f2SKenneth D. Merry mprsas_target_reset_complete(struct mpr_softc *sc, struct mpr_command *tm)
1326991554f2SKenneth D. Merry {
1327991554f2SKenneth D. Merry 	MPI2_SCSI_TASK_MANAGE_REPLY *reply;
1328991554f2SKenneth D. Merry 	MPI2_SCSI_TASK_MANAGE_REQUEST *req;
1329991554f2SKenneth D. Merry 	struct mprsas_target *targ;
1330991554f2SKenneth D. Merry 
1331991554f2SKenneth D. Merry 	callout_stop(&tm->cm_callout);
1332991554f2SKenneth D. Merry 
1333991554f2SKenneth D. Merry 	req = (MPI2_SCSI_TASK_MANAGE_REQUEST *)tm->cm_req;
1334991554f2SKenneth D. Merry 	reply = (MPI2_SCSI_TASK_MANAGE_REPLY *)tm->cm_reply;
1335991554f2SKenneth D. Merry 	targ = tm->cm_targ;
1336991554f2SKenneth D. Merry 
1337991554f2SKenneth D. Merry 	/*
1338991554f2SKenneth D. Merry 	 * Currently there should be no way we can hit this case.  It only
1339991554f2SKenneth D. Merry 	 * happens when we have a failure to allocate chain frames, and
1340991554f2SKenneth D. Merry 	 * task management commands don't have S/G lists.
1341991554f2SKenneth D. Merry 	 */
1342991554f2SKenneth D. Merry 	if ((tm->cm_flags & MPR_CM_FLAGS_ERROR_MASK) != 0) {
1343a2c14879SStephen McConnell 		mpr_dprint(sc, MPR_ERROR, "%s: cm_flags = %#x for target "
1344a2c14879SStephen McConnell 		    "reset! This should not happen!\n", __func__, tm->cm_flags);
1345991554f2SKenneth D. Merry 		mprsas_free_tm(sc, tm);
1346991554f2SKenneth D. Merry 		return;
1347991554f2SKenneth D. Merry 	}
1348991554f2SKenneth D. Merry 
1349991554f2SKenneth D. Merry 	if (reply == NULL) {
13506eea4f46SScott Long 		mpr_dprint(sc, MPR_RECOVERY,
13516eea4f46SScott Long 		    "NULL target reset reply for tm %p TaskMID %u\n",
13526eea4f46SScott Long 		    tm, le16toh(req->TaskMID));
1353991554f2SKenneth D. Merry 		if ((sc->mpr_flags & MPR_FLAGS_DIAGRESET) != 0) {
1354991554f2SKenneth D. Merry 			/* this completion was due to a reset, just cleanup */
13556eea4f46SScott Long 			mpr_dprint(sc, MPR_RECOVERY, "Hardware undergoing "
13566eea4f46SScott Long 			    "reset, ignoring NULL target reset reply\n");
1357991554f2SKenneth D. Merry 			targ->tm = NULL;
1358991554f2SKenneth D. Merry 			mprsas_free_tm(sc, tm);
1359991554f2SKenneth D. Merry 		}
1360991554f2SKenneth D. Merry 		else {
1361991554f2SKenneth D. Merry 			/* we should have gotten a reply. */
13626eea4f46SScott Long 			mpr_dprint(sc, MPR_INFO|MPR_RECOVERY, "NULL reply on "
13636eea4f46SScott Long 			    "target reset attempt, resetting controller\n");
1364991554f2SKenneth D. Merry 			mpr_reinit(sc);
1365991554f2SKenneth D. Merry 		}
1366991554f2SKenneth D. Merry 		return;
1367991554f2SKenneth D. Merry 	}
1368991554f2SKenneth D. Merry 
13696eea4f46SScott Long 	mpr_dprint(sc, MPR_RECOVERY,
1370991554f2SKenneth D. Merry 	    "target reset status 0x%x code 0x%x count %u\n",
1371991554f2SKenneth D. Merry 	    le16toh(reply->IOCStatus), le32toh(reply->ResponseCode),
1372991554f2SKenneth D. Merry 	    le32toh(reply->TerminationCount));
1373991554f2SKenneth D. Merry 
1374991554f2SKenneth D. Merry 	if (targ->outstanding == 0) {
13756eea4f46SScott Long 		/*
13766eea4f46SScott Long 		 * We've finished recovery for this target and all
1377991554f2SKenneth D. Merry 		 * of its logical units.
1378991554f2SKenneth D. Merry 		 */
13796eea4f46SScott Long 		mpr_dprint(sc, MPR_RECOVERY|MPR_INFO,
13806eea4f46SScott Long 		    "Finished reset recovery for target %u\n", targ->tid);
1381991554f2SKenneth D. Merry 
1382991554f2SKenneth D. Merry 		mprsas_announce_reset(sc, AC_SENT_BDR, tm->cm_targ->tid,
1383991554f2SKenneth D. Merry 		    CAM_LUN_WILDCARD);
1384991554f2SKenneth D. Merry 
1385991554f2SKenneth D. Merry 		targ->tm = NULL;
1386991554f2SKenneth D. Merry 		mprsas_free_tm(sc, tm);
13876eea4f46SScott Long 	} else {
13886eea4f46SScott Long 		/*
13896eea4f46SScott Long 		 * After a target reset, if this target still has
1390991554f2SKenneth D. Merry 		 * outstanding commands, the reset effectively failed,
1391991554f2SKenneth D. Merry 		 * regardless of the status reported.  escalate.
1392991554f2SKenneth D. Merry 		 */
13936eea4f46SScott Long 		mpr_dprint(sc, MPR_INFO|MPR_RECOVERY,
13946eea4f46SScott Long 		    "Target reset complete for target %u, but still have %u "
13956eea4f46SScott Long 		    "command(s), resetting controller\n", targ->tid,
13966eea4f46SScott Long 		    targ->outstanding);
1397991554f2SKenneth D. Merry 		mpr_reinit(sc);
1398991554f2SKenneth D. Merry 	}
1399991554f2SKenneth D. Merry }
1400991554f2SKenneth D. Merry 
1401991554f2SKenneth D. Merry #define MPR_RESET_TIMEOUT 30
1402991554f2SKenneth D. Merry 
1403a2c14879SStephen McConnell int
1404991554f2SKenneth D. Merry mprsas_send_reset(struct mpr_softc *sc, struct mpr_command *tm, uint8_t type)
1405991554f2SKenneth D. Merry {
1406991554f2SKenneth D. Merry 	MPI2_SCSI_TASK_MANAGE_REQUEST *req;
1407991554f2SKenneth D. Merry 	struct mprsas_target *target;
140889d1c21fSKashyap D Desai 	int err, timeout;
1409991554f2SKenneth D. Merry 
1410991554f2SKenneth D. Merry 	target = tm->cm_targ;
1411991554f2SKenneth D. Merry 	if (target->handle == 0) {
1412a2c14879SStephen McConnell 		mpr_dprint(sc, MPR_ERROR, "%s null devhandle for target_id "
1413a2c14879SStephen McConnell 		    "%d\n", __func__, target->tid);
1414991554f2SKenneth D. Merry 		return -1;
1415991554f2SKenneth D. Merry 	}
1416991554f2SKenneth D. Merry 
1417991554f2SKenneth D. Merry 	req = (MPI2_SCSI_TASK_MANAGE_REQUEST *)tm->cm_req;
1418991554f2SKenneth D. Merry 	req->DevHandle = htole16(target->handle);
1419991554f2SKenneth D. Merry 	req->TaskType = type;
1420991554f2SKenneth D. Merry 
142189d1c21fSKashyap D Desai 	if (!target->is_nvme || sc->custom_nvme_tm_handling) {
142289d1c21fSKashyap D Desai 		timeout = MPR_RESET_TIMEOUT;
142389d1c21fSKashyap D Desai 		/*
142489d1c21fSKashyap D Desai 		 * Target reset method =
142589d1c21fSKashyap D Desai 		 *     SAS Hard Link Reset / SATA Link Reset
142689d1c21fSKashyap D Desai 		 */
142789d1c21fSKashyap D Desai 		req->MsgFlags = MPI2_SCSITASKMGMT_MSGFLAGS_LINK_RESET;
142889d1c21fSKashyap D Desai 	} else {
142989d1c21fSKashyap D Desai 		timeout = (target->controller_reset_timeout) ? (
143089d1c21fSKashyap D Desai 		    target->controller_reset_timeout) : (MPR_RESET_TIMEOUT);
143189d1c21fSKashyap D Desai 		/* PCIe Protocol Level Reset*/
143289d1c21fSKashyap D Desai 		req->MsgFlags =
143389d1c21fSKashyap D Desai 		    MPI26_SCSITASKMGMT_MSGFLAGS_PROTOCOL_LVL_RST_PCIE;
143489d1c21fSKashyap D Desai 	}
143589d1c21fSKashyap D Desai 
1436991554f2SKenneth D. Merry 	if (type == MPI2_SCSITASKMGMT_TASKTYPE_LOGICAL_UNIT_RESET) {
1437991554f2SKenneth D. Merry 		/* XXX Need to handle invalid LUNs */
1438991554f2SKenneth D. Merry 		MPR_SET_LUN(req->LUN, tm->cm_lun);
1439991554f2SKenneth D. Merry 		tm->cm_targ->logical_unit_resets++;
14406eea4f46SScott Long 		mpr_dprint(sc, MPR_RECOVERY|MPR_INFO,
14416eea4f46SScott Long 		    "Sending logical unit reset to target %u lun %d\n",
14426eea4f46SScott Long 		    target->tid, tm->cm_lun);
1443991554f2SKenneth D. Merry 		tm->cm_complete = mprsas_logical_unit_reset_complete;
1444a2c14879SStephen McConnell 		mprsas_prepare_for_tm(sc, tm, target, tm->cm_lun);
14456eea4f46SScott Long 	} else if (type == MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET) {
1446991554f2SKenneth D. Merry 		tm->cm_targ->target_resets++;
14476eea4f46SScott Long 		mpr_dprint(sc, MPR_RECOVERY|MPR_INFO,
14486eea4f46SScott Long 		    "Sending target reset to target %u\n", target->tid);
1449991554f2SKenneth D. Merry 		tm->cm_complete = mprsas_target_reset_complete;
1450a2c14879SStephen McConnell 		mprsas_prepare_for_tm(sc, tm, target, CAM_LUN_WILDCARD);
1451991554f2SKenneth D. Merry 	}
1452991554f2SKenneth D. Merry 	else {
1453991554f2SKenneth D. Merry 		mpr_dprint(sc, MPR_ERROR, "unexpected reset type 0x%x\n", type);
1454991554f2SKenneth D. Merry 		return -1;
1455991554f2SKenneth D. Merry 	}
1456991554f2SKenneth D. Merry 
1457991554f2SKenneth D. Merry 	if (target->encl_level_valid) {
14586eea4f46SScott Long 		mpr_dprint(sc, MPR_RECOVERY|MPR_INFO,
14596eea4f46SScott Long 		    "At enclosure level %d, slot %d, connector name (%4s)\n",
14606eea4f46SScott Long 		    target->encl_level, target->encl_slot,
14616eea4f46SScott Long 		    target->connector_name);
1462991554f2SKenneth D. Merry 	}
1463991554f2SKenneth D. Merry 
1464991554f2SKenneth D. Merry 	tm->cm_data = NULL;
1465991554f2SKenneth D. Merry 	tm->cm_complete_data = (void *)tm;
1466991554f2SKenneth D. Merry 
146789d1c21fSKashyap D Desai 	callout_reset(&tm->cm_callout, timeout * hz,
1468991554f2SKenneth D. Merry 	    mprsas_tm_timeout, tm);
1469991554f2SKenneth D. Merry 
1470991554f2SKenneth D. Merry 	err = mpr_map_command(sc, tm);
1471991554f2SKenneth D. Merry 	if (err)
14726eea4f46SScott Long 		mpr_dprint(sc, MPR_ERROR|MPR_RECOVERY,
1473a2c14879SStephen McConnell 		    "error %d sending reset type %u\n", err, type);
1474991554f2SKenneth D. Merry 
1475991554f2SKenneth D. Merry 	return err;
1476991554f2SKenneth D. Merry }
1477991554f2SKenneth D. Merry 
1478991554f2SKenneth D. Merry static void
1479991554f2SKenneth D. Merry mprsas_abort_complete(struct mpr_softc *sc, struct mpr_command *tm)
1480991554f2SKenneth D. Merry {
1481991554f2SKenneth D. Merry 	struct mpr_command *cm;
1482991554f2SKenneth D. Merry 	MPI2_SCSI_TASK_MANAGE_REPLY *reply;
1483991554f2SKenneth D. Merry 	MPI2_SCSI_TASK_MANAGE_REQUEST *req;
1484991554f2SKenneth D. Merry 	struct mprsas_target *targ;
1485991554f2SKenneth D. Merry 
1486991554f2SKenneth D. Merry 	callout_stop(&tm->cm_callout);
1487991554f2SKenneth D. Merry 
1488991554f2SKenneth D. Merry 	req = (MPI2_SCSI_TASK_MANAGE_REQUEST *)tm->cm_req;
1489991554f2SKenneth D. Merry 	reply = (MPI2_SCSI_TASK_MANAGE_REPLY *)tm->cm_reply;
1490991554f2SKenneth D. Merry 	targ = tm->cm_targ;
1491991554f2SKenneth D. Merry 
1492991554f2SKenneth D. Merry 	/*
1493991554f2SKenneth D. Merry 	 * Currently there should be no way we can hit this case.  It only
1494991554f2SKenneth D. Merry 	 * happens when we have a failure to allocate chain frames, and
1495991554f2SKenneth D. Merry 	 * task management commands don't have S/G lists.
1496991554f2SKenneth D. Merry 	 */
1497991554f2SKenneth D. Merry 	if ((tm->cm_flags & MPR_CM_FLAGS_ERROR_MASK) != 0) {
14986eea4f46SScott Long 		mpr_dprint(sc, MPR_RECOVERY|MPR_ERROR,
1499991554f2SKenneth D. Merry 		    "cm_flags = %#x for abort %p TaskMID %u!\n",
1500991554f2SKenneth D. Merry 		    tm->cm_flags, tm, le16toh(req->TaskMID));
1501991554f2SKenneth D. Merry 		mprsas_free_tm(sc, tm);
1502991554f2SKenneth D. Merry 		return;
1503991554f2SKenneth D. Merry 	}
1504991554f2SKenneth D. Merry 
1505991554f2SKenneth D. Merry 	if (reply == NULL) {
15066eea4f46SScott Long 		mpr_dprint(sc, MPR_RECOVERY,
1507991554f2SKenneth D. Merry 		    "NULL abort reply for tm %p TaskMID %u\n",
1508991554f2SKenneth D. Merry 		    tm, le16toh(req->TaskMID));
1509991554f2SKenneth D. Merry 		if ((sc->mpr_flags & MPR_FLAGS_DIAGRESET) != 0) {
1510991554f2SKenneth D. Merry 			/* this completion was due to a reset, just cleanup */
15116eea4f46SScott Long 			mpr_dprint(sc, MPR_RECOVERY, "Hardware undergoing "
15126eea4f46SScott Long 			    "reset, ignoring NULL abort reply\n");
1513991554f2SKenneth D. Merry 			targ->tm = NULL;
1514991554f2SKenneth D. Merry 			mprsas_free_tm(sc, tm);
15156eea4f46SScott Long 		} else {
1516991554f2SKenneth D. Merry 			/* we should have gotten a reply. */
15176eea4f46SScott Long 			mpr_dprint(sc, MPR_INFO|MPR_RECOVERY, "NULL reply on "
15186eea4f46SScott Long 			    "abort attempt, resetting controller\n");
1519991554f2SKenneth D. Merry 			mpr_reinit(sc);
1520991554f2SKenneth D. Merry 		}
1521991554f2SKenneth D. Merry 		return;
1522991554f2SKenneth D. Merry 	}
1523991554f2SKenneth D. Merry 
15246eea4f46SScott Long 	mpr_dprint(sc, MPR_RECOVERY,
1525991554f2SKenneth D. Merry 	    "abort TaskMID %u status 0x%x code 0x%x count %u\n",
1526991554f2SKenneth D. Merry 	    le16toh(req->TaskMID),
1527991554f2SKenneth D. Merry 	    le16toh(reply->IOCStatus), le32toh(reply->ResponseCode),
1528991554f2SKenneth D. Merry 	    le32toh(reply->TerminationCount));
1529991554f2SKenneth D. Merry 
1530991554f2SKenneth D. Merry 	cm = TAILQ_FIRST(&tm->cm_targ->timedout_commands);
1531991554f2SKenneth D. Merry 	if (cm == NULL) {
15326eea4f46SScott Long 		/*
15336eea4f46SScott Long 		 * if there are no more timedout commands, we're done with
1534991554f2SKenneth D. Merry 		 * error recovery for this target.
1535991554f2SKenneth D. Merry 		 */
15366eea4f46SScott Long 		mpr_dprint(sc, MPR_INFO|MPR_RECOVERY,
15376eea4f46SScott Long 		    "Finished abort recovery for target %u\n", targ->tid);
1538991554f2SKenneth D. Merry 		targ->tm = NULL;
1539991554f2SKenneth D. Merry 		mprsas_free_tm(sc, tm);
15406eea4f46SScott Long 	} else if (le16toh(req->TaskMID) != cm->cm_desc.Default.SMID) {
1541991554f2SKenneth D. Merry 		/* abort success, but we have more timedout commands to abort */
15426eea4f46SScott Long 		mpr_dprint(sc, MPR_INFO|MPR_RECOVERY,
15436eea4f46SScott Long 		    "Continuing abort recovery for target %u\n", targ->tid);
1544991554f2SKenneth D. Merry 		mprsas_send_abort(sc, tm, cm);
15456eea4f46SScott Long 	} else {
15466eea4f46SScott Long 		/*
15476eea4f46SScott Long 		 * we didn't get a command completion, so the abort
1548991554f2SKenneth D. Merry 		 * failed as far as we're concerned.  escalate.
1549991554f2SKenneth D. Merry 		 */
15506eea4f46SScott Long 		mpr_dprint(sc, MPR_INFO|MPR_RECOVERY,
15516eea4f46SScott Long 		    "Abort failed for target %u, sending logical unit reset\n",
15526eea4f46SScott Long 		    targ->tid);
1553991554f2SKenneth D. Merry 
1554991554f2SKenneth D. Merry 		mprsas_send_reset(sc, tm,
1555991554f2SKenneth D. Merry 		    MPI2_SCSITASKMGMT_TASKTYPE_LOGICAL_UNIT_RESET);
1556991554f2SKenneth D. Merry 	}
1557991554f2SKenneth D. Merry }
1558991554f2SKenneth D. Merry 
1559991554f2SKenneth D. Merry #define MPR_ABORT_TIMEOUT 5
1560991554f2SKenneth D. Merry 
1561991554f2SKenneth D. Merry static int
1562991554f2SKenneth D. Merry mprsas_send_abort(struct mpr_softc *sc, struct mpr_command *tm,
1563991554f2SKenneth D. Merry     struct mpr_command *cm)
1564991554f2SKenneth D. Merry {
1565991554f2SKenneth D. Merry 	MPI2_SCSI_TASK_MANAGE_REQUEST *req;
1566991554f2SKenneth D. Merry 	struct mprsas_target *targ;
156789d1c21fSKashyap D Desai 	int err, timeout;
1568991554f2SKenneth D. Merry 
1569991554f2SKenneth D. Merry 	targ = cm->cm_targ;
1570991554f2SKenneth D. Merry 	if (targ->handle == 0) {
15716eea4f46SScott Long 		mpr_dprint(sc, MPR_ERROR|MPR_RECOVERY,
15726eea4f46SScott Long 		   "%s null devhandle for target_id %d\n",
1573991554f2SKenneth D. Merry 		    __func__, cm->cm_ccb->ccb_h.target_id);
1574991554f2SKenneth D. Merry 		return -1;
1575991554f2SKenneth D. Merry 	}
1576991554f2SKenneth D. Merry 
1577855fe445SScott Long 	mprsas_log_command(cm, MPR_RECOVERY|MPR_INFO,
1578991554f2SKenneth D. Merry 	    "Aborting command %p\n", cm);
1579991554f2SKenneth D. Merry 
1580991554f2SKenneth D. Merry 	req = (MPI2_SCSI_TASK_MANAGE_REQUEST *)tm->cm_req;
1581991554f2SKenneth D. Merry 	req->DevHandle = htole16(targ->handle);
1582991554f2SKenneth D. Merry 	req->TaskType = MPI2_SCSITASKMGMT_TASKTYPE_ABORT_TASK;
1583991554f2SKenneth D. Merry 
1584991554f2SKenneth D. Merry 	/* XXX Need to handle invalid LUNs */
1585991554f2SKenneth D. Merry 	MPR_SET_LUN(req->LUN, cm->cm_ccb->ccb_h.target_lun);
1586991554f2SKenneth D. Merry 
1587991554f2SKenneth D. Merry 	req->TaskMID = htole16(cm->cm_desc.Default.SMID);
1588991554f2SKenneth D. Merry 
1589991554f2SKenneth D. Merry 	tm->cm_data = NULL;
1590991554f2SKenneth D. Merry 	tm->cm_complete = mprsas_abort_complete;
1591991554f2SKenneth D. Merry 	tm->cm_complete_data = (void *)tm;
1592991554f2SKenneth D. Merry 	tm->cm_targ = cm->cm_targ;
1593991554f2SKenneth D. Merry 	tm->cm_lun = cm->cm_lun;
1594991554f2SKenneth D. Merry 
159589d1c21fSKashyap D Desai 	if (!targ->is_nvme || sc->custom_nvme_tm_handling)
159689d1c21fSKashyap D Desai 		timeout	= MPR_ABORT_TIMEOUT;
159789d1c21fSKashyap D Desai 	else
159889d1c21fSKashyap D Desai 		timeout = sc->nvme_abort_timeout;
159989d1c21fSKashyap D Desai 
160089d1c21fSKashyap D Desai 	callout_reset(&tm->cm_callout, timeout * hz,
1601991554f2SKenneth D. Merry 	    mprsas_tm_timeout, tm);
1602991554f2SKenneth D. Merry 
1603991554f2SKenneth D. Merry 	targ->aborts++;
1604991554f2SKenneth D. Merry 
1605a2c14879SStephen McConnell 	mprsas_prepare_for_tm(sc, tm, targ, tm->cm_lun);
1606a2c14879SStephen McConnell 
1607991554f2SKenneth D. Merry 	err = mpr_map_command(sc, tm);
1608991554f2SKenneth D. Merry 	if (err)
16096eea4f46SScott Long 		mpr_dprint(sc, MPR_ERROR|MPR_RECOVERY,
1610991554f2SKenneth D. Merry 		    "error %d sending abort for cm %p SMID %u\n",
1611991554f2SKenneth D. Merry 		    err, cm, req->TaskMID);
1612991554f2SKenneth D. Merry 	return err;
1613991554f2SKenneth D. Merry }
1614991554f2SKenneth D. Merry 
1615991554f2SKenneth D. Merry static void
1616991554f2SKenneth D. Merry mprsas_scsiio_timeout(void *data)
1617991554f2SKenneth D. Merry {
16186eea4f46SScott Long 	sbintime_t elapsed, now;
16196eea4f46SScott Long 	union ccb *ccb;
1620991554f2SKenneth D. Merry 	struct mpr_softc *sc;
1621991554f2SKenneth D. Merry 	struct mpr_command *cm;
1622991554f2SKenneth D. Merry 	struct mprsas_target *targ;
1623991554f2SKenneth D. Merry 
1624991554f2SKenneth D. Merry 	cm = (struct mpr_command *)data;
1625991554f2SKenneth D. Merry 	sc = cm->cm_sc;
16266eea4f46SScott Long 	ccb = cm->cm_ccb;
16276eea4f46SScott Long 	now = sbinuptime();
1628991554f2SKenneth D. Merry 
1629991554f2SKenneth D. Merry 	MPR_FUNCTRACE(sc);
1630991554f2SKenneth D. Merry 	mtx_assert(&sc->mpr_mtx, MA_OWNED);
1631991554f2SKenneth D. Merry 
16326eea4f46SScott Long 	mpr_dprint(sc, MPR_XINFO|MPR_RECOVERY, "Timeout checking cm %p\n", cm);
1633991554f2SKenneth D. Merry 
1634991554f2SKenneth D. Merry 	/*
1635991554f2SKenneth D. Merry 	 * Run the interrupt handler to make sure it's not pending.  This
1636991554f2SKenneth D. Merry 	 * isn't perfect because the command could have already completed
1637991554f2SKenneth D. Merry 	 * and been re-used, though this is unlikely.
1638991554f2SKenneth D. Merry 	 */
1639991554f2SKenneth D. Merry 	mpr_intr_locked(sc);
16408fe7bf06SWarner Losh 	if (cm->cm_flags & MPR_CM_FLAGS_ON_RECOVERY) {
1641991554f2SKenneth D. Merry 		mprsas_log_command(cm, MPR_XINFO,
1642991554f2SKenneth D. Merry 		    "SCSI command %p almost timed out\n", cm);
1643991554f2SKenneth D. Merry 		return;
1644991554f2SKenneth D. Merry 	}
1645991554f2SKenneth D. Merry 
1646991554f2SKenneth D. Merry 	if (cm->cm_ccb == NULL) {
1647991554f2SKenneth D. Merry 		mpr_dprint(sc, MPR_ERROR, "command timeout with NULL ccb\n");
1648991554f2SKenneth D. Merry 		return;
1649991554f2SKenneth D. Merry 	}
1650991554f2SKenneth D. Merry 
1651991554f2SKenneth D. Merry 	targ = cm->cm_targ;
1652991554f2SKenneth D. Merry 	targ->timeouts++;
1653991554f2SKenneth D. Merry 
16546eea4f46SScott Long 	elapsed = now - ccb->ccb_h.qos.sim_data;
16556eea4f46SScott Long 	mprsas_log_command(cm, MPR_INFO|MPR_RECOVERY,
16566eea4f46SScott Long 	    "Command timeout on target %u(0x%04x), %d set, %d.%d elapsed\n",
16576eea4f46SScott Long 	    targ->tid, targ->handle, ccb->ccb_h.timeout,
16586eea4f46SScott Long 	    sbintime_getsec(elapsed), elapsed & 0xffffffff);
1659991554f2SKenneth D. Merry 	if (targ->encl_level_valid) {
16606eea4f46SScott Long 		mpr_dprint(sc, MPR_INFO|MPR_RECOVERY,
16616eea4f46SScott Long 		    "At enclosure level %d, slot %d, connector name (%4s)\n",
16626eea4f46SScott Long 		    targ->encl_level, targ->encl_slot, targ->connector_name);
1663991554f2SKenneth D. Merry 	}
1664991554f2SKenneth D. Merry 
1665991554f2SKenneth D. Merry 	/* XXX first, check the firmware state, to see if it's still
1666991554f2SKenneth D. Merry 	 * operational.  if not, do a diag reset.
1667991554f2SKenneth D. Merry 	 */
1668a2c14879SStephen McConnell 	mprsas_set_ccbstatus(cm->cm_ccb, CAM_CMD_TIMEOUT);
16698fe7bf06SWarner Losh 	cm->cm_flags |= MPR_CM_FLAGS_ON_RECOVERY | MPR_CM_FLAGS_TIMEDOUT;
1670991554f2SKenneth D. Merry 	TAILQ_INSERT_TAIL(&targ->timedout_commands, cm, cm_recovery);
1671991554f2SKenneth D. Merry 
1672991554f2SKenneth D. Merry 	if (targ->tm != NULL) {
1673991554f2SKenneth D. Merry 		/* target already in recovery, just queue up another
1674991554f2SKenneth D. Merry 		 * timedout command to be processed later.
1675991554f2SKenneth D. Merry 		 */
16762bbaed4dSWarner Losh 		mpr_dprint(sc, MPR_RECOVERY,
16772bbaed4dSWarner Losh 		    "queued timedout cm %p for processing by tm %p\n",
16782bbaed4dSWarner Losh 		    cm, targ->tm);
16792bbaed4dSWarner Losh 	} else if ((targ->tm = mprsas_alloc_tm(sc)) != NULL) {
16806eea4f46SScott Long 		mpr_dprint(sc, MPR_RECOVERY|MPR_INFO,
16816eea4f46SScott Long 		    "Sending abort to target %u for SMID %d\n", targ->tid,
16826eea4f46SScott Long 		    cm->cm_desc.Default.SMID);
16836eea4f46SScott Long 		mpr_dprint(sc, MPR_RECOVERY, "timedout cm %p allocated tm %p\n",
16846eea4f46SScott Long 		    cm, targ->tm);
16852bbaed4dSWarner Losh 
16862bbaed4dSWarner Losh 		/* start recovery by aborting the first timedout command */
1687991554f2SKenneth D. Merry 		mprsas_send_abort(sc, targ->tm, cm);
16882bbaed4dSWarner Losh 	} else {
1689991554f2SKenneth D. Merry 		/* XXX queue this target up for recovery once a TM becomes
1690991554f2SKenneth D. Merry 		 * available.  The firmware only has a limited number of
1691991554f2SKenneth D. Merry 		 * HighPriority credits for the high priority requests used
1692991554f2SKenneth D. Merry 		 * for task management, and we ran out.
1693991554f2SKenneth D. Merry 		 *
1694991554f2SKenneth D. Merry 		 * Isilon: don't worry about this for now, since we have
1695991554f2SKenneth D. Merry 		 * more credits than disks in an enclosure, and limit
1696991554f2SKenneth D. Merry 		 * ourselves to one TM per target for recovery.
1697991554f2SKenneth D. Merry 		 */
16986eea4f46SScott Long 		mpr_dprint(sc, MPR_ERROR|MPR_RECOVERY,
16996eea4f46SScott Long 		    "timedout cm %p failed to allocate a tm\n", cm);
1700991554f2SKenneth D. Merry 	}
1701991554f2SKenneth D. Merry }
1702991554f2SKenneth D. Merry 
170367feec50SStephen McConnell /**
170467feec50SStephen McConnell  * mprsas_build_nvme_unmap - Build Native NVMe DSM command equivalent
170567feec50SStephen McConnell  *			     to SCSI Unmap.
170667feec50SStephen McConnell  * Return 0 - for success,
170767feec50SStephen McConnell  *	  1 - to immediately return back the command with success status to CAM
170867feec50SStephen McConnell  *	  negative value - to fallback to firmware path i.e. issue scsi unmap
170967feec50SStephen McConnell  *			   to FW without any translation.
171067feec50SStephen McConnell  */
171167feec50SStephen McConnell static int
171267feec50SStephen McConnell mprsas_build_nvme_unmap(struct mpr_softc *sc, struct mpr_command *cm,
171367feec50SStephen McConnell     union ccb *ccb, struct mprsas_target *targ)
171467feec50SStephen McConnell {
171567feec50SStephen McConnell 	Mpi26NVMeEncapsulatedRequest_t *req = NULL;
171667feec50SStephen McConnell 	struct ccb_scsiio *csio;
171767feec50SStephen McConnell 	struct unmap_parm_list *plist;
171867feec50SStephen McConnell 	struct nvme_dsm_range *nvme_dsm_ranges = NULL;
171967feec50SStephen McConnell 	struct nvme_command *c;
172067feec50SStephen McConnell 	int i, res;
172167feec50SStephen McConnell 	uint16_t ndesc, list_len, data_length;
172267feec50SStephen McConnell 	struct mpr_prp_page *prp_page_info;
172367feec50SStephen McConnell 	uint64_t nvme_dsm_ranges_dma_handle;
172467feec50SStephen McConnell 
172567feec50SStephen McConnell 	csio = &ccb->csio;
172667feec50SStephen McConnell 	list_len = (scsiio_cdb_ptr(csio)[7] << 8 | scsiio_cdb_ptr(csio)[8]);
172767feec50SStephen McConnell 	if (!list_len) {
172867feec50SStephen McConnell 		mpr_dprint(sc, MPR_ERROR, "Parameter list length is Zero\n");
172967feec50SStephen McConnell 		return -EINVAL;
173067feec50SStephen McConnell 	}
173167feec50SStephen McConnell 
173267feec50SStephen McConnell 	plist = malloc(csio->dxfer_len, M_MPR, M_ZERO|M_NOWAIT);
173367feec50SStephen McConnell 	if (!plist) {
173467feec50SStephen McConnell 		mpr_dprint(sc, MPR_ERROR, "Unable to allocate memory to "
173567feec50SStephen McConnell 		    "save UNMAP data\n");
173667feec50SStephen McConnell 		return -ENOMEM;
173767feec50SStephen McConnell 	}
173867feec50SStephen McConnell 
173967feec50SStephen McConnell 	/* Copy SCSI unmap data to a local buffer */
174067feec50SStephen McConnell 	bcopy(csio->data_ptr, plist, csio->dxfer_len);
174167feec50SStephen McConnell 
174267feec50SStephen McConnell 	/* return back the unmap command to CAM with success status,
174367feec50SStephen McConnell 	 * if number of descripts is zero.
174467feec50SStephen McConnell 	 */
174567feec50SStephen McConnell 	ndesc = be16toh(plist->unmap_blk_desc_data_len) >> 4;
174667feec50SStephen McConnell 	if (!ndesc) {
174767feec50SStephen McConnell 		mpr_dprint(sc, MPR_XINFO, "Number of descriptors in "
174867feec50SStephen McConnell 		    "UNMAP cmd is Zero\n");
174967feec50SStephen McConnell 		res = 1;
175067feec50SStephen McConnell 		goto out;
175167feec50SStephen McConnell 	}
175267feec50SStephen McConnell 
175367feec50SStephen McConnell 	data_length = ndesc * sizeof(struct nvme_dsm_range);
175467feec50SStephen McConnell 	if (data_length > targ->MDTS) {
175567feec50SStephen McConnell 		mpr_dprint(sc, MPR_ERROR, "data length: %d is greater than "
175667feec50SStephen McConnell 		    "Device's MDTS: %d\n", data_length, targ->MDTS);
175767feec50SStephen McConnell 		res = -EINVAL;
175867feec50SStephen McConnell 		goto out;
175967feec50SStephen McConnell 	}
176067feec50SStephen McConnell 
176167feec50SStephen McConnell 	prp_page_info = mpr_alloc_prp_page(sc);
176267feec50SStephen McConnell 	KASSERT(prp_page_info != NULL, ("%s: There is no PRP Page for "
176367feec50SStephen McConnell 	    "UNMAP command.\n", __func__));
176467feec50SStephen McConnell 
176567feec50SStephen McConnell 	/*
176667feec50SStephen McConnell 	 * Insert the allocated PRP page into the command's PRP page list. This
176767feec50SStephen McConnell 	 * will be freed when the command is freed.
176867feec50SStephen McConnell 	 */
176967feec50SStephen McConnell 	TAILQ_INSERT_TAIL(&cm->cm_prp_page_list, prp_page_info, prp_page_link);
177067feec50SStephen McConnell 
177167feec50SStephen McConnell 	nvme_dsm_ranges = (struct nvme_dsm_range *)prp_page_info->prp_page;
177267feec50SStephen McConnell 	nvme_dsm_ranges_dma_handle = prp_page_info->prp_page_busaddr;
177367feec50SStephen McConnell 
177467feec50SStephen McConnell 	bzero(nvme_dsm_ranges, data_length);
177567feec50SStephen McConnell 
177667feec50SStephen McConnell 	/* Convert SCSI unmap's descriptor data to NVMe DSM specific Range data
177767feec50SStephen McConnell 	 * for each descriptors contained in SCSI UNMAP data.
177867feec50SStephen McConnell 	 */
177967feec50SStephen McConnell 	for (i = 0; i < ndesc; i++) {
178067feec50SStephen McConnell 		nvme_dsm_ranges[i].length =
178167feec50SStephen McConnell 		    htole32(be32toh(plist->desc[i].nlb));
178267feec50SStephen McConnell 		nvme_dsm_ranges[i].starting_lba =
178367feec50SStephen McConnell 		    htole64(be64toh(plist->desc[i].slba));
178467feec50SStephen McConnell 		nvme_dsm_ranges[i].attributes = 0;
178567feec50SStephen McConnell 	}
178667feec50SStephen McConnell 
178767feec50SStephen McConnell 	/* Build MPI2.6's NVMe Encapsulated Request Message */
178867feec50SStephen McConnell 	req = (Mpi26NVMeEncapsulatedRequest_t *)cm->cm_req;
178967feec50SStephen McConnell 	bzero(req, sizeof(*req));
179067feec50SStephen McConnell 	req->DevHandle = htole16(targ->handle);
179167feec50SStephen McConnell 	req->Function = MPI2_FUNCTION_NVME_ENCAPSULATED;
179267feec50SStephen McConnell 	req->Flags = MPI26_NVME_FLAGS_WRITE;
179367feec50SStephen McConnell 	req->ErrorResponseBaseAddress.High =
179467feec50SStephen McConnell 	    htole32((uint32_t)((uint64_t)cm->cm_sense_busaddr >> 32));
179567feec50SStephen McConnell 	req->ErrorResponseBaseAddress.Low =
179667feec50SStephen McConnell 	    htole32(cm->cm_sense_busaddr);
179767feec50SStephen McConnell 	req->ErrorResponseAllocationLength =
179867feec50SStephen McConnell 	    htole16(sizeof(struct nvme_completion));
179967feec50SStephen McConnell 	req->EncapsulatedCommandLength =
180067feec50SStephen McConnell 	    htole16(sizeof(struct nvme_command));
180167feec50SStephen McConnell 	req->DataLength = htole32(data_length);
180267feec50SStephen McConnell 
180367feec50SStephen McConnell 	/* Build NVMe DSM command */
180467feec50SStephen McConnell 	c = (struct nvme_command *) req->NVMe_Command;
18059544e6dcSChuck Tuffli 	c->opc = NVME_OPC_DATASET_MANAGEMENT;
180667feec50SStephen McConnell 	c->nsid = htole32(csio->ccb_h.target_lun + 1);
180767feec50SStephen McConnell 	c->cdw10 = htole32(ndesc - 1);
180867feec50SStephen McConnell 	c->cdw11 = htole32(NVME_DSM_ATTR_DEALLOCATE);
180967feec50SStephen McConnell 
181067feec50SStephen McConnell 	cm->cm_length = data_length;
181167feec50SStephen McConnell 	cm->cm_data = NULL;
181267feec50SStephen McConnell 
181367feec50SStephen McConnell 	cm->cm_complete = mprsas_scsiio_complete;
181467feec50SStephen McConnell 	cm->cm_complete_data = ccb;
181567feec50SStephen McConnell 	cm->cm_targ = targ;
181667feec50SStephen McConnell 	cm->cm_lun = csio->ccb_h.target_lun;
181767feec50SStephen McConnell 	cm->cm_ccb = ccb;
181867feec50SStephen McConnell 
181967feec50SStephen McConnell 	cm->cm_desc.Default.RequestFlags =
182067feec50SStephen McConnell 	    MPI26_REQ_DESCRIPT_FLAGS_PCIE_ENCAPSULATED;
182167feec50SStephen McConnell 
18226eea4f46SScott Long 	csio->ccb_h.qos.sim_data = sbinuptime();
182367feec50SStephen McConnell 	callout_reset_sbt(&cm->cm_callout, SBT_1MS * ccb->ccb_h.timeout, 0,
182467feec50SStephen McConnell 	    mprsas_scsiio_timeout, cm, 0);
182567feec50SStephen McConnell 
182667feec50SStephen McConnell 	targ->issued++;
182767feec50SStephen McConnell 	targ->outstanding++;
182867feec50SStephen McConnell 	TAILQ_INSERT_TAIL(&targ->commands, cm, cm_link);
182967feec50SStephen McConnell 	ccb->ccb_h.status |= CAM_SIM_QUEUED;
183067feec50SStephen McConnell 
183167feec50SStephen McConnell 	mprsas_log_command(cm, MPR_XINFO, "%s cm %p ccb %p outstanding %u\n",
183267feec50SStephen McConnell 	    __func__, cm, ccb, targ->outstanding);
183367feec50SStephen McConnell 
183418982e8fSStephen McConnell 	mpr_build_nvme_prp(sc, cm, req,
183518982e8fSStephen McConnell 	    (void *)(uintptr_t)nvme_dsm_ranges_dma_handle, 0, data_length);
183667feec50SStephen McConnell 	mpr_map_command(sc, cm);
183761f17c5fSScott Long 	res = 0;
183867feec50SStephen McConnell 
183967feec50SStephen McConnell out:
184067feec50SStephen McConnell 	free(plist, M_MPR);
184161f17c5fSScott Long 	return (res);
184267feec50SStephen McConnell }
184367feec50SStephen McConnell 
1844991554f2SKenneth D. Merry static void
1845991554f2SKenneth D. Merry mprsas_action_scsiio(struct mprsas_softc *sassc, union ccb *ccb)
1846991554f2SKenneth D. Merry {
1847991554f2SKenneth D. Merry 	MPI2_SCSI_IO_REQUEST *req;
1848991554f2SKenneth D. Merry 	struct ccb_scsiio *csio;
1849991554f2SKenneth D. Merry 	struct mpr_softc *sc;
1850991554f2SKenneth D. Merry 	struct mprsas_target *targ;
1851991554f2SKenneth D. Merry 	struct mprsas_lun *lun;
1852991554f2SKenneth D. Merry 	struct mpr_command *cm;
185367feec50SStephen McConnell 	uint8_t i, lba_byte, *ref_tag_addr, scsi_opcode;
1854991554f2SKenneth D. Merry 	uint16_t eedp_flags;
1855991554f2SKenneth D. Merry 	uint32_t mpi_control;
185667feec50SStephen McConnell 	int rc;
1857991554f2SKenneth D. Merry 
1858991554f2SKenneth D. Merry 	sc = sassc->sc;
1859991554f2SKenneth D. Merry 	MPR_FUNCTRACE(sc);
1860991554f2SKenneth D. Merry 	mtx_assert(&sc->mpr_mtx, MA_OWNED);
1861991554f2SKenneth D. Merry 
1862991554f2SKenneth D. Merry 	csio = &ccb->csio;
1863a2c14879SStephen McConnell 	KASSERT(csio->ccb_h.target_id < sassc->maxtargets,
1864a2c14879SStephen McConnell 	    ("Target %d out of bounds in XPT_SCSI_IO\n",
1865a2c14879SStephen McConnell 	     csio->ccb_h.target_id));
1866991554f2SKenneth D. Merry 	targ = &sassc->targets[csio->ccb_h.target_id];
1867991554f2SKenneth D. Merry 	mpr_dprint(sc, MPR_TRACE, "ccb %p target flag %x\n", ccb, targ->flags);
1868991554f2SKenneth D. Merry 	if (targ->handle == 0x0) {
1869e35816c1SWarner Losh 		if (targ->flags & MPRSAS_TARGET_INDIAGRESET) {
1870e35816c1SWarner Losh 			mpr_dprint(sc, MPR_ERROR,
1871e35816c1SWarner Losh 			    "%s NULL handle for target %u in diag reset freezing queue\n",
1872e35816c1SWarner Losh 			    __func__, csio->ccb_h.target_id);
1873e35816c1SWarner Losh 			ccb->ccb_h.status = CAM_REQUEUE_REQ | CAM_DEV_QFRZN;
1874e35816c1SWarner Losh 			xpt_freeze_devq(ccb->ccb_h.path, 1);
1875e35816c1SWarner Losh 			xpt_done(ccb);
1876e35816c1SWarner Losh 			return;
1877e35816c1SWarner Losh 		}
1878991554f2SKenneth D. Merry 		mpr_dprint(sc, MPR_ERROR, "%s NULL handle for target %u\n",
1879991554f2SKenneth D. Merry 		    __func__, csio->ccb_h.target_id);
1880a2c14879SStephen McConnell 		mprsas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE);
1881991554f2SKenneth D. Merry 		xpt_done(ccb);
1882991554f2SKenneth D. Merry 		return;
1883991554f2SKenneth D. Merry 	}
1884991554f2SKenneth D. Merry 	if (targ->flags & MPR_TARGET_FLAGS_RAID_COMPONENT) {
1885a2c14879SStephen McConnell 		mpr_dprint(sc, MPR_ERROR, "%s Raid component no SCSI IO "
1886991554f2SKenneth D. Merry 		    "supported %u\n", __func__, csio->ccb_h.target_id);
1887a2c14879SStephen McConnell 		mprsas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE);
1888991554f2SKenneth D. Merry 		xpt_done(ccb);
1889991554f2SKenneth D. Merry 		return;
1890991554f2SKenneth D. Merry 	}
1891991554f2SKenneth D. Merry 	/*
1892991554f2SKenneth D. Merry 	 * Sometimes, it is possible to get a command that is not "In
1893991554f2SKenneth D. Merry 	 * Progress" and was actually aborted by the upper layer.  Check for
1894991554f2SKenneth D. Merry 	 * this here and complete the command without error.
1895991554f2SKenneth D. Merry 	 */
1896a2c14879SStephen McConnell 	if (mprsas_get_ccbstatus(ccb) != CAM_REQ_INPROG) {
1897991554f2SKenneth D. Merry 		mpr_dprint(sc, MPR_TRACE, "%s Command is not in progress for "
1898991554f2SKenneth D. Merry 		    "target %u\n", __func__, csio->ccb_h.target_id);
1899991554f2SKenneth D. Merry 		xpt_done(ccb);
1900991554f2SKenneth D. Merry 		return;
1901991554f2SKenneth D. Merry 	}
1902991554f2SKenneth D. Merry 	/*
1903991554f2SKenneth D. Merry 	 * If devinfo is 0 this will be a volume.  In that case don't tell CAM
1904991554f2SKenneth D. Merry 	 * that the volume has timed out.  We want volumes to be enumerated
19054c1cdd4aSWarner Losh 	 * until they are deleted/removed, not just failed. In either event,
19064c1cdd4aSWarner Losh 	 * we're removing the target due to a firmware event telling us
19074c1cdd4aSWarner Losh 	 * the device is now gone (as opposed to some transient event). Since
19084c1cdd4aSWarner Losh 	 * we're opting to remove failed devices from the OS's view, we need
19094c1cdd4aSWarner Losh 	 * to propagate that status up the stack.
1910991554f2SKenneth D. Merry 	 */
1911991554f2SKenneth D. Merry 	if (targ->flags & MPRSAS_TARGET_INREMOVAL) {
1912991554f2SKenneth D. Merry 		if (targ->devinfo == 0)
1913a2c14879SStephen McConnell 			mprsas_set_ccbstatus(ccb, CAM_REQ_CMP);
1914991554f2SKenneth D. Merry 		else
19154c1cdd4aSWarner Losh 			mprsas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE);
1916991554f2SKenneth D. Merry 		xpt_done(ccb);
1917991554f2SKenneth D. Merry 		return;
1918991554f2SKenneth D. Merry 	}
1919991554f2SKenneth D. Merry 
1920991554f2SKenneth D. Merry 	if ((sc->mpr_flags & MPR_FLAGS_SHUTDOWN) != 0) {
1921a2c14879SStephen McConnell 		mpr_dprint(sc, MPR_INFO, "%s shutting down\n", __func__);
1922a2c14879SStephen McConnell 		mprsas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE);
1923a2c14879SStephen McConnell 		xpt_done(ccb);
1924a2c14879SStephen McConnell 		return;
1925a2c14879SStephen McConnell 	}
1926a2c14879SStephen McConnell 
1927a2c14879SStephen McConnell 	/*
19289781c28cSAlexander Motin 	 * If target has a reset in progress, the devq should be frozen.
19299781c28cSAlexander Motin 	 * Geting here we likely hit a race, so just requeue.
1930a2c14879SStephen McConnell 	 */
1931a2c14879SStephen McConnell 	if (targ->flags & MPRSAS_TARGET_INRESET) {
19329781c28cSAlexander Motin 		ccb->ccb_h.status = CAM_REQUEUE_REQ | CAM_DEV_QFRZN;
1933a8837c77SWarner Losh 		mpr_dprint(sc, MPR_XINFO | MPR_RECOVERY,
1934a8837c77SWarner Losh 		    "%s: Freezing devq for target ID %d\n",
1935a8837c77SWarner Losh 		    __func__, targ->tid);
1936a2c14879SStephen McConnell 		xpt_freeze_devq(ccb->ccb_h.path, 1);
1937991554f2SKenneth D. Merry 		xpt_done(ccb);
1938991554f2SKenneth D. Merry 		return;
1939991554f2SKenneth D. Merry 	}
1940991554f2SKenneth D. Merry 
1941991554f2SKenneth D. Merry 	cm = mpr_alloc_command(sc);
1942991554f2SKenneth D. Merry 	if (cm == NULL || (sc->mpr_flags & MPR_FLAGS_DIAGRESET)) {
1943991554f2SKenneth D. Merry 		if (cm != NULL) {
1944991554f2SKenneth D. Merry 			mpr_free_command(sc, cm);
1945991554f2SKenneth D. Merry 		}
1946991554f2SKenneth D. Merry 		if ((sassc->flags & MPRSAS_QUEUE_FROZEN) == 0) {
1947991554f2SKenneth D. Merry 			xpt_freeze_simq(sassc->sim, 1);
1948991554f2SKenneth D. Merry 			sassc->flags |= MPRSAS_QUEUE_FROZEN;
1949991554f2SKenneth D. Merry 		}
1950991554f2SKenneth D. Merry 		ccb->ccb_h.status &= ~CAM_SIM_QUEUED;
1951991554f2SKenneth D. Merry 		ccb->ccb_h.status |= CAM_REQUEUE_REQ;
1952991554f2SKenneth D. Merry 		xpt_done(ccb);
1953991554f2SKenneth D. Merry 		return;
1954991554f2SKenneth D. Merry 	}
1955991554f2SKenneth D. Merry 
195667feec50SStephen McConnell 	/* For NVME device's issue UNMAP command directly to NVME drives by
195767feec50SStephen McConnell 	 * constructing equivalent native NVMe DataSetManagement command.
195867feec50SStephen McConnell 	 */
195967feec50SStephen McConnell 	scsi_opcode = scsiio_cdb_ptr(csio)[0];
196067feec50SStephen McConnell 	if (scsi_opcode == UNMAP &&
196167feec50SStephen McConnell 	    targ->is_nvme &&
196267feec50SStephen McConnell 	    (csio->ccb_h.flags & CAM_DATA_MASK) == CAM_DATA_VADDR) {
196367feec50SStephen McConnell 		rc = mprsas_build_nvme_unmap(sc, cm, ccb, targ);
196467feec50SStephen McConnell 		if (rc == 1) { /* return command to CAM with success status */
196567feec50SStephen McConnell 			mpr_free_command(sc, cm);
196667feec50SStephen McConnell 			mprsas_set_ccbstatus(ccb, CAM_REQ_CMP);
196767feec50SStephen McConnell 			xpt_done(ccb);
196867feec50SStephen McConnell 			return;
196967feec50SStephen McConnell 		} else if (!rc) /* Issued NVMe Encapsulated Request Message */
197067feec50SStephen McConnell 			return;
197167feec50SStephen McConnell 	}
197267feec50SStephen McConnell 
1973991554f2SKenneth D. Merry 	req = (MPI2_SCSI_IO_REQUEST *)cm->cm_req;
1974991554f2SKenneth D. Merry 	bzero(req, sizeof(*req));
1975991554f2SKenneth D. Merry 	req->DevHandle = htole16(targ->handle);
1976991554f2SKenneth D. Merry 	req->Function = MPI2_FUNCTION_SCSI_IO_REQUEST;
1977991554f2SKenneth D. Merry 	req->MsgFlags = 0;
1978991554f2SKenneth D. Merry 	req->SenseBufferLowAddress = htole32(cm->cm_sense_busaddr);
1979991554f2SKenneth D. Merry 	req->SenseBufferLength = MPR_SENSE_LEN;
1980991554f2SKenneth D. Merry 	req->SGLFlags = 0;
1981991554f2SKenneth D. Merry 	req->ChainOffset = 0;
1982991554f2SKenneth D. Merry 	req->SGLOffset0 = 24;	/* 32bit word offset to the SGL */
1983991554f2SKenneth D. Merry 	req->SGLOffset1= 0;
1984991554f2SKenneth D. Merry 	req->SGLOffset2= 0;
1985991554f2SKenneth D. Merry 	req->SGLOffset3= 0;
1986991554f2SKenneth D. Merry 	req->SkipCount = 0;
1987991554f2SKenneth D. Merry 	req->DataLength = htole32(csio->dxfer_len);
1988991554f2SKenneth D. Merry 	req->BidirectionalDataLength = 0;
1989991554f2SKenneth D. Merry 	req->IoFlags = htole16(csio->cdb_len);
1990991554f2SKenneth D. Merry 	req->EEDPFlags = 0;
1991991554f2SKenneth D. Merry 
1992991554f2SKenneth D. Merry 	/* Note: BiDirectional transfers are not supported */
1993991554f2SKenneth D. Merry 	switch (csio->ccb_h.flags & CAM_DIR_MASK) {
1994991554f2SKenneth D. Merry 	case CAM_DIR_IN:
1995991554f2SKenneth D. Merry 		mpi_control = MPI2_SCSIIO_CONTROL_READ;
1996991554f2SKenneth D. Merry 		cm->cm_flags |= MPR_CM_FLAGS_DATAIN;
1997991554f2SKenneth D. Merry 		break;
1998991554f2SKenneth D. Merry 	case CAM_DIR_OUT:
1999991554f2SKenneth D. Merry 		mpi_control = MPI2_SCSIIO_CONTROL_WRITE;
2000991554f2SKenneth D. Merry 		cm->cm_flags |= MPR_CM_FLAGS_DATAOUT;
2001991554f2SKenneth D. Merry 		break;
2002991554f2SKenneth D. Merry 	case CAM_DIR_NONE:
2003991554f2SKenneth D. Merry 	default:
2004991554f2SKenneth D. Merry 		mpi_control = MPI2_SCSIIO_CONTROL_NODATATRANSFER;
2005991554f2SKenneth D. Merry 		break;
2006991554f2SKenneth D. Merry 	}
2007991554f2SKenneth D. Merry 
2008991554f2SKenneth D. Merry 	if (csio->cdb_len == 32)
2009991554f2SKenneth D. Merry 		mpi_control |= 4 << MPI2_SCSIIO_CONTROL_ADDCDBLEN_SHIFT;
2010991554f2SKenneth D. Merry 	/*
2011991554f2SKenneth D. Merry 	 * It looks like the hardware doesn't require an explicit tag
2012991554f2SKenneth D. Merry 	 * number for each transaction.  SAM Task Management not supported
2013991554f2SKenneth D. Merry 	 * at the moment.
2014991554f2SKenneth D. Merry 	 */
2015991554f2SKenneth D. Merry 	switch (csio->tag_action) {
2016991554f2SKenneth D. Merry 	case MSG_HEAD_OF_Q_TAG:
2017991554f2SKenneth D. Merry 		mpi_control |= MPI2_SCSIIO_CONTROL_HEADOFQ;
2018991554f2SKenneth D. Merry 		break;
2019991554f2SKenneth D. Merry 	case MSG_ORDERED_Q_TAG:
2020991554f2SKenneth D. Merry 		mpi_control |= MPI2_SCSIIO_CONTROL_ORDEREDQ;
2021991554f2SKenneth D. Merry 		break;
2022991554f2SKenneth D. Merry 	case MSG_ACA_TASK:
2023991554f2SKenneth D. Merry 		mpi_control |= MPI2_SCSIIO_CONTROL_ACAQ;
2024991554f2SKenneth D. Merry 		break;
2025991554f2SKenneth D. Merry 	case CAM_TAG_ACTION_NONE:
2026991554f2SKenneth D. Merry 	case MSG_SIMPLE_Q_TAG:
2027991554f2SKenneth D. Merry 	default:
2028991554f2SKenneth D. Merry 		mpi_control |= MPI2_SCSIIO_CONTROL_SIMPLEQ;
2029991554f2SKenneth D. Merry 		break;
2030991554f2SKenneth D. Merry 	}
203188364968SAlexander Motin 	mpi_control |= (csio->priority << MPI2_SCSIIO_CONTROL_CMDPRI_SHIFT) &
203288364968SAlexander Motin 	    MPI2_SCSIIO_CONTROL_CMDPRI_MASK;
2033991554f2SKenneth D. Merry 	mpi_control |= sc->mapping_table[csio->ccb_h.target_id].TLR_bits;
2034991554f2SKenneth D. Merry 	req->Control = htole32(mpi_control);
2035991554f2SKenneth D. Merry 
2036991554f2SKenneth D. Merry 	if (MPR_SET_LUN(req->LUN, csio->ccb_h.target_lun) != 0) {
2037991554f2SKenneth D. Merry 		mpr_free_command(sc, cm);
2038a2c14879SStephen McConnell 		mprsas_set_ccbstatus(ccb, CAM_LUN_INVALID);
2039991554f2SKenneth D. Merry 		xpt_done(ccb);
2040991554f2SKenneth D. Merry 		return;
2041991554f2SKenneth D. Merry 	}
2042991554f2SKenneth D. Merry 
2043991554f2SKenneth D. Merry 	if (csio->ccb_h.flags & CAM_CDB_POINTER)
2044991554f2SKenneth D. Merry 		bcopy(csio->cdb_io.cdb_ptr, &req->CDB.CDB32[0], csio->cdb_len);
2045fa699bb2SAlan Somers 	else {
2046fa699bb2SAlan Somers 		KASSERT(csio->cdb_len <= IOCDBLEN,
204767feec50SStephen McConnell 		    ("cdb_len %d is greater than IOCDBLEN but CAM_CDB_POINTER "
204867feec50SStephen McConnell 		    "is not set", csio->cdb_len));
2049991554f2SKenneth D. Merry 		bcopy(csio->cdb_io.cdb_bytes, &req->CDB.CDB32[0],csio->cdb_len);
2050fa699bb2SAlan Somers 	}
2051991554f2SKenneth D. Merry 	req->IoFlags = htole16(csio->cdb_len);
2052991554f2SKenneth D. Merry 
2053991554f2SKenneth D. Merry 	/*
2054991554f2SKenneth D. Merry 	 * Check if EEDP is supported and enabled.  If it is then check if the
2055991554f2SKenneth D. Merry 	 * SCSI opcode could be using EEDP.  If so, make sure the LUN exists and
2056991554f2SKenneth D. Merry 	 * is formatted for EEDP support.  If all of this is true, set CDB up
2057991554f2SKenneth D. Merry 	 * for EEDP transfer.
2058991554f2SKenneth D. Merry 	 */
2059991554f2SKenneth D. Merry 	eedp_flags = op_code_prot[req->CDB.CDB32[0]];
2060991554f2SKenneth D. Merry 	if (sc->eedp_enabled && eedp_flags) {
2061991554f2SKenneth D. Merry 		SLIST_FOREACH(lun, &targ->luns, lun_link) {
2062991554f2SKenneth D. Merry 			if (lun->lun_id == csio->ccb_h.target_lun) {
2063991554f2SKenneth D. Merry 				break;
2064991554f2SKenneth D. Merry 			}
2065991554f2SKenneth D. Merry 		}
2066991554f2SKenneth D. Merry 
2067991554f2SKenneth D. Merry 		if ((lun != NULL) && (lun->eedp_formatted)) {
206871900a79SAlfredo Dal'Ava Junior 			req->EEDPBlockSize = htole32(lun->eedp_block_size);
2069991554f2SKenneth D. Merry 			eedp_flags |= (MPI2_SCSIIO_EEDPFLAGS_INC_PRI_REFTAG |
2070991554f2SKenneth D. Merry 			    MPI2_SCSIIO_EEDPFLAGS_CHECK_REFTAG |
2071991554f2SKenneth D. Merry 			    MPI2_SCSIIO_EEDPFLAGS_CHECK_GUARD);
207267feec50SStephen McConnell 			if (sc->mpr_flags & MPR_FLAGS_GEN35_IOC) {
207367feec50SStephen McConnell 				eedp_flags |=
207467feec50SStephen McConnell 				    MPI25_SCSIIO_EEDPFLAGS_APPTAG_DISABLE_MODE;
207567feec50SStephen McConnell 			}
2076991554f2SKenneth D. Merry 			req->EEDPFlags = htole16(eedp_flags);
2077991554f2SKenneth D. Merry 
2078991554f2SKenneth D. Merry 			/*
2079991554f2SKenneth D. Merry 			 * If CDB less than 32, fill in Primary Ref Tag with
2080991554f2SKenneth D. Merry 			 * low 4 bytes of LBA.  If CDB is 32, tag stuff is
2081991554f2SKenneth D. Merry 			 * already there.  Also, set protection bit.  FreeBSD
2082991554f2SKenneth D. Merry 			 * currently does not support CDBs bigger than 16, but
2083991554f2SKenneth D. Merry 			 * the code doesn't hurt, and will be here for the
2084991554f2SKenneth D. Merry 			 * future.
2085991554f2SKenneth D. Merry 			 */
2086991554f2SKenneth D. Merry 			if (csio->cdb_len != 32) {
2087991554f2SKenneth D. Merry 				lba_byte = (csio->cdb_len == 16) ? 6 : 2;
2088991554f2SKenneth D. Merry 				ref_tag_addr = (uint8_t *)&req->CDB.EEDP32.
2089991554f2SKenneth D. Merry 				    PrimaryReferenceTag;
2090991554f2SKenneth D. Merry 				for (i = 0; i < 4; i++) {
2091991554f2SKenneth D. Merry 					*ref_tag_addr =
2092991554f2SKenneth D. Merry 					    req->CDB.CDB32[lba_byte + i];
2093991554f2SKenneth D. Merry 					ref_tag_addr++;
2094991554f2SKenneth D. Merry 				}
2095991554f2SKenneth D. Merry 				req->CDB.EEDP32.PrimaryReferenceTag =
2096991554f2SKenneth D. Merry 				    htole32(req->
2097991554f2SKenneth D. Merry 				    CDB.EEDP32.PrimaryReferenceTag);
2098991554f2SKenneth D. Merry 				req->CDB.EEDP32.PrimaryApplicationTagMask =
2099991554f2SKenneth D. Merry 				    0xFFFF;
21008881681bSKenneth D. Merry 				req->CDB.CDB32[1] =
21018881681bSKenneth D. Merry 				    (req->CDB.CDB32[1] & 0x1F) | 0x20;
2102991554f2SKenneth D. Merry 			} else {
2103991554f2SKenneth D. Merry 				eedp_flags |=
2104991554f2SKenneth D. Merry 				    MPI2_SCSIIO_EEDPFLAGS_INC_PRI_APPTAG;
2105991554f2SKenneth D. Merry 				req->EEDPFlags = htole16(eedp_flags);
2106991554f2SKenneth D. Merry 				req->CDB.CDB32[10] = (req->CDB.CDB32[10] &
2107991554f2SKenneth D. Merry 				    0x1F) | 0x20;
2108991554f2SKenneth D. Merry 			}
2109991554f2SKenneth D. Merry 		}
2110991554f2SKenneth D. Merry 	}
2111991554f2SKenneth D. Merry 
2112991554f2SKenneth D. Merry 	cm->cm_length = csio->dxfer_len;
2113991554f2SKenneth D. Merry 	if (cm->cm_length != 0) {
2114991554f2SKenneth D. Merry 		cm->cm_data = ccb;
2115991554f2SKenneth D. Merry 		cm->cm_flags |= MPR_CM_FLAGS_USE_CCB;
2116991554f2SKenneth D. Merry 	} else {
2117991554f2SKenneth D. Merry 		cm->cm_data = NULL;
2118991554f2SKenneth D. Merry 	}
2119991554f2SKenneth D. Merry 	cm->cm_sge = &req->SGL;
2120991554f2SKenneth D. Merry 	cm->cm_sglsize = (32 - 24) * 4;
2121991554f2SKenneth D. Merry 	cm->cm_complete = mprsas_scsiio_complete;
2122991554f2SKenneth D. Merry 	cm->cm_complete_data = ccb;
2123991554f2SKenneth D. Merry 	cm->cm_targ = targ;
2124991554f2SKenneth D. Merry 	cm->cm_lun = csio->ccb_h.target_lun;
2125991554f2SKenneth D. Merry 	cm->cm_ccb = ccb;
2126991554f2SKenneth D. Merry 	/*
2127991554f2SKenneth D. Merry 	 * If using FP desc type, need to set a bit in IoFlags (SCSI IO is 0)
2128991554f2SKenneth D. Merry 	 * and set descriptor type.
2129991554f2SKenneth D. Merry 	 */
2130991554f2SKenneth D. Merry 	if (targ->scsi_req_desc_type ==
2131991554f2SKenneth D. Merry 	    MPI25_REQ_DESCRIPT_FLAGS_FAST_PATH_SCSI_IO) {
2132991554f2SKenneth D. Merry 		req->IoFlags |= MPI25_SCSIIO_IOFLAGS_FAST_PATH;
2133991554f2SKenneth D. Merry 		cm->cm_desc.FastPathSCSIIO.RequestFlags =
2134991554f2SKenneth D. Merry 		    MPI25_REQ_DESCRIPT_FLAGS_FAST_PATH_SCSI_IO;
213567feec50SStephen McConnell 		if (!sc->atomic_desc_capable) {
213667feec50SStephen McConnell 			cm->cm_desc.FastPathSCSIIO.DevHandle =
213767feec50SStephen McConnell 			    htole16(targ->handle);
213867feec50SStephen McConnell 		}
2139991554f2SKenneth D. Merry 	} else {
2140991554f2SKenneth D. Merry 		cm->cm_desc.SCSIIO.RequestFlags =
2141991554f2SKenneth D. Merry 		    MPI2_REQ_DESCRIPT_FLAGS_SCSI_IO;
214267feec50SStephen McConnell 		if (!sc->atomic_desc_capable)
2143991554f2SKenneth D. Merry 			cm->cm_desc.SCSIIO.DevHandle = htole16(targ->handle);
2144991554f2SKenneth D. Merry 	}
2145991554f2SKenneth D. Merry 
21466eea4f46SScott Long 	csio->ccb_h.qos.sim_data = sbinuptime();
214785c9dd9dSSteven Hartland 	callout_reset_sbt(&cm->cm_callout, SBT_1MS * ccb->ccb_h.timeout, 0,
214885c9dd9dSSteven Hartland 	    mprsas_scsiio_timeout, cm, 0);
2149991554f2SKenneth D. Merry 
2150991554f2SKenneth D. Merry 	targ->issued++;
2151991554f2SKenneth D. Merry 	targ->outstanding++;
2152991554f2SKenneth D. Merry 	TAILQ_INSERT_TAIL(&targ->commands, cm, cm_link);
2153991554f2SKenneth D. Merry 	ccb->ccb_h.status |= CAM_SIM_QUEUED;
2154991554f2SKenneth D. Merry 
2155991554f2SKenneth D. Merry 	mprsas_log_command(cm, MPR_XINFO, "%s cm %p ccb %p outstanding %u\n",
2156991554f2SKenneth D. Merry 	    __func__, cm, ccb, targ->outstanding);
2157991554f2SKenneth D. Merry 
2158991554f2SKenneth D. Merry 	mpr_map_command(sc, cm);
2159991554f2SKenneth D. Merry 	return;
2160991554f2SKenneth D. Merry }
2161991554f2SKenneth D. Merry 
2162991554f2SKenneth D. Merry /**
2163991554f2SKenneth D. Merry  * mpr_sc_failed_io_info - translated non-succesfull SCSI_IO request
2164991554f2SKenneth D. Merry  */
2165991554f2SKenneth D. Merry static void
2166991554f2SKenneth D. Merry mpr_sc_failed_io_info(struct mpr_softc *sc, struct ccb_scsiio *csio,
2167991554f2SKenneth D. Merry     Mpi2SCSIIOReply_t *mpi_reply, struct mprsas_target *targ)
2168991554f2SKenneth D. Merry {
2169991554f2SKenneth D. Merry 	u32 response_info;
2170991554f2SKenneth D. Merry 	u8 *response_bytes;
2171991554f2SKenneth D. Merry 	u16 ioc_status = le16toh(mpi_reply->IOCStatus) &
2172991554f2SKenneth D. Merry 	    MPI2_IOCSTATUS_MASK;
2173991554f2SKenneth D. Merry 	u8 scsi_state = mpi_reply->SCSIState;
2174991554f2SKenneth D. Merry 	u8 scsi_status = mpi_reply->SCSIStatus;
2175991554f2SKenneth D. Merry 	char *desc_ioc_state = NULL;
2176991554f2SKenneth D. Merry 	char *desc_scsi_status = NULL;
2177991554f2SKenneth D. Merry 	u32 log_info = le32toh(mpi_reply->IOCLogInfo);
2178991554f2SKenneth D. Merry 
2179991554f2SKenneth D. Merry 	if (log_info == 0x31170000)
2180991554f2SKenneth D. Merry 		return;
2181991554f2SKenneth D. Merry 
21822bf620cbSScott Long 	desc_ioc_state = mpr_describe_table(mpr_iocstatus_string,
21832bf620cbSScott Long 	     ioc_status);
21842bf620cbSScott Long 	desc_scsi_status = mpr_describe_table(mpr_scsi_status_string,
21852bf620cbSScott Long 	    scsi_status);
2186991554f2SKenneth D. Merry 
2187991554f2SKenneth D. Merry 	mpr_dprint(sc, MPR_XINFO, "\thandle(0x%04x), ioc_status(%s)(0x%04x)\n",
2188991554f2SKenneth D. Merry 	    le16toh(mpi_reply->DevHandle), desc_ioc_state, ioc_status);
2189991554f2SKenneth D. Merry 	if (targ->encl_level_valid) {
2190991554f2SKenneth D. Merry 		mpr_dprint(sc, MPR_XINFO, "At enclosure level %d, slot %d, "
2191991554f2SKenneth D. Merry 		    "connector name (%4s)\n", targ->encl_level, targ->encl_slot,
2192991554f2SKenneth D. Merry 		    targ->connector_name);
2193991554f2SKenneth D. Merry 	}
21942bf620cbSScott Long 
21952bf620cbSScott Long 	/*
21962bf620cbSScott Long 	 * We can add more detail about underflow data here
2197991554f2SKenneth D. Merry 	 * TO-DO
21982bf620cbSScott Long 	 */
2199991554f2SKenneth D. Merry 	mpr_dprint(sc, MPR_XINFO, "\tscsi_status(%s)(0x%02x), "
22002bf620cbSScott Long 	    "scsi_state %b\n", desc_scsi_status, scsi_status,
22012bf620cbSScott Long 	    scsi_state, "\20" "\1AutosenseValid" "\2AutosenseFailed"
22022bf620cbSScott Long 	    "\3NoScsiStatus" "\4Terminated" "\5Response InfoValid");
2203991554f2SKenneth D. Merry 
2204991554f2SKenneth D. Merry 	if (sc->mpr_debug & MPR_XINFO &&
2205991554f2SKenneth D. Merry 	    scsi_state & MPI2_SCSI_STATE_AUTOSENSE_VALID) {
2206991554f2SKenneth D. Merry 		mpr_dprint(sc, MPR_XINFO, "-> Sense Buffer Data : Start :\n");
2207991554f2SKenneth D. Merry 		scsi_sense_print(csio);
2208991554f2SKenneth D. Merry 		mpr_dprint(sc, MPR_XINFO, "-> Sense Buffer Data : End :\n");
2209991554f2SKenneth D. Merry 	}
2210991554f2SKenneth D. Merry 
2211991554f2SKenneth D. Merry 	if (scsi_state & MPI2_SCSI_STATE_RESPONSE_INFO_VALID) {
2212991554f2SKenneth D. Merry 		response_info = le32toh(mpi_reply->ResponseInfo);
2213991554f2SKenneth D. Merry 		response_bytes = (u8 *)&response_info;
22142bf620cbSScott Long 		mpr_dprint(sc, MPR_XINFO, "response code(0x%01x): %s\n",
22152bf620cbSScott Long 		    response_bytes[0],
22162bf620cbSScott Long 		    mpr_describe_table(mpr_scsi_taskmgmt_string,
22172bf620cbSScott Long 		    response_bytes[0]));
2218991554f2SKenneth D. Merry 	}
2219991554f2SKenneth D. Merry }
2220991554f2SKenneth D. Merry 
222167feec50SStephen McConnell /** mprsas_nvme_trans_status_code
222267feec50SStephen McConnell  *
222367feec50SStephen McConnell  * Convert Native NVMe command error status to
222467feec50SStephen McConnell  * equivalent SCSI error status.
222567feec50SStephen McConnell  *
222667feec50SStephen McConnell  * Returns appropriate scsi_status
222767feec50SStephen McConnell  */
222867feec50SStephen McConnell static u8
22290d787e9bSWojciech Macek mprsas_nvme_trans_status_code(uint16_t nvme_status,
223067feec50SStephen McConnell     struct mpr_command *cm)
223167feec50SStephen McConnell {
223267feec50SStephen McConnell 	u8 status = MPI2_SCSI_STATUS_GOOD;
223367feec50SStephen McConnell 	int skey, asc, ascq;
223467feec50SStephen McConnell 	union ccb *ccb = cm->cm_complete_data;
223567feec50SStephen McConnell 	int returned_sense_len;
22360d787e9bSWojciech Macek 	uint8_t sct, sc;
22370d787e9bSWojciech Macek 
22380d787e9bSWojciech Macek 	sct = NVME_STATUS_GET_SCT(nvme_status);
22390d787e9bSWojciech Macek 	sc = NVME_STATUS_GET_SC(nvme_status);
224067feec50SStephen McConnell 
224167feec50SStephen McConnell 	status = MPI2_SCSI_STATUS_CHECK_CONDITION;
224267feec50SStephen McConnell 	skey = SSD_KEY_ILLEGAL_REQUEST;
224367feec50SStephen McConnell 	asc = SCSI_ASC_NO_SENSE;
224467feec50SStephen McConnell 	ascq = SCSI_ASCQ_CAUSE_NOT_REPORTABLE;
224567feec50SStephen McConnell 
22460d787e9bSWojciech Macek 	switch (sct) {
224767feec50SStephen McConnell 	case NVME_SCT_GENERIC:
22480d787e9bSWojciech Macek 		switch (sc) {
224967feec50SStephen McConnell 		case NVME_SC_SUCCESS:
225067feec50SStephen McConnell 			status = MPI2_SCSI_STATUS_GOOD;
225167feec50SStephen McConnell 			skey = SSD_KEY_NO_SENSE;
225267feec50SStephen McConnell 			asc = SCSI_ASC_NO_SENSE;
225367feec50SStephen McConnell 			ascq = SCSI_ASCQ_CAUSE_NOT_REPORTABLE;
225467feec50SStephen McConnell 			break;
225567feec50SStephen McConnell 		case NVME_SC_INVALID_OPCODE:
225667feec50SStephen McConnell 			status = MPI2_SCSI_STATUS_CHECK_CONDITION;
225767feec50SStephen McConnell 			skey = SSD_KEY_ILLEGAL_REQUEST;
225867feec50SStephen McConnell 			asc = SCSI_ASC_ILLEGAL_COMMAND;
225967feec50SStephen McConnell 			ascq = SCSI_ASCQ_CAUSE_NOT_REPORTABLE;
226067feec50SStephen McConnell 			break;
226167feec50SStephen McConnell 		case NVME_SC_INVALID_FIELD:
226267feec50SStephen McConnell 			status = MPI2_SCSI_STATUS_CHECK_CONDITION;
226367feec50SStephen McConnell 			skey = SSD_KEY_ILLEGAL_REQUEST;
226467feec50SStephen McConnell 			asc = SCSI_ASC_INVALID_CDB;
226567feec50SStephen McConnell 			ascq = SCSI_ASCQ_CAUSE_NOT_REPORTABLE;
226667feec50SStephen McConnell 			break;
226767feec50SStephen McConnell 		case NVME_SC_DATA_TRANSFER_ERROR:
226867feec50SStephen McConnell 			status = MPI2_SCSI_STATUS_CHECK_CONDITION;
226967feec50SStephen McConnell 			skey = SSD_KEY_MEDIUM_ERROR;
227067feec50SStephen McConnell 			asc = SCSI_ASC_NO_SENSE;
227167feec50SStephen McConnell 			ascq = SCSI_ASCQ_CAUSE_NOT_REPORTABLE;
227267feec50SStephen McConnell 			break;
227367feec50SStephen McConnell 		case NVME_SC_ABORTED_POWER_LOSS:
227467feec50SStephen McConnell 			status = MPI2_SCSI_STATUS_TASK_ABORTED;
227567feec50SStephen McConnell 			skey = SSD_KEY_ABORTED_COMMAND;
227667feec50SStephen McConnell 			asc = SCSI_ASC_WARNING;
227767feec50SStephen McConnell 			ascq = SCSI_ASCQ_POWER_LOSS_EXPECTED;
227867feec50SStephen McConnell 			break;
227967feec50SStephen McConnell 		case NVME_SC_INTERNAL_DEVICE_ERROR:
228067feec50SStephen McConnell 			status = MPI2_SCSI_STATUS_CHECK_CONDITION;
228167feec50SStephen McConnell 			skey = SSD_KEY_HARDWARE_ERROR;
228267feec50SStephen McConnell 			asc = SCSI_ASC_INTERNAL_TARGET_FAILURE;
228367feec50SStephen McConnell 			ascq = SCSI_ASCQ_CAUSE_NOT_REPORTABLE;
228467feec50SStephen McConnell 			break;
228567feec50SStephen McConnell 		case NVME_SC_ABORTED_BY_REQUEST:
228667feec50SStephen McConnell 		case NVME_SC_ABORTED_SQ_DELETION:
228767feec50SStephen McConnell 		case NVME_SC_ABORTED_FAILED_FUSED:
228867feec50SStephen McConnell 		case NVME_SC_ABORTED_MISSING_FUSED:
228967feec50SStephen McConnell 			status = MPI2_SCSI_STATUS_TASK_ABORTED;
229067feec50SStephen McConnell 			skey = SSD_KEY_ABORTED_COMMAND;
229167feec50SStephen McConnell 			asc = SCSI_ASC_NO_SENSE;
229267feec50SStephen McConnell 			ascq = SCSI_ASCQ_CAUSE_NOT_REPORTABLE;
229367feec50SStephen McConnell 			break;
229467feec50SStephen McConnell 		case NVME_SC_INVALID_NAMESPACE_OR_FORMAT:
229567feec50SStephen McConnell 			status = MPI2_SCSI_STATUS_CHECK_CONDITION;
229667feec50SStephen McConnell 			skey = SSD_KEY_ILLEGAL_REQUEST;
229767feec50SStephen McConnell 			asc = SCSI_ASC_ACCESS_DENIED_INVALID_LUN_ID;
229867feec50SStephen McConnell 			ascq = SCSI_ASCQ_INVALID_LUN_ID;
229967feec50SStephen McConnell 			break;
230067feec50SStephen McConnell 		case NVME_SC_LBA_OUT_OF_RANGE:
230167feec50SStephen McConnell 			status = MPI2_SCSI_STATUS_CHECK_CONDITION;
230267feec50SStephen McConnell 			skey = SSD_KEY_ILLEGAL_REQUEST;
230367feec50SStephen McConnell 			asc = SCSI_ASC_ILLEGAL_BLOCK;
230467feec50SStephen McConnell 			ascq = SCSI_ASCQ_CAUSE_NOT_REPORTABLE;
230567feec50SStephen McConnell 			break;
230667feec50SStephen McConnell 		case NVME_SC_CAPACITY_EXCEEDED:
230767feec50SStephen McConnell 			status = MPI2_SCSI_STATUS_CHECK_CONDITION;
230867feec50SStephen McConnell 			skey = SSD_KEY_MEDIUM_ERROR;
230967feec50SStephen McConnell 			asc = SCSI_ASC_NO_SENSE;
231067feec50SStephen McConnell 			ascq = SCSI_ASCQ_CAUSE_NOT_REPORTABLE;
231167feec50SStephen McConnell 			break;
231267feec50SStephen McConnell 		case NVME_SC_NAMESPACE_NOT_READY:
231367feec50SStephen McConnell 			status = MPI2_SCSI_STATUS_CHECK_CONDITION;
231467feec50SStephen McConnell 			skey = SSD_KEY_NOT_READY;
231567feec50SStephen McConnell 			asc = SCSI_ASC_LUN_NOT_READY;
231667feec50SStephen McConnell 			ascq = SCSI_ASCQ_CAUSE_NOT_REPORTABLE;
231767feec50SStephen McConnell 			break;
231867feec50SStephen McConnell 		}
231967feec50SStephen McConnell 		break;
232067feec50SStephen McConnell 	case NVME_SCT_COMMAND_SPECIFIC:
23210d787e9bSWojciech Macek 		switch (sc) {
232267feec50SStephen McConnell 		case NVME_SC_INVALID_FORMAT:
232367feec50SStephen McConnell 			status = MPI2_SCSI_STATUS_CHECK_CONDITION;
232467feec50SStephen McConnell 			skey = SSD_KEY_ILLEGAL_REQUEST;
232567feec50SStephen McConnell 			asc = SCSI_ASC_FORMAT_COMMAND_FAILED;
232667feec50SStephen McConnell 			ascq = SCSI_ASCQ_FORMAT_COMMAND_FAILED;
232767feec50SStephen McConnell 			break;
232867feec50SStephen McConnell 		case NVME_SC_CONFLICTING_ATTRIBUTES:
232967feec50SStephen McConnell 			status = MPI2_SCSI_STATUS_CHECK_CONDITION;
233067feec50SStephen McConnell 			skey = SSD_KEY_ILLEGAL_REQUEST;
233167feec50SStephen McConnell 			asc = SCSI_ASC_INVALID_CDB;
233267feec50SStephen McConnell 			ascq = SCSI_ASCQ_CAUSE_NOT_REPORTABLE;
233367feec50SStephen McConnell 			break;
233467feec50SStephen McConnell 		}
233567feec50SStephen McConnell 		break;
233667feec50SStephen McConnell 	case NVME_SCT_MEDIA_ERROR:
23370d787e9bSWojciech Macek 		switch (sc) {
233867feec50SStephen McConnell 		case NVME_SC_WRITE_FAULTS:
233967feec50SStephen McConnell 			status = MPI2_SCSI_STATUS_CHECK_CONDITION;
234067feec50SStephen McConnell 			skey = SSD_KEY_MEDIUM_ERROR;
234167feec50SStephen McConnell 			asc = SCSI_ASC_PERIPHERAL_DEV_WRITE_FAULT;
234267feec50SStephen McConnell 			ascq = SCSI_ASCQ_CAUSE_NOT_REPORTABLE;
234367feec50SStephen McConnell 			break;
234467feec50SStephen McConnell 		case NVME_SC_UNRECOVERED_READ_ERROR:
234567feec50SStephen McConnell 			status = MPI2_SCSI_STATUS_CHECK_CONDITION;
234667feec50SStephen McConnell 			skey = SSD_KEY_MEDIUM_ERROR;
234767feec50SStephen McConnell 			asc = SCSI_ASC_UNRECOVERED_READ_ERROR;
234867feec50SStephen McConnell 			ascq = SCSI_ASCQ_CAUSE_NOT_REPORTABLE;
234967feec50SStephen McConnell 			break;
235067feec50SStephen McConnell 		case NVME_SC_GUARD_CHECK_ERROR:
235167feec50SStephen McConnell 			status = MPI2_SCSI_STATUS_CHECK_CONDITION;
235267feec50SStephen McConnell 			skey = SSD_KEY_MEDIUM_ERROR;
235367feec50SStephen McConnell 			asc = SCSI_ASC_LOG_BLOCK_GUARD_CHECK_FAILED;
235467feec50SStephen McConnell 			ascq = SCSI_ASCQ_LOG_BLOCK_GUARD_CHECK_FAILED;
235567feec50SStephen McConnell 			break;
235667feec50SStephen McConnell 		case NVME_SC_APPLICATION_TAG_CHECK_ERROR:
235767feec50SStephen McConnell 			status = MPI2_SCSI_STATUS_CHECK_CONDITION;
235867feec50SStephen McConnell 			skey = SSD_KEY_MEDIUM_ERROR;
235967feec50SStephen McConnell 			asc = SCSI_ASC_LOG_BLOCK_APPTAG_CHECK_FAILED;
236067feec50SStephen McConnell 			ascq = SCSI_ASCQ_LOG_BLOCK_APPTAG_CHECK_FAILED;
236167feec50SStephen McConnell 			break;
236267feec50SStephen McConnell 		case NVME_SC_REFERENCE_TAG_CHECK_ERROR:
236367feec50SStephen McConnell 			status = MPI2_SCSI_STATUS_CHECK_CONDITION;
236467feec50SStephen McConnell 			skey = SSD_KEY_MEDIUM_ERROR;
236567feec50SStephen McConnell 			asc = SCSI_ASC_LOG_BLOCK_REFTAG_CHECK_FAILED;
236667feec50SStephen McConnell 			ascq = SCSI_ASCQ_LOG_BLOCK_REFTAG_CHECK_FAILED;
236767feec50SStephen McConnell 			break;
236867feec50SStephen McConnell 		case NVME_SC_COMPARE_FAILURE:
236967feec50SStephen McConnell 			status = MPI2_SCSI_STATUS_CHECK_CONDITION;
237067feec50SStephen McConnell 			skey = SSD_KEY_MISCOMPARE;
237167feec50SStephen McConnell 			asc = SCSI_ASC_MISCOMPARE_DURING_VERIFY;
237267feec50SStephen McConnell 			ascq = SCSI_ASCQ_CAUSE_NOT_REPORTABLE;
237367feec50SStephen McConnell 			break;
237467feec50SStephen McConnell 		case NVME_SC_ACCESS_DENIED:
237567feec50SStephen McConnell 			status = MPI2_SCSI_STATUS_CHECK_CONDITION;
237667feec50SStephen McConnell 			skey = SSD_KEY_ILLEGAL_REQUEST;
237767feec50SStephen McConnell 			asc = SCSI_ASC_ACCESS_DENIED_INVALID_LUN_ID;
237867feec50SStephen McConnell 			ascq = SCSI_ASCQ_INVALID_LUN_ID;
237967feec50SStephen McConnell 			break;
238067feec50SStephen McConnell 		}
238167feec50SStephen McConnell 		break;
238267feec50SStephen McConnell 	}
238367feec50SStephen McConnell 
238467feec50SStephen McConnell 	returned_sense_len = sizeof(struct scsi_sense_data);
238567feec50SStephen McConnell 	if (returned_sense_len < ccb->csio.sense_len)
238667feec50SStephen McConnell 		ccb->csio.sense_resid = ccb->csio.sense_len -
238767feec50SStephen McConnell 		    returned_sense_len;
238867feec50SStephen McConnell 	else
238967feec50SStephen McConnell 		ccb->csio.sense_resid = 0;
239067feec50SStephen McConnell 
239167feec50SStephen McConnell 	scsi_set_sense_data(&ccb->csio.sense_data, SSD_TYPE_FIXED,
239267feec50SStephen McConnell 	    1, skey, asc, ascq, SSD_ELEM_NONE);
239367feec50SStephen McConnell 	ccb->ccb_h.status |= CAM_AUTOSNS_VALID;
239467feec50SStephen McConnell 
239567feec50SStephen McConnell 	return status;
239667feec50SStephen McConnell }
239767feec50SStephen McConnell 
239867feec50SStephen McConnell /** mprsas_complete_nvme_unmap
239967feec50SStephen McConnell  *
240067feec50SStephen McConnell  * Complete native NVMe command issued using NVMe Encapsulated
240167feec50SStephen McConnell  * Request Message.
240267feec50SStephen McConnell  */
240367feec50SStephen McConnell static u8
240467feec50SStephen McConnell mprsas_complete_nvme_unmap(struct mpr_softc *sc, struct mpr_command *cm)
240567feec50SStephen McConnell {
240667feec50SStephen McConnell 	Mpi26NVMeEncapsulatedErrorReply_t *mpi_reply;
240767feec50SStephen McConnell 	struct nvme_completion *nvme_completion = NULL;
240867feec50SStephen McConnell 	u8 scsi_status = MPI2_SCSI_STATUS_GOOD;
240967feec50SStephen McConnell 
241067feec50SStephen McConnell 	mpi_reply =(Mpi26NVMeEncapsulatedErrorReply_t *)cm->cm_reply;
241167feec50SStephen McConnell 	if (le16toh(mpi_reply->ErrorResponseCount)){
241267feec50SStephen McConnell 		nvme_completion = (struct nvme_completion *)cm->cm_sense;
241367feec50SStephen McConnell 		scsi_status = mprsas_nvme_trans_status_code(
241467feec50SStephen McConnell 		    nvme_completion->status, cm);
241567feec50SStephen McConnell 	}
241667feec50SStephen McConnell 	return scsi_status;
241767feec50SStephen McConnell }
241867feec50SStephen McConnell 
2419991554f2SKenneth D. Merry static void
2420991554f2SKenneth D. Merry mprsas_scsiio_complete(struct mpr_softc *sc, struct mpr_command *cm)
2421991554f2SKenneth D. Merry {
2422991554f2SKenneth D. Merry 	MPI2_SCSI_IO_REPLY *rep;
2423991554f2SKenneth D. Merry 	union ccb *ccb;
2424991554f2SKenneth D. Merry 	struct ccb_scsiio *csio;
2425991554f2SKenneth D. Merry 	struct mprsas_softc *sassc;
2426991554f2SKenneth D. Merry 	struct scsi_vpd_supported_page_list *vpd_list = NULL;
242767feec50SStephen McConnell 	u8 *TLR_bits, TLR_on, *scsi_cdb;
2428991554f2SKenneth D. Merry 	int dir = 0, i;
2429991554f2SKenneth D. Merry 	u16 alloc_len;
2430a2c14879SStephen McConnell 	struct mprsas_target *target;
2431a2c14879SStephen McConnell 	target_id_t target_id;
2432991554f2SKenneth D. Merry 
2433991554f2SKenneth D. Merry 	MPR_FUNCTRACE(sc);
2434991554f2SKenneth D. Merry 
2435991554f2SKenneth D. Merry 	callout_stop(&cm->cm_callout);
2436991554f2SKenneth D. Merry 	mtx_assert(&sc->mpr_mtx, MA_OWNED);
2437991554f2SKenneth D. Merry 
2438991554f2SKenneth D. Merry 	sassc = sc->sassc;
2439991554f2SKenneth D. Merry 	ccb = cm->cm_complete_data;
2440991554f2SKenneth D. Merry 	csio = &ccb->csio;
2441a2c14879SStephen McConnell 	target_id = csio->ccb_h.target_id;
2442991554f2SKenneth D. Merry 	rep = (MPI2_SCSI_IO_REPLY *)cm->cm_reply;
244371900a79SAlfredo Dal'Ava Junior 	mpr_dprint(sc, MPR_TRACE,
244471900a79SAlfredo Dal'Ava Junior 	    "cm %p SMID %u ccb %p reply %p outstanding %u csio->scsi_status 0x%x,"
244571900a79SAlfredo Dal'Ava Junior 	    "csio->dxfer_len 0x%x, csio->msg_le 0x%xn\n", cm,
244671900a79SAlfredo Dal'Ava Junior 	    cm->cm_desc.Default.SMID, cm->cm_ccb, cm->cm_reply,
244771900a79SAlfredo Dal'Ava Junior 	    cm->cm_targ->outstanding, csio->scsi_status,
244871900a79SAlfredo Dal'Ava Junior 	    csio->dxfer_len, csio->msg_len);
2449991554f2SKenneth D. Merry 	/*
2450991554f2SKenneth D. Merry 	 * XXX KDM if the chain allocation fails, does it matter if we do
2451991554f2SKenneth D. Merry 	 * the sync and unload here?  It is simpler to do it in every case,
2452991554f2SKenneth D. Merry 	 * assuming it doesn't cause problems.
2453991554f2SKenneth D. Merry 	 */
2454991554f2SKenneth D. Merry 	if (cm->cm_data != NULL) {
2455991554f2SKenneth D. Merry 		if (cm->cm_flags & MPR_CM_FLAGS_DATAIN)
2456991554f2SKenneth D. Merry 			dir = BUS_DMASYNC_POSTREAD;
2457991554f2SKenneth D. Merry 		else if (cm->cm_flags & MPR_CM_FLAGS_DATAOUT)
2458991554f2SKenneth D. Merry 			dir = BUS_DMASYNC_POSTWRITE;
2459991554f2SKenneth D. Merry 		bus_dmamap_sync(sc->buffer_dmat, cm->cm_dmamap, dir);
2460991554f2SKenneth D. Merry 		bus_dmamap_unload(sc->buffer_dmat, cm->cm_dmamap);
2461991554f2SKenneth D. Merry 	}
2462991554f2SKenneth D. Merry 
2463991554f2SKenneth D. Merry 	cm->cm_targ->completed++;
2464991554f2SKenneth D. Merry 	cm->cm_targ->outstanding--;
2465991554f2SKenneth D. Merry 	TAILQ_REMOVE(&cm->cm_targ->commands, cm, cm_link);
2466991554f2SKenneth D. Merry 	ccb->ccb_h.status &= ~(CAM_STATUS_MASK | CAM_SIM_QUEUED);
2467991554f2SKenneth D. Merry 
24688fe7bf06SWarner Losh 	if (cm->cm_flags & MPR_CM_FLAGS_ON_RECOVERY) {
2469991554f2SKenneth D. Merry 		TAILQ_REMOVE(&cm->cm_targ->timedout_commands, cm, cm_recovery);
24708fe7bf06SWarner Losh 		KASSERT(cm->cm_state == MPR_CM_STATE_BUSY,
2471175ad3d0SKenneth D. Merry 		    ("Not busy for CM_FLAGS_TIMEDOUT: %u\n", cm->cm_state));
24728fe7bf06SWarner Losh 		cm->cm_flags &= ~MPR_CM_FLAGS_ON_RECOVERY;
2473991554f2SKenneth D. Merry 		if (cm->cm_reply != NULL)
2474991554f2SKenneth D. Merry 			mprsas_log_command(cm, MPR_RECOVERY,
2475991554f2SKenneth D. Merry 			    "completed timedout cm %p ccb %p during recovery "
2476991554f2SKenneth D. Merry 			    "ioc %x scsi %x state %x xfer %u\n", cm, cm->cm_ccb,
2477991554f2SKenneth D. Merry 			    le16toh(rep->IOCStatus), rep->SCSIStatus,
2478991554f2SKenneth D. Merry 			    rep->SCSIState, le32toh(rep->TransferCount));
2479991554f2SKenneth D. Merry 		else
2480991554f2SKenneth D. Merry 			mprsas_log_command(cm, MPR_RECOVERY,
2481991554f2SKenneth D. Merry 			    "completed timedout cm %p ccb %p during recovery\n",
2482991554f2SKenneth D. Merry 			    cm, cm->cm_ccb);
2483991554f2SKenneth D. Merry 	} else if (cm->cm_targ->tm != NULL) {
2484991554f2SKenneth D. Merry 		if (cm->cm_reply != NULL)
2485991554f2SKenneth D. Merry 			mprsas_log_command(cm, MPR_RECOVERY,
2486991554f2SKenneth D. Merry 			    "completed cm %p ccb %p during recovery "
2487991554f2SKenneth D. Merry 			    "ioc %x scsi %x state %x xfer %u\n",
2488991554f2SKenneth D. Merry 			    cm, cm->cm_ccb, le16toh(rep->IOCStatus),
2489991554f2SKenneth D. Merry 			    rep->SCSIStatus, rep->SCSIState,
2490991554f2SKenneth D. Merry 			    le32toh(rep->TransferCount));
2491991554f2SKenneth D. Merry 		else
2492991554f2SKenneth D. Merry 			mprsas_log_command(cm, MPR_RECOVERY,
2493991554f2SKenneth D. Merry 			    "completed cm %p ccb %p during recovery\n",
2494991554f2SKenneth D. Merry 			    cm, cm->cm_ccb);
2495991554f2SKenneth D. Merry 	} else if ((sc->mpr_flags & MPR_FLAGS_DIAGRESET) != 0) {
2496991554f2SKenneth D. Merry 		mprsas_log_command(cm, MPR_RECOVERY,
2497991554f2SKenneth D. Merry 		    "reset completed cm %p ccb %p\n", cm, cm->cm_ccb);
2498991554f2SKenneth D. Merry 	}
2499991554f2SKenneth D. Merry 
2500991554f2SKenneth D. Merry 	if ((cm->cm_flags & MPR_CM_FLAGS_ERROR_MASK) != 0) {
2501991554f2SKenneth D. Merry 		/*
2502991554f2SKenneth D. Merry 		 * We ran into an error after we tried to map the command,
2503991554f2SKenneth D. Merry 		 * so we're getting a callback without queueing the command
2504991554f2SKenneth D. Merry 		 * to the hardware.  So we set the status here, and it will
2505991554f2SKenneth D. Merry 		 * be retained below.  We'll go through the "fast path",
2506991554f2SKenneth D. Merry 		 * because there can be no reply when we haven't actually
2507991554f2SKenneth D. Merry 		 * gone out to the hardware.
2508991554f2SKenneth D. Merry 		 */
2509a2c14879SStephen McConnell 		mprsas_set_ccbstatus(ccb, CAM_REQUEUE_REQ);
2510991554f2SKenneth D. Merry 
2511991554f2SKenneth D. Merry 		/*
2512991554f2SKenneth D. Merry 		 * Currently the only error included in the mask is
2513991554f2SKenneth D. Merry 		 * MPR_CM_FLAGS_CHAIN_FAILED, which means we're out of
2514991554f2SKenneth D. Merry 		 * chain frames.  We need to freeze the queue until we get
2515991554f2SKenneth D. Merry 		 * a command that completed without this error, which will
2516991554f2SKenneth D. Merry 		 * hopefully have some chain frames attached that we can
2517991554f2SKenneth D. Merry 		 * use.  If we wanted to get smarter about it, we would
2518991554f2SKenneth D. Merry 		 * only unfreeze the queue in this condition when we're
2519991554f2SKenneth D. Merry 		 * sure that we're getting some chain frames back.  That's
2520991554f2SKenneth D. Merry 		 * probably unnecessary.
2521991554f2SKenneth D. Merry 		 */
2522991554f2SKenneth D. Merry 		if ((sassc->flags & MPRSAS_QUEUE_FROZEN) == 0) {
2523991554f2SKenneth D. Merry 			xpt_freeze_simq(sassc->sim, 1);
2524991554f2SKenneth D. Merry 			sassc->flags |= MPRSAS_QUEUE_FROZEN;
2525a8837c77SWarner Losh 			mpr_dprint(sc, MPR_XINFO | MPR_RECOVERY,
2526a8837c77SWarner Losh 			    "Error sending command, freezing SIM queue\n");
2527991554f2SKenneth D. Merry 		}
2528991554f2SKenneth D. Merry 	}
2529991554f2SKenneth D. Merry 
2530991554f2SKenneth D. Merry 	/*
253167feec50SStephen McConnell 	 * Point to the SCSI CDB, which is dependent on the CAM_CDB_POINTER
253267feec50SStephen McConnell 	 * flag, and use it in a few places in the rest of this function for
253367feec50SStephen McConnell 	 * convenience. Use the macro if available.
253467feec50SStephen McConnell 	 */
253567feec50SStephen McConnell 	scsi_cdb = scsiio_cdb_ptr(csio);
253667feec50SStephen McConnell 
253767feec50SStephen McConnell 	/*
2538991554f2SKenneth D. Merry 	 * If this is a Start Stop Unit command and it was issued by the driver
2539991554f2SKenneth D. Merry 	 * during shutdown, decrement the refcount to account for all of the
2540991554f2SKenneth D. Merry 	 * commands that were sent.  All SSU commands should be completed before
2541991554f2SKenneth D. Merry 	 * shutdown completes, meaning SSU_refcount will be 0 after SSU_started
2542991554f2SKenneth D. Merry 	 * is TRUE.
2543991554f2SKenneth D. Merry 	 */
254467feec50SStephen McConnell 	if (sc->SSU_started && (scsi_cdb[0] == START_STOP_UNIT)) {
2545991554f2SKenneth D. Merry 		mpr_dprint(sc, MPR_INFO, "Decrementing SSU count.\n");
2546991554f2SKenneth D. Merry 		sc->SSU_refcount--;
2547991554f2SKenneth D. Merry 	}
2548991554f2SKenneth D. Merry 
2549991554f2SKenneth D. Merry 	/* Take the fast path to completion */
2550991554f2SKenneth D. Merry 	if (cm->cm_reply == NULL) {
2551a2c14879SStephen McConnell 		if (mprsas_get_ccbstatus(ccb) == CAM_REQ_INPROG) {
2552991554f2SKenneth D. Merry 			if ((sc->mpr_flags & MPR_FLAGS_DIAGRESET) != 0)
2553a2c14879SStephen McConnell 				mprsas_set_ccbstatus(ccb, CAM_SCSI_BUS_RESET);
2554991554f2SKenneth D. Merry 			else {
2555a2c14879SStephen McConnell 				mprsas_set_ccbstatus(ccb, CAM_REQ_CMP);
2556a2c14879SStephen McConnell 				csio->scsi_status = SCSI_STATUS_OK;
2557991554f2SKenneth D. Merry 			}
2558991554f2SKenneth D. Merry 			if (sassc->flags & MPRSAS_QUEUE_FROZEN) {
2559991554f2SKenneth D. Merry 				ccb->ccb_h.status |= CAM_RELEASE_SIMQ;
2560991554f2SKenneth D. Merry 				sassc->flags &= ~MPRSAS_QUEUE_FROZEN;
2561a8837c77SWarner Losh 				mpr_dprint(sc, MPR_XINFO | MPR_RECOVERY,
2562991554f2SKenneth D. Merry 				    "Unfreezing SIM queue\n");
2563991554f2SKenneth D. Merry 			}
2564991554f2SKenneth D. Merry 		}
2565991554f2SKenneth D. Merry 
2566991554f2SKenneth D. Merry 		/*
2567991554f2SKenneth D. Merry 		 * There are two scenarios where the status won't be
2568991554f2SKenneth D. Merry 		 * CAM_REQ_CMP.  The first is if MPR_CM_FLAGS_ERROR_MASK is
2569991554f2SKenneth D. Merry 		 * set, the second is in the MPR_FLAGS_DIAGRESET above.
2570991554f2SKenneth D. Merry 		 */
2571a2c14879SStephen McConnell 		if (mprsas_get_ccbstatus(ccb) != CAM_REQ_CMP) {
2572991554f2SKenneth D. Merry 			/*
2573991554f2SKenneth D. Merry 			 * Freeze the dev queue so that commands are
2574a2c14879SStephen McConnell 			 * executed in the correct order after error
2575991554f2SKenneth D. Merry 			 * recovery.
2576991554f2SKenneth D. Merry 			 */
2577991554f2SKenneth D. Merry 			ccb->ccb_h.status |= CAM_DEV_QFRZN;
2578991554f2SKenneth D. Merry 			xpt_freeze_devq(ccb->ccb_h.path, /*count*/ 1);
2579991554f2SKenneth D. Merry 		}
2580991554f2SKenneth D. Merry 		mpr_free_command(sc, cm);
2581991554f2SKenneth D. Merry 		xpt_done(ccb);
2582991554f2SKenneth D. Merry 		return;
2583991554f2SKenneth D. Merry 	}
2584991554f2SKenneth D. Merry 
258567feec50SStephen McConnell 	target = &sassc->targets[target_id];
258667feec50SStephen McConnell 	if (scsi_cdb[0] == UNMAP &&
258767feec50SStephen McConnell 	    target->is_nvme &&
258867feec50SStephen McConnell 	    (csio->ccb_h.flags & CAM_DATA_MASK) == CAM_DATA_VADDR) {
258967feec50SStephen McConnell 		rep->SCSIStatus = mprsas_complete_nvme_unmap(sc, cm);
259067feec50SStephen McConnell 		csio->scsi_status = rep->SCSIStatus;
259167feec50SStephen McConnell 	}
259267feec50SStephen McConnell 
2593991554f2SKenneth D. Merry 	mprsas_log_command(cm, MPR_XINFO,
2594991554f2SKenneth D. Merry 	    "ioc %x scsi %x state %x xfer %u\n",
2595991554f2SKenneth D. Merry 	    le16toh(rep->IOCStatus), rep->SCSIStatus, rep->SCSIState,
2596991554f2SKenneth D. Merry 	    le32toh(rep->TransferCount));
2597991554f2SKenneth D. Merry 
2598991554f2SKenneth D. Merry 	switch (le16toh(rep->IOCStatus) & MPI2_IOCSTATUS_MASK) {
2599991554f2SKenneth D. Merry 	case MPI2_IOCSTATUS_SCSI_DATA_UNDERRUN:
2600991554f2SKenneth D. Merry 		csio->resid = cm->cm_length - le32toh(rep->TransferCount);
2601991554f2SKenneth D. Merry 		/* FALLTHROUGH */
2602991554f2SKenneth D. Merry 	case MPI2_IOCSTATUS_SUCCESS:
2603991554f2SKenneth D. Merry 	case MPI2_IOCSTATUS_SCSI_RECOVERED_ERROR:
2604991554f2SKenneth D. Merry 		if ((le16toh(rep->IOCStatus) & MPI2_IOCSTATUS_MASK) ==
2605991554f2SKenneth D. Merry 		    MPI2_IOCSTATUS_SCSI_RECOVERED_ERROR)
2606991554f2SKenneth D. Merry 			mprsas_log_command(cm, MPR_XINFO, "recovered error\n");
2607991554f2SKenneth D. Merry 
2608991554f2SKenneth D. Merry 		/* Completion failed at the transport level. */
2609991554f2SKenneth D. Merry 		if (rep->SCSIState & (MPI2_SCSI_STATE_NO_SCSI_STATUS |
2610991554f2SKenneth D. Merry 		    MPI2_SCSI_STATE_TERMINATED)) {
2611a2c14879SStephen McConnell 			mprsas_set_ccbstatus(ccb, CAM_REQ_CMP_ERR);
2612991554f2SKenneth D. Merry 			break;
2613991554f2SKenneth D. Merry 		}
2614991554f2SKenneth D. Merry 
2615991554f2SKenneth D. Merry 		/* In a modern packetized environment, an autosense failure
2616991554f2SKenneth D. Merry 		 * implies that there's not much else that can be done to
2617991554f2SKenneth D. Merry 		 * recover the command.
2618991554f2SKenneth D. Merry 		 */
2619991554f2SKenneth D. Merry 		if (rep->SCSIState & MPI2_SCSI_STATE_AUTOSENSE_FAILED) {
2620a2c14879SStephen McConnell 			mprsas_set_ccbstatus(ccb, CAM_AUTOSENSE_FAIL);
2621991554f2SKenneth D. Merry 			break;
2622991554f2SKenneth D. Merry 		}
2623991554f2SKenneth D. Merry 
2624991554f2SKenneth D. Merry 		/*
2625991554f2SKenneth D. Merry 		 * CAM doesn't care about SAS Response Info data, but if this is
2626991554f2SKenneth D. Merry 		 * the state check if TLR should be done.  If not, clear the
2627991554f2SKenneth D. Merry 		 * TLR_bits for the target.
2628991554f2SKenneth D. Merry 		 */
2629991554f2SKenneth D. Merry 		if ((rep->SCSIState & MPI2_SCSI_STATE_RESPONSE_INFO_VALID) &&
2630991554f2SKenneth D. Merry 		    ((le32toh(rep->ResponseInfo) & MPI2_SCSI_RI_MASK_REASONCODE)
2631991554f2SKenneth D. Merry 		    == MPR_SCSI_RI_INVALID_FRAME)) {
2632a2c14879SStephen McConnell 			sc->mapping_table[target_id].TLR_bits =
2633991554f2SKenneth D. Merry 			    (u8)MPI2_SCSIIO_CONTROL_NO_TLR;
2634991554f2SKenneth D. Merry 		}
2635991554f2SKenneth D. Merry 
2636991554f2SKenneth D. Merry 		/*
2637991554f2SKenneth D. Merry 		 * Intentionally override the normal SCSI status reporting
2638991554f2SKenneth D. Merry 		 * for these two cases.  These are likely to happen in a
2639991554f2SKenneth D. Merry 		 * multi-initiator environment, and we want to make sure that
2640991554f2SKenneth D. Merry 		 * CAM retries these commands rather than fail them.
2641991554f2SKenneth D. Merry 		 */
2642991554f2SKenneth D. Merry 		if ((rep->SCSIStatus == MPI2_SCSI_STATUS_COMMAND_TERMINATED) ||
2643991554f2SKenneth D. Merry 		    (rep->SCSIStatus == MPI2_SCSI_STATUS_TASK_ABORTED)) {
2644a2c14879SStephen McConnell 			mprsas_set_ccbstatus(ccb, CAM_REQ_ABORTED);
2645991554f2SKenneth D. Merry 			break;
2646991554f2SKenneth D. Merry 		}
2647991554f2SKenneth D. Merry 
2648991554f2SKenneth D. Merry 		/* Handle normal status and sense */
2649991554f2SKenneth D. Merry 		csio->scsi_status = rep->SCSIStatus;
2650991554f2SKenneth D. Merry 		if (rep->SCSIStatus == MPI2_SCSI_STATUS_GOOD)
2651a2c14879SStephen McConnell 			mprsas_set_ccbstatus(ccb, CAM_REQ_CMP);
2652991554f2SKenneth D. Merry 		else
2653a2c14879SStephen McConnell 			mprsas_set_ccbstatus(ccb, CAM_SCSI_STATUS_ERROR);
2654991554f2SKenneth D. Merry 
2655991554f2SKenneth D. Merry 		if (rep->SCSIState & MPI2_SCSI_STATE_AUTOSENSE_VALID) {
2656991554f2SKenneth D. Merry 			int sense_len, returned_sense_len;
2657991554f2SKenneth D. Merry 
2658991554f2SKenneth D. Merry 			returned_sense_len = min(le32toh(rep->SenseCount),
2659991554f2SKenneth D. Merry 			    sizeof(struct scsi_sense_data));
2660991554f2SKenneth D. Merry 			if (returned_sense_len < csio->sense_len)
2661991554f2SKenneth D. Merry 				csio->sense_resid = csio->sense_len -
2662991554f2SKenneth D. Merry 				    returned_sense_len;
2663991554f2SKenneth D. Merry 			else
2664991554f2SKenneth D. Merry 				csio->sense_resid = 0;
2665991554f2SKenneth D. Merry 
2666991554f2SKenneth D. Merry 			sense_len = min(returned_sense_len,
2667991554f2SKenneth D. Merry 			    csio->sense_len - csio->sense_resid);
2668991554f2SKenneth D. Merry 			bzero(&csio->sense_data, sizeof(csio->sense_data));
2669991554f2SKenneth D. Merry 			bcopy(cm->cm_sense, &csio->sense_data, sense_len);
2670991554f2SKenneth D. Merry 			ccb->ccb_h.status |= CAM_AUTOSNS_VALID;
2671991554f2SKenneth D. Merry 		}
2672991554f2SKenneth D. Merry 
2673991554f2SKenneth D. Merry 		/*
2674991554f2SKenneth D. Merry 		 * Check if this is an INQUIRY command.  If it's a VPD inquiry,
2675991554f2SKenneth D. Merry 		 * and it's page code 0 (Supported Page List), and there is
2676991554f2SKenneth D. Merry 		 * inquiry data, and this is for a sequential access device, and
2677991554f2SKenneth D. Merry 		 * the device is an SSP target, and TLR is supported by the
2678991554f2SKenneth D. Merry 		 * controller, turn the TLR_bits value ON if page 0x90 is
2679991554f2SKenneth D. Merry 		 * supported.
2680991554f2SKenneth D. Merry 		 */
268167feec50SStephen McConnell 		if ((scsi_cdb[0] == INQUIRY) &&
268267feec50SStephen McConnell 		    (scsi_cdb[1] & SI_EVPD) &&
268367feec50SStephen McConnell 		    (scsi_cdb[2] == SVPD_SUPPORTED_PAGE_LIST) &&
2684991554f2SKenneth D. Merry 		    ((csio->ccb_h.flags & CAM_DATA_MASK) == CAM_DATA_VADDR) &&
2685d2b4e18bSKenneth D. Merry 		    (csio->data_ptr != NULL) &&
2686d2b4e18bSKenneth D. Merry 		    ((csio->data_ptr[0] & 0x1f) == T_SEQUENTIAL) &&
2687d2b4e18bSKenneth D. Merry 		    (sc->control_TLR) &&
2688a2c14879SStephen McConnell 		    (sc->mapping_table[target_id].device_info &
2689991554f2SKenneth D. Merry 		    MPI2_SAS_DEVICE_INFO_SSP_TARGET)) {
2690991554f2SKenneth D. Merry 			vpd_list = (struct scsi_vpd_supported_page_list *)
2691991554f2SKenneth D. Merry 			    csio->data_ptr;
2692a2c14879SStephen McConnell 			TLR_bits = &sc->mapping_table[target_id].TLR_bits;
2693991554f2SKenneth D. Merry 			*TLR_bits = (u8)MPI2_SCSIIO_CONTROL_NO_TLR;
2694991554f2SKenneth D. Merry 			TLR_on = (u8)MPI2_SCSIIO_CONTROL_TLR_ON;
269567feec50SStephen McConnell 			alloc_len = ((u16)scsi_cdb[3] << 8) + scsi_cdb[4];
2696d2b4e18bSKenneth D. Merry 			alloc_len -= csio->resid;
2697991554f2SKenneth D. Merry 			for (i = 0; i < MIN(vpd_list->length, alloc_len); i++) {
2698991554f2SKenneth D. Merry 				if (vpd_list->list[i] == 0x90) {
2699991554f2SKenneth D. Merry 					*TLR_bits = TLR_on;
2700991554f2SKenneth D. Merry 					break;
2701991554f2SKenneth D. Merry 				}
2702991554f2SKenneth D. Merry 			}
2703991554f2SKenneth D. Merry 		}
2704a2c14879SStephen McConnell 
2705a2c14879SStephen McConnell 		/*
2706a2c14879SStephen McConnell 		 * If this is a SATA direct-access end device, mark it so that
2707a2c14879SStephen McConnell 		 * a SCSI StartStopUnit command will be sent to it when the
2708a2c14879SStephen McConnell 		 * driver is being shutdown.
2709a2c14879SStephen McConnell 		 */
271067feec50SStephen McConnell 		if ((scsi_cdb[0] == INQUIRY) &&
2711fa699bb2SAlan Somers 		    (csio->data_ptr != NULL) &&
2712a2c14879SStephen McConnell 		    ((csio->data_ptr[0] & 0x1f) == T_DIRECT) &&
2713a2c14879SStephen McConnell 		    (sc->mapping_table[target_id].device_info &
2714a2c14879SStephen McConnell 		    MPI2_SAS_DEVICE_INFO_SATA_DEVICE) &&
2715a2c14879SStephen McConnell 		    ((sc->mapping_table[target_id].device_info &
2716a2c14879SStephen McConnell 		    MPI2_SAS_DEVICE_INFO_MASK_DEVICE_TYPE) ==
2717a2c14879SStephen McConnell 		    MPI2_SAS_DEVICE_INFO_END_DEVICE)) {
2718a2c14879SStephen McConnell 			target = &sassc->targets[target_id];
2719a2c14879SStephen McConnell 			target->supports_SSU = TRUE;
2720a2c14879SStephen McConnell 			mpr_dprint(sc, MPR_XINFO, "Target %d supports SSU\n",
2721a2c14879SStephen McConnell 			    target_id);
2722a2c14879SStephen McConnell 		}
2723991554f2SKenneth D. Merry 		break;
2724991554f2SKenneth D. Merry 	case MPI2_IOCSTATUS_SCSI_INVALID_DEVHANDLE:
2725991554f2SKenneth D. Merry 	case MPI2_IOCSTATUS_SCSI_DEVICE_NOT_THERE:
2726991554f2SKenneth D. Merry 		/*
2727991554f2SKenneth D. Merry 		 * If devinfo is 0 this will be a volume.  In that case don't
2728991554f2SKenneth D. Merry 		 * tell CAM that the volume is not there.  We want volumes to
2729991554f2SKenneth D. Merry 		 * be enumerated until they are deleted/removed, not just
2730991554f2SKenneth D. Merry 		 * failed.
2731991554f2SKenneth D. Merry 		 */
2732991554f2SKenneth D. Merry 		if (cm->cm_targ->devinfo == 0)
2733a2c14879SStephen McConnell 			mprsas_set_ccbstatus(ccb, CAM_REQ_CMP);
2734991554f2SKenneth D. Merry 		else
2735a2c14879SStephen McConnell 			mprsas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE);
2736991554f2SKenneth D. Merry 		break;
2737991554f2SKenneth D. Merry 	case MPI2_IOCSTATUS_INVALID_SGL:
2738991554f2SKenneth D. Merry 		mpr_print_scsiio_cmd(sc, cm);
2739a2c14879SStephen McConnell 		mprsas_set_ccbstatus(ccb, CAM_UNREC_HBA_ERROR);
2740991554f2SKenneth D. Merry 		break;
2741991554f2SKenneth D. Merry 	case MPI2_IOCSTATUS_SCSI_TASK_TERMINATED:
2742991554f2SKenneth D. Merry 		/*
2743991554f2SKenneth D. Merry 		 * This is one of the responses that comes back when an I/O
2744991554f2SKenneth D. Merry 		 * has been aborted.  If it is because of a timeout that we
2745991554f2SKenneth D. Merry 		 * initiated, just set the status to CAM_CMD_TIMEOUT.
2746991554f2SKenneth D. Merry 		 * Otherwise set it to CAM_REQ_ABORTED.  The effect on the
2747991554f2SKenneth D. Merry 		 * command is the same (it gets retried, subject to the
2748991554f2SKenneth D. Merry 		 * retry counter), the only difference is what gets printed
2749991554f2SKenneth D. Merry 		 * on the console.
2750991554f2SKenneth D. Merry 		 */
27518fe7bf06SWarner Losh 		if (cm->cm_flags & MPR_CM_FLAGS_TIMEDOUT)
2752a2c14879SStephen McConnell 			mprsas_set_ccbstatus(ccb, CAM_CMD_TIMEOUT);
2753991554f2SKenneth D. Merry 		else
2754a2c14879SStephen McConnell 			mprsas_set_ccbstatus(ccb, CAM_REQ_ABORTED);
2755991554f2SKenneth D. Merry 		break;
2756991554f2SKenneth D. Merry 	case MPI2_IOCSTATUS_SCSI_DATA_OVERRUN:
2757991554f2SKenneth D. Merry 		/* resid is ignored for this condition */
2758991554f2SKenneth D. Merry 		csio->resid = 0;
2759a2c14879SStephen McConnell 		mprsas_set_ccbstatus(ccb, CAM_DATA_RUN_ERR);
2760991554f2SKenneth D. Merry 		break;
2761991554f2SKenneth D. Merry 	case MPI2_IOCSTATUS_SCSI_IOC_TERMINATED:
2762991554f2SKenneth D. Merry 	case MPI2_IOCSTATUS_SCSI_EXT_TERMINATED:
2763991554f2SKenneth D. Merry 		/*
27646adfa7edSAlan Somers 		 * These can sometimes be transient transport-related
27656adfa7edSAlan Somers 		 * errors, and sometimes persistent drive-related errors.
27666adfa7edSAlan Somers 		 * We used to retry these without decrementing the retry
27676adfa7edSAlan Somers 		 * count by returning CAM_REQUEUE_REQ.  Unfortunately, if
27686adfa7edSAlan Somers 		 * we hit a persistent drive problem that returns one of
27696adfa7edSAlan Somers 		 * these error codes, we would retry indefinitely.  So,
27706adfa7edSAlan Somers 		 * return CAM_REQ_CMP_ERROR so that we decrement the retry
27716adfa7edSAlan Somers 		 * count and avoid infinite retries.  We're taking the
27726adfa7edSAlan Somers 		 * potential risk of flagging false failures in the event
27736adfa7edSAlan Somers 		 * of a topology-related error (e.g. a SAS expander problem
27746adfa7edSAlan Somers 		 * causes a command addressed to a drive to fail), but
27754c1cdd4aSWarner Losh 		 * avoiding getting into an infinite retry loop. However,
27764c1cdd4aSWarner Losh 		 * if we get them while were moving a device, we should
27774c1cdd4aSWarner Losh 		 * fail the request as 'not there' because the device
27784c1cdd4aSWarner Losh 		 * is effectively gone.
2779991554f2SKenneth D. Merry 		 */
27804c1cdd4aSWarner Losh 		if (cm->cm_targ->flags & MPRSAS_TARGET_INREMOVAL)
27814c1cdd4aSWarner Losh 			mprsas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE);
27824c1cdd4aSWarner Losh 		else
27836adfa7edSAlan Somers 			mprsas_set_ccbstatus(ccb, CAM_REQ_CMP_ERR);
27846eea4f46SScott Long 		mpr_dprint(sc, MPR_INFO,
27854c1cdd4aSWarner Losh 		    "Controller reported %s tgt %u SMID %u loginfo %x%s\n",
27862bf620cbSScott Long 		    mpr_describe_table(mpr_iocstatus_string,
27872bf620cbSScott Long 		    le16toh(rep->IOCStatus) & MPI2_IOCSTATUS_MASK),
27882bf620cbSScott Long 		    target_id, cm->cm_desc.Default.SMID,
27894c1cdd4aSWarner Losh 		    le32toh(rep->IOCLogInfo),
27904c1cdd4aSWarner Losh 		    (cm->cm_targ->flags & MPRSAS_TARGET_INREMOVAL) ? " departing" : "");
27916eea4f46SScott Long 		mpr_dprint(sc, MPR_XINFO,
27926eea4f46SScott Long 		    "SCSIStatus %x SCSIState %x xfercount %u\n",
2793694cb8b8SScott Long 		    rep->SCSIStatus, rep->SCSIState,
2794991554f2SKenneth D. Merry 		    le32toh(rep->TransferCount));
2795991554f2SKenneth D. Merry 		break;
2796991554f2SKenneth D. Merry 	case MPI2_IOCSTATUS_INVALID_FUNCTION:
2797991554f2SKenneth D. Merry 	case MPI2_IOCSTATUS_INTERNAL_ERROR:
2798991554f2SKenneth D. Merry 	case MPI2_IOCSTATUS_INVALID_VPID:
2799991554f2SKenneth D. Merry 	case MPI2_IOCSTATUS_INVALID_FIELD:
2800991554f2SKenneth D. Merry 	case MPI2_IOCSTATUS_INVALID_STATE:
2801991554f2SKenneth D. Merry 	case MPI2_IOCSTATUS_OP_STATE_NOT_SUPPORTED:
2802991554f2SKenneth D. Merry 	case MPI2_IOCSTATUS_SCSI_IO_DATA_ERROR:
2803991554f2SKenneth D. Merry 	case MPI2_IOCSTATUS_SCSI_PROTOCOL_ERROR:
2804991554f2SKenneth D. Merry 	case MPI2_IOCSTATUS_SCSI_RESIDUAL_MISMATCH:
2805991554f2SKenneth D. Merry 	case MPI2_IOCSTATUS_SCSI_TASK_MGMT_FAILED:
2806991554f2SKenneth D. Merry 	default:
2807991554f2SKenneth D. Merry 		mprsas_log_command(cm, MPR_XINFO,
2808694cb8b8SScott Long 		    "completed ioc %x loginfo %x scsi %x state %x xfer %u\n",
2809694cb8b8SScott Long 		    le16toh(rep->IOCStatus), le32toh(rep->IOCLogInfo),
2810694cb8b8SScott Long 		    rep->SCSIStatus, rep->SCSIState,
2811991554f2SKenneth D. Merry 		    le32toh(rep->TransferCount));
2812991554f2SKenneth D. Merry 		csio->resid = cm->cm_length;
281367feec50SStephen McConnell 
281467feec50SStephen McConnell 		if (scsi_cdb[0] == UNMAP &&
281567feec50SStephen McConnell 		    target->is_nvme &&
281667feec50SStephen McConnell 		    (csio->ccb_h.flags & CAM_DATA_MASK) == CAM_DATA_VADDR)
281767feec50SStephen McConnell 			mprsas_set_ccbstatus(ccb, CAM_REQ_CMP);
281867feec50SStephen McConnell 		else
2819a2c14879SStephen McConnell 			mprsas_set_ccbstatus(ccb, CAM_REQ_CMP_ERR);
282067feec50SStephen McConnell 
2821991554f2SKenneth D. Merry 		break;
2822991554f2SKenneth D. Merry 	}
2823991554f2SKenneth D. Merry 
2824991554f2SKenneth D. Merry 	mpr_sc_failed_io_info(sc, csio, rep, cm->cm_targ);
2825991554f2SKenneth D. Merry 
2826991554f2SKenneth D. Merry 	if (sassc->flags & MPRSAS_QUEUE_FROZEN) {
2827991554f2SKenneth D. Merry 		ccb->ccb_h.status |= CAM_RELEASE_SIMQ;
2828991554f2SKenneth D. Merry 		sassc->flags &= ~MPRSAS_QUEUE_FROZEN;
2829a8837c77SWarner Losh 		mpr_dprint(sc, MPR_INFO, "Command completed, unfreezing SIM "
2830991554f2SKenneth D. Merry 		    "queue\n");
2831991554f2SKenneth D. Merry 	}
2832991554f2SKenneth D. Merry 
2833a2c14879SStephen McConnell 	if (mprsas_get_ccbstatus(ccb) != CAM_REQ_CMP) {
2834991554f2SKenneth D. Merry 		ccb->ccb_h.status |= CAM_DEV_QFRZN;
2835991554f2SKenneth D. Merry 		xpt_freeze_devq(ccb->ccb_h.path, /*count*/ 1);
2836991554f2SKenneth D. Merry 	}
2837991554f2SKenneth D. Merry 
28384c1cdd4aSWarner Losh 	/*
28394c1cdd4aSWarner Losh 	 * Check to see if we're removing the device. If so, and this is the
28404c1cdd4aSWarner Losh 	 * last command on the queue, proceed with the deferred removal of the
28414c1cdd4aSWarner Losh 	 * device.  Note, for removing a volume, this won't trigger because
28424c1cdd4aSWarner Losh 	 * pending_remove_tm will be NULL.
28434c1cdd4aSWarner Losh 	 */
28444c1cdd4aSWarner Losh 	if (cm->cm_targ->flags & MPRSAS_TARGET_INREMOVAL) {
28454c1cdd4aSWarner Losh 		if (TAILQ_FIRST(&cm->cm_targ->commands) == NULL &&
28464c1cdd4aSWarner Losh 		    cm->cm_targ->pending_remove_tm != NULL) {
2847*ca420b4eSWarner Losh 			mpr_dprint(sc, MPR_INFO,
2848*ca420b4eSWarner Losh 			    "Last pending command complete: starting remove_device target %u handle 0x%04x\n",
2849*ca420b4eSWarner Losh 			    cm->cm_targ->tid, cm->cm_targ->handle);
28504c1cdd4aSWarner Losh 			mpr_map_command(sc, cm->cm_targ->pending_remove_tm);
28514c1cdd4aSWarner Losh 			cm->cm_targ->pending_remove_tm = NULL;
28524c1cdd4aSWarner Losh 		}
28534c1cdd4aSWarner Losh 	}
28544c1cdd4aSWarner Losh 
2855991554f2SKenneth D. Merry 	mpr_free_command(sc, cm);
2856991554f2SKenneth D. Merry 	xpt_done(ccb);
2857991554f2SKenneth D. Merry }
2858991554f2SKenneth D. Merry 
2859991554f2SKenneth D. Merry static void
2860991554f2SKenneth D. Merry mprsas_smpio_complete(struct mpr_softc *sc, struct mpr_command *cm)
2861991554f2SKenneth D. Merry {
2862991554f2SKenneth D. Merry 	MPI2_SMP_PASSTHROUGH_REPLY *rpl;
2863991554f2SKenneth D. Merry 	MPI2_SMP_PASSTHROUGH_REQUEST *req;
2864991554f2SKenneth D. Merry 	uint64_t sasaddr;
2865991554f2SKenneth D. Merry 	union ccb *ccb;
2866991554f2SKenneth D. Merry 
2867991554f2SKenneth D. Merry 	ccb = cm->cm_complete_data;
2868991554f2SKenneth D. Merry 
2869991554f2SKenneth D. Merry 	/*
2870991554f2SKenneth D. Merry 	 * Currently there should be no way we can hit this case.  It only
2871991554f2SKenneth D. Merry 	 * happens when we have a failure to allocate chain frames, and SMP
2872991554f2SKenneth D. Merry 	 * commands require two S/G elements only.  That should be handled
2873991554f2SKenneth D. Merry 	 * in the standard request size.
2874991554f2SKenneth D. Merry 	 */
2875991554f2SKenneth D. Merry 	if ((cm->cm_flags & MPR_CM_FLAGS_ERROR_MASK) != 0) {
2876a2c14879SStephen McConnell 		mpr_dprint(sc, MPR_ERROR, "%s: cm_flags = %#x on SMP "
2877a2c14879SStephen McConnell 		    "request!\n", __func__, cm->cm_flags);
2878a2c14879SStephen McConnell 		mprsas_set_ccbstatus(ccb, CAM_REQ_CMP_ERR);
2879991554f2SKenneth D. Merry 		goto bailout;
2880991554f2SKenneth D. Merry         }
2881991554f2SKenneth D. Merry 
2882991554f2SKenneth D. Merry 	rpl = (MPI2_SMP_PASSTHROUGH_REPLY *)cm->cm_reply;
2883991554f2SKenneth D. Merry 	if (rpl == NULL) {
2884991554f2SKenneth D. Merry 		mpr_dprint(sc, MPR_ERROR, "%s: NULL cm_reply!\n", __func__);
2885a2c14879SStephen McConnell 		mprsas_set_ccbstatus(ccb, CAM_REQ_CMP_ERR);
2886991554f2SKenneth D. Merry 		goto bailout;
2887991554f2SKenneth D. Merry 	}
2888991554f2SKenneth D. Merry 
2889991554f2SKenneth D. Merry 	req = (MPI2_SMP_PASSTHROUGH_REQUEST *)cm->cm_req;
2890991554f2SKenneth D. Merry 	sasaddr = le32toh(req->SASAddress.Low);
2891991554f2SKenneth D. Merry 	sasaddr |= ((uint64_t)(le32toh(req->SASAddress.High))) << 32;
2892991554f2SKenneth D. Merry 
2893991554f2SKenneth D. Merry 	if ((le16toh(rpl->IOCStatus) & MPI2_IOCSTATUS_MASK) !=
2894991554f2SKenneth D. Merry 	    MPI2_IOCSTATUS_SUCCESS ||
2895991554f2SKenneth D. Merry 	    rpl->SASStatus != MPI2_SASSTATUS_SUCCESS) {
2896991554f2SKenneth D. Merry 		mpr_dprint(sc, MPR_XINFO, "%s: IOCStatus %04x SASStatus %02x\n",
2897991554f2SKenneth D. Merry 		    __func__, le16toh(rpl->IOCStatus), rpl->SASStatus);
2898a2c14879SStephen McConnell 		mprsas_set_ccbstatus(ccb, CAM_REQ_CMP_ERR);
2899991554f2SKenneth D. Merry 		goto bailout;
2900991554f2SKenneth D. Merry 	}
2901991554f2SKenneth D. Merry 
2902a2c14879SStephen McConnell 	mpr_dprint(sc, MPR_XINFO, "%s: SMP request to SAS address %#jx "
2903a2c14879SStephen McConnell 	    "completed successfully\n", __func__, (uintmax_t)sasaddr);
2904991554f2SKenneth D. Merry 
2905991554f2SKenneth D. Merry 	if (ccb->smpio.smp_response[2] == SMP_FR_ACCEPTED)
2906a2c14879SStephen McConnell 		mprsas_set_ccbstatus(ccb, CAM_REQ_CMP);
2907991554f2SKenneth D. Merry 	else
2908a2c14879SStephen McConnell 		mprsas_set_ccbstatus(ccb, CAM_SMP_STATUS_ERROR);
2909991554f2SKenneth D. Merry 
2910991554f2SKenneth D. Merry bailout:
2911991554f2SKenneth D. Merry 	/*
2912991554f2SKenneth D. Merry 	 * We sync in both directions because we had DMAs in the S/G list
2913991554f2SKenneth D. Merry 	 * in both directions.
2914991554f2SKenneth D. Merry 	 */
2915991554f2SKenneth D. Merry 	bus_dmamap_sync(sc->buffer_dmat, cm->cm_dmamap,
2916991554f2SKenneth D. Merry 			BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
2917991554f2SKenneth D. Merry 	bus_dmamap_unload(sc->buffer_dmat, cm->cm_dmamap);
2918991554f2SKenneth D. Merry 	mpr_free_command(sc, cm);
2919991554f2SKenneth D. Merry 	xpt_done(ccb);
2920991554f2SKenneth D. Merry }
2921991554f2SKenneth D. Merry 
2922991554f2SKenneth D. Merry static void
29237a2a6a1aSStephen McConnell mprsas_send_smpcmd(struct mprsas_softc *sassc, union ccb *ccb, uint64_t sasaddr)
2924991554f2SKenneth D. Merry {
2925991554f2SKenneth D. Merry 	struct mpr_command *cm;
2926991554f2SKenneth D. Merry 	uint8_t *request, *response;
2927991554f2SKenneth D. Merry 	MPI2_SMP_PASSTHROUGH_REQUEST *req;
2928991554f2SKenneth D. Merry 	struct mpr_softc *sc;
2929991554f2SKenneth D. Merry 	int error;
2930991554f2SKenneth D. Merry 
2931991554f2SKenneth D. Merry 	sc = sassc->sc;
2932991554f2SKenneth D. Merry 	error = 0;
2933991554f2SKenneth D. Merry 
2934991554f2SKenneth D. Merry 	switch (ccb->ccb_h.flags & CAM_DATA_MASK) {
2935991554f2SKenneth D. Merry 	case CAM_DATA_PADDR:
2936991554f2SKenneth D. Merry 	case CAM_DATA_SG_PADDR:
2937991554f2SKenneth D. Merry 		/*
2938991554f2SKenneth D. Merry 		 * XXX We don't yet support physical addresses here.
2939991554f2SKenneth D. Merry 		 */
2940991554f2SKenneth D. Merry 		mpr_dprint(sc, MPR_ERROR, "%s: physical addresses not "
2941991554f2SKenneth D. Merry 		    "supported\n", __func__);
2942a2c14879SStephen McConnell 		mprsas_set_ccbstatus(ccb, CAM_REQ_INVALID);
2943991554f2SKenneth D. Merry 		xpt_done(ccb);
2944991554f2SKenneth D. Merry 		return;
2945991554f2SKenneth D. Merry 	case CAM_DATA_SG:
2946991554f2SKenneth D. Merry 		/*
2947991554f2SKenneth D. Merry 		 * The chip does not support more than one buffer for the
2948991554f2SKenneth D. Merry 		 * request or response.
2949991554f2SKenneth D. Merry 		 */
2950991554f2SKenneth D. Merry 		if ((ccb->smpio.smp_request_sglist_cnt > 1)
2951991554f2SKenneth D. Merry 		    || (ccb->smpio.smp_response_sglist_cnt > 1)) {
29527a2a6a1aSStephen McConnell 			mpr_dprint(sc, MPR_ERROR, "%s: multiple request or "
29537a2a6a1aSStephen McConnell 			    "response buffer segments not supported for SMP\n",
29547a2a6a1aSStephen McConnell 			    __func__);
2955a2c14879SStephen McConnell 			mprsas_set_ccbstatus(ccb, CAM_REQ_INVALID);
2956991554f2SKenneth D. Merry 			xpt_done(ccb);
2957991554f2SKenneth D. Merry 			return;
2958991554f2SKenneth D. Merry 		}
2959991554f2SKenneth D. Merry 
2960991554f2SKenneth D. Merry 		/*
2961991554f2SKenneth D. Merry 		 * The CAM_SCATTER_VALID flag was originally implemented
2962991554f2SKenneth D. Merry 		 * for the XPT_SCSI_IO CCB, which only has one data pointer.
2963991554f2SKenneth D. Merry 		 * We have two.  So, just take that flag to mean that we
2964991554f2SKenneth D. Merry 		 * might have S/G lists, and look at the S/G segment count
2965991554f2SKenneth D. Merry 		 * to figure out whether that is the case for each individual
2966991554f2SKenneth D. Merry 		 * buffer.
2967991554f2SKenneth D. Merry 		 */
2968991554f2SKenneth D. Merry 		if (ccb->smpio.smp_request_sglist_cnt != 0) {
2969991554f2SKenneth D. Merry 			bus_dma_segment_t *req_sg;
2970991554f2SKenneth D. Merry 
2971991554f2SKenneth D. Merry 			req_sg = (bus_dma_segment_t *)ccb->smpio.smp_request;
2972991554f2SKenneth D. Merry 			request = (uint8_t *)(uintptr_t)req_sg[0].ds_addr;
2973991554f2SKenneth D. Merry 		} else
2974991554f2SKenneth D. Merry 			request = ccb->smpio.smp_request;
2975991554f2SKenneth D. Merry 
2976991554f2SKenneth D. Merry 		if (ccb->smpio.smp_response_sglist_cnt != 0) {
2977991554f2SKenneth D. Merry 			bus_dma_segment_t *rsp_sg;
2978991554f2SKenneth D. Merry 
2979991554f2SKenneth D. Merry 			rsp_sg = (bus_dma_segment_t *)ccb->smpio.smp_response;
2980991554f2SKenneth D. Merry 			response = (uint8_t *)(uintptr_t)rsp_sg[0].ds_addr;
2981991554f2SKenneth D. Merry 		} else
2982991554f2SKenneth D. Merry 			response = ccb->smpio.smp_response;
2983991554f2SKenneth D. Merry 		break;
2984991554f2SKenneth D. Merry 	case CAM_DATA_VADDR:
2985991554f2SKenneth D. Merry 		request = ccb->smpio.smp_request;
2986991554f2SKenneth D. Merry 		response = ccb->smpio.smp_response;
2987991554f2SKenneth D. Merry 		break;
2988991554f2SKenneth D. Merry 	default:
2989a2c14879SStephen McConnell 		mprsas_set_ccbstatus(ccb, CAM_REQ_INVALID);
2990991554f2SKenneth D. Merry 		xpt_done(ccb);
2991991554f2SKenneth D. Merry 		return;
2992991554f2SKenneth D. Merry 	}
2993991554f2SKenneth D. Merry 
2994991554f2SKenneth D. Merry 	cm = mpr_alloc_command(sc);
2995991554f2SKenneth D. Merry 	if (cm == NULL) {
29967a2a6a1aSStephen McConnell 		mpr_dprint(sc, MPR_ERROR, "%s: cannot allocate command\n",
29977a2a6a1aSStephen McConnell 		    __func__);
2998a2c14879SStephen McConnell 		mprsas_set_ccbstatus(ccb, CAM_RESRC_UNAVAIL);
2999991554f2SKenneth D. Merry 		xpt_done(ccb);
3000991554f2SKenneth D. Merry 		return;
3001991554f2SKenneth D. Merry 	}
3002991554f2SKenneth D. Merry 
3003991554f2SKenneth D. Merry 	req = (MPI2_SMP_PASSTHROUGH_REQUEST *)cm->cm_req;
3004991554f2SKenneth D. Merry 	bzero(req, sizeof(*req));
3005991554f2SKenneth D. Merry 	req->Function = MPI2_FUNCTION_SMP_PASSTHROUGH;
3006991554f2SKenneth D. Merry 
3007991554f2SKenneth D. Merry 	/* Allow the chip to use any route to this SAS address. */
3008991554f2SKenneth D. Merry 	req->PhysicalPort = 0xff;
3009991554f2SKenneth D. Merry 
3010991554f2SKenneth D. Merry 	req->RequestDataLength = htole16(ccb->smpio.smp_request_len);
3011991554f2SKenneth D. Merry 	req->SGLFlags =
3012991554f2SKenneth D. Merry 	    MPI2_SGLFLAGS_SYSTEM_ADDRESS_SPACE | MPI2_SGLFLAGS_SGL_TYPE_MPI;
3013991554f2SKenneth D. Merry 
3014991554f2SKenneth D. Merry 	mpr_dprint(sc, MPR_XINFO, "%s: sending SMP request to SAS address "
3015991554f2SKenneth D. Merry 	    "%#jx\n", __func__, (uintmax_t)sasaddr);
3016991554f2SKenneth D. Merry 
3017991554f2SKenneth D. Merry 	mpr_init_sge(cm, req, &req->SGL);
3018991554f2SKenneth D. Merry 
3019991554f2SKenneth D. Merry 	/*
3020991554f2SKenneth D. Merry 	 * Set up a uio to pass into mpr_map_command().  This allows us to
3021991554f2SKenneth D. Merry 	 * do one map command, and one busdma call in there.
3022991554f2SKenneth D. Merry 	 */
3023991554f2SKenneth D. Merry 	cm->cm_uio.uio_iov = cm->cm_iovec;
3024991554f2SKenneth D. Merry 	cm->cm_uio.uio_iovcnt = 2;
3025991554f2SKenneth D. Merry 	cm->cm_uio.uio_segflg = UIO_SYSSPACE;
3026991554f2SKenneth D. Merry 
3027991554f2SKenneth D. Merry 	/*
3028991554f2SKenneth D. Merry 	 * The read/write flag isn't used by busdma, but set it just in
3029991554f2SKenneth D. Merry 	 * case.  This isn't exactly accurate, either, since we're going in
3030991554f2SKenneth D. Merry 	 * both directions.
3031991554f2SKenneth D. Merry 	 */
3032991554f2SKenneth D. Merry 	cm->cm_uio.uio_rw = UIO_WRITE;
3033991554f2SKenneth D. Merry 
3034991554f2SKenneth D. Merry 	cm->cm_iovec[0].iov_base = request;
3035991554f2SKenneth D. Merry 	cm->cm_iovec[0].iov_len = le16toh(req->RequestDataLength);
3036991554f2SKenneth D. Merry 	cm->cm_iovec[1].iov_base = response;
3037991554f2SKenneth D. Merry 	cm->cm_iovec[1].iov_len = ccb->smpio.smp_response_len;
3038991554f2SKenneth D. Merry 
3039991554f2SKenneth D. Merry 	cm->cm_uio.uio_resid = cm->cm_iovec[0].iov_len +
3040991554f2SKenneth D. Merry 			       cm->cm_iovec[1].iov_len;
3041991554f2SKenneth D. Merry 
3042991554f2SKenneth D. Merry 	/*
3043991554f2SKenneth D. Merry 	 * Trigger a warning message in mpr_data_cb() for the user if we
3044991554f2SKenneth D. Merry 	 * wind up exceeding two S/G segments.  The chip expects one
3045991554f2SKenneth D. Merry 	 * segment for the request and another for the response.
3046991554f2SKenneth D. Merry 	 */
3047991554f2SKenneth D. Merry 	cm->cm_max_segs = 2;
3048991554f2SKenneth D. Merry 
3049991554f2SKenneth D. Merry 	cm->cm_desc.Default.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_DEFAULT_TYPE;
3050991554f2SKenneth D. Merry 	cm->cm_complete = mprsas_smpio_complete;
3051991554f2SKenneth D. Merry 	cm->cm_complete_data = ccb;
3052991554f2SKenneth D. Merry 
3053991554f2SKenneth D. Merry 	/*
3054991554f2SKenneth D. Merry 	 * Tell the mapping code that we're using a uio, and that this is
3055991554f2SKenneth D. Merry 	 * an SMP passthrough request.  There is a little special-case
3056991554f2SKenneth D. Merry 	 * logic there (in mpr_data_cb()) to handle the bidirectional
3057991554f2SKenneth D. Merry 	 * transfer.
3058991554f2SKenneth D. Merry 	 */
3059991554f2SKenneth D. Merry 	cm->cm_flags |= MPR_CM_FLAGS_USE_UIO | MPR_CM_FLAGS_SMP_PASS |
3060991554f2SKenneth D. Merry 			MPR_CM_FLAGS_DATAIN | MPR_CM_FLAGS_DATAOUT;
3061991554f2SKenneth D. Merry 
3062991554f2SKenneth D. Merry 	/* The chip data format is little endian. */
3063991554f2SKenneth D. Merry 	req->SASAddress.High = htole32(sasaddr >> 32);
3064991554f2SKenneth D. Merry 	req->SASAddress.Low = htole32(sasaddr);
3065991554f2SKenneth D. Merry 
3066991554f2SKenneth D. Merry 	/*
3067991554f2SKenneth D. Merry 	 * XXX Note that we don't have a timeout/abort mechanism here.
3068991554f2SKenneth D. Merry 	 * From the manual, it looks like task management requests only
3069991554f2SKenneth D. Merry 	 * work for SCSI IO and SATA passthrough requests.  We may need to
3070991554f2SKenneth D. Merry 	 * have a mechanism to retry requests in the event of a chip reset
3071991554f2SKenneth D. Merry 	 * at least.  Hopefully the chip will insure that any errors short
3072991554f2SKenneth D. Merry 	 * of that are relayed back to the driver.
3073991554f2SKenneth D. Merry 	 */
3074991554f2SKenneth D. Merry 	error = mpr_map_command(sc, cm);
3075991554f2SKenneth D. Merry 	if ((error != 0) && (error != EINPROGRESS)) {
3076991554f2SKenneth D. Merry 		mpr_dprint(sc, MPR_ERROR, "%s: error %d returned from "
3077991554f2SKenneth D. Merry 		    "mpr_map_command()\n", __func__, error);
3078991554f2SKenneth D. Merry 		goto bailout_error;
3079991554f2SKenneth D. Merry 	}
3080991554f2SKenneth D. Merry 
3081991554f2SKenneth D. Merry 	return;
3082991554f2SKenneth D. Merry 
3083991554f2SKenneth D. Merry bailout_error:
3084991554f2SKenneth D. Merry 	mpr_free_command(sc, cm);
3085a2c14879SStephen McConnell 	mprsas_set_ccbstatus(ccb, CAM_RESRC_UNAVAIL);
3086991554f2SKenneth D. Merry 	xpt_done(ccb);
3087991554f2SKenneth D. Merry 	return;
3088991554f2SKenneth D. Merry }
3089991554f2SKenneth D. Merry 
3090991554f2SKenneth D. Merry static void
3091991554f2SKenneth D. Merry mprsas_action_smpio(struct mprsas_softc *sassc, union ccb *ccb)
3092991554f2SKenneth D. Merry {
3093991554f2SKenneth D. Merry 	struct mpr_softc *sc;
3094991554f2SKenneth D. Merry 	struct mprsas_target *targ;
3095991554f2SKenneth D. Merry 	uint64_t sasaddr = 0;
3096991554f2SKenneth D. Merry 
3097991554f2SKenneth D. Merry 	sc = sassc->sc;
3098991554f2SKenneth D. Merry 
3099991554f2SKenneth D. Merry 	/*
3100991554f2SKenneth D. Merry 	 * Make sure the target exists.
3101991554f2SKenneth D. Merry 	 */
3102991554f2SKenneth D. Merry 	KASSERT(ccb->ccb_h.target_id < sassc->maxtargets,
3103991554f2SKenneth D. Merry 	    ("Target %d out of bounds in XPT_SMP_IO\n", ccb->ccb_h.target_id));
3104991554f2SKenneth D. Merry 	targ = &sassc->targets[ccb->ccb_h.target_id];
3105991554f2SKenneth D. Merry 	if (targ->handle == 0x0) {
3106991554f2SKenneth D. Merry 		mpr_dprint(sc, MPR_ERROR, "%s: target %d does not exist!\n",
3107991554f2SKenneth D. Merry 		    __func__, ccb->ccb_h.target_id);
3108a2c14879SStephen McConnell 		mprsas_set_ccbstatus(ccb, CAM_SEL_TIMEOUT);
3109991554f2SKenneth D. Merry 		xpt_done(ccb);
3110991554f2SKenneth D. Merry 		return;
3111991554f2SKenneth D. Merry 	}
3112991554f2SKenneth D. Merry 
3113991554f2SKenneth D. Merry 	/*
3114991554f2SKenneth D. Merry 	 * If this device has an embedded SMP target, we'll talk to it
3115991554f2SKenneth D. Merry 	 * directly.
3116991554f2SKenneth D. Merry 	 * figure out what the expander's address is.
3117991554f2SKenneth D. Merry 	 */
3118991554f2SKenneth D. Merry 	if ((targ->devinfo & MPI2_SAS_DEVICE_INFO_SMP_TARGET) != 0)
3119991554f2SKenneth D. Merry 		sasaddr = targ->sasaddr;
3120991554f2SKenneth D. Merry 
3121991554f2SKenneth D. Merry 	/*
3122991554f2SKenneth D. Merry 	 * If we don't have a SAS address for the expander yet, try
3123991554f2SKenneth D. Merry 	 * grabbing it from the page 0x83 information cached in the
3124991554f2SKenneth D. Merry 	 * transport layer for this target.  LSI expanders report the
3125991554f2SKenneth D. Merry 	 * expander SAS address as the port-associated SAS address in
3126991554f2SKenneth D. Merry 	 * Inquiry VPD page 0x83.  Maxim expanders don't report it in page
3127991554f2SKenneth D. Merry 	 * 0x83.
3128991554f2SKenneth D. Merry 	 *
3129991554f2SKenneth D. Merry 	 * XXX KDM disable this for now, but leave it commented out so that
3130991554f2SKenneth D. Merry 	 * it is obvious that this is another possible way to get the SAS
3131991554f2SKenneth D. Merry 	 * address.
3132991554f2SKenneth D. Merry 	 *
3133991554f2SKenneth D. Merry 	 * The parent handle method below is a little more reliable, and
3134991554f2SKenneth D. Merry 	 * the other benefit is that it works for devices other than SES
3135991554f2SKenneth D. Merry 	 * devices.  So you can send a SMP request to a da(4) device and it
3136991554f2SKenneth D. Merry 	 * will get routed to the expander that device is attached to.
3137991554f2SKenneth D. Merry 	 * (Assuming the da(4) device doesn't contain an SMP target...)
3138991554f2SKenneth D. Merry 	 */
3139991554f2SKenneth D. Merry #if 0
3140991554f2SKenneth D. Merry 	if (sasaddr == 0)
3141991554f2SKenneth D. Merry 		sasaddr = xpt_path_sas_addr(ccb->ccb_h.path);
3142991554f2SKenneth D. Merry #endif
3143991554f2SKenneth D. Merry 
3144991554f2SKenneth D. Merry 	/*
3145991554f2SKenneth D. Merry 	 * If we still don't have a SAS address for the expander, look for
3146991554f2SKenneth D. Merry 	 * the parent device of this device, which is probably the expander.
3147991554f2SKenneth D. Merry 	 */
3148991554f2SKenneth D. Merry 	if (sasaddr == 0) {
3149991554f2SKenneth D. Merry #ifdef OLD_MPR_PROBE
3150991554f2SKenneth D. Merry 		struct mprsas_target *parent_target;
3151991554f2SKenneth D. Merry #endif
3152991554f2SKenneth D. Merry 
3153991554f2SKenneth D. Merry 		if (targ->parent_handle == 0x0) {
3154991554f2SKenneth D. Merry 			mpr_dprint(sc, MPR_ERROR, "%s: handle %d does not have "
3155991554f2SKenneth D. Merry 			    "a valid parent handle!\n", __func__, targ->handle);
3156a2c14879SStephen McConnell 			mprsas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE);
3157991554f2SKenneth D. Merry 			goto bailout;
3158991554f2SKenneth D. Merry 		}
3159991554f2SKenneth D. Merry #ifdef OLD_MPR_PROBE
3160991554f2SKenneth D. Merry 		parent_target = mprsas_find_target_by_handle(sassc, 0,
3161991554f2SKenneth D. Merry 		    targ->parent_handle);
3162991554f2SKenneth D. Merry 
3163991554f2SKenneth D. Merry 		if (parent_target == NULL) {
3164991554f2SKenneth D. Merry 			mpr_dprint(sc, MPR_ERROR, "%s: handle %d does not have "
3165991554f2SKenneth D. Merry 			    "a valid parent target!\n", __func__, targ->handle);
3166a2c14879SStephen McConnell 			mprsas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE);
3167991554f2SKenneth D. Merry 			goto bailout;
3168991554f2SKenneth D. Merry 		}
3169991554f2SKenneth D. Merry 
3170991554f2SKenneth D. Merry 		if ((parent_target->devinfo &
3171991554f2SKenneth D. Merry 		     MPI2_SAS_DEVICE_INFO_SMP_TARGET) == 0) {
3172991554f2SKenneth D. Merry 			mpr_dprint(sc, MPR_ERROR, "%s: handle %d parent %d "
3173991554f2SKenneth D. Merry 			    "does not have an SMP target!\n", __func__,
3174991554f2SKenneth D. Merry 			    targ->handle, parent_target->handle);
3175a2c14879SStephen McConnell 			mprsas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE);
3176991554f2SKenneth D. Merry 			goto bailout;
3177991554f2SKenneth D. Merry 		}
3178991554f2SKenneth D. Merry 
3179991554f2SKenneth D. Merry 		sasaddr = parent_target->sasaddr;
3180991554f2SKenneth D. Merry #else /* OLD_MPR_PROBE */
3181991554f2SKenneth D. Merry 		if ((targ->parent_devinfo &
3182991554f2SKenneth D. Merry 		     MPI2_SAS_DEVICE_INFO_SMP_TARGET) == 0) {
3183991554f2SKenneth D. Merry 			mpr_dprint(sc, MPR_ERROR, "%s: handle %d parent %d "
3184991554f2SKenneth D. Merry 			    "does not have an SMP target!\n", __func__,
3185991554f2SKenneth D. Merry 			    targ->handle, targ->parent_handle);
3186a2c14879SStephen McConnell 			mprsas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE);
3187991554f2SKenneth D. Merry 			goto bailout;
3188991554f2SKenneth D. Merry 		}
3189991554f2SKenneth D. Merry 		if (targ->parent_sasaddr == 0x0) {
3190991554f2SKenneth D. Merry 			mpr_dprint(sc, MPR_ERROR, "%s: handle %d parent handle "
3191991554f2SKenneth D. Merry 			    "%d does not have a valid SAS address!\n", __func__,
3192991554f2SKenneth D. Merry 			    targ->handle, targ->parent_handle);
3193a2c14879SStephen McConnell 			mprsas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE);
3194991554f2SKenneth D. Merry 			goto bailout;
3195991554f2SKenneth D. Merry 		}
3196991554f2SKenneth D. Merry 
3197991554f2SKenneth D. Merry 		sasaddr = targ->parent_sasaddr;
3198991554f2SKenneth D. Merry #endif /* OLD_MPR_PROBE */
3199991554f2SKenneth D. Merry 	}
3200991554f2SKenneth D. Merry 
3201991554f2SKenneth D. Merry 	if (sasaddr == 0) {
3202991554f2SKenneth D. Merry 		mpr_dprint(sc, MPR_INFO, "%s: unable to find SAS address for "
3203991554f2SKenneth D. Merry 		    "handle %d\n", __func__, targ->handle);
3204a2c14879SStephen McConnell 		mprsas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE);
3205991554f2SKenneth D. Merry 		goto bailout;
3206991554f2SKenneth D. Merry 	}
3207991554f2SKenneth D. Merry 	mprsas_send_smpcmd(sassc, ccb, sasaddr);
3208991554f2SKenneth D. Merry 
3209991554f2SKenneth D. Merry 	return;
3210991554f2SKenneth D. Merry 
3211991554f2SKenneth D. Merry bailout:
3212991554f2SKenneth D. Merry 	xpt_done(ccb);
3213991554f2SKenneth D. Merry 
3214991554f2SKenneth D. Merry }
3215991554f2SKenneth D. Merry 
3216991554f2SKenneth D. Merry static void
3217991554f2SKenneth D. Merry mprsas_action_resetdev(struct mprsas_softc *sassc, union ccb *ccb)
3218991554f2SKenneth D. Merry {
3219991554f2SKenneth D. Merry 	MPI2_SCSI_TASK_MANAGE_REQUEST *req;
3220991554f2SKenneth D. Merry 	struct mpr_softc *sc;
3221991554f2SKenneth D. Merry 	struct mpr_command *tm;
3222991554f2SKenneth D. Merry 	struct mprsas_target *targ;
3223991554f2SKenneth D. Merry 
3224991554f2SKenneth D. Merry 	MPR_FUNCTRACE(sassc->sc);
3225991554f2SKenneth D. Merry 	mtx_assert(&sassc->sc->mpr_mtx, MA_OWNED);
3226991554f2SKenneth D. Merry 
32277a2a6a1aSStephen McConnell 	KASSERT(ccb->ccb_h.target_id < sassc->maxtargets, ("Target %d out of "
32287a2a6a1aSStephen McConnell 	    "bounds in XPT_RESET_DEV\n", ccb->ccb_h.target_id));
3229991554f2SKenneth D. Merry 	sc = sassc->sc;
32303921a9f7SScott Long 	tm = mprsas_alloc_tm(sc);
3231991554f2SKenneth D. Merry 	if (tm == NULL) {
32327a2a6a1aSStephen McConnell 		mpr_dprint(sc, MPR_ERROR, "command alloc failure in "
32337a2a6a1aSStephen McConnell 		    "mprsas_action_resetdev\n");
3234a2c14879SStephen McConnell 		mprsas_set_ccbstatus(ccb, CAM_RESRC_UNAVAIL);
3235991554f2SKenneth D. Merry 		xpt_done(ccb);
3236991554f2SKenneth D. Merry 		return;
3237991554f2SKenneth D. Merry 	}
3238991554f2SKenneth D. Merry 
3239991554f2SKenneth D. Merry 	targ = &sassc->targets[ccb->ccb_h.target_id];
3240991554f2SKenneth D. Merry 	req = (MPI2_SCSI_TASK_MANAGE_REQUEST *)tm->cm_req;
3241991554f2SKenneth D. Merry 	req->DevHandle = htole16(targ->handle);
3242991554f2SKenneth D. Merry 	req->TaskType = MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET;
3243991554f2SKenneth D. Merry 
324489d1c21fSKashyap D Desai 	if (!targ->is_nvme || sc->custom_nvme_tm_handling) {
3245991554f2SKenneth D. Merry 		/* SAS Hard Link Reset / SATA Link Reset */
3246991554f2SKenneth D. Merry 		req->MsgFlags = MPI2_SCSITASKMGMT_MSGFLAGS_LINK_RESET;
324789d1c21fSKashyap D Desai 	} else {
324889d1c21fSKashyap D Desai 		/* PCIe Protocol Level Reset*/
324989d1c21fSKashyap D Desai 		req->MsgFlags =
325089d1c21fSKashyap D Desai 		    MPI26_SCSITASKMGMT_MSGFLAGS_PROTOCOL_LVL_RST_PCIE;
325189d1c21fSKashyap D Desai 	}
3252991554f2SKenneth D. Merry 
3253991554f2SKenneth D. Merry 	tm->cm_data = NULL;
3254991554f2SKenneth D. Merry 	tm->cm_complete = mprsas_resetdev_complete;
3255991554f2SKenneth D. Merry 	tm->cm_complete_data = ccb;
3256a2c14879SStephen McConnell 
3257a2c14879SStephen McConnell 	mpr_dprint(sc, MPR_INFO, "%s: Sending reset for target ID %d\n",
3258a2c14879SStephen McConnell 	    __func__, targ->tid);
3259991554f2SKenneth D. Merry 	tm->cm_targ = targ;
3260a2c14879SStephen McConnell 
326146b9415fSScott Long 	mprsas_prepare_for_tm(sc, tm, targ, CAM_LUN_WILDCARD);
3262991554f2SKenneth D. Merry 	mpr_map_command(sc, tm);
3263991554f2SKenneth D. Merry }
3264991554f2SKenneth D. Merry 
3265991554f2SKenneth D. Merry static void
3266991554f2SKenneth D. Merry mprsas_resetdev_complete(struct mpr_softc *sc, struct mpr_command *tm)
3267991554f2SKenneth D. Merry {
3268991554f2SKenneth D. Merry 	MPI2_SCSI_TASK_MANAGE_REPLY *resp;
3269991554f2SKenneth D. Merry 	union ccb *ccb;
3270991554f2SKenneth D. Merry 
3271991554f2SKenneth D. Merry 	MPR_FUNCTRACE(sc);
3272991554f2SKenneth D. Merry 	mtx_assert(&sc->mpr_mtx, MA_OWNED);
3273991554f2SKenneth D. Merry 
3274991554f2SKenneth D. Merry 	resp = (MPI2_SCSI_TASK_MANAGE_REPLY *)tm->cm_reply;
3275991554f2SKenneth D. Merry 	ccb = tm->cm_complete_data;
3276991554f2SKenneth D. Merry 
3277991554f2SKenneth D. Merry 	/*
3278991554f2SKenneth D. Merry 	 * Currently there should be no way we can hit this case.  It only
3279991554f2SKenneth D. Merry 	 * happens when we have a failure to allocate chain frames, and
3280991554f2SKenneth D. Merry 	 * task management commands don't have S/G lists.
3281991554f2SKenneth D. Merry 	 */
3282991554f2SKenneth D. Merry 	if ((tm->cm_flags & MPR_CM_FLAGS_ERROR_MASK) != 0) {
3283991554f2SKenneth D. Merry 		MPI2_SCSI_TASK_MANAGE_REQUEST *req;
3284991554f2SKenneth D. Merry 
3285991554f2SKenneth D. Merry 		req = (MPI2_SCSI_TASK_MANAGE_REQUEST *)tm->cm_req;
3286991554f2SKenneth D. Merry 
3287991554f2SKenneth D. Merry 		mpr_dprint(sc, MPR_ERROR, "%s: cm_flags = %#x for reset of "
3288991554f2SKenneth D. Merry 		    "handle %#04x! This should not happen!\n", __func__,
3289991554f2SKenneth D. Merry 		    tm->cm_flags, req->DevHandle);
3290a2c14879SStephen McConnell 		mprsas_set_ccbstatus(ccb, CAM_REQ_CMP_ERR);
3291991554f2SKenneth D. Merry 		goto bailout;
3292991554f2SKenneth D. Merry 	}
3293991554f2SKenneth D. Merry 
32947a2a6a1aSStephen McConnell 	mpr_dprint(sc, MPR_XINFO, "%s: IOCStatus = 0x%x ResponseCode = 0x%x\n",
32957a2a6a1aSStephen McConnell 	    __func__, le16toh(resp->IOCStatus), le32toh(resp->ResponseCode));
3296991554f2SKenneth D. Merry 
3297991554f2SKenneth D. Merry 	if (le32toh(resp->ResponseCode) == MPI2_SCSITASKMGMT_RSP_TM_COMPLETE) {
3298a2c14879SStephen McConnell 		mprsas_set_ccbstatus(ccb, CAM_REQ_CMP);
3299991554f2SKenneth D. Merry 		mprsas_announce_reset(sc, AC_SENT_BDR, tm->cm_targ->tid,
3300991554f2SKenneth D. Merry 		    CAM_LUN_WILDCARD);
3301991554f2SKenneth D. Merry 	}
3302991554f2SKenneth D. Merry 	else
3303a2c14879SStephen McConnell 		mprsas_set_ccbstatus(ccb, CAM_REQ_CMP_ERR);
3304991554f2SKenneth D. Merry 
3305991554f2SKenneth D. Merry bailout:
3306991554f2SKenneth D. Merry 
3307991554f2SKenneth D. Merry 	mprsas_free_tm(sc, tm);
3308991554f2SKenneth D. Merry 	xpt_done(ccb);
3309991554f2SKenneth D. Merry }
3310991554f2SKenneth D. Merry 
3311991554f2SKenneth D. Merry static void
3312991554f2SKenneth D. Merry mprsas_poll(struct cam_sim *sim)
3313991554f2SKenneth D. Merry {
3314991554f2SKenneth D. Merry 	struct mprsas_softc *sassc;
3315991554f2SKenneth D. Merry 
3316991554f2SKenneth D. Merry 	sassc = cam_sim_softc(sim);
3317991554f2SKenneth D. Merry 
3318991554f2SKenneth D. Merry 	if (sassc->sc->mpr_debug & MPR_TRACE) {
3319991554f2SKenneth D. Merry 		/* frequent debug messages during a panic just slow
3320991554f2SKenneth D. Merry 		 * everything down too much.
3321991554f2SKenneth D. Merry 		 */
3322a2c14879SStephen McConnell 		mpr_dprint(sassc->sc, MPR_XINFO, "%s clearing MPR_TRACE\n",
3323a2c14879SStephen McConnell 		    __func__);
3324991554f2SKenneth D. Merry 		sassc->sc->mpr_debug &= ~MPR_TRACE;
3325991554f2SKenneth D. Merry 	}
3326991554f2SKenneth D. Merry 
3327991554f2SKenneth D. Merry 	mpr_intr_locked(sassc->sc);
3328991554f2SKenneth D. Merry }
3329991554f2SKenneth D. Merry 
3330991554f2SKenneth D. Merry static void
3331991554f2SKenneth D. Merry mprsas_async(void *callback_arg, uint32_t code, struct cam_path *path,
3332991554f2SKenneth D. Merry     void *arg)
3333991554f2SKenneth D. Merry {
3334991554f2SKenneth D. Merry 	struct mpr_softc *sc;
3335991554f2SKenneth D. Merry 
3336991554f2SKenneth D. Merry 	sc = (struct mpr_softc *)callback_arg;
3337991554f2SKenneth D. Merry 
333802d81940SAlexander Motin 	mpr_lock(sc);
3339991554f2SKenneth D. Merry 	switch (code) {
3340991554f2SKenneth D. Merry 	case AC_ADVINFO_CHANGED: {
3341991554f2SKenneth D. Merry 		struct mprsas_target *target;
3342991554f2SKenneth D. Merry 		struct mprsas_softc *sassc;
3343991554f2SKenneth D. Merry 		struct scsi_read_capacity_data_long rcap_buf;
3344991554f2SKenneth D. Merry 		struct ccb_dev_advinfo cdai;
3345991554f2SKenneth D. Merry 		struct mprsas_lun *lun;
3346991554f2SKenneth D. Merry 		lun_id_t lunid;
3347991554f2SKenneth D. Merry 		int found_lun;
3348991554f2SKenneth D. Merry 		uintptr_t buftype;
3349991554f2SKenneth D. Merry 
3350991554f2SKenneth D. Merry 		buftype = (uintptr_t)arg;
3351991554f2SKenneth D. Merry 
3352991554f2SKenneth D. Merry 		found_lun = 0;
3353991554f2SKenneth D. Merry 		sassc = sc->sassc;
3354991554f2SKenneth D. Merry 
3355991554f2SKenneth D. Merry 		/*
3356991554f2SKenneth D. Merry 		 * We're only interested in read capacity data changes.
3357991554f2SKenneth D. Merry 		 */
3358991554f2SKenneth D. Merry 		if (buftype != CDAI_TYPE_RCAPLONG)
3359991554f2SKenneth D. Merry 			break;
3360991554f2SKenneth D. Merry 
3361991554f2SKenneth D. Merry 		/*
3362991554f2SKenneth D. Merry 		 * We should have a handle for this, but check to make sure.
3363991554f2SKenneth D. Merry 		 */
3364991554f2SKenneth D. Merry 		KASSERT(xpt_path_target_id(path) < sassc->maxtargets,
3365991554f2SKenneth D. Merry 		    ("Target %d out of bounds in mprsas_async\n",
3366991554f2SKenneth D. Merry 		    xpt_path_target_id(path)));
3367991554f2SKenneth D. Merry 		target = &sassc->targets[xpt_path_target_id(path)];
3368991554f2SKenneth D. Merry 		if (target->handle == 0)
3369991554f2SKenneth D. Merry 			break;
3370991554f2SKenneth D. Merry 
3371991554f2SKenneth D. Merry 		lunid = xpt_path_lun_id(path);
3372991554f2SKenneth D. Merry 
3373991554f2SKenneth D. Merry 		SLIST_FOREACH(lun, &target->luns, lun_link) {
3374991554f2SKenneth D. Merry 			if (lun->lun_id == lunid) {
3375991554f2SKenneth D. Merry 				found_lun = 1;
3376991554f2SKenneth D. Merry 				break;
3377991554f2SKenneth D. Merry 			}
3378991554f2SKenneth D. Merry 		}
3379991554f2SKenneth D. Merry 
3380991554f2SKenneth D. Merry 		if (found_lun == 0) {
3381991554f2SKenneth D. Merry 			lun = malloc(sizeof(struct mprsas_lun), M_MPR,
3382991554f2SKenneth D. Merry 			    M_NOWAIT | M_ZERO);
3383991554f2SKenneth D. Merry 			if (lun == NULL) {
3384991554f2SKenneth D. Merry 				mpr_dprint(sc, MPR_ERROR, "Unable to alloc "
3385991554f2SKenneth D. Merry 				    "LUN for EEDP support.\n");
3386991554f2SKenneth D. Merry 				break;
3387991554f2SKenneth D. Merry 			}
3388991554f2SKenneth D. Merry 			lun->lun_id = lunid;
3389991554f2SKenneth D. Merry 			SLIST_INSERT_HEAD(&target->luns, lun, lun_link);
3390991554f2SKenneth D. Merry 		}
3391991554f2SKenneth D. Merry 
3392991554f2SKenneth D. Merry 		bzero(&rcap_buf, sizeof(rcap_buf));
33937608b98cSEdward Tomasz Napierala 		bzero(&cdai, sizeof(cdai));
3394991554f2SKenneth D. Merry 		xpt_setup_ccb(&cdai.ccb_h, path, CAM_PRIORITY_NORMAL);
3395991554f2SKenneth D. Merry 		cdai.ccb_h.func_code = XPT_DEV_ADVINFO;
3396991554f2SKenneth D. Merry 		cdai.ccb_h.flags = CAM_DIR_IN;
3397991554f2SKenneth D. Merry 		cdai.buftype = CDAI_TYPE_RCAPLONG;
3398e8577fb4SKenneth D. Merry 		cdai.flags = CDAI_FLAG_NONE;
3399991554f2SKenneth D. Merry 		cdai.bufsiz = sizeof(rcap_buf);
3400991554f2SKenneth D. Merry 		cdai.buf = (uint8_t *)&rcap_buf;
3401991554f2SKenneth D. Merry 		xpt_action((union ccb *)&cdai);
3402991554f2SKenneth D. Merry 		if ((cdai.ccb_h.status & CAM_DEV_QFRZN) != 0)
3403991554f2SKenneth D. Merry 			cam_release_devq(cdai.ccb_h.path, 0, 0, 0, FALSE);
3404991554f2SKenneth D. Merry 
3405a2c14879SStephen McConnell 		if ((mprsas_get_ccbstatus((union ccb *)&cdai) == CAM_REQ_CMP)
3406991554f2SKenneth D. Merry 		    && (rcap_buf.prot & SRC16_PROT_EN)) {
34078881681bSKenneth D. Merry 			switch (rcap_buf.prot & SRC16_P_TYPE) {
34088881681bSKenneth D. Merry 			case SRC16_PTYPE_1:
34098881681bSKenneth D. Merry 			case SRC16_PTYPE_3:
3410991554f2SKenneth D. Merry 				lun->eedp_formatted = TRUE;
34118881681bSKenneth D. Merry 				lun->eedp_block_size =
34128881681bSKenneth D. Merry 				    scsi_4btoul(rcap_buf.length);
34138881681bSKenneth D. Merry 				break;
34148881681bSKenneth D. Merry 			case SRC16_PTYPE_2:
34158881681bSKenneth D. Merry 			default:
34168881681bSKenneth D. Merry 				lun->eedp_formatted = FALSE;
34178881681bSKenneth D. Merry 				lun->eedp_block_size = 0;
34188881681bSKenneth D. Merry 				break;
34198881681bSKenneth D. Merry 			}
3420991554f2SKenneth D. Merry 		} else {
3421991554f2SKenneth D. Merry 			lun->eedp_formatted = FALSE;
3422991554f2SKenneth D. Merry 			lun->eedp_block_size = 0;
3423991554f2SKenneth D. Merry 		}
3424991554f2SKenneth D. Merry 		break;
3425991554f2SKenneth D. Merry 	}
3426991554f2SKenneth D. Merry 	default:
3427991554f2SKenneth D. Merry 		break;
3428991554f2SKenneth D. Merry 	}
342902d81940SAlexander Motin 	mpr_unlock(sc);
3430991554f2SKenneth D. Merry }
3431991554f2SKenneth D. Merry 
3432a2c14879SStephen McConnell /*
34339781c28cSAlexander Motin  * Freeze the devq and set the INRESET flag so that no I/O will be sent to
34349781c28cSAlexander Motin  * the target until the reset has completed.  The CCB holds the path which
34359781c28cSAlexander Motin  * is used to release the devq.  The devq is released and the CCB is freed
3436a2c14879SStephen McConnell  * when the TM completes.
3437a8837c77SWarner Losh  * We only need to do this when we're entering reset, not at each time we
3438a8837c77SWarner Losh  * need to send an abort (which will happen if multiple commands timeout
3439a8837c77SWarner Losh  * while we're sending the abort). We do not release the queue for each
3440a8837c77SWarner Losh  * command we complete (just at the end when we free the tm), so freezing
3441a8837c77SWarner Losh  * it each time doesn't make sense.
3442a2c14879SStephen McConnell  */
3443b7f1ee79SScott Long void
3444b7f1ee79SScott Long mprsas_prepare_for_tm(struct mpr_softc *sc, struct mpr_command *tm,
3445b7f1ee79SScott Long     struct mprsas_target *target, lun_id_t lun_id)
3446b7f1ee79SScott Long {
3447b7f1ee79SScott Long 	union ccb *ccb;
3448b7f1ee79SScott Long 	path_id_t path_id;
3449b7f1ee79SScott Long 
3450a2c14879SStephen McConnell 	ccb = xpt_alloc_ccb_nowait();
3451a2c14879SStephen McConnell 	if (ccb) {
3452a2c14879SStephen McConnell 		path_id = cam_sim_path(sc->sassc->sim);
3453a2c14879SStephen McConnell 		if (xpt_create_path(&ccb->ccb_h.path, xpt_periph, path_id,
3454a2c14879SStephen McConnell 		    target->tid, lun_id) != CAM_REQ_CMP) {
3455a2c14879SStephen McConnell 			xpt_free_ccb(ccb);
3456a2c14879SStephen McConnell 		} else {
3457a8837c77SWarner Losh 			tm->cm_ccb = ccb;
3458a8837c77SWarner Losh 			tm->cm_targ = target;
3459a8837c77SWarner Losh 			if ((target->flags & MPRSAS_TARGET_INRESET) == 0) {
3460a8837c77SWarner Losh 				mpr_dprint(sc, MPR_XINFO | MPR_RECOVERY,
34619781c28cSAlexander Motin 				    "%s: Freezing devq for target ID %d\n",
34629781c28cSAlexander Motin 				    __func__, target->tid);
34639781c28cSAlexander Motin 				xpt_freeze_devq(ccb->ccb_h.path, 1);
3464a2c14879SStephen McConnell 				target->flags |= MPRSAS_TARGET_INRESET;
3465a2c14879SStephen McConnell 			}
3466a2c14879SStephen McConnell 		}
3467a2c14879SStephen McConnell 	}
3468a8837c77SWarner Losh }
3469a2c14879SStephen McConnell 
3470991554f2SKenneth D. Merry int
3471991554f2SKenneth D. Merry mprsas_startup(struct mpr_softc *sc)
3472991554f2SKenneth D. Merry {
3473991554f2SKenneth D. Merry 	/*
3474991554f2SKenneth D. Merry 	 * Send the port enable message and set the wait_for_port_enable flag.
3475991554f2SKenneth D. Merry 	 * This flag helps to keep the simq frozen until all discovery events
3476991554f2SKenneth D. Merry 	 * are processed.
3477991554f2SKenneth D. Merry 	 */
3478991554f2SKenneth D. Merry 	sc->wait_for_port_enable = 1;
3479991554f2SKenneth D. Merry 	mprsas_send_portenable(sc);
3480991554f2SKenneth D. Merry 	return (0);
3481991554f2SKenneth D. Merry }
3482991554f2SKenneth D. Merry 
3483991554f2SKenneth D. Merry static int
3484991554f2SKenneth D. Merry mprsas_send_portenable(struct mpr_softc *sc)
3485991554f2SKenneth D. Merry {
3486991554f2SKenneth D. Merry 	MPI2_PORT_ENABLE_REQUEST *request;
3487991554f2SKenneth D. Merry 	struct mpr_command *cm;
3488991554f2SKenneth D. Merry 
3489991554f2SKenneth D. Merry 	MPR_FUNCTRACE(sc);
3490991554f2SKenneth D. Merry 
3491991554f2SKenneth D. Merry 	if ((cm = mpr_alloc_command(sc)) == NULL)
3492991554f2SKenneth D. Merry 		return (EBUSY);
3493991554f2SKenneth D. Merry 	request = (MPI2_PORT_ENABLE_REQUEST *)cm->cm_req;
3494991554f2SKenneth D. Merry 	request->Function = MPI2_FUNCTION_PORT_ENABLE;
3495991554f2SKenneth D. Merry 	request->MsgFlags = 0;
3496991554f2SKenneth D. Merry 	request->VP_ID = 0;
3497991554f2SKenneth D. Merry 	cm->cm_desc.Default.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_DEFAULT_TYPE;
3498991554f2SKenneth D. Merry 	cm->cm_complete = mprsas_portenable_complete;
3499991554f2SKenneth D. Merry 	cm->cm_data = NULL;
3500991554f2SKenneth D. Merry 	cm->cm_sge = NULL;
3501991554f2SKenneth D. Merry 
3502991554f2SKenneth D. Merry 	mpr_map_command(sc, cm);
3503991554f2SKenneth D. Merry 	mpr_dprint(sc, MPR_XINFO,
3504991554f2SKenneth D. Merry 	    "mpr_send_portenable finished cm %p req %p complete %p\n",
3505991554f2SKenneth D. Merry 	    cm, cm->cm_req, cm->cm_complete);
3506991554f2SKenneth D. Merry 	return (0);
3507991554f2SKenneth D. Merry }
3508991554f2SKenneth D. Merry 
3509991554f2SKenneth D. Merry static void
3510991554f2SKenneth D. Merry mprsas_portenable_complete(struct mpr_softc *sc, struct mpr_command *cm)
3511991554f2SKenneth D. Merry {
3512991554f2SKenneth D. Merry 	MPI2_PORT_ENABLE_REPLY *reply;
3513991554f2SKenneth D. Merry 	struct mprsas_softc *sassc;
3514991554f2SKenneth D. Merry 
3515991554f2SKenneth D. Merry 	MPR_FUNCTRACE(sc);
3516991554f2SKenneth D. Merry 	sassc = sc->sassc;
3517991554f2SKenneth D. Merry 
3518991554f2SKenneth D. Merry 	/*
3519991554f2SKenneth D. Merry 	 * Currently there should be no way we can hit this case.  It only
3520991554f2SKenneth D. Merry 	 * happens when we have a failure to allocate chain frames, and
3521991554f2SKenneth D. Merry 	 * port enable commands don't have S/G lists.
3522991554f2SKenneth D. Merry 	 */
3523991554f2SKenneth D. Merry 	if ((cm->cm_flags & MPR_CM_FLAGS_ERROR_MASK) != 0) {
3524991554f2SKenneth D. Merry 		mpr_dprint(sc, MPR_ERROR, "%s: cm_flags = %#x for port enable! "
3525991554f2SKenneth D. Merry 		    "This should not happen!\n", __func__, cm->cm_flags);
3526991554f2SKenneth D. Merry 	}
3527991554f2SKenneth D. Merry 
3528991554f2SKenneth D. Merry 	reply = (MPI2_PORT_ENABLE_REPLY *)cm->cm_reply;
3529991554f2SKenneth D. Merry 	if (reply == NULL)
3530991554f2SKenneth D. Merry 		mpr_dprint(sc, MPR_FAULT, "Portenable NULL reply\n");
353171900a79SAlfredo Dal'Ava Junior 	else if ((le16toh(reply->IOCStatus) & MPI2_IOCSTATUS_MASK) !=
3532991554f2SKenneth D. Merry 	    MPI2_IOCSTATUS_SUCCESS)
3533991554f2SKenneth D. Merry 		mpr_dprint(sc, MPR_FAULT, "Portenable failed\n");
3534991554f2SKenneth D. Merry 
3535991554f2SKenneth D. Merry 	mpr_free_command(sc, cm);
3536991554f2SKenneth D. Merry 	/*
3537991554f2SKenneth D. Merry 	 * Done waiting for port enable to complete.  Decrement the refcount.
3538991554f2SKenneth D. Merry 	 * If refcount is 0, discovery is complete and a rescan of the bus can
3539991554f2SKenneth D. Merry 	 * take place.
3540991554f2SKenneth D. Merry 	 */
3541991554f2SKenneth D. Merry 	sc->wait_for_port_enable = 0;
3542991554f2SKenneth D. Merry 	sc->port_enable_complete = 1;
3543991554f2SKenneth D. Merry 	wakeup(&sc->port_enable_complete);
3544991554f2SKenneth D. Merry 	mprsas_startup_decrement(sassc);
3545991554f2SKenneth D. Merry }
3546991554f2SKenneth D. Merry 
3547991554f2SKenneth D. Merry int
3548991554f2SKenneth D. Merry mprsas_check_id(struct mprsas_softc *sassc, int id)
3549991554f2SKenneth D. Merry {
3550991554f2SKenneth D. Merry 	struct mpr_softc *sc = sassc->sc;
3551991554f2SKenneth D. Merry 	char *ids;
3552991554f2SKenneth D. Merry 	char *name;
3553991554f2SKenneth D. Merry 
3554991554f2SKenneth D. Merry 	ids = &sc->exclude_ids[0];
3555991554f2SKenneth D. Merry 	while((name = strsep(&ids, ",")) != NULL) {
3556991554f2SKenneth D. Merry 		if (name[0] == '\0')
3557991554f2SKenneth D. Merry 			continue;
3558991554f2SKenneth D. Merry 		if (strtol(name, NULL, 0) == (long)id)
3559991554f2SKenneth D. Merry 			return (1);
3560991554f2SKenneth D. Merry 	}
3561991554f2SKenneth D. Merry 
3562991554f2SKenneth D. Merry 	return (0);
3563991554f2SKenneth D. Merry }
3564a2c14879SStephen McConnell 
3565a2c14879SStephen McConnell void
3566a2c14879SStephen McConnell mprsas_realloc_targets(struct mpr_softc *sc, int maxtargets)
3567a2c14879SStephen McConnell {
3568a2c14879SStephen McConnell 	struct mprsas_softc *sassc;
3569a2c14879SStephen McConnell 	struct mprsas_lun *lun, *lun_tmp;
3570a2c14879SStephen McConnell 	struct mprsas_target *targ;
3571a2c14879SStephen McConnell 	int i;
3572a2c14879SStephen McConnell 
3573a2c14879SStephen McConnell 	sassc = sc->sassc;
3574a2c14879SStephen McConnell 	/*
3575a2c14879SStephen McConnell 	 * The number of targets is based on IOC Facts, so free all of
3576a2c14879SStephen McConnell 	 * the allocated LUNs for each target and then the target buffer
3577a2c14879SStephen McConnell 	 * itself.
3578a2c14879SStephen McConnell 	 */
3579a2c14879SStephen McConnell 	for (i=0; i< maxtargets; i++) {
3580a2c14879SStephen McConnell 		targ = &sassc->targets[i];
3581a2c14879SStephen McConnell 		SLIST_FOREACH_SAFE(lun, &targ->luns, lun_link, lun_tmp) {
3582a2c14879SStephen McConnell 			free(lun, M_MPR);
3583a2c14879SStephen McConnell 		}
3584a2c14879SStephen McConnell 	}
3585a2c14879SStephen McConnell 	free(sassc->targets, M_MPR);
3586a2c14879SStephen McConnell 
3587a2c14879SStephen McConnell 	sassc->targets = malloc(sizeof(struct mprsas_target) * maxtargets,
3588a2c14879SStephen McConnell 	    M_MPR, M_WAITOK|M_ZERO);
3589a2c14879SStephen McConnell }
3590