xref: /freebsd/sys/dev/mps/mps_sas.c (revision e9e8876a4d6afc1ad5315faaa191b25121a813d7)
1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3  *
4  * Copyright (c) 2009 Yahoo! Inc.
5  * Copyright (c) 2011-2015 LSI Corp.
6  * Copyright (c) 2013-2015 Avago Technologies
7  * All rights reserved.
8  *
9  * Redistribution and use in source and binary forms, with or without
10  * modification, are permitted provided that the following conditions
11  * are met:
12  * 1. Redistributions of source code must retain the above copyright
13  *    notice, this list of conditions and the following disclaimer.
14  * 2. Redistributions in binary form must reproduce the above copyright
15  *    notice, this list of conditions and the following disclaimer in the
16  *    documentation and/or other materials provided with the distribution.
17  *
18  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
19  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
22  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28  * SUCH DAMAGE.
29  *
30  * Avago Technologies (LSI) MPT-Fusion Host Adapter FreeBSD
31  *
32  * $FreeBSD$
33  */
34 
35 #include <sys/cdefs.h>
36 __FBSDID("$FreeBSD$");
37 
38 /* Communications core for Avago Technologies (LSI) MPT2 */
39 
40 /* TODO Move headers to mpsvar */
41 #include <sys/types.h>
42 #include <sys/param.h>
43 #include <sys/systm.h>
44 #include <sys/kernel.h>
45 #include <sys/selinfo.h>
46 #include <sys/module.h>
47 #include <sys/bus.h>
48 #include <sys/conf.h>
49 #include <sys/bio.h>
50 #include <sys/malloc.h>
51 #include <sys/uio.h>
52 #include <sys/sysctl.h>
53 #include <sys/endian.h>
54 #include <sys/queue.h>
55 #include <sys/kthread.h>
56 #include <sys/taskqueue.h>
57 #include <sys/sbuf.h>
58 
59 #include <machine/bus.h>
60 #include <machine/resource.h>
61 #include <sys/rman.h>
62 
63 #include <machine/stdarg.h>
64 
65 #include <cam/cam.h>
66 #include <cam/cam_ccb.h>
67 #include <cam/cam_xpt.h>
68 #include <cam/cam_debug.h>
69 #include <cam/cam_sim.h>
70 #include <cam/cam_xpt_sim.h>
71 #include <cam/cam_xpt_periph.h>
72 #include <cam/cam_periph.h>
73 #include <cam/scsi/scsi_all.h>
74 #include <cam/scsi/scsi_message.h>
75 #include <cam/scsi/smp_all.h>
76 
77 #include <dev/mps/mpi/mpi2_type.h>
78 #include <dev/mps/mpi/mpi2.h>
79 #include <dev/mps/mpi/mpi2_ioc.h>
80 #include <dev/mps/mpi/mpi2_sas.h>
81 #include <dev/mps/mpi/mpi2_cnfg.h>
82 #include <dev/mps/mpi/mpi2_init.h>
83 #include <dev/mps/mpi/mpi2_tool.h>
84 #include <dev/mps/mps_ioctl.h>
85 #include <dev/mps/mpsvar.h>
86 #include <dev/mps/mps_table.h>
87 #include <dev/mps/mps_sas.h>
88 
89 #define MPSSAS_DISCOVERY_TIMEOUT	20
90 #define MPSSAS_MAX_DISCOVERY_TIMEOUTS	10 /* 200 seconds */
91 
92 /*
93  * static array to check SCSI OpCode for EEDP protection bits
94  */
95 #define	PRO_R MPI2_SCSIIO_EEDPFLAGS_CHECK_REMOVE_OP
96 #define	PRO_W MPI2_SCSIIO_EEDPFLAGS_INSERT_OP
97 #define	PRO_V MPI2_SCSIIO_EEDPFLAGS_INSERT_OP
98 static uint8_t op_code_prot[256] = {
99 	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
100 	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
101 	0, 0, 0, 0, 0, 0, 0, 0, PRO_R, 0, PRO_W, 0, 0, 0, PRO_W, PRO_V,
102 	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
103 	0, PRO_W, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
104 	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
105 	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
106 	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
107 	0, 0, 0, 0, 0, 0, 0, 0, PRO_R, 0, PRO_W, 0, 0, 0, PRO_W, PRO_V,
108 	0, 0, 0, PRO_W, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
109 	0, 0, 0, 0, 0, 0, 0, 0, PRO_R, 0, PRO_W, 0, 0, 0, PRO_W, PRO_V,
110 	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
111 	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
112 	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
113 	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
114 	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
115 };
116 
117 MALLOC_DEFINE(M_MPSSAS, "MPSSAS", "MPS SAS memory");
118 
119 static void mpssas_remove_device(struct mps_softc *, struct mps_command *);
120 static void mpssas_remove_complete(struct mps_softc *, struct mps_command *);
121 static void mpssas_action(struct cam_sim *sim, union ccb *ccb);
122 static void mpssas_poll(struct cam_sim *sim);
123 static int mpssas_send_abort(struct mps_softc *sc, struct mps_command *tm,
124     struct mps_command *cm);
125 static void mpssas_scsiio_timeout(void *data);
126 static void mpssas_abort_complete(struct mps_softc *sc, struct mps_command *cm);
127 static void mpssas_direct_drive_io(struct mpssas_softc *sassc,
128     struct mps_command *cm, union ccb *ccb);
129 static void mpssas_action_scsiio(struct mpssas_softc *, union ccb *);
130 static void mpssas_scsiio_complete(struct mps_softc *, struct mps_command *);
131 static void mpssas_action_resetdev(struct mpssas_softc *, union ccb *);
132 static void mpssas_smpio_complete(struct mps_softc *sc, struct mps_command *cm);
133 static void mpssas_send_smpcmd(struct mpssas_softc *sassc, union ccb *ccb,
134 			       uint64_t sasaddr);
135 static void mpssas_action_smpio(struct mpssas_softc *sassc, union ccb *ccb);
136 static void mpssas_resetdev_complete(struct mps_softc *, struct mps_command *);
137 static void mpssas_async(void *callback_arg, uint32_t code,
138 			 struct cam_path *path, void *arg);
139 static int mpssas_send_portenable(struct mps_softc *sc);
140 static void mpssas_portenable_complete(struct mps_softc *sc,
141     struct mps_command *cm);
142 
143 struct mpssas_target *
144 mpssas_find_target_by_handle(struct mpssas_softc *sassc, int start, uint16_t handle)
145 {
146 	struct mpssas_target *target;
147 	int i;
148 
149 	for (i = start; i < sassc->maxtargets; i++) {
150 		target = &sassc->targets[i];
151 		if (target->handle == handle)
152 			return (target);
153 	}
154 
155 	return (NULL);
156 }
157 
158 /* we need to freeze the simq during attach and diag reset, to avoid failing
159  * commands before device handles have been found by discovery.  Since
160  * discovery involves reading config pages and possibly sending commands,
161  * discovery actions may continue even after we receive the end of discovery
162  * event, so refcount discovery actions instead of assuming we can unfreeze
163  * the simq when we get the event.
164  */
165 void
166 mpssas_startup_increment(struct mpssas_softc *sassc)
167 {
168 	MPS_FUNCTRACE(sassc->sc);
169 
170 	if ((sassc->flags & MPSSAS_IN_STARTUP) != 0) {
171 		if (sassc->startup_refcount++ == 0) {
172 			/* just starting, freeze the simq */
173 			mps_dprint(sassc->sc, MPS_INIT,
174 			    "%s freezing simq\n", __func__);
175 			xpt_hold_boot();
176 			xpt_freeze_simq(sassc->sim, 1);
177 		}
178 		mps_dprint(sassc->sc, MPS_INIT, "%s refcount %u\n", __func__,
179 		    sassc->startup_refcount);
180 	}
181 }
182 
183 void
184 mpssas_release_simq_reinit(struct mpssas_softc *sassc)
185 {
186 	if (sassc->flags & MPSSAS_QUEUE_FROZEN) {
187 		sassc->flags &= ~MPSSAS_QUEUE_FROZEN;
188 		xpt_release_simq(sassc->sim, 1);
189 		mps_dprint(sassc->sc, MPS_INFO, "Unfreezing SIM queue\n");
190 	}
191 }
192 
193 void
194 mpssas_startup_decrement(struct mpssas_softc *sassc)
195 {
196 	MPS_FUNCTRACE(sassc->sc);
197 
198 	if ((sassc->flags & MPSSAS_IN_STARTUP) != 0) {
199 		if (--sassc->startup_refcount == 0) {
200 			/* finished all discovery-related actions, release
201 			 * the simq and rescan for the latest topology.
202 			 */
203 			mps_dprint(sassc->sc, MPS_INIT,
204 			    "%s releasing simq\n", __func__);
205 			sassc->flags &= ~MPSSAS_IN_STARTUP;
206 			xpt_release_simq(sassc->sim, 1);
207 			xpt_release_boot();
208 		}
209 		mps_dprint(sassc->sc, MPS_INIT, "%s refcount %u\n", __func__,
210 		    sassc->startup_refcount);
211 	}
212 }
213 
214 /*
215  * The firmware requires us to stop sending commands when we're doing task
216  * management.
217  * XXX The logic for serializing the device has been made lazy and moved to
218  * mpssas_prepare_for_tm().
219  */
220 struct mps_command *
221 mpssas_alloc_tm(struct mps_softc *sc)
222 {
223 	MPI2_SCSI_TASK_MANAGE_REQUEST *req;
224 	struct mps_command *tm;
225 
226 	tm = mps_alloc_high_priority_command(sc);
227 	if (tm == NULL)
228 		return (NULL);
229 
230 	req = (MPI2_SCSI_TASK_MANAGE_REQUEST *)tm->cm_req;
231 	req->Function = MPI2_FUNCTION_SCSI_TASK_MGMT;
232 	return tm;
233 }
234 
235 void
236 mpssas_free_tm(struct mps_softc *sc, struct mps_command *tm)
237 {
238 	if (tm == NULL)
239 		return;
240 
241 	/*
242 	 * For TM's the devq is frozen for the device.  Unfreeze it here and
243 	 * free the resources used for freezing the devq.  Must clear the
244 	 * INRESET flag as well or scsi I/O will not work.
245 	 */
246 	if (tm->cm_ccb) {
247 		mps_dprint(sc, MPS_XINFO | MPS_RECOVERY,
248 		    "Unfreezing devq for target ID %d\n",
249 		    tm->cm_targ->tid);
250 		tm->cm_targ->flags &= ~MPSSAS_TARGET_INRESET;
251 		xpt_release_devq(tm->cm_ccb->ccb_h.path, 1, TRUE);
252 		xpt_free_path(tm->cm_ccb->ccb_h.path);
253 		xpt_free_ccb(tm->cm_ccb);
254 	}
255 
256 	mps_free_high_priority_command(sc, tm);
257 }
258 
259 void
260 mpssas_rescan_target(struct mps_softc *sc, struct mpssas_target *targ)
261 {
262 	struct mpssas_softc *sassc = sc->sassc;
263 	path_id_t pathid;
264 	target_id_t targetid;
265 	union ccb *ccb;
266 
267 	MPS_FUNCTRACE(sc);
268 	pathid = cam_sim_path(sassc->sim);
269 	if (targ == NULL)
270 		targetid = CAM_TARGET_WILDCARD;
271 	else
272 		targetid = targ - sassc->targets;
273 
274 	/*
275 	 * Allocate a CCB and schedule a rescan.
276 	 */
277 	ccb = xpt_alloc_ccb_nowait();
278 	if (ccb == NULL) {
279 		mps_dprint(sc, MPS_ERROR, "unable to alloc CCB for rescan\n");
280 		return;
281 	}
282 
283 	if (xpt_create_path(&ccb->ccb_h.path, NULL, pathid,
284 	    targetid, CAM_LUN_WILDCARD) != CAM_REQ_CMP) {
285 		mps_dprint(sc, MPS_ERROR, "unable to create path for rescan\n");
286 		xpt_free_ccb(ccb);
287 		return;
288 	}
289 
290 	if (targetid == CAM_TARGET_WILDCARD)
291 		ccb->ccb_h.func_code = XPT_SCAN_BUS;
292 	else
293 		ccb->ccb_h.func_code = XPT_SCAN_TGT;
294 
295 	mps_dprint(sc, MPS_TRACE, "%s targetid %u\n", __func__, targetid);
296 	xpt_rescan(ccb);
297 }
298 
299 static void
300 mpssas_log_command(struct mps_command *cm, u_int level, const char *fmt, ...)
301 {
302 	struct sbuf sb;
303 	va_list ap;
304 	char str[224];
305 	char path_str[64];
306 
307 	if (cm == NULL)
308 		return;
309 
310 	/* No need to be in here if debugging isn't enabled */
311 	if ((cm->cm_sc->mps_debug & level) == 0)
312 		return;
313 
314 	sbuf_new(&sb, str, sizeof(str), 0);
315 
316 	va_start(ap, fmt);
317 
318 	if (cm->cm_ccb != NULL) {
319 		xpt_path_string(cm->cm_ccb->csio.ccb_h.path, path_str,
320 				sizeof(path_str));
321 		sbuf_cat(&sb, path_str);
322 		if (cm->cm_ccb->ccb_h.func_code == XPT_SCSI_IO) {
323 			scsi_command_string(&cm->cm_ccb->csio, &sb);
324 			sbuf_printf(&sb, "length %d ",
325 				    cm->cm_ccb->csio.dxfer_len);
326 		}
327 	}
328 	else {
329 		sbuf_printf(&sb, "(noperiph:%s%d:%u:%u:%u): ",
330 		    cam_sim_name(cm->cm_sc->sassc->sim),
331 		    cam_sim_unit(cm->cm_sc->sassc->sim),
332 		    cam_sim_bus(cm->cm_sc->sassc->sim),
333 		    cm->cm_targ ? cm->cm_targ->tid : 0xFFFFFFFF,
334 		    cm->cm_lun);
335 	}
336 
337 	sbuf_printf(&sb, "SMID %u ", cm->cm_desc.Default.SMID);
338 	sbuf_vprintf(&sb, fmt, ap);
339 	sbuf_finish(&sb);
340 	mps_print_field(cm->cm_sc, "%s", sbuf_data(&sb));
341 
342 	va_end(ap);
343 }
344 
345 static void
346 mpssas_remove_volume(struct mps_softc *sc, struct mps_command *tm)
347 {
348 	MPI2_SCSI_TASK_MANAGE_REPLY *reply;
349 	struct mpssas_target *targ;
350 	uint16_t handle;
351 
352 	MPS_FUNCTRACE(sc);
353 
354 	reply = (MPI2_SCSI_TASK_MANAGE_REPLY *)tm->cm_reply;
355 	handle = (uint16_t)(uintptr_t)tm->cm_complete_data;
356 	targ = tm->cm_targ;
357 
358 	if (reply == NULL) {
359 		/* XXX retry the remove after the diag reset completes? */
360 		mps_dprint(sc, MPS_FAULT,
361 		    "%s NULL reply resetting device 0x%04x\n", __func__,
362 		    handle);
363 		mpssas_free_tm(sc, tm);
364 		return;
365 	}
366 
367 	if ((le16toh(reply->IOCStatus) & MPI2_IOCSTATUS_MASK) !=
368 	    MPI2_IOCSTATUS_SUCCESS) {
369 		mps_dprint(sc, MPS_ERROR,
370 		   "IOCStatus = 0x%x while resetting device 0x%x\n",
371 		   le16toh(reply->IOCStatus), handle);
372 	}
373 
374 	mps_dprint(sc, MPS_XINFO,
375 	    "Reset aborted %u commands\n", reply->TerminationCount);
376 	mps_free_reply(sc, tm->cm_reply_data);
377 	tm->cm_reply = NULL;	/* Ensures the reply won't get re-freed */
378 
379 	mps_dprint(sc, MPS_XINFO,
380 	    "clearing target %u handle 0x%04x\n", targ->tid, handle);
381 
382 	/*
383 	 * Don't clear target if remove fails because things will get confusing.
384 	 * Leave the devname and sasaddr intact so that we know to avoid reusing
385 	 * this target id if possible, and so we can assign the same target id
386 	 * to this device if it comes back in the future.
387 	 */
388 	if ((le16toh(reply->IOCStatus) & MPI2_IOCSTATUS_MASK) ==
389 	    MPI2_IOCSTATUS_SUCCESS) {
390 		targ = tm->cm_targ;
391 		targ->handle = 0x0;
392 		targ->encl_handle = 0x0;
393 		targ->encl_slot = 0x0;
394 		targ->exp_dev_handle = 0x0;
395 		targ->phy_num = 0x0;
396 		targ->linkrate = 0x0;
397 		targ->devinfo = 0x0;
398 		targ->flags = 0x0;
399 	}
400 
401 	mpssas_free_tm(sc, tm);
402 }
403 
404 /*
405  * No Need to call "MPI2_SAS_OP_REMOVE_DEVICE" For Volume removal.
406  * Otherwise Volume Delete is same as Bare Drive Removal.
407  */
408 void
409 mpssas_prepare_volume_remove(struct mpssas_softc *sassc, uint16_t handle)
410 {
411 	MPI2_SCSI_TASK_MANAGE_REQUEST *req;
412 	struct mps_softc *sc;
413 	struct mps_command *tm;
414 	struct mpssas_target *targ = NULL;
415 
416 	MPS_FUNCTRACE(sassc->sc);
417 	sc = sassc->sc;
418 
419 #ifdef WD_SUPPORT
420 	/*
421 	 * If this is a WD controller, determine if the disk should be exposed
422 	 * to the OS or not.  If disk should be exposed, return from this
423 	 * function without doing anything.
424 	 */
425 	if (sc->WD_available && (sc->WD_hide_expose ==
426 	    MPS_WD_EXPOSE_ALWAYS)) {
427 		return;
428 	}
429 #endif //WD_SUPPORT
430 
431 	targ = mpssas_find_target_by_handle(sassc, 0, handle);
432 	if (targ == NULL) {
433 		/* FIXME: what is the action? */
434 		/* We don't know about this device? */
435 		mps_dprint(sc, MPS_ERROR,
436 		   "%s %d : invalid handle 0x%x \n", __func__,__LINE__, handle);
437 		return;
438 	}
439 
440 	targ->flags |= MPSSAS_TARGET_INREMOVAL;
441 
442 	tm = mpssas_alloc_tm(sc);
443 	if (tm == NULL) {
444 		mps_dprint(sc, MPS_ERROR,
445 		    "%s: command alloc failure\n", __func__);
446 		return;
447 	}
448 
449 	mpssas_rescan_target(sc, targ);
450 
451 	req = (MPI2_SCSI_TASK_MANAGE_REQUEST *)tm->cm_req;
452 	req->DevHandle = targ->handle;
453 	req->TaskType = MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET;
454 
455 	/* SAS Hard Link Reset / SATA Link Reset */
456 	req->MsgFlags = MPI2_SCSITASKMGMT_MSGFLAGS_LINK_RESET;
457 
458 	tm->cm_targ = targ;
459 	tm->cm_data = NULL;
460 	tm->cm_complete = mpssas_remove_volume;
461 	tm->cm_complete_data = (void *)(uintptr_t)handle;
462 
463 	mps_dprint(sc, MPS_INFO, "%s: Sending reset for target ID %d\n",
464 	    __func__, targ->tid);
465 	mpssas_prepare_for_tm(sc, tm, targ, CAM_LUN_WILDCARD);
466 
467 	mps_map_command(sc, tm);
468 }
469 
470 /*
471  * The MPT2 firmware performs debounce on the link to avoid transient link
472  * errors and false removals.  When it does decide that link has been lost
473  * and a device need to go away, it expects that the host will perform a
474  * target reset and then an op remove.  The reset has the side-effect of
475  * aborting any outstanding requests for the device, which is required for
476  * the op-remove to succeed.  It's not clear if the host should check for
477  * the device coming back alive after the reset.
478  */
479 void
480 mpssas_prepare_remove(struct mpssas_softc *sassc, uint16_t handle)
481 {
482 	MPI2_SCSI_TASK_MANAGE_REQUEST *req;
483 	struct mps_softc *sc;
484 	struct mps_command *cm;
485 	struct mpssas_target *targ = NULL;
486 
487 	MPS_FUNCTRACE(sassc->sc);
488 
489 	sc = sassc->sc;
490 
491 	targ = mpssas_find_target_by_handle(sassc, 0, handle);
492 	if (targ == NULL) {
493 		/* FIXME: what is the action? */
494 		/* We don't know about this device? */
495 		mps_dprint(sc, MPS_ERROR,
496 		    "%s : invalid handle 0x%x \n", __func__, handle);
497 		return;
498 	}
499 
500 	targ->flags |= MPSSAS_TARGET_INREMOVAL;
501 
502 	cm = mpssas_alloc_tm(sc);
503 	if (cm == NULL) {
504 		mps_dprint(sc, MPS_ERROR,
505 		    "%s: command alloc failure\n", __func__);
506 		return;
507 	}
508 
509 	mpssas_rescan_target(sc, targ);
510 
511 	req = (MPI2_SCSI_TASK_MANAGE_REQUEST *)cm->cm_req;
512 	req->DevHandle = htole16(targ->handle);
513 	req->TaskType = MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET;
514 
515 	/* SAS Hard Link Reset / SATA Link Reset */
516 	req->MsgFlags = MPI2_SCSITASKMGMT_MSGFLAGS_LINK_RESET;
517 
518 	cm->cm_targ = targ;
519 	cm->cm_data = NULL;
520 	cm->cm_complete = mpssas_remove_device;
521 	cm->cm_complete_data = (void *)(uintptr_t)handle;
522 
523 	mps_dprint(sc, MPS_INFO, "%s: Sending reset for target ID %d\n",
524 	    __func__, targ->tid);
525 	mpssas_prepare_for_tm(sc, cm, targ, CAM_LUN_WILDCARD);
526 
527 	mps_map_command(sc, cm);
528 }
529 
530 static void
531 mpssas_remove_device(struct mps_softc *sc, struct mps_command *tm)
532 {
533 	MPI2_SCSI_TASK_MANAGE_REPLY *reply;
534 	MPI2_SAS_IOUNIT_CONTROL_REQUEST *req;
535 	struct mpssas_target *targ;
536 	uint16_t handle;
537 
538 	MPS_FUNCTRACE(sc);
539 
540 	reply = (MPI2_SCSI_TASK_MANAGE_REPLY *)tm->cm_reply;
541 	handle = (uint16_t)(uintptr_t)tm->cm_complete_data;
542 	targ = tm->cm_targ;
543 
544 	/*
545 	 * Currently there should be no way we can hit this case.  It only
546 	 * happens when we have a failure to allocate chain frames, and
547 	 * task management commands don't have S/G lists.
548 	 */
549 	if ((tm->cm_flags & MPS_CM_FLAGS_ERROR_MASK) != 0) {
550 		mps_dprint(sc, MPS_ERROR,
551 		    "%s: cm_flags = %#x for remove of handle %#04x! "
552 		    "This should not happen!\n", __func__, tm->cm_flags,
553 		    handle);
554 	}
555 
556 	if (reply == NULL) {
557 		/* XXX retry the remove after the diag reset completes? */
558 		mps_dprint(sc, MPS_FAULT,
559 		    "%s NULL reply resetting device 0x%04x\n", __func__,
560 		    handle);
561 		mpssas_free_tm(sc, tm);
562 		return;
563 	}
564 
565 	if ((le16toh(reply->IOCStatus) & MPI2_IOCSTATUS_MASK) !=
566 	    MPI2_IOCSTATUS_SUCCESS) {
567 		mps_dprint(sc, MPS_ERROR,
568 		   "IOCStatus = 0x%x while resetting device 0x%x\n",
569 		   le16toh(reply->IOCStatus), handle);
570 	}
571 
572 	mps_dprint(sc, MPS_XINFO, "Reset aborted %u commands\n",
573 	    le32toh(reply->TerminationCount));
574 	mps_free_reply(sc, tm->cm_reply_data);
575 	tm->cm_reply = NULL;	/* Ensures the reply won't get re-freed */
576 
577 	/* Reuse the existing command */
578 	req = (MPI2_SAS_IOUNIT_CONTROL_REQUEST *)tm->cm_req;
579 	memset(req, 0, sizeof(*req));
580 	req->Function = MPI2_FUNCTION_SAS_IO_UNIT_CONTROL;
581 	req->Operation = MPI2_SAS_OP_REMOVE_DEVICE;
582 	req->DevHandle = htole16(handle);
583 	tm->cm_data = NULL;
584 	tm->cm_desc.Default.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_DEFAULT_TYPE;
585 	tm->cm_complete = mpssas_remove_complete;
586 	tm->cm_complete_data = (void *)(uintptr_t)handle;
587 
588 	/*
589 	 * Wait to send the REMOVE_DEVICE until all the commands have cleared.
590 	 * They should be aborted or time out and we'll kick thus off there
591 	 * if so.
592 	 */
593 	if (TAILQ_FIRST(&targ->commands) == NULL) {
594 		mps_dprint(sc, MPS_INFO, "No pending commands: starting remove_device\n");
595 		mps_map_command(sc, tm);
596 		targ->pending_remove_tm = NULL;
597 	} else {
598 		targ->pending_remove_tm = tm;
599 	}
600 
601 	mps_dprint(sc, MPS_XINFO, "clearing target %u handle 0x%04x\n",
602 		   targ->tid, handle);
603 }
604 
605 static void
606 mpssas_remove_complete(struct mps_softc *sc, struct mps_command *tm)
607 {
608 	MPI2_SAS_IOUNIT_CONTROL_REPLY *reply;
609 	uint16_t handle;
610 	struct mpssas_target *targ;
611 	struct mpssas_lun *lun;
612 
613 	MPS_FUNCTRACE(sc);
614 
615 	reply = (MPI2_SAS_IOUNIT_CONTROL_REPLY *)tm->cm_reply;
616 	handle = (uint16_t)(uintptr_t)tm->cm_complete_data;
617 	targ = tm->cm_targ;
618 
619 	/*
620 	 * At this point, we should have no pending commands for the target.
621 	 * The remove target has just completed.
622 	 */
623 	KASSERT(TAILQ_FIRST(&targ->commands) == NULL,
624 	    ("%s: no commands should be pending\n", __func__));
625 
626 	/*
627 	 * Currently there should be no way we can hit this case.  It only
628 	 * happens when we have a failure to allocate chain frames, and
629 	 * task management commands don't have S/G lists.
630 	 */
631 	if ((tm->cm_flags & MPS_CM_FLAGS_ERROR_MASK) != 0) {
632 		mps_dprint(sc, MPS_XINFO,
633 			   "%s: cm_flags = %#x for remove of handle %#04x! "
634 			   "This should not happen!\n", __func__, tm->cm_flags,
635 			   handle);
636 		mpssas_free_tm(sc, tm);
637 		return;
638 	}
639 
640 	if (reply == NULL) {
641 		/* most likely a chip reset */
642 		mps_dprint(sc, MPS_FAULT,
643 		    "%s NULL reply removing device 0x%04x\n", __func__, handle);
644 		mpssas_free_tm(sc, tm);
645 		return;
646 	}
647 
648 	mps_dprint(sc, MPS_XINFO,
649 	    "%s on handle 0x%04x, IOCStatus= 0x%x\n", __func__,
650 	    handle, le16toh(reply->IOCStatus));
651 
652 	/*
653 	 * Don't clear target if remove fails because things will get confusing.
654 	 * Leave the devname and sasaddr intact so that we know to avoid reusing
655 	 * this target id if possible, and so we can assign the same target id
656 	 * to this device if it comes back in the future.
657 	 */
658 	if ((le16toh(reply->IOCStatus) & MPI2_IOCSTATUS_MASK) ==
659 	    MPI2_IOCSTATUS_SUCCESS) {
660 		targ->handle = 0x0;
661 		targ->encl_handle = 0x0;
662 		targ->encl_slot = 0x0;
663 		targ->exp_dev_handle = 0x0;
664 		targ->phy_num = 0x0;
665 		targ->linkrate = 0x0;
666 		targ->devinfo = 0x0;
667 		targ->flags = 0x0;
668 
669 		while(!SLIST_EMPTY(&targ->luns)) {
670 			lun = SLIST_FIRST(&targ->luns);
671 			SLIST_REMOVE_HEAD(&targ->luns, lun_link);
672 			free(lun, M_MPT2);
673 		}
674 	}
675 
676 	mpssas_free_tm(sc, tm);
677 }
678 
679 static int
680 mpssas_register_events(struct mps_softc *sc)
681 {
682 	u32 events[MPI2_EVENT_NOTIFY_EVENTMASK_WORDS];
683 
684 	bzero(events, 16);
685 	setbit(events, MPI2_EVENT_SAS_DEVICE_STATUS_CHANGE);
686 	setbit(events, MPI2_EVENT_SAS_DISCOVERY);
687 	setbit(events, MPI2_EVENT_SAS_BROADCAST_PRIMITIVE);
688 	setbit(events, MPI2_EVENT_SAS_INIT_DEVICE_STATUS_CHANGE);
689 	setbit(events, MPI2_EVENT_SAS_INIT_TABLE_OVERFLOW);
690 	setbit(events, MPI2_EVENT_SAS_TOPOLOGY_CHANGE_LIST);
691 	setbit(events, MPI2_EVENT_SAS_ENCL_DEVICE_STATUS_CHANGE);
692 	setbit(events, MPI2_EVENT_IR_CONFIGURATION_CHANGE_LIST);
693 	setbit(events, MPI2_EVENT_IR_VOLUME);
694 	setbit(events, MPI2_EVENT_IR_PHYSICAL_DISK);
695 	setbit(events, MPI2_EVENT_IR_OPERATION_STATUS);
696 	setbit(events, MPI2_EVENT_LOG_ENTRY_ADDED);
697 
698 	mps_register_events(sc, events, mpssas_evt_handler, NULL,
699 	    &sc->sassc->mpssas_eh);
700 
701 	return (0);
702 }
703 
704 int
705 mps_attach_sas(struct mps_softc *sc)
706 {
707 	struct mpssas_softc *sassc;
708 	cam_status status;
709 	int unit, error = 0, reqs;
710 
711 	MPS_FUNCTRACE(sc);
712 	mps_dprint(sc, MPS_INIT, "%s entered\n", __func__);
713 
714 	sassc = malloc(sizeof(struct mpssas_softc), M_MPT2, M_WAITOK|M_ZERO);
715 
716 	/*
717 	 * XXX MaxTargets could change during a reinit.  Since we don't
718 	 * resize the targets[] array during such an event, cache the value
719 	 * of MaxTargets here so that we don't get into trouble later.  This
720 	 * should move into the reinit logic.
721 	 */
722 	sassc->maxtargets = sc->facts->MaxTargets + sc->facts->MaxVolumes;
723 	sassc->targets = malloc(sizeof(struct mpssas_target) *
724 	    sassc->maxtargets, M_MPT2, M_WAITOK|M_ZERO);
725 	sc->sassc = sassc;
726 	sassc->sc = sc;
727 
728 	reqs = sc->num_reqs - sc->num_prireqs - 1;
729 	if ((sassc->devq = cam_simq_alloc(reqs)) == NULL) {
730 		mps_dprint(sc, MPS_ERROR, "Cannot allocate SIMQ\n");
731 		error = ENOMEM;
732 		goto out;
733 	}
734 
735 	unit = device_get_unit(sc->mps_dev);
736 	sassc->sim = cam_sim_alloc(mpssas_action, mpssas_poll, "mps", sassc,
737 	    unit, &sc->mps_mtx, reqs, reqs, sassc->devq);
738 	if (sassc->sim == NULL) {
739 		mps_dprint(sc, MPS_INIT|MPS_ERROR, "Cannot allocate SIM\n");
740 		error = EINVAL;
741 		goto out;
742 	}
743 
744 	TAILQ_INIT(&sassc->ev_queue);
745 
746 	/* Initialize taskqueue for Event Handling */
747 	TASK_INIT(&sassc->ev_task, 0, mpssas_firmware_event_work, sc);
748 	sassc->ev_tq = taskqueue_create("mps_taskq", M_NOWAIT | M_ZERO,
749 	    taskqueue_thread_enqueue, &sassc->ev_tq);
750 	taskqueue_start_threads(&sassc->ev_tq, 1, PRIBIO, "%s taskq",
751 	    device_get_nameunit(sc->mps_dev));
752 
753 	mps_lock(sc);
754 
755 	/*
756 	 * XXX There should be a bus for every port on the adapter, but since
757 	 * we're just going to fake the topology for now, we'll pretend that
758 	 * everything is just a target on a single bus.
759 	 */
760 	if ((error = xpt_bus_register(sassc->sim, sc->mps_dev, 0)) != 0) {
761 		mps_dprint(sc, MPS_INIT|MPS_ERROR,
762 		    "Error %d registering SCSI bus\n", error);
763 		mps_unlock(sc);
764 		goto out;
765 	}
766 
767 	/*
768 	 * Assume that discovery events will start right away.
769 	 *
770 	 * Hold off boot until discovery is complete.
771 	 */
772 	sassc->flags |= MPSSAS_IN_STARTUP | MPSSAS_IN_DISCOVERY;
773 	sc->sassc->startup_refcount = 0;
774 	mpssas_startup_increment(sassc);
775 
776 	callout_init(&sassc->discovery_callout, 1 /*mpsafe*/);
777 
778 	mps_unlock(sc);
779 
780 	/*
781 	 * Register for async events so we can determine the EEDP
782 	 * capabilities of devices.
783 	 */
784 	status = xpt_create_path(&sassc->path, /*periph*/NULL,
785 	    cam_sim_path(sc->sassc->sim), CAM_TARGET_WILDCARD,
786 	    CAM_LUN_WILDCARD);
787 	if (status != CAM_REQ_CMP) {
788 		mps_dprint(sc, MPS_ERROR|MPS_INIT,
789 		    "Error %#x creating sim path\n", status);
790 		sassc->path = NULL;
791 	} else {
792 		int event;
793 
794 		event = AC_ADVINFO_CHANGED;
795 		status = xpt_register_async(event, mpssas_async, sc,
796 					    sassc->path);
797 		if (status != CAM_REQ_CMP) {
798 			mps_dprint(sc, MPS_ERROR,
799 			    "Error %#x registering async handler for "
800 			    "AC_ADVINFO_CHANGED events\n", status);
801 			xpt_free_path(sassc->path);
802 			sassc->path = NULL;
803 		}
804 	}
805 	if (status != CAM_REQ_CMP) {
806 		/*
807 		 * EEDP use is the exception, not the rule.
808 		 * Warn the user, but do not fail to attach.
809 		 */
810 		mps_printf(sc, "EEDP capabilities disabled.\n");
811 	}
812 
813 	mpssas_register_events(sc);
814 out:
815 	if (error)
816 		mps_detach_sas(sc);
817 
818 	mps_dprint(sc, MPS_INIT, "%s exit error= %d\n", __func__, error);
819 	return (error);
820 }
821 
822 int
823 mps_detach_sas(struct mps_softc *sc)
824 {
825 	struct mpssas_softc *sassc;
826 	struct mpssas_lun *lun, *lun_tmp;
827 	struct mpssas_target *targ;
828 	int i;
829 
830 	MPS_FUNCTRACE(sc);
831 
832 	if (sc->sassc == NULL)
833 		return (0);
834 
835 	sassc = sc->sassc;
836 	mps_deregister_events(sc, sassc->mpssas_eh);
837 
838 	/*
839 	 * Drain and free the event handling taskqueue with the lock
840 	 * unheld so that any parallel processing tasks drain properly
841 	 * without deadlocking.
842 	 */
843 	if (sassc->ev_tq != NULL)
844 		taskqueue_free(sassc->ev_tq);
845 
846 	/* Deregister our async handler */
847 	if (sassc->path != NULL) {
848 		xpt_register_async(0, mpssas_async, sc, sassc->path);
849 		xpt_free_path(sassc->path);
850 		sassc->path = NULL;
851 	}
852 
853 	/* Make sure CAM doesn't wedge if we had to bail out early. */
854 	mps_lock(sc);
855 
856 	while (sassc->startup_refcount != 0)
857 		mpssas_startup_decrement(sassc);
858 
859 	if (sassc->flags & MPSSAS_IN_STARTUP)
860 		xpt_release_simq(sassc->sim, 1);
861 
862 	if (sassc->sim != NULL) {
863 		xpt_bus_deregister(cam_sim_path(sassc->sim));
864 		cam_sim_free(sassc->sim, FALSE);
865 	}
866 
867 	mps_unlock(sc);
868 
869 	if (sassc->devq != NULL)
870 		cam_simq_free(sassc->devq);
871 
872 	for(i=0; i< sassc->maxtargets ;i++) {
873 		targ = &sassc->targets[i];
874 		SLIST_FOREACH_SAFE(lun, &targ->luns, lun_link, lun_tmp) {
875 			free(lun, M_MPT2);
876 		}
877 	}
878 	free(sassc->targets, M_MPT2);
879 	free(sassc, M_MPT2);
880 	sc->sassc = NULL;
881 
882 	return (0);
883 }
884 
885 void
886 mpssas_discovery_end(struct mpssas_softc *sassc)
887 {
888 	struct mps_softc *sc = sassc->sc;
889 
890 	MPS_FUNCTRACE(sc);
891 
892 	if (sassc->flags & MPSSAS_DISCOVERY_TIMEOUT_PENDING)
893 		callout_stop(&sassc->discovery_callout);
894 
895 	/*
896 	 * After discovery has completed, check the mapping table for any
897 	 * missing devices and update their missing counts. Only do this once
898 	 * whenever the driver is initialized so that missing counts aren't
899 	 * updated unnecessarily. Note that just because discovery has
900 	 * completed doesn't mean that events have been processed yet. The
901 	 * check_devices function is a callout timer that checks if ALL devices
902 	 * are missing. If so, it will wait a little longer for events to
903 	 * complete and keep resetting itself until some device in the mapping
904 	 * table is not missing, meaning that event processing has started.
905 	 */
906 	if (sc->track_mapping_events) {
907 		mps_dprint(sc, MPS_XINFO | MPS_MAPPING, "Discovery has "
908 		    "completed. Check for missing devices in the mapping "
909 		    "table.\n");
910 		callout_reset(&sc->device_check_callout,
911 		    MPS_MISSING_CHECK_DELAY * hz, mps_mapping_check_devices,
912 		    sc);
913 	}
914 }
915 
916 static void
917 mpssas_action(struct cam_sim *sim, union ccb *ccb)
918 {
919 	struct mpssas_softc *sassc;
920 
921 	sassc = cam_sim_softc(sim);
922 
923 	MPS_FUNCTRACE(sassc->sc);
924 	mps_dprint(sassc->sc, MPS_TRACE, "ccb func_code 0x%x\n",
925 	    ccb->ccb_h.func_code);
926 	mtx_assert(&sassc->sc->mps_mtx, MA_OWNED);
927 
928 	switch (ccb->ccb_h.func_code) {
929 	case XPT_PATH_INQ:
930 	{
931 		struct ccb_pathinq *cpi = &ccb->cpi;
932 		struct mps_softc *sc = sassc->sc;
933 
934 		cpi->version_num = 1;
935 		cpi->hba_inquiry = PI_SDTR_ABLE|PI_TAG_ABLE|PI_WIDE_16;
936 		cpi->target_sprt = 0;
937 		cpi->hba_misc = PIM_NOBUSRESET | PIM_UNMAPPED | PIM_NOSCAN;
938 		cpi->hba_eng_cnt = 0;
939 		cpi->max_target = sassc->maxtargets - 1;
940 		cpi->max_lun = 255;
941 
942 		/*
943 		 * initiator_id is set here to an ID outside the set of valid
944 		 * target IDs (including volumes).
945 		 */
946 		cpi->initiator_id = sassc->maxtargets;
947 		strlcpy(cpi->sim_vid, "FreeBSD", SIM_IDLEN);
948 		strlcpy(cpi->hba_vid, "Avago Tech", HBA_IDLEN);
949 		strlcpy(cpi->dev_name, cam_sim_name(sim), DEV_IDLEN);
950 		cpi->unit_number = cam_sim_unit(sim);
951 		cpi->bus_id = cam_sim_bus(sim);
952 		cpi->base_transfer_speed = 150000;
953 		cpi->transport = XPORT_SAS;
954 		cpi->transport_version = 0;
955 		cpi->protocol = PROTO_SCSI;
956 		cpi->protocol_version = SCSI_REV_SPC;
957 		cpi->maxio = sc->maxio;
958 		mpssas_set_ccbstatus(ccb, CAM_REQ_CMP);
959 		break;
960 	}
961 	case XPT_GET_TRAN_SETTINGS:
962 	{
963 		struct ccb_trans_settings	*cts;
964 		struct ccb_trans_settings_sas	*sas;
965 		struct ccb_trans_settings_scsi	*scsi;
966 		struct mpssas_target *targ;
967 
968 		cts = &ccb->cts;
969 		sas = &cts->xport_specific.sas;
970 		scsi = &cts->proto_specific.scsi;
971 
972 		KASSERT(cts->ccb_h.target_id < sassc->maxtargets,
973 		    ("Target %d out of bounds in XPT_GET_TRANS_SETTINGS\n",
974 		    cts->ccb_h.target_id));
975 		targ = &sassc->targets[cts->ccb_h.target_id];
976 		if (targ->handle == 0x0) {
977 			mpssas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE);
978 			break;
979 		}
980 
981 		cts->protocol_version = SCSI_REV_SPC2;
982 		cts->transport = XPORT_SAS;
983 		cts->transport_version = 0;
984 
985 		sas->valid = CTS_SAS_VALID_SPEED;
986 		switch (targ->linkrate) {
987 		case 0x08:
988 			sas->bitrate = 150000;
989 			break;
990 		case 0x09:
991 			sas->bitrate = 300000;
992 			break;
993 		case 0x0a:
994 			sas->bitrate = 600000;
995 			break;
996 		default:
997 			sas->valid = 0;
998 		}
999 
1000 		cts->protocol = PROTO_SCSI;
1001 		scsi->valid = CTS_SCSI_VALID_TQ;
1002 		scsi->flags = CTS_SCSI_FLAGS_TAG_ENB;
1003 
1004 		mpssas_set_ccbstatus(ccb, CAM_REQ_CMP);
1005 		break;
1006 	}
1007 	case XPT_CALC_GEOMETRY:
1008 		cam_calc_geometry(&ccb->ccg, /*extended*/1);
1009 		mpssas_set_ccbstatus(ccb, CAM_REQ_CMP);
1010 		break;
1011 	case XPT_RESET_DEV:
1012 		mps_dprint(sassc->sc, MPS_XINFO, "mpssas_action XPT_RESET_DEV\n");
1013 		mpssas_action_resetdev(sassc, ccb);
1014 		return;
1015 	case XPT_RESET_BUS:
1016 	case XPT_ABORT:
1017 	case XPT_TERM_IO:
1018 		mps_dprint(sassc->sc, MPS_XINFO,
1019 		    "mpssas_action faking success for abort or reset\n");
1020 		mpssas_set_ccbstatus(ccb, CAM_REQ_CMP);
1021 		break;
1022 	case XPT_SCSI_IO:
1023 		mpssas_action_scsiio(sassc, ccb);
1024 		return;
1025 	case XPT_SMP_IO:
1026 		mpssas_action_smpio(sassc, ccb);
1027 		return;
1028 	default:
1029 		mpssas_set_ccbstatus(ccb, CAM_FUNC_NOTAVAIL);
1030 		break;
1031 	}
1032 	xpt_done(ccb);
1033 
1034 }
1035 
1036 static void
1037 mpssas_announce_reset(struct mps_softc *sc, uint32_t ac_code,
1038     target_id_t target_id, lun_id_t lun_id)
1039 {
1040 	path_id_t path_id = cam_sim_path(sc->sassc->sim);
1041 	struct cam_path *path;
1042 
1043 	mps_dprint(sc, MPS_XINFO, "%s code %x target %d lun %jx\n", __func__,
1044 	    ac_code, target_id, (uintmax_t)lun_id);
1045 
1046 	if (xpt_create_path(&path, NULL,
1047 		path_id, target_id, lun_id) != CAM_REQ_CMP) {
1048 		mps_dprint(sc, MPS_ERROR, "unable to create path for reset "
1049 			   "notification\n");
1050 		return;
1051 	}
1052 
1053 	xpt_async(ac_code, path, NULL);
1054 	xpt_free_path(path);
1055 }
1056 
1057 static void
1058 mpssas_complete_all_commands(struct mps_softc *sc)
1059 {
1060 	struct mps_command *cm;
1061 	int i;
1062 	int completed;
1063 
1064 	MPS_FUNCTRACE(sc);
1065 	mtx_assert(&sc->mps_mtx, MA_OWNED);
1066 
1067 	/* complete all commands with a NULL reply */
1068 	for (i = 1; i < sc->num_reqs; i++) {
1069 		cm = &sc->commands[i];
1070 		if (cm->cm_state == MPS_CM_STATE_FREE)
1071 			continue;
1072 
1073 		cm->cm_state = MPS_CM_STATE_BUSY;
1074 		cm->cm_reply = NULL;
1075 		completed = 0;
1076 
1077 		if (cm->cm_flags & MPS_CM_FLAGS_SATA_ID_TIMEOUT) {
1078 			MPASS(cm->cm_data);
1079 			free(cm->cm_data, M_MPT2);
1080 			cm->cm_data = NULL;
1081 		}
1082 
1083 		if (cm->cm_flags & MPS_CM_FLAGS_POLLED)
1084 			cm->cm_flags |= MPS_CM_FLAGS_COMPLETE;
1085 
1086 		if (cm->cm_complete != NULL) {
1087 			mpssas_log_command(cm, MPS_RECOVERY,
1088 			    "completing cm %p state %x ccb %p for diag reset\n",
1089 			    cm, cm->cm_state, cm->cm_ccb);
1090 
1091 			cm->cm_complete(sc, cm);
1092 			completed = 1;
1093 		} else if (cm->cm_flags & MPS_CM_FLAGS_WAKEUP) {
1094 			mpssas_log_command(cm, MPS_RECOVERY,
1095 			    "waking up cm %p state %x ccb %p for diag reset\n",
1096 			    cm, cm->cm_state, cm->cm_ccb);
1097 			wakeup(cm);
1098 			completed = 1;
1099 		}
1100 
1101 		if ((completed == 0) && (cm->cm_state != MPS_CM_STATE_FREE)) {
1102 			/* this should never happen, but if it does, log */
1103 			mpssas_log_command(cm, MPS_RECOVERY,
1104 			    "cm %p state %x flags 0x%x ccb %p during diag "
1105 			    "reset\n", cm, cm->cm_state, cm->cm_flags,
1106 			    cm->cm_ccb);
1107 		}
1108 	}
1109 
1110 	sc->io_cmds_active = 0;
1111 }
1112 
1113 void
1114 mpssas_handle_reinit(struct mps_softc *sc)
1115 {
1116 	int i;
1117 
1118 	/* Go back into startup mode and freeze the simq, so that CAM
1119 	 * doesn't send any commands until after we've rediscovered all
1120 	 * targets and found the proper device handles for them.
1121 	 *
1122 	 * After the reset, portenable will trigger discovery, and after all
1123 	 * discovery-related activities have finished, the simq will be
1124 	 * released.
1125 	 */
1126 	mps_dprint(sc, MPS_INIT, "%s startup\n", __func__);
1127 	sc->sassc->flags |= MPSSAS_IN_STARTUP;
1128 	sc->sassc->flags |= MPSSAS_IN_DISCOVERY;
1129 	mpssas_startup_increment(sc->sassc);
1130 
1131 	/* notify CAM of a bus reset */
1132 	mpssas_announce_reset(sc, AC_BUS_RESET, CAM_TARGET_WILDCARD,
1133 	    CAM_LUN_WILDCARD);
1134 
1135 	/* complete and cleanup after all outstanding commands */
1136 	mpssas_complete_all_commands(sc);
1137 
1138 	mps_dprint(sc, MPS_INIT,
1139 	    "%s startup %u after command completion\n", __func__,
1140 	    sc->sassc->startup_refcount);
1141 
1142 	/* zero all the target handles, since they may change after the
1143 	 * reset, and we have to rediscover all the targets and use the new
1144 	 * handles.
1145 	 */
1146 	for (i = 0; i < sc->sassc->maxtargets; i++) {
1147 		if (sc->sassc->targets[i].outstanding != 0)
1148 			mps_dprint(sc, MPS_INIT, "target %u outstanding %u\n",
1149 			    i, sc->sassc->targets[i].outstanding);
1150 		sc->sassc->targets[i].handle = 0x0;
1151 		sc->sassc->targets[i].exp_dev_handle = 0x0;
1152 		sc->sassc->targets[i].outstanding = 0;
1153 		sc->sassc->targets[i].flags = MPSSAS_TARGET_INDIAGRESET;
1154 	}
1155 }
1156 
1157 static void
1158 mpssas_tm_timeout(void *data)
1159 {
1160 	struct mps_command *tm = data;
1161 	struct mps_softc *sc = tm->cm_sc;
1162 
1163 	mtx_assert(&sc->mps_mtx, MA_OWNED);
1164 
1165 	mpssas_log_command(tm, MPS_INFO|MPS_RECOVERY,
1166 	    "task mgmt %p timed out\n", tm);
1167 
1168 	KASSERT(tm->cm_state == MPS_CM_STATE_INQUEUE,
1169 	    ("command not inqueue, state = %u\n", tm->cm_state));
1170 
1171 	tm->cm_state = MPS_CM_STATE_BUSY;
1172 	mps_reinit(sc);
1173 }
1174 
1175 static void
1176 mpssas_logical_unit_reset_complete(struct mps_softc *sc, struct mps_command *tm)
1177 {
1178 	MPI2_SCSI_TASK_MANAGE_REPLY *reply;
1179 	unsigned int cm_count = 0;
1180 	struct mps_command *cm;
1181 	struct mpssas_target *targ;
1182 
1183 	callout_stop(&tm->cm_callout);
1184 
1185 	reply = (MPI2_SCSI_TASK_MANAGE_REPLY *)tm->cm_reply;
1186 	targ = tm->cm_targ;
1187 
1188 	/*
1189 	 * Currently there should be no way we can hit this case.  It only
1190 	 * happens when we have a failure to allocate chain frames, and
1191 	 * task management commands don't have S/G lists.
1192 	 * XXXSL So should it be an assertion?
1193 	 */
1194 	if ((tm->cm_flags & MPS_CM_FLAGS_ERROR_MASK) != 0) {
1195 		mps_dprint(sc, MPS_RECOVERY|MPS_ERROR,
1196 		    "%s: cm_flags = %#x for LUN reset! "
1197 		   "This should not happen!\n", __func__, tm->cm_flags);
1198 		mpssas_free_tm(sc, tm);
1199 		return;
1200 	}
1201 
1202 	if (reply == NULL) {
1203 		mps_dprint(sc, MPS_RECOVERY, "NULL reset reply for tm %p\n",
1204 		    tm);
1205 		if ((sc->mps_flags & MPS_FLAGS_DIAGRESET) != 0) {
1206 			/* this completion was due to a reset, just cleanup */
1207 			mps_dprint(sc, MPS_RECOVERY, "Hardware undergoing "
1208 			    "reset, ignoring NULL LUN reset reply\n");
1209 			targ->tm = NULL;
1210 			mpssas_free_tm(sc, tm);
1211 		}
1212 		else {
1213 			/* we should have gotten a reply. */
1214 			mps_dprint(sc, MPS_INFO|MPS_RECOVERY, "NULL reply on "
1215 			    "LUN reset attempt, resetting controller\n");
1216 			mps_reinit(sc);
1217 		}
1218 		return;
1219 	}
1220 
1221 	mps_dprint(sc, MPS_RECOVERY,
1222 	    "logical unit reset status 0x%x code 0x%x count %u\n",
1223 	    le16toh(reply->IOCStatus), le32toh(reply->ResponseCode),
1224 	    le32toh(reply->TerminationCount));
1225 
1226 	/*
1227 	 * See if there are any outstanding commands for this LUN.
1228 	 * This could be made more efficient by using a per-LU data
1229 	 * structure of some sort.
1230 	 */
1231 	TAILQ_FOREACH(cm, &targ->commands, cm_link) {
1232 		if (cm->cm_lun == tm->cm_lun)
1233 			cm_count++;
1234 	}
1235 
1236 	if (cm_count == 0) {
1237 		mps_dprint(sc, MPS_RECOVERY|MPS_INFO,
1238 		    "Finished recovery after LUN reset for target %u\n",
1239 		    targ->tid);
1240 
1241 		mpssas_announce_reset(sc, AC_SENT_BDR, targ->tid, tm->cm_lun);
1242 
1243 		/*
1244 		 * We've finished recovery for this logical unit.  check and
1245 		 * see if some other logical unit has a timedout command
1246 		 * that needs to be processed.
1247 		 */
1248 		cm = TAILQ_FIRST(&targ->timedout_commands);
1249 		if (cm) {
1250 			mps_dprint(sc, MPS_INFO|MPS_RECOVERY,
1251 			    "More commands to abort for target %u\n",
1252 			    targ->tid);
1253 			mpssas_send_abort(sc, tm, cm);
1254 		} else {
1255 			targ->tm = NULL;
1256 			mpssas_free_tm(sc, tm);
1257 		}
1258 	} else {
1259 		/*
1260 		 * If we still have commands for this LUN, the reset
1261 		 * effectively failed, regardless of the status reported.
1262 		 * Escalate to a target reset.
1263 		 */
1264 		mps_dprint(sc, MPS_INFO|MPS_RECOVERY,
1265 		    "logical unit reset complete for target %u, but still "
1266 		    "have %u command(s), sending target reset\n", targ->tid,
1267 		    cm_count);
1268 		mpssas_send_reset(sc, tm,
1269 		    MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET);
1270 	}
1271 }
1272 
1273 static void
1274 mpssas_target_reset_complete(struct mps_softc *sc, struct mps_command *tm)
1275 {
1276 	MPI2_SCSI_TASK_MANAGE_REPLY *reply;
1277 	MPI2_SCSI_TASK_MANAGE_REQUEST *req;
1278 	struct mpssas_target *targ;
1279 
1280 	callout_stop(&tm->cm_callout);
1281 
1282 	req = (MPI2_SCSI_TASK_MANAGE_REQUEST *)tm->cm_req;
1283 	reply = (MPI2_SCSI_TASK_MANAGE_REPLY *)tm->cm_reply;
1284 	targ = tm->cm_targ;
1285 
1286 	/*
1287 	 * Currently there should be no way we can hit this case.  It only
1288 	 * happens when we have a failure to allocate chain frames, and
1289 	 * task management commands don't have S/G lists.
1290 	 */
1291 	if ((tm->cm_flags & MPS_CM_FLAGS_ERROR_MASK) != 0) {
1292 		mps_dprint(sc, MPS_ERROR,"%s: cm_flags = %#x for target reset! "
1293 			   "This should not happen!\n", __func__, tm->cm_flags);
1294 		mpssas_free_tm(sc, tm);
1295 		return;
1296 	}
1297 
1298 	if (reply == NULL) {
1299 		mps_dprint(sc, MPS_RECOVERY,
1300 		    "NULL target reset reply for tm %pi TaskMID %u\n",
1301 		    tm, le16toh(req->TaskMID));
1302 		if ((sc->mps_flags & MPS_FLAGS_DIAGRESET) != 0) {
1303 			/* this completion was due to a reset, just cleanup */
1304 			mps_dprint(sc, MPS_RECOVERY, "Hardware undergoing "
1305 			    "reset, ignoring NULL target reset reply\n");
1306 			targ->tm = NULL;
1307 			mpssas_free_tm(sc, tm);
1308 		} else {
1309 			/* we should have gotten a reply. */
1310 			mps_dprint(sc, MPS_INFO|MPS_RECOVERY, "NULL reply on "
1311 			    "target reset attempt, resetting controller\n");
1312 			mps_reinit(sc);
1313 		}
1314 		return;
1315 	}
1316 
1317 	mps_dprint(sc, MPS_RECOVERY,
1318 	    "target reset status 0x%x code 0x%x count %u\n",
1319 	    le16toh(reply->IOCStatus), le32toh(reply->ResponseCode),
1320 	    le32toh(reply->TerminationCount));
1321 
1322 	if (targ->outstanding == 0) {
1323 		/* we've finished recovery for this target and all
1324 		 * of its logical units.
1325 		 */
1326 		mps_dprint(sc, MPS_RECOVERY|MPS_INFO,
1327 		    "Finished reset recovery for target %u\n", targ->tid);
1328 
1329 		mpssas_announce_reset(sc, AC_SENT_BDR, tm->cm_targ->tid,
1330 		    CAM_LUN_WILDCARD);
1331 
1332 		targ->tm = NULL;
1333 		mpssas_free_tm(sc, tm);
1334 	} else {
1335 		/*
1336 		 * After a target reset, if this target still has
1337 		 * outstanding commands, the reset effectively failed,
1338 		 * regardless of the status reported.  escalate.
1339 		 */
1340 		mps_dprint(sc, MPS_INFO|MPS_RECOVERY,
1341 		    "Target reset complete for target %u, but still have %u "
1342 		    "command(s), resetting controller\n", targ->tid,
1343 		    targ->outstanding);
1344 		mps_reinit(sc);
1345 	}
1346 }
1347 
1348 #define MPS_RESET_TIMEOUT 30
1349 
1350 int
1351 mpssas_send_reset(struct mps_softc *sc, struct mps_command *tm, uint8_t type)
1352 {
1353 	MPI2_SCSI_TASK_MANAGE_REQUEST *req;
1354 	struct mpssas_target *target;
1355 	int err;
1356 
1357 	target = tm->cm_targ;
1358 	if (target->handle == 0) {
1359 		mps_dprint(sc, MPS_ERROR,"%s null devhandle for target_id %d\n",
1360 		    __func__, target->tid);
1361 		return -1;
1362 	}
1363 
1364 	req = (MPI2_SCSI_TASK_MANAGE_REQUEST *)tm->cm_req;
1365 	req->DevHandle = htole16(target->handle);
1366 	req->TaskType = type;
1367 
1368 	if (type == MPI2_SCSITASKMGMT_TASKTYPE_LOGICAL_UNIT_RESET) {
1369 		/* XXX Need to handle invalid LUNs */
1370 		MPS_SET_LUN(req->LUN, tm->cm_lun);
1371 		tm->cm_targ->logical_unit_resets++;
1372 		mps_dprint(sc, MPS_RECOVERY|MPS_INFO,
1373 		    "Sending logical unit reset to target %u lun %d\n",
1374 		    target->tid, tm->cm_lun);
1375 		tm->cm_complete = mpssas_logical_unit_reset_complete;
1376 		mpssas_prepare_for_tm(sc, tm, target, tm->cm_lun);
1377 	} else if (type == MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET) {
1378 		/*
1379 		 * Target reset method =
1380 		 * 	SAS Hard Link Reset / SATA Link Reset
1381 		 */
1382 		req->MsgFlags = MPI2_SCSITASKMGMT_MSGFLAGS_LINK_RESET;
1383 		tm->cm_targ->target_resets++;
1384 		mps_dprint(sc, MPS_RECOVERY|MPS_INFO,
1385 		    "Sending target reset to target %u\n", target->tid);
1386 		tm->cm_complete = mpssas_target_reset_complete;
1387 		mpssas_prepare_for_tm(sc, tm, target, CAM_LUN_WILDCARD);
1388 	} else {
1389 		mps_dprint(sc, MPS_ERROR, "unexpected reset type 0x%x\n", type);
1390 		return -1;
1391 	}
1392 
1393 	tm->cm_data = NULL;
1394 	tm->cm_complete_data = (void *)tm;
1395 
1396 	callout_reset(&tm->cm_callout, MPS_RESET_TIMEOUT * hz,
1397 	    mpssas_tm_timeout, tm);
1398 
1399 	err = mps_map_command(sc, tm);
1400 	if (err)
1401 		mps_dprint(sc, MPS_ERROR|MPS_RECOVERY,
1402 		    "error %d sending reset type %u\n",
1403 		    err, type);
1404 
1405 	return err;
1406 }
1407 
1408 static void
1409 mpssas_abort_complete(struct mps_softc *sc, struct mps_command *tm)
1410 {
1411 	struct mps_command *cm;
1412 	MPI2_SCSI_TASK_MANAGE_REPLY *reply;
1413 	MPI2_SCSI_TASK_MANAGE_REQUEST *req;
1414 	struct mpssas_target *targ;
1415 
1416 	callout_stop(&tm->cm_callout);
1417 
1418 	req = (MPI2_SCSI_TASK_MANAGE_REQUEST *)tm->cm_req;
1419 	reply = (MPI2_SCSI_TASK_MANAGE_REPLY *)tm->cm_reply;
1420 	targ = tm->cm_targ;
1421 
1422 	/*
1423 	 * Currently there should be no way we can hit this case.  It only
1424 	 * happens when we have a failure to allocate chain frames, and
1425 	 * task management commands don't have S/G lists.
1426 	 */
1427 	if ((tm->cm_flags & MPS_CM_FLAGS_ERROR_MASK) != 0) {
1428 		mps_dprint(sc, MPS_RECOVERY,
1429 		    "cm_flags = %#x for abort %p TaskMID %u!\n",
1430 		    tm->cm_flags, tm, le16toh(req->TaskMID));
1431 		mpssas_free_tm(sc, tm);
1432 		return;
1433 	}
1434 
1435 	if (reply == NULL) {
1436 		mps_dprint(sc, MPS_RECOVERY,
1437 		    "NULL abort reply for tm %p TaskMID %u\n",
1438 		    tm, le16toh(req->TaskMID));
1439 		if ((sc->mps_flags & MPS_FLAGS_DIAGRESET) != 0) {
1440 			/* this completion was due to a reset, just cleanup */
1441 			mps_dprint(sc, MPS_RECOVERY, "Hardware undergoing "
1442 			    "reset, ignoring NULL abort reply\n");
1443 			targ->tm = NULL;
1444 			mpssas_free_tm(sc, tm);
1445 		} else {
1446 			/* we should have gotten a reply. */
1447 			mps_dprint(sc, MPS_INFO|MPS_RECOVERY, "NULL reply on "
1448 			    "abort attempt, resetting controller\n");
1449 			mps_reinit(sc);
1450 		}
1451 		return;
1452 	}
1453 
1454 	mps_dprint(sc, MPS_RECOVERY,
1455 	    "abort TaskMID %u status 0x%x code 0x%x count %u\n",
1456 	    le16toh(req->TaskMID),
1457 	    le16toh(reply->IOCStatus), le32toh(reply->ResponseCode),
1458 	    le32toh(reply->TerminationCount));
1459 
1460 	cm = TAILQ_FIRST(&tm->cm_targ->timedout_commands);
1461 	if (cm == NULL) {
1462 		/*
1463 		 * If there are no more timedout commands, we're done with
1464 		 * error recovery for this target.
1465 		 */
1466 		mps_dprint(sc, MPS_INFO|MPS_RECOVERY,
1467 		    "Finished abort recovery for target %u\n", targ->tid);
1468 
1469 		targ->tm = NULL;
1470 		mpssas_free_tm(sc, tm);
1471 	} else if (le16toh(req->TaskMID) != cm->cm_desc.Default.SMID) {
1472 		/* abort success, but we have more timedout commands to abort */
1473 		mps_dprint(sc, MPS_INFO|MPS_RECOVERY,
1474 		    "Continuing abort recovery for target %u\n", targ->tid);
1475 
1476 		mpssas_send_abort(sc, tm, cm);
1477 	} else {
1478 		/* we didn't get a command completion, so the abort
1479 		 * failed as far as we're concerned.  escalate.
1480 		 */
1481 		mps_dprint(sc, MPS_RECOVERY,
1482 		    "Abort failed for target %u, sending logical unit reset\n",
1483 		    targ->tid);
1484 
1485 		mpssas_send_reset(sc, tm,
1486 		    MPI2_SCSITASKMGMT_TASKTYPE_LOGICAL_UNIT_RESET);
1487 	}
1488 }
1489 
1490 #define MPS_ABORT_TIMEOUT 5
1491 
1492 static int
1493 mpssas_send_abort(struct mps_softc *sc, struct mps_command *tm, struct mps_command *cm)
1494 {
1495 	MPI2_SCSI_TASK_MANAGE_REQUEST *req;
1496 	struct mpssas_target *targ;
1497 	int err;
1498 
1499 	targ = cm->cm_targ;
1500 	if (targ->handle == 0) {
1501 		mps_dprint(sc, MPS_ERROR|MPS_RECOVERY,
1502 		    "%s null devhandle for target_id %d\n",
1503 		    __func__, cm->cm_ccb->ccb_h.target_id);
1504 		return -1;
1505 	}
1506 
1507 	mpssas_log_command(cm, MPS_RECOVERY|MPS_INFO,
1508 	    "Aborting command %p\n", cm);
1509 
1510 	req = (MPI2_SCSI_TASK_MANAGE_REQUEST *)tm->cm_req;
1511 	req->DevHandle = htole16(targ->handle);
1512 	req->TaskType = MPI2_SCSITASKMGMT_TASKTYPE_ABORT_TASK;
1513 
1514 	/* XXX Need to handle invalid LUNs */
1515 	MPS_SET_LUN(req->LUN, cm->cm_ccb->ccb_h.target_lun);
1516 
1517 	req->TaskMID = htole16(cm->cm_desc.Default.SMID);
1518 
1519 	tm->cm_data = NULL;
1520 	tm->cm_complete = mpssas_abort_complete;
1521 	tm->cm_complete_data = (void *)tm;
1522 	tm->cm_targ = cm->cm_targ;
1523 	tm->cm_lun = cm->cm_lun;
1524 
1525 	callout_reset(&tm->cm_callout, MPS_ABORT_TIMEOUT * hz,
1526 	    mpssas_tm_timeout, tm);
1527 
1528 	targ->aborts++;
1529 
1530 	mpssas_prepare_for_tm(sc, tm, targ, tm->cm_lun);
1531 
1532 	err = mps_map_command(sc, tm);
1533 	if (err)
1534 		mps_dprint(sc, MPS_ERROR|MPS_RECOVERY,
1535 		    "error %d sending abort for cm %p SMID %u\n",
1536 		    err, cm, req->TaskMID);
1537 	return err;
1538 }
1539 
1540 static void
1541 mpssas_scsiio_timeout(void *data)
1542 {
1543 	sbintime_t elapsed, now;
1544 	union ccb *ccb;
1545 	struct mps_softc *sc;
1546 	struct mps_command *cm;
1547 	struct mpssas_target *targ;
1548 
1549 	cm = (struct mps_command *)data;
1550 	sc = cm->cm_sc;
1551 	ccb = cm->cm_ccb;
1552 	now = sbinuptime();
1553 
1554 	MPS_FUNCTRACE(sc);
1555 	mtx_assert(&sc->mps_mtx, MA_OWNED);
1556 
1557 	mps_dprint(sc, MPS_XINFO|MPS_RECOVERY, "Timeout checking cm %p\n", cm);
1558 
1559 	/*
1560 	 * Run the interrupt handler to make sure it's not pending.  This
1561 	 * isn't perfect because the command could have already completed
1562 	 * and been re-used, though this is unlikely.
1563 	 */
1564 	mps_intr_locked(sc);
1565 	if (cm->cm_flags & MPS_CM_FLAGS_ON_RECOVERY) {
1566 		mpssas_log_command(cm, MPS_XINFO,
1567 		    "SCSI command %p almost timed out\n", cm);
1568 		return;
1569 	}
1570 
1571 	if (cm->cm_ccb == NULL) {
1572 		mps_dprint(sc, MPS_ERROR, "command timeout with NULL ccb\n");
1573 		return;
1574 	}
1575 
1576 	targ = cm->cm_targ;
1577 	targ->timeouts++;
1578 
1579 	elapsed = now - ccb->ccb_h.qos.sim_data;
1580 	mpssas_log_command(cm, MPS_INFO|MPS_RECOVERY,
1581 	    "Command timeout on target %u(0x%04x) %d set, %d.%d elapsed\n",
1582 	    targ->tid, targ->handle, ccb->ccb_h.timeout,
1583 	    sbintime_getsec(elapsed), elapsed & 0xffffffff);
1584 
1585 	/* XXX first, check the firmware state, to see if it's still
1586 	 * operational.  if not, do a diag reset.
1587 	 */
1588 	mpssas_set_ccbstatus(cm->cm_ccb, CAM_CMD_TIMEOUT);
1589 	cm->cm_flags |= MPS_CM_FLAGS_ON_RECOVERY | MPS_CM_FLAGS_TIMEDOUT;
1590 	TAILQ_INSERT_TAIL(&targ->timedout_commands, cm, cm_recovery);
1591 
1592 	if (targ->tm != NULL) {
1593 		/* target already in recovery, just queue up another
1594 		 * timedout command to be processed later.
1595 		 */
1596 		mps_dprint(sc, MPS_RECOVERY,
1597 		    "queued timedout cm %p for processing by tm %p\n",
1598 		    cm, targ->tm);
1599 	} else if ((targ->tm = mpssas_alloc_tm(sc)) != NULL) {
1600 		mps_dprint(sc, MPS_RECOVERY|MPS_INFO,
1601 		    "Sending abort to target %u for SMID %d\n", targ->tid,
1602 		    cm->cm_desc.Default.SMID);
1603 		mps_dprint(sc, MPS_RECOVERY, "timedout cm %p allocated tm %p\n",
1604 		    cm, targ->tm);
1605 
1606 		/* start recovery by aborting the first timedout command */
1607 		mpssas_send_abort(sc, targ->tm, cm);
1608 	} else {
1609 		/* XXX queue this target up for recovery once a TM becomes
1610 		 * available.  The firmware only has a limited number of
1611 		 * HighPriority credits for the high priority requests used
1612 		 * for task management, and we ran out.
1613 		 *
1614 		 * Isilon: don't worry about this for now, since we have
1615 		 * more credits than disks in an enclosure, and limit
1616 		 * ourselves to one TM per target for recovery.
1617 		 */
1618 		mps_dprint(sc, MPS_ERROR|MPS_RECOVERY,
1619 		    "timedout cm %p failed to allocate a tm\n", cm);
1620 	}
1621 
1622 }
1623 
1624 static void
1625 mpssas_action_scsiio(struct mpssas_softc *sassc, union ccb *ccb)
1626 {
1627 	MPI2_SCSI_IO_REQUEST *req;
1628 	struct ccb_scsiio *csio;
1629 	struct mps_softc *sc;
1630 	struct mpssas_target *targ;
1631 	struct mpssas_lun *lun;
1632 	struct mps_command *cm;
1633 	uint8_t i, lba_byte, *ref_tag_addr;
1634 	uint16_t eedp_flags;
1635 	uint32_t mpi_control;
1636 
1637 	sc = sassc->sc;
1638 	MPS_FUNCTRACE(sc);
1639 	mtx_assert(&sc->mps_mtx, MA_OWNED);
1640 
1641 	csio = &ccb->csio;
1642 	KASSERT(csio->ccb_h.target_id < sassc->maxtargets,
1643 	    ("Target %d out of bounds in XPT_SCSI_IO\n",
1644 	     csio->ccb_h.target_id));
1645 	targ = &sassc->targets[csio->ccb_h.target_id];
1646 	mps_dprint(sc, MPS_TRACE, "ccb %p target flag %x\n", ccb, targ->flags);
1647 	if (targ->handle == 0x0) {
1648 		mps_dprint(sc, MPS_ERROR, "%s NULL handle for target %u\n",
1649 		    __func__, csio->ccb_h.target_id);
1650 		mpssas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE);
1651 		xpt_done(ccb);
1652 		return;
1653 	}
1654 	if (targ->flags & MPS_TARGET_FLAGS_RAID_COMPONENT) {
1655 		mps_dprint(sc, MPS_ERROR, "%s Raid component no SCSI IO "
1656 		    "supported %u\n", __func__, csio->ccb_h.target_id);
1657 		mpssas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE);
1658 		xpt_done(ccb);
1659 		return;
1660 	}
1661 	/*
1662 	 * Sometimes, it is possible to get a command that is not "In
1663 	 * Progress" and was actually aborted by the upper layer.  Check for
1664 	 * this here and complete the command without error.
1665 	 */
1666 	if (mpssas_get_ccbstatus(ccb) != CAM_REQ_INPROG) {
1667 		mps_dprint(sc, MPS_TRACE, "%s Command is not in progress for "
1668 		    "target %u\n", __func__, csio->ccb_h.target_id);
1669 		xpt_done(ccb);
1670 		return;
1671 	}
1672 	/*
1673 	 * If devinfo is 0 this will be a volume.  In that case don't tell CAM
1674 	 * that the volume has timed out.  We want volumes to be enumerated
1675 	 * until they are deleted/removed, not just failed. In either event,
1676 	 * we're removing the target due to a firmware event telling us
1677 	 * the device is now gone (as opposed to some transient event). Since
1678 	 * we're opting to remove failed devices from the OS's view, we need
1679 	 * to propagate that status up the stack.
1680 	 */
1681 	if (targ->flags & MPSSAS_TARGET_INREMOVAL) {
1682 		if (targ->devinfo == 0)
1683 			mpssas_set_ccbstatus(ccb, CAM_REQ_CMP);
1684 		else
1685 			mpssas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE);
1686 		xpt_done(ccb);
1687 		return;
1688 	}
1689 
1690 	if ((sc->mps_flags & MPS_FLAGS_SHUTDOWN) != 0) {
1691 		mps_dprint(sc, MPS_INFO, "%s shutting down\n", __func__);
1692 		mpssas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE);
1693 		xpt_done(ccb);
1694 		return;
1695 	}
1696 
1697 	/*
1698 	 * If target has a reset in progress, the devq should be frozen.
1699 	 * Geting here we likely hit a race, so just requeue.
1700 	 */
1701 	if (targ->flags & MPSSAS_TARGET_INRESET) {
1702 		ccb->ccb_h.status = CAM_REQUEUE_REQ | CAM_DEV_QFRZN;
1703 		mps_dprint(sc, MPS_INFO, "%s: Freezing devq for target ID %d\n",
1704 		    __func__, targ->tid);
1705 		xpt_freeze_devq(ccb->ccb_h.path, 1);
1706 		xpt_done(ccb);
1707 		return;
1708 	}
1709 
1710 	cm = mps_alloc_command(sc);
1711 	if (cm == NULL || (sc->mps_flags & MPS_FLAGS_DIAGRESET)) {
1712 		if (cm != NULL) {
1713 			mps_free_command(sc, cm);
1714 		}
1715 		if ((sassc->flags & MPSSAS_QUEUE_FROZEN) == 0) {
1716 			xpt_freeze_simq(sassc->sim, 1);
1717 			sassc->flags |= MPSSAS_QUEUE_FROZEN;
1718 		}
1719 		ccb->ccb_h.status &= ~CAM_SIM_QUEUED;
1720 		ccb->ccb_h.status |= CAM_REQUEUE_REQ;
1721 		xpt_done(ccb);
1722 		return;
1723 	}
1724 
1725 	req = (MPI2_SCSI_IO_REQUEST *)cm->cm_req;
1726 	bzero(req, sizeof(*req));
1727 	req->DevHandle = htole16(targ->handle);
1728 	req->Function = MPI2_FUNCTION_SCSI_IO_REQUEST;
1729 	req->MsgFlags = 0;
1730 	req->SenseBufferLowAddress = htole32(cm->cm_sense_busaddr);
1731 	req->SenseBufferLength = MPS_SENSE_LEN;
1732 	req->SGLFlags = 0;
1733 	req->ChainOffset = 0;
1734 	req->SGLOffset0 = 24;	/* 32bit word offset to the SGL */
1735 	req->SGLOffset1= 0;
1736 	req->SGLOffset2= 0;
1737 	req->SGLOffset3= 0;
1738 	req->SkipCount = 0;
1739 	req->DataLength = htole32(csio->dxfer_len);
1740 	req->BidirectionalDataLength = 0;
1741 	req->IoFlags = htole16(csio->cdb_len);
1742 	req->EEDPFlags = 0;
1743 
1744 	/* Note: BiDirectional transfers are not supported */
1745 	switch (csio->ccb_h.flags & CAM_DIR_MASK) {
1746 	case CAM_DIR_IN:
1747 		mpi_control = MPI2_SCSIIO_CONTROL_READ;
1748 		cm->cm_flags |= MPS_CM_FLAGS_DATAIN;
1749 		break;
1750 	case CAM_DIR_OUT:
1751 		mpi_control = MPI2_SCSIIO_CONTROL_WRITE;
1752 		cm->cm_flags |= MPS_CM_FLAGS_DATAOUT;
1753 		break;
1754 	case CAM_DIR_NONE:
1755 	default:
1756 		mpi_control = MPI2_SCSIIO_CONTROL_NODATATRANSFER;
1757 		break;
1758 	}
1759 
1760 	if (csio->cdb_len == 32)
1761                 mpi_control |= 4 << MPI2_SCSIIO_CONTROL_ADDCDBLEN_SHIFT;
1762 	/*
1763 	 * It looks like the hardware doesn't require an explicit tag
1764 	 * number for each transaction.  SAM Task Management not supported
1765 	 * at the moment.
1766 	 */
1767 	switch (csio->tag_action) {
1768 	case MSG_HEAD_OF_Q_TAG:
1769 		mpi_control |= MPI2_SCSIIO_CONTROL_HEADOFQ;
1770 		break;
1771 	case MSG_ORDERED_Q_TAG:
1772 		mpi_control |= MPI2_SCSIIO_CONTROL_ORDEREDQ;
1773 		break;
1774 	case MSG_ACA_TASK:
1775 		mpi_control |= MPI2_SCSIIO_CONTROL_ACAQ;
1776 		break;
1777 	case CAM_TAG_ACTION_NONE:
1778 	case MSG_SIMPLE_Q_TAG:
1779 	default:
1780 		mpi_control |= MPI2_SCSIIO_CONTROL_SIMPLEQ;
1781 		break;
1782 	}
1783 	mpi_control |= (csio->priority << MPI2_SCSIIO_CONTROL_TASKPRI_SHIFT) &
1784 	    MPI2_SCSIIO_CONTROL_TASKPRI_MASK;
1785 	mpi_control |= sc->mapping_table[csio->ccb_h.target_id].TLR_bits;
1786 	req->Control = htole32(mpi_control);
1787 	if (MPS_SET_LUN(req->LUN, csio->ccb_h.target_lun) != 0) {
1788 		mps_free_command(sc, cm);
1789 		mpssas_set_ccbstatus(ccb, CAM_LUN_INVALID);
1790 		xpt_done(ccb);
1791 		return;
1792 	}
1793 
1794 	if (csio->ccb_h.flags & CAM_CDB_POINTER)
1795 		bcopy(csio->cdb_io.cdb_ptr, &req->CDB.CDB32[0], csio->cdb_len);
1796 	else
1797 		bcopy(csio->cdb_io.cdb_bytes, &req->CDB.CDB32[0],csio->cdb_len);
1798 	req->IoFlags = htole16(csio->cdb_len);
1799 
1800 	/*
1801 	 * Check if EEDP is supported and enabled.  If it is then check if the
1802 	 * SCSI opcode could be using EEDP.  If so, make sure the LUN exists and
1803 	 * is formatted for EEDP support.  If all of this is true, set CDB up
1804 	 * for EEDP transfer.
1805 	 */
1806 	eedp_flags = op_code_prot[req->CDB.CDB32[0]];
1807 	if (sc->eedp_enabled && eedp_flags) {
1808 		SLIST_FOREACH(lun, &targ->luns, lun_link) {
1809 			if (lun->lun_id == csio->ccb_h.target_lun) {
1810 				break;
1811 			}
1812 		}
1813 
1814 		if ((lun != NULL) && (lun->eedp_formatted)) {
1815 			req->EEDPBlockSize = htole16(lun->eedp_block_size);
1816 			eedp_flags |= (MPI2_SCSIIO_EEDPFLAGS_INC_PRI_REFTAG |
1817 			    MPI2_SCSIIO_EEDPFLAGS_CHECK_REFTAG |
1818 			    MPI2_SCSIIO_EEDPFLAGS_CHECK_GUARD);
1819 			req->EEDPFlags = htole16(eedp_flags);
1820 
1821 			/*
1822 			 * If CDB less than 32, fill in Primary Ref Tag with
1823 			 * low 4 bytes of LBA.  If CDB is 32, tag stuff is
1824 			 * already there.  Also, set protection bit.  FreeBSD
1825 			 * currently does not support CDBs bigger than 16, but
1826 			 * the code doesn't hurt, and will be here for the
1827 			 * future.
1828 			 */
1829 			if (csio->cdb_len != 32) {
1830 				lba_byte = (csio->cdb_len == 16) ? 6 : 2;
1831 				ref_tag_addr = (uint8_t *)&req->CDB.EEDP32.
1832 				    PrimaryReferenceTag;
1833 				for (i = 0; i < 4; i++) {
1834 					*ref_tag_addr =
1835 					    req->CDB.CDB32[lba_byte + i];
1836 					ref_tag_addr++;
1837 				}
1838 				req->CDB.EEDP32.PrimaryReferenceTag =
1839 					htole32(req->CDB.EEDP32.PrimaryReferenceTag);
1840 				req->CDB.EEDP32.PrimaryApplicationTagMask =
1841 				    0xFFFF;
1842 				req->CDB.CDB32[1] = (req->CDB.CDB32[1] & 0x1F) |
1843 				    0x20;
1844 			} else {
1845 				eedp_flags |=
1846 				    MPI2_SCSIIO_EEDPFLAGS_INC_PRI_APPTAG;
1847 				req->EEDPFlags = htole16(eedp_flags);
1848 				req->CDB.CDB32[10] = (req->CDB.CDB32[10] &
1849 				    0x1F) | 0x20;
1850 			}
1851 		}
1852 	}
1853 
1854 	cm->cm_length = csio->dxfer_len;
1855 	if (cm->cm_length != 0) {
1856 		cm->cm_data = ccb;
1857 		cm->cm_flags |= MPS_CM_FLAGS_USE_CCB;
1858 	} else {
1859 		cm->cm_data = NULL;
1860 	}
1861 	cm->cm_sge = &req->SGL;
1862 	cm->cm_sglsize = (32 - 24) * 4;
1863 	cm->cm_desc.SCSIIO.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_SCSI_IO;
1864 	cm->cm_desc.SCSIIO.DevHandle = htole16(targ->handle);
1865 	cm->cm_complete = mpssas_scsiio_complete;
1866 	cm->cm_complete_data = ccb;
1867 	cm->cm_targ = targ;
1868 	cm->cm_lun = csio->ccb_h.target_lun;
1869 	cm->cm_ccb = ccb;
1870 
1871 	/*
1872 	 * If HBA is a WD and the command is not for a retry, try to build a
1873 	 * direct I/O message. If failed, or the command is for a retry, send
1874 	 * the I/O to the IR volume itself.
1875 	 */
1876 	if (sc->WD_valid_config) {
1877 		if (ccb->ccb_h.sim_priv.entries[0].field == MPS_WD_RETRY) {
1878 			mpssas_direct_drive_io(sassc, cm, ccb);
1879 		} else {
1880 			mpssas_set_ccbstatus(ccb, CAM_REQ_INPROG);
1881 		}
1882 	}
1883 
1884 #if defined(BUF_TRACKING) || defined(FULL_BUF_TRACKING)
1885 	if (csio->bio != NULL)
1886 		biotrack(csio->bio, __func__);
1887 #endif
1888 	csio->ccb_h.qos.sim_data = sbinuptime();
1889 	callout_reset_sbt(&cm->cm_callout, SBT_1MS * ccb->ccb_h.timeout, 0,
1890 	    mpssas_scsiio_timeout, cm, 0);
1891 
1892 	targ->issued++;
1893 	targ->outstanding++;
1894 	TAILQ_INSERT_TAIL(&targ->commands, cm, cm_link);
1895 	ccb->ccb_h.status |= CAM_SIM_QUEUED;
1896 
1897 	mpssas_log_command(cm, MPS_XINFO, "%s cm %p ccb %p outstanding %u\n",
1898 	    __func__, cm, ccb, targ->outstanding);
1899 
1900 	mps_map_command(sc, cm);
1901 	return;
1902 }
1903 
1904 /**
1905  * mps_sc_failed_io_info - translated non-succesfull SCSI_IO request
1906  */
1907 static void
1908 mps_sc_failed_io_info(struct mps_softc *sc, struct ccb_scsiio *csio,
1909     Mpi2SCSIIOReply_t *mpi_reply)
1910 {
1911 	u32 response_info;
1912 	u8 *response_bytes;
1913 	u16 ioc_status = le16toh(mpi_reply->IOCStatus) &
1914 	    MPI2_IOCSTATUS_MASK;
1915 	u8 scsi_state = mpi_reply->SCSIState;
1916 	u8 scsi_status = mpi_reply->SCSIStatus;
1917 	u32 log_info = le32toh(mpi_reply->IOCLogInfo);
1918 	const char *desc_ioc_state, *desc_scsi_status;
1919 
1920 	if (log_info == 0x31170000)
1921 		return;
1922 
1923 	desc_ioc_state = mps_describe_table(mps_iocstatus_string,
1924 	    ioc_status);
1925 	desc_scsi_status = mps_describe_table(mps_scsi_status_string,
1926 	    scsi_status);
1927 
1928 	mps_dprint(sc, MPS_XINFO, "\thandle(0x%04x), ioc_status(%s)(0x%04x)\n",
1929 	    le16toh(mpi_reply->DevHandle), desc_ioc_state, ioc_status);
1930 
1931 	/*
1932 	 *We can add more detail about underflow data here
1933 	 * TO-DO
1934 	 */
1935 	mps_dprint(sc, MPS_XINFO, "\tscsi_status(%s)(0x%02x), "
1936 	    "scsi_state %b\n", desc_scsi_status, scsi_status,
1937 	    scsi_state, "\20" "\1AutosenseValid" "\2AutosenseFailed"
1938 	    "\3NoScsiStatus" "\4Terminated" "\5Response InfoValid");
1939 
1940 	if (sc->mps_debug & MPS_XINFO &&
1941 		scsi_state & MPI2_SCSI_STATE_AUTOSENSE_VALID) {
1942 		mps_dprint(sc, MPS_XINFO, "-> Sense Buffer Data : Start :\n");
1943 		scsi_sense_print(csio);
1944 		mps_dprint(sc, MPS_XINFO, "-> Sense Buffer Data : End :\n");
1945 	}
1946 
1947 	if (scsi_state & MPI2_SCSI_STATE_RESPONSE_INFO_VALID) {
1948 		response_info = le32toh(mpi_reply->ResponseInfo);
1949 		response_bytes = (u8 *)&response_info;
1950 		mps_dprint(sc, MPS_XINFO, "response code(0x%1x): %s\n",
1951 		    response_bytes[0],
1952 		    mps_describe_table(mps_scsi_taskmgmt_string,
1953 		    response_bytes[0]));
1954 	}
1955 }
1956 
1957 static void
1958 mpssas_scsiio_complete(struct mps_softc *sc, struct mps_command *cm)
1959 {
1960 	MPI2_SCSI_IO_REPLY *rep;
1961 	union ccb *ccb;
1962 	struct ccb_scsiio *csio;
1963 	struct mpssas_softc *sassc;
1964 	struct scsi_vpd_supported_page_list *vpd_list = NULL;
1965 	u8 *TLR_bits, TLR_on;
1966 	int dir = 0, i;
1967 	u16 alloc_len;
1968 	struct mpssas_target *target;
1969 	target_id_t target_id;
1970 
1971 	MPS_FUNCTRACE(sc);
1972 	mps_dprint(sc, MPS_TRACE,
1973 	    "cm %p SMID %u ccb %p reply %p outstanding %u\n", cm,
1974 	    cm->cm_desc.Default.SMID, cm->cm_ccb, cm->cm_reply,
1975 	    cm->cm_targ->outstanding);
1976 
1977 	callout_stop(&cm->cm_callout);
1978 	mtx_assert(&sc->mps_mtx, MA_OWNED);
1979 
1980 	sassc = sc->sassc;
1981 	ccb = cm->cm_complete_data;
1982 	csio = &ccb->csio;
1983 	target_id = csio->ccb_h.target_id;
1984 	rep = (MPI2_SCSI_IO_REPLY *)cm->cm_reply;
1985 	/*
1986 	 * XXX KDM if the chain allocation fails, does it matter if we do
1987 	 * the sync and unload here?  It is simpler to do it in every case,
1988 	 * assuming it doesn't cause problems.
1989 	 */
1990 	if (cm->cm_data != NULL) {
1991 		if (cm->cm_flags & MPS_CM_FLAGS_DATAIN)
1992 			dir = BUS_DMASYNC_POSTREAD;
1993 		else if (cm->cm_flags & MPS_CM_FLAGS_DATAOUT)
1994 			dir = BUS_DMASYNC_POSTWRITE;
1995 		bus_dmamap_sync(sc->buffer_dmat, cm->cm_dmamap, dir);
1996 		bus_dmamap_unload(sc->buffer_dmat, cm->cm_dmamap);
1997 	}
1998 
1999 	cm->cm_targ->completed++;
2000 	cm->cm_targ->outstanding--;
2001 	TAILQ_REMOVE(&cm->cm_targ->commands, cm, cm_link);
2002 	ccb->ccb_h.status &= ~(CAM_STATUS_MASK | CAM_SIM_QUEUED);
2003 
2004 #if defined(BUF_TRACKING) || defined(FULL_BUF_TRACKING)
2005 	if (ccb->csio.bio != NULL)
2006 		biotrack(ccb->csio.bio, __func__);
2007 #endif
2008 
2009 	if (cm->cm_flags & MPS_CM_FLAGS_ON_RECOVERY) {
2010 		TAILQ_REMOVE(&cm->cm_targ->timedout_commands, cm, cm_recovery);
2011 		KASSERT(cm->cm_state == MPS_CM_STATE_BUSY,
2012 		    ("Not busy for CM_FLAGS_TIMEDOUT: %u\n", cm->cm_state));
2013 		cm->cm_flags &= ~MPS_CM_FLAGS_ON_RECOVERY;
2014 		if (cm->cm_reply != NULL)
2015 			mpssas_log_command(cm, MPS_RECOVERY,
2016 			    "completed timedout cm %p ccb %p during recovery "
2017 			    "ioc %x scsi %x state %x xfer %u\n",
2018 			    cm, cm->cm_ccb, le16toh(rep->IOCStatus),
2019 			    rep->SCSIStatus, rep->SCSIState,
2020 			    le32toh(rep->TransferCount));
2021 		else
2022 			mpssas_log_command(cm, MPS_RECOVERY,
2023 			    "completed timedout cm %p ccb %p during recovery\n",
2024 			    cm, cm->cm_ccb);
2025 	} else if (cm->cm_targ->tm != NULL) {
2026 		if (cm->cm_reply != NULL)
2027 			mpssas_log_command(cm, MPS_RECOVERY,
2028 			    "completed cm %p ccb %p during recovery "
2029 			    "ioc %x scsi %x state %x xfer %u\n",
2030 			    cm, cm->cm_ccb, le16toh(rep->IOCStatus),
2031 			    rep->SCSIStatus, rep->SCSIState,
2032 			    le32toh(rep->TransferCount));
2033 		else
2034 			mpssas_log_command(cm, MPS_RECOVERY,
2035 			    "completed cm %p ccb %p during recovery\n",
2036 			    cm, cm->cm_ccb);
2037 	} else if ((sc->mps_flags & MPS_FLAGS_DIAGRESET) != 0) {
2038 		mpssas_log_command(cm, MPS_RECOVERY,
2039 		    "reset completed cm %p ccb %p\n",
2040 		    cm, cm->cm_ccb);
2041 	}
2042 
2043 	if ((cm->cm_flags & MPS_CM_FLAGS_ERROR_MASK) != 0) {
2044 		/*
2045 		 * We ran into an error after we tried to map the command,
2046 		 * so we're getting a callback without queueing the command
2047 		 * to the hardware.  So we set the status here, and it will
2048 		 * be retained below.  We'll go through the "fast path",
2049 		 * because there can be no reply when we haven't actually
2050 		 * gone out to the hardware.
2051 		 */
2052 		mpssas_set_ccbstatus(ccb, CAM_REQUEUE_REQ);
2053 
2054 		/*
2055 		 * Currently the only error included in the mask is
2056 		 * MPS_CM_FLAGS_CHAIN_FAILED, which means we're out of
2057 		 * chain frames.  We need to freeze the queue until we get
2058 		 * a command that completed without this error, which will
2059 		 * hopefully have some chain frames attached that we can
2060 		 * use.  If we wanted to get smarter about it, we would
2061 		 * only unfreeze the queue in this condition when we're
2062 		 * sure that we're getting some chain frames back.  That's
2063 		 * probably unnecessary.
2064 		 */
2065 		if ((sassc->flags & MPSSAS_QUEUE_FROZEN) == 0) {
2066 			xpt_freeze_simq(sassc->sim, 1);
2067 			sassc->flags |= MPSSAS_QUEUE_FROZEN;
2068 			mps_dprint(sc, MPS_XINFO, "Error sending command, "
2069 				   "freezing SIM queue\n");
2070 		}
2071 	}
2072 
2073 	/*
2074 	 * If this is a Start Stop Unit command and it was issued by the driver
2075 	 * during shutdown, decrement the refcount to account for all of the
2076 	 * commands that were sent.  All SSU commands should be completed before
2077 	 * shutdown completes, meaning SSU_refcount will be 0 after SSU_started
2078 	 * is TRUE.
2079 	 */
2080 	if (sc->SSU_started && (csio->cdb_io.cdb_bytes[0] == START_STOP_UNIT)) {
2081 		mps_dprint(sc, MPS_INFO, "Decrementing SSU count.\n");
2082 		sc->SSU_refcount--;
2083 	}
2084 
2085 	/* Take the fast path to completion */
2086 	if (cm->cm_reply == NULL) {
2087 		if (mpssas_get_ccbstatus(ccb) == CAM_REQ_INPROG) {
2088 			if ((sc->mps_flags & MPS_FLAGS_DIAGRESET) != 0)
2089 				mpssas_set_ccbstatus(ccb, CAM_SCSI_BUS_RESET);
2090 			else {
2091 				mpssas_set_ccbstatus(ccb, CAM_REQ_CMP);
2092 				ccb->csio.scsi_status = SCSI_STATUS_OK;
2093 			}
2094 			if (sassc->flags & MPSSAS_QUEUE_FROZEN) {
2095 				ccb->ccb_h.status |= CAM_RELEASE_SIMQ;
2096 				sassc->flags &= ~MPSSAS_QUEUE_FROZEN;
2097 				mps_dprint(sc, MPS_XINFO,
2098 				    "Unfreezing SIM queue\n");
2099 			}
2100 		}
2101 
2102 		/*
2103 		 * There are two scenarios where the status won't be
2104 		 * CAM_REQ_CMP.  The first is if MPS_CM_FLAGS_ERROR_MASK is
2105 		 * set, the second is in the MPS_FLAGS_DIAGRESET above.
2106 		 */
2107 		if (mpssas_get_ccbstatus(ccb) != CAM_REQ_CMP) {
2108 			/*
2109 			 * Freeze the dev queue so that commands are
2110 			 * executed in the correct order after error
2111 			 * recovery.
2112 			 */
2113 			ccb->ccb_h.status |= CAM_DEV_QFRZN;
2114 			xpt_freeze_devq(ccb->ccb_h.path, /*count*/ 1);
2115 		}
2116 		mps_free_command(sc, cm);
2117 		xpt_done(ccb);
2118 		return;
2119 	}
2120 
2121 	mpssas_log_command(cm, MPS_XINFO,
2122 	    "ioc %x scsi %x state %x xfer %u\n",
2123 	    le16toh(rep->IOCStatus), rep->SCSIStatus, rep->SCSIState,
2124 	    le32toh(rep->TransferCount));
2125 
2126 	/*
2127 	 * If this is a Direct Drive I/O, reissue the I/O to the original IR
2128 	 * Volume if an error occurred (normal I/O retry).  Use the original
2129 	 * CCB, but set a flag that this will be a retry so that it's sent to
2130 	 * the original volume.  Free the command but reuse the CCB.
2131 	 */
2132 	if (cm->cm_flags & MPS_CM_FLAGS_DD_IO) {
2133 		mps_free_command(sc, cm);
2134 		ccb->ccb_h.sim_priv.entries[0].field = MPS_WD_RETRY;
2135 		mpssas_action_scsiio(sassc, ccb);
2136 		return;
2137 	} else
2138 		ccb->ccb_h.sim_priv.entries[0].field = 0;
2139 
2140 	switch (le16toh(rep->IOCStatus) & MPI2_IOCSTATUS_MASK) {
2141 	case MPI2_IOCSTATUS_SCSI_DATA_UNDERRUN:
2142 		csio->resid = cm->cm_length - le32toh(rep->TransferCount);
2143 		/* FALLTHROUGH */
2144 	case MPI2_IOCSTATUS_SUCCESS:
2145 	case MPI2_IOCSTATUS_SCSI_RECOVERED_ERROR:
2146 
2147 		if ((le16toh(rep->IOCStatus) & MPI2_IOCSTATUS_MASK) ==
2148 		    MPI2_IOCSTATUS_SCSI_RECOVERED_ERROR)
2149 			mpssas_log_command(cm, MPS_XINFO, "recovered error\n");
2150 
2151 		/* Completion failed at the transport level. */
2152 		if (rep->SCSIState & (MPI2_SCSI_STATE_NO_SCSI_STATUS |
2153 		    MPI2_SCSI_STATE_TERMINATED)) {
2154 			mpssas_set_ccbstatus(ccb, CAM_REQ_CMP_ERR);
2155 			break;
2156 		}
2157 
2158 		/* In a modern packetized environment, an autosense failure
2159 		 * implies that there's not much else that can be done to
2160 		 * recover the command.
2161 		 */
2162 		if (rep->SCSIState & MPI2_SCSI_STATE_AUTOSENSE_FAILED) {
2163 			mpssas_set_ccbstatus(ccb, CAM_AUTOSENSE_FAIL);
2164 			break;
2165 		}
2166 
2167 		/*
2168 		 * CAM doesn't care about SAS Response Info data, but if this is
2169 		 * the state check if TLR should be done.  If not, clear the
2170 		 * TLR_bits for the target.
2171 		 */
2172 		if ((rep->SCSIState & MPI2_SCSI_STATE_RESPONSE_INFO_VALID) &&
2173 		    ((le32toh(rep->ResponseInfo) &
2174 		    MPI2_SCSI_RI_MASK_REASONCODE) ==
2175 		    MPS_SCSI_RI_INVALID_FRAME)) {
2176 			sc->mapping_table[target_id].TLR_bits =
2177 			    (u8)MPI2_SCSIIO_CONTROL_NO_TLR;
2178 		}
2179 
2180 		/*
2181 		 * Intentionally override the normal SCSI status reporting
2182 		 * for these two cases.  These are likely to happen in a
2183 		 * multi-initiator environment, and we want to make sure that
2184 		 * CAM retries these commands rather than fail them.
2185 		 */
2186 		if ((rep->SCSIStatus == MPI2_SCSI_STATUS_COMMAND_TERMINATED) ||
2187 		    (rep->SCSIStatus == MPI2_SCSI_STATUS_TASK_ABORTED)) {
2188 			mpssas_set_ccbstatus(ccb, CAM_REQ_ABORTED);
2189 			break;
2190 		}
2191 
2192 		/* Handle normal status and sense */
2193 		csio->scsi_status = rep->SCSIStatus;
2194 		if (rep->SCSIStatus == MPI2_SCSI_STATUS_GOOD)
2195 			mpssas_set_ccbstatus(ccb, CAM_REQ_CMP);
2196 		else
2197 			mpssas_set_ccbstatus(ccb, CAM_SCSI_STATUS_ERROR);
2198 
2199 		if (rep->SCSIState & MPI2_SCSI_STATE_AUTOSENSE_VALID) {
2200 			int sense_len, returned_sense_len;
2201 
2202 			returned_sense_len = min(le32toh(rep->SenseCount),
2203 			    sizeof(struct scsi_sense_data));
2204 			if (returned_sense_len < ccb->csio.sense_len)
2205 				ccb->csio.sense_resid = ccb->csio.sense_len -
2206 					returned_sense_len;
2207 			else
2208 				ccb->csio.sense_resid = 0;
2209 
2210 			sense_len = min(returned_sense_len,
2211 			    ccb->csio.sense_len - ccb->csio.sense_resid);
2212 			bzero(&ccb->csio.sense_data,
2213 			      sizeof(ccb->csio.sense_data));
2214 			bcopy(cm->cm_sense, &ccb->csio.sense_data, sense_len);
2215 			ccb->ccb_h.status |= CAM_AUTOSNS_VALID;
2216 		}
2217 
2218 		/*
2219 		 * Check if this is an INQUIRY command.  If it's a VPD inquiry,
2220 		 * and it's page code 0 (Supported Page List), and there is
2221 		 * inquiry data, and this is for a sequential access device, and
2222 		 * the device is an SSP target, and TLR is supported by the
2223 		 * controller, turn the TLR_bits value ON if page 0x90 is
2224 		 * supported.
2225 		 */
2226 		if ((csio->cdb_io.cdb_bytes[0] == INQUIRY) &&
2227 		    (csio->cdb_io.cdb_bytes[1] & SI_EVPD) &&
2228 		    (csio->cdb_io.cdb_bytes[2] == SVPD_SUPPORTED_PAGE_LIST) &&
2229 		    ((csio->ccb_h.flags & CAM_DATA_MASK) == CAM_DATA_VADDR) &&
2230 		    (csio->data_ptr != NULL) &&
2231 		    ((csio->data_ptr[0] & 0x1f) == T_SEQUENTIAL) &&
2232 		    (sc->control_TLR) &&
2233 		    (sc->mapping_table[target_id].device_info &
2234 		    MPI2_SAS_DEVICE_INFO_SSP_TARGET)) {
2235 			vpd_list = (struct scsi_vpd_supported_page_list *)
2236 			    csio->data_ptr;
2237 			TLR_bits = &sc->mapping_table[target_id].TLR_bits;
2238 			*TLR_bits = (u8)MPI2_SCSIIO_CONTROL_NO_TLR;
2239 			TLR_on = (u8)MPI2_SCSIIO_CONTROL_TLR_ON;
2240 			alloc_len = ((u16)csio->cdb_io.cdb_bytes[3] << 8) +
2241 			    csio->cdb_io.cdb_bytes[4];
2242 			alloc_len -= csio->resid;
2243 			for (i = 0; i < MIN(vpd_list->length, alloc_len); i++) {
2244 				if (vpd_list->list[i] == 0x90) {
2245 					*TLR_bits = TLR_on;
2246 					break;
2247 				}
2248 			}
2249 		}
2250 
2251 		/*
2252 		 * If this is a SATA direct-access end device, mark it so that
2253 		 * a SCSI StartStopUnit command will be sent to it when the
2254 		 * driver is being shutdown.
2255 		 */
2256 		if ((csio->cdb_io.cdb_bytes[0] == INQUIRY) &&
2257 		    ((csio->data_ptr[0] & 0x1f) == T_DIRECT) &&
2258 		    (sc->mapping_table[target_id].device_info &
2259 		    MPI2_SAS_DEVICE_INFO_SATA_DEVICE) &&
2260 		    ((sc->mapping_table[target_id].device_info &
2261 		    MPI2_SAS_DEVICE_INFO_MASK_DEVICE_TYPE) ==
2262 		    MPI2_SAS_DEVICE_INFO_END_DEVICE)) {
2263 			target = &sassc->targets[target_id];
2264 			target->supports_SSU = TRUE;
2265 			mps_dprint(sc, MPS_XINFO, "Target %d supports SSU\n",
2266 			    target_id);
2267 		}
2268 		break;
2269 	case MPI2_IOCSTATUS_SCSI_INVALID_DEVHANDLE:
2270 	case MPI2_IOCSTATUS_SCSI_DEVICE_NOT_THERE:
2271 		/*
2272 		 * If devinfo is 0 this will be a volume.  In that case don't
2273 		 * tell CAM that the volume is not there.  We want volumes to
2274 		 * be enumerated until they are deleted/removed, not just
2275 		 * failed.
2276 		 */
2277 		if (cm->cm_targ->devinfo == 0)
2278 			mpssas_set_ccbstatus(ccb, CAM_REQ_CMP);
2279 		else
2280 			mpssas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE);
2281 		break;
2282 	case MPI2_IOCSTATUS_INVALID_SGL:
2283 		mps_print_scsiio_cmd(sc, cm);
2284 		mpssas_set_ccbstatus(ccb, CAM_UNREC_HBA_ERROR);
2285 		break;
2286 	case MPI2_IOCSTATUS_SCSI_TASK_TERMINATED:
2287 		/*
2288 		 * This is one of the responses that comes back when an I/O
2289 		 * has been aborted.  If it is because of a timeout that we
2290 		 * initiated, just set the status to CAM_CMD_TIMEOUT.
2291 		 * Otherwise set it to CAM_REQ_ABORTED.  The effect on the
2292 		 * command is the same (it gets retried, subject to the
2293 		 * retry counter), the only difference is what gets printed
2294 		 * on the console.
2295 		 */
2296 		if (cm->cm_flags & MPS_CM_FLAGS_TIMEDOUT)
2297 			mpssas_set_ccbstatus(ccb, CAM_CMD_TIMEOUT);
2298 		else
2299 			mpssas_set_ccbstatus(ccb, CAM_REQ_ABORTED);
2300 		break;
2301 	case MPI2_IOCSTATUS_SCSI_DATA_OVERRUN:
2302 		/* resid is ignored for this condition */
2303 		csio->resid = 0;
2304 		mpssas_set_ccbstatus(ccb, CAM_DATA_RUN_ERR);
2305 		break;
2306 	case MPI2_IOCSTATUS_SCSI_IOC_TERMINATED:
2307 	case MPI2_IOCSTATUS_SCSI_EXT_TERMINATED:
2308 		/*
2309 		 * These can sometimes be transient transport-related
2310 		 * errors, and sometimes persistent drive-related errors.
2311 		 * We used to retry these without decrementing the retry
2312 		 * count by returning CAM_REQUEUE_REQ.  Unfortunately, if
2313 		 * we hit a persistent drive problem that returns one of
2314 		 * these error codes, we would retry indefinitely.  So,
2315 		 * return CAM_REQ_CMP_ERROR so that we decrement the retry
2316 		 * count and avoid infinite retries.  We're taking the
2317 		 * potential risk of flagging false failures in the event
2318 		 * of a topology-related error (e.g. a SAS expander problem
2319 		 * causes a command addressed to a drive to fail), but
2320 		 * avoiding getting into an infinite retry loop. However,
2321 		 * if we get them while were moving a device, we should
2322 		 * fail the request as 'not there' because the device
2323 		 * is effectively gone.
2324 		 */
2325 		if (cm->cm_targ->flags & MPSSAS_TARGET_INREMOVAL)
2326 			mpssas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE);
2327 		else
2328 			mpssas_set_ccbstatus(ccb, CAM_REQ_CMP_ERR);
2329 		mps_dprint(sc, MPS_INFO,
2330 		    "Controller reported %s tgt %u SMID %u loginfo %x%s\n",
2331 		    mps_describe_table(mps_iocstatus_string,
2332 		    le16toh(rep->IOCStatus) & MPI2_IOCSTATUS_MASK),
2333 		    target_id, cm->cm_desc.Default.SMID,
2334 		    le32toh(rep->IOCLogInfo),
2335 		    (cm->cm_targ->flags & MPSSAS_TARGET_INREMOVAL) ? " departing" : "");
2336 		mps_dprint(sc, MPS_XINFO,
2337 		    "SCSIStatus %x SCSIState %x xfercount %u\n",
2338 		    rep->SCSIStatus, rep->SCSIState,
2339 		    le32toh(rep->TransferCount));
2340 		break;
2341 	case MPI2_IOCSTATUS_INVALID_FUNCTION:
2342 	case MPI2_IOCSTATUS_INTERNAL_ERROR:
2343 	case MPI2_IOCSTATUS_INVALID_VPID:
2344 	case MPI2_IOCSTATUS_INVALID_FIELD:
2345 	case MPI2_IOCSTATUS_INVALID_STATE:
2346 	case MPI2_IOCSTATUS_OP_STATE_NOT_SUPPORTED:
2347 	case MPI2_IOCSTATUS_SCSI_IO_DATA_ERROR:
2348 	case MPI2_IOCSTATUS_SCSI_PROTOCOL_ERROR:
2349 	case MPI2_IOCSTATUS_SCSI_RESIDUAL_MISMATCH:
2350 	case MPI2_IOCSTATUS_SCSI_TASK_MGMT_FAILED:
2351 	default:
2352 		mpssas_log_command(cm, MPS_XINFO,
2353 		    "completed ioc %x loginfo %x scsi %x state %x xfer %u\n",
2354 		    le16toh(rep->IOCStatus), le32toh(rep->IOCLogInfo),
2355 		    rep->SCSIStatus, rep->SCSIState,
2356 		    le32toh(rep->TransferCount));
2357 		csio->resid = cm->cm_length;
2358 		mpssas_set_ccbstatus(ccb, CAM_REQ_CMP_ERR);
2359 		break;
2360 	}
2361 
2362 	mps_sc_failed_io_info(sc,csio,rep);
2363 
2364 	if (sassc->flags & MPSSAS_QUEUE_FROZEN) {
2365 		ccb->ccb_h.status |= CAM_RELEASE_SIMQ;
2366 		sassc->flags &= ~MPSSAS_QUEUE_FROZEN;
2367 		mps_dprint(sc, MPS_XINFO, "Command completed, "
2368 		    "unfreezing SIM queue\n");
2369 	}
2370 
2371 	if (mpssas_get_ccbstatus(ccb) != CAM_REQ_CMP) {
2372 		ccb->ccb_h.status |= CAM_DEV_QFRZN;
2373 		xpt_freeze_devq(ccb->ccb_h.path, /*count*/ 1);
2374 	}
2375 
2376 	/*
2377 	 * Check to see if we're removing the device. If so, and this is the
2378 	 * last command on the queue, proceed with the deferred removal of the
2379 	 * device.  Note, for removing a volume, this won't trigger because
2380 	 * pending_remove_tm will be NULL.
2381 	 */
2382 	if (cm->cm_targ->flags & MPSSAS_TARGET_INREMOVAL) {
2383 		if (TAILQ_FIRST(&cm->cm_targ->commands) == NULL &&
2384 		    cm->cm_targ->pending_remove_tm != NULL) {
2385 			mps_dprint(sc, MPS_INFO, "Last pending command complete: starting remove_device\n");
2386 			mps_map_command(sc, cm->cm_targ->pending_remove_tm);
2387 			cm->cm_targ->pending_remove_tm = NULL;
2388 		}
2389 	}
2390 
2391 	mps_free_command(sc, cm);
2392 	xpt_done(ccb);
2393 }
2394 
2395 /* All Request reached here are Endian safe */
2396 static void
2397 mpssas_direct_drive_io(struct mpssas_softc *sassc, struct mps_command *cm,
2398     union ccb *ccb) {
2399 	pMpi2SCSIIORequest_t	pIO_req;
2400 	struct mps_softc	*sc = sassc->sc;
2401 	uint64_t		virtLBA;
2402 	uint32_t		physLBA, stripe_offset, stripe_unit;
2403 	uint32_t		io_size, column;
2404 	uint8_t			*ptrLBA, lba_idx, physLBA_byte, *CDB;
2405 
2406 	/*
2407 	 * If this is a valid SCSI command (Read6, Read10, Read16, Write6,
2408 	 * Write10, or Write16), build a direct I/O message.  Otherwise, the I/O
2409 	 * will be sent to the IR volume itself.  Since Read6 and Write6 are a
2410 	 * bit different than the 10/16 CDBs, handle them separately.
2411 	 */
2412 	pIO_req = (pMpi2SCSIIORequest_t)cm->cm_req;
2413 	CDB = pIO_req->CDB.CDB32;
2414 
2415 	/*
2416 	 * Handle 6 byte CDBs.
2417 	 */
2418 	if ((pIO_req->DevHandle == sc->DD_dev_handle) && ((CDB[0] == READ_6) ||
2419 	    (CDB[0] == WRITE_6))) {
2420 		/*
2421 		 * Get the transfer size in blocks.
2422 		 */
2423 		io_size = (cm->cm_length >> sc->DD_block_exponent);
2424 
2425 		/*
2426 		 * Get virtual LBA given in the CDB.
2427 		 */
2428 		virtLBA = ((uint64_t)(CDB[1] & 0x1F) << 16) |
2429 		    ((uint64_t)CDB[2] << 8) | (uint64_t)CDB[3];
2430 
2431 		/*
2432 		 * Check that LBA range for I/O does not exceed volume's
2433 		 * MaxLBA.
2434 		 */
2435 		if ((virtLBA + (uint64_t)io_size - 1) <=
2436 		    sc->DD_max_lba) {
2437 			/*
2438 			 * Check if the I/O crosses a stripe boundary.  If not,
2439 			 * translate the virtual LBA to a physical LBA and set
2440 			 * the DevHandle for the PhysDisk to be used.  If it
2441 			 * does cross a boundary, do normal I/O.  To get the
2442 			 * right DevHandle to use, get the map number for the
2443 			 * column, then use that map number to look up the
2444 			 * DevHandle of the PhysDisk.
2445 			 */
2446 			stripe_offset = (uint32_t)virtLBA &
2447 			    (sc->DD_stripe_size - 1);
2448 			if ((stripe_offset + io_size) <= sc->DD_stripe_size) {
2449 				physLBA = (uint32_t)virtLBA >>
2450 				    sc->DD_stripe_exponent;
2451 				stripe_unit = physLBA / sc->DD_num_phys_disks;
2452 				column = physLBA % sc->DD_num_phys_disks;
2453 				pIO_req->DevHandle =
2454 				    htole16(sc->DD_column_map[column].dev_handle);
2455 				/* ???? Is this endian safe*/
2456 				cm->cm_desc.SCSIIO.DevHandle =
2457 				    pIO_req->DevHandle;
2458 
2459 				physLBA = (stripe_unit <<
2460 				    sc->DD_stripe_exponent) + stripe_offset;
2461 				ptrLBA = &pIO_req->CDB.CDB32[1];
2462 				physLBA_byte = (uint8_t)(physLBA >> 16);
2463 				*ptrLBA = physLBA_byte;
2464 				ptrLBA = &pIO_req->CDB.CDB32[2];
2465 				physLBA_byte = (uint8_t)(physLBA >> 8);
2466 				*ptrLBA = physLBA_byte;
2467 				ptrLBA = &pIO_req->CDB.CDB32[3];
2468 				physLBA_byte = (uint8_t)physLBA;
2469 				*ptrLBA = physLBA_byte;
2470 
2471 				/*
2472 				 * Set flag that Direct Drive I/O is
2473 				 * being done.
2474 				 */
2475 				cm->cm_flags |= MPS_CM_FLAGS_DD_IO;
2476 			}
2477 		}
2478 		return;
2479 	}
2480 
2481 	/*
2482 	 * Handle 10, 12 or 16 byte CDBs.
2483 	 */
2484 	if ((pIO_req->DevHandle == sc->DD_dev_handle) && ((CDB[0] == READ_10) ||
2485 	    (CDB[0] == WRITE_10) || (CDB[0] == READ_16) ||
2486 	    (CDB[0] == WRITE_16) || (CDB[0] == READ_12) ||
2487 	    (CDB[0] == WRITE_12))) {
2488 		/*
2489 		 * For 16-byte CDB's, verify that the upper 4 bytes of the CDB
2490 		 * are 0.  If not, this is accessing beyond 2TB so handle it in
2491 		 * the else section.  10-byte and 12-byte CDB's are OK.
2492 		 * FreeBSD sends very rare 12 byte READ/WRITE, but driver is
2493 		 * ready to accept 12byte CDB for Direct IOs.
2494 		 */
2495 		if ((CDB[0] == READ_10 || CDB[0] == WRITE_10) ||
2496 		    (CDB[0] == READ_12 || CDB[0] == WRITE_12) ||
2497 		    !(CDB[2] | CDB[3] | CDB[4] | CDB[5])) {
2498 			/*
2499 			 * Get the transfer size in blocks.
2500 			 */
2501 			io_size = (cm->cm_length >> sc->DD_block_exponent);
2502 
2503 			/*
2504 			 * Get virtual LBA.  Point to correct lower 4 bytes of
2505 			 * LBA in the CDB depending on command.
2506 			 */
2507 			lba_idx = ((CDB[0] == READ_12) ||
2508 				(CDB[0] == WRITE_12) ||
2509 				(CDB[0] == READ_10) ||
2510 				(CDB[0] == WRITE_10))? 2 : 6;
2511 			virtLBA = ((uint64_t)CDB[lba_idx] << 24) |
2512 			    ((uint64_t)CDB[lba_idx + 1] << 16) |
2513 			    ((uint64_t)CDB[lba_idx + 2] << 8) |
2514 			    (uint64_t)CDB[lba_idx + 3];
2515 
2516 			/*
2517 			 * Check that LBA range for I/O does not exceed volume's
2518 			 * MaxLBA.
2519 			 */
2520 			if ((virtLBA + (uint64_t)io_size - 1) <=
2521 			    sc->DD_max_lba) {
2522 				/*
2523 				 * Check if the I/O crosses a stripe boundary.
2524 				 * If not, translate the virtual LBA to a
2525 				 * physical LBA and set the DevHandle for the
2526 				 * PhysDisk to be used.  If it does cross a
2527 				 * boundary, do normal I/O.  To get the right
2528 				 * DevHandle to use, get the map number for the
2529 				 * column, then use that map number to look up
2530 				 * the DevHandle of the PhysDisk.
2531 				 */
2532 				stripe_offset = (uint32_t)virtLBA &
2533 				    (sc->DD_stripe_size - 1);
2534 				if ((stripe_offset + io_size) <=
2535 				    sc->DD_stripe_size) {
2536 					physLBA = (uint32_t)virtLBA >>
2537 					    sc->DD_stripe_exponent;
2538 					stripe_unit = physLBA /
2539 					    sc->DD_num_phys_disks;
2540 					column = physLBA %
2541 					    sc->DD_num_phys_disks;
2542 					pIO_req->DevHandle =
2543 					    htole16(sc->DD_column_map[column].
2544 					    dev_handle);
2545 					cm->cm_desc.SCSIIO.DevHandle =
2546 					    pIO_req->DevHandle;
2547 
2548 					physLBA = (stripe_unit <<
2549 					    sc->DD_stripe_exponent) +
2550 					    stripe_offset;
2551 					ptrLBA =
2552 					    &pIO_req->CDB.CDB32[lba_idx];
2553 					physLBA_byte = (uint8_t)(physLBA >> 24);
2554 					*ptrLBA = physLBA_byte;
2555 					ptrLBA =
2556 					    &pIO_req->CDB.CDB32[lba_idx + 1];
2557 					physLBA_byte = (uint8_t)(physLBA >> 16);
2558 					*ptrLBA = physLBA_byte;
2559 					ptrLBA =
2560 					    &pIO_req->CDB.CDB32[lba_idx + 2];
2561 					physLBA_byte = (uint8_t)(physLBA >> 8);
2562 					*ptrLBA = physLBA_byte;
2563 					ptrLBA =
2564 					    &pIO_req->CDB.CDB32[lba_idx + 3];
2565 					physLBA_byte = (uint8_t)physLBA;
2566 					*ptrLBA = physLBA_byte;
2567 
2568 					/*
2569 					 * Set flag that Direct Drive I/O is
2570 					 * being done.
2571 					 */
2572 					cm->cm_flags |= MPS_CM_FLAGS_DD_IO;
2573 				}
2574 			}
2575 		} else {
2576 			/*
2577 			 * 16-byte CDB and the upper 4 bytes of the CDB are not
2578 			 * 0.  Get the transfer size in blocks.
2579 			 */
2580 			io_size = (cm->cm_length >> sc->DD_block_exponent);
2581 
2582 			/*
2583 			 * Get virtual LBA.
2584 			 */
2585 			virtLBA = ((uint64_t)CDB[2] << 54) |
2586 			    ((uint64_t)CDB[3] << 48) |
2587 			    ((uint64_t)CDB[4] << 40) |
2588 			    ((uint64_t)CDB[5] << 32) |
2589 			    ((uint64_t)CDB[6] << 24) |
2590 			    ((uint64_t)CDB[7] << 16) |
2591 			    ((uint64_t)CDB[8] << 8) |
2592 			    (uint64_t)CDB[9];
2593 
2594 			/*
2595 			 * Check that LBA range for I/O does not exceed volume's
2596 			 * MaxLBA.
2597 			 */
2598 			if ((virtLBA + (uint64_t)io_size - 1) <=
2599 			    sc->DD_max_lba) {
2600 				/*
2601 				 * Check if the I/O crosses a stripe boundary.
2602 				 * If not, translate the virtual LBA to a
2603 				 * physical LBA and set the DevHandle for the
2604 				 * PhysDisk to be used.  If it does cross a
2605 				 * boundary, do normal I/O.  To get the right
2606 				 * DevHandle to use, get the map number for the
2607 				 * column, then use that map number to look up
2608 				 * the DevHandle of the PhysDisk.
2609 				 */
2610 				stripe_offset = (uint32_t)virtLBA &
2611 				    (sc->DD_stripe_size - 1);
2612 				if ((stripe_offset + io_size) <=
2613 				    sc->DD_stripe_size) {
2614 					physLBA = (uint32_t)(virtLBA >>
2615 					    sc->DD_stripe_exponent);
2616 					stripe_unit = physLBA /
2617 					    sc->DD_num_phys_disks;
2618 					column = physLBA %
2619 					    sc->DD_num_phys_disks;
2620 					pIO_req->DevHandle =
2621 					    htole16(sc->DD_column_map[column].
2622 					    dev_handle);
2623 					cm->cm_desc.SCSIIO.DevHandle =
2624 					    pIO_req->DevHandle;
2625 
2626 					physLBA = (stripe_unit <<
2627 					    sc->DD_stripe_exponent) +
2628 					    stripe_offset;
2629 
2630 					/*
2631 					 * Set upper 4 bytes of LBA to 0.  We
2632 					 * assume that the phys disks are less
2633 					 * than 2 TB's in size.  Then, set the
2634 					 * lower 4 bytes.
2635 					 */
2636 					pIO_req->CDB.CDB32[2] = 0;
2637 					pIO_req->CDB.CDB32[3] = 0;
2638 					pIO_req->CDB.CDB32[4] = 0;
2639 					pIO_req->CDB.CDB32[5] = 0;
2640 					ptrLBA = &pIO_req->CDB.CDB32[6];
2641 					physLBA_byte = (uint8_t)(physLBA >> 24);
2642 					*ptrLBA = physLBA_byte;
2643 					ptrLBA = &pIO_req->CDB.CDB32[7];
2644 					physLBA_byte = (uint8_t)(physLBA >> 16);
2645 					*ptrLBA = physLBA_byte;
2646 					ptrLBA = &pIO_req->CDB.CDB32[8];
2647 					physLBA_byte = (uint8_t)(physLBA >> 8);
2648 					*ptrLBA = physLBA_byte;
2649 					ptrLBA = &pIO_req->CDB.CDB32[9];
2650 					physLBA_byte = (uint8_t)physLBA;
2651 					*ptrLBA = physLBA_byte;
2652 
2653 					/*
2654 					 * Set flag that Direct Drive I/O is
2655 					 * being done.
2656 					 */
2657 					cm->cm_flags |= MPS_CM_FLAGS_DD_IO;
2658 				}
2659 			}
2660 		}
2661 	}
2662 }
2663 
2664 static void
2665 mpssas_smpio_complete(struct mps_softc *sc, struct mps_command *cm)
2666 {
2667 	MPI2_SMP_PASSTHROUGH_REPLY *rpl;
2668 	MPI2_SMP_PASSTHROUGH_REQUEST *req;
2669 	uint64_t sasaddr;
2670 	union ccb *ccb;
2671 
2672 	ccb = cm->cm_complete_data;
2673 
2674 	/*
2675 	 * Currently there should be no way we can hit this case.  It only
2676 	 * happens when we have a failure to allocate chain frames, and SMP
2677 	 * commands require two S/G elements only.  That should be handled
2678 	 * in the standard request size.
2679 	 */
2680 	if ((cm->cm_flags & MPS_CM_FLAGS_ERROR_MASK) != 0) {
2681 		mps_dprint(sc, MPS_ERROR,"%s: cm_flags = %#x on SMP request!\n",
2682 			   __func__, cm->cm_flags);
2683 		mpssas_set_ccbstatus(ccb, CAM_REQ_CMP_ERR);
2684 		goto bailout;
2685         }
2686 
2687 	rpl = (MPI2_SMP_PASSTHROUGH_REPLY *)cm->cm_reply;
2688 	if (rpl == NULL) {
2689 		mps_dprint(sc, MPS_ERROR, "%s: NULL cm_reply!\n", __func__);
2690 		mpssas_set_ccbstatus(ccb, CAM_REQ_CMP_ERR);
2691 		goto bailout;
2692 	}
2693 
2694 	req = (MPI2_SMP_PASSTHROUGH_REQUEST *)cm->cm_req;
2695 	sasaddr = le32toh(req->SASAddress.Low);
2696 	sasaddr |= ((uint64_t)(le32toh(req->SASAddress.High))) << 32;
2697 
2698 	if ((le16toh(rpl->IOCStatus) & MPI2_IOCSTATUS_MASK) !=
2699 	    MPI2_IOCSTATUS_SUCCESS ||
2700 	    rpl->SASStatus != MPI2_SASSTATUS_SUCCESS) {
2701 		mps_dprint(sc, MPS_XINFO, "%s: IOCStatus %04x SASStatus %02x\n",
2702 		    __func__, le16toh(rpl->IOCStatus), rpl->SASStatus);
2703 		mpssas_set_ccbstatus(ccb, CAM_REQ_CMP_ERR);
2704 		goto bailout;
2705 	}
2706 
2707 	mps_dprint(sc, MPS_XINFO, "%s: SMP request to SAS address "
2708 		   "%#jx completed successfully\n", __func__,
2709 		   (uintmax_t)sasaddr);
2710 
2711 	if (ccb->smpio.smp_response[2] == SMP_FR_ACCEPTED)
2712 		mpssas_set_ccbstatus(ccb, CAM_REQ_CMP);
2713 	else
2714 		mpssas_set_ccbstatus(ccb, CAM_SMP_STATUS_ERROR);
2715 
2716 bailout:
2717 	/*
2718 	 * We sync in both directions because we had DMAs in the S/G list
2719 	 * in both directions.
2720 	 */
2721 	bus_dmamap_sync(sc->buffer_dmat, cm->cm_dmamap,
2722 			BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
2723 	bus_dmamap_unload(sc->buffer_dmat, cm->cm_dmamap);
2724 	mps_free_command(sc, cm);
2725 	xpt_done(ccb);
2726 }
2727 
2728 static void
2729 mpssas_send_smpcmd(struct mpssas_softc *sassc, union ccb *ccb, uint64_t sasaddr)
2730 {
2731 	struct mps_command *cm;
2732 	uint8_t *request, *response;
2733 	MPI2_SMP_PASSTHROUGH_REQUEST *req;
2734 	struct mps_softc *sc;
2735 	int error;
2736 
2737 	sc = sassc->sc;
2738 	error = 0;
2739 
2740 	/*
2741 	 * XXX We don't yet support physical addresses here.
2742 	 */
2743 	switch ((ccb->ccb_h.flags & CAM_DATA_MASK)) {
2744 	case CAM_DATA_PADDR:
2745 	case CAM_DATA_SG_PADDR:
2746 		mps_dprint(sc, MPS_ERROR,
2747 			   "%s: physical addresses not supported\n", __func__);
2748 		mpssas_set_ccbstatus(ccb, CAM_REQ_INVALID);
2749 		xpt_done(ccb);
2750 		return;
2751 	case CAM_DATA_SG:
2752 		/*
2753 		 * The chip does not support more than one buffer for the
2754 		 * request or response.
2755 		 */
2756 	 	if ((ccb->smpio.smp_request_sglist_cnt > 1)
2757 		  || (ccb->smpio.smp_response_sglist_cnt > 1)) {
2758 			mps_dprint(sc, MPS_ERROR,
2759 				   "%s: multiple request or response "
2760 				   "buffer segments not supported for SMP\n",
2761 				   __func__);
2762 			mpssas_set_ccbstatus(ccb, CAM_REQ_INVALID);
2763 			xpt_done(ccb);
2764 			return;
2765 		}
2766 
2767 		/*
2768 		 * The CAM_SCATTER_VALID flag was originally implemented
2769 		 * for the XPT_SCSI_IO CCB, which only has one data pointer.
2770 		 * We have two.  So, just take that flag to mean that we
2771 		 * might have S/G lists, and look at the S/G segment count
2772 		 * to figure out whether that is the case for each individual
2773 		 * buffer.
2774 		 */
2775 		if (ccb->smpio.smp_request_sglist_cnt != 0) {
2776 			bus_dma_segment_t *req_sg;
2777 
2778 			req_sg = (bus_dma_segment_t *)ccb->smpio.smp_request;
2779 			request = (uint8_t *)(uintptr_t)req_sg[0].ds_addr;
2780 		} else
2781 			request = ccb->smpio.smp_request;
2782 
2783 		if (ccb->smpio.smp_response_sglist_cnt != 0) {
2784 			bus_dma_segment_t *rsp_sg;
2785 
2786 			rsp_sg = (bus_dma_segment_t *)ccb->smpio.smp_response;
2787 			response = (uint8_t *)(uintptr_t)rsp_sg[0].ds_addr;
2788 		} else
2789 			response = ccb->smpio.smp_response;
2790 		break;
2791 	case CAM_DATA_VADDR:
2792 		request = ccb->smpio.smp_request;
2793 		response = ccb->smpio.smp_response;
2794 		break;
2795 	default:
2796 		mpssas_set_ccbstatus(ccb, CAM_REQ_INVALID);
2797 		xpt_done(ccb);
2798 		return;
2799 	}
2800 
2801 	cm = mps_alloc_command(sc);
2802 	if (cm == NULL) {
2803 		mps_dprint(sc, MPS_ERROR,
2804 		    "%s: cannot allocate command\n", __func__);
2805 		mpssas_set_ccbstatus(ccb, CAM_RESRC_UNAVAIL);
2806 		xpt_done(ccb);
2807 		return;
2808 	}
2809 
2810 	req = (MPI2_SMP_PASSTHROUGH_REQUEST *)cm->cm_req;
2811 	bzero(req, sizeof(*req));
2812 	req->Function = MPI2_FUNCTION_SMP_PASSTHROUGH;
2813 
2814 	/* Allow the chip to use any route to this SAS address. */
2815 	req->PhysicalPort = 0xff;
2816 
2817 	req->RequestDataLength = htole16(ccb->smpio.smp_request_len);
2818 	req->SGLFlags =
2819 	    MPI2_SGLFLAGS_SYSTEM_ADDRESS_SPACE | MPI2_SGLFLAGS_SGL_TYPE_MPI;
2820 
2821 	mps_dprint(sc, MPS_XINFO, "%s: sending SMP request to SAS "
2822 	    "address %#jx\n", __func__, (uintmax_t)sasaddr);
2823 
2824 	mpi_init_sge(cm, req, &req->SGL);
2825 
2826 	/*
2827 	 * Set up a uio to pass into mps_map_command().  This allows us to
2828 	 * do one map command, and one busdma call in there.
2829 	 */
2830 	cm->cm_uio.uio_iov = cm->cm_iovec;
2831 	cm->cm_uio.uio_iovcnt = 2;
2832 	cm->cm_uio.uio_segflg = UIO_SYSSPACE;
2833 
2834 	/*
2835 	 * The read/write flag isn't used by busdma, but set it just in
2836 	 * case.  This isn't exactly accurate, either, since we're going in
2837 	 * both directions.
2838 	 */
2839 	cm->cm_uio.uio_rw = UIO_WRITE;
2840 
2841 	cm->cm_iovec[0].iov_base = request;
2842 	cm->cm_iovec[0].iov_len = le16toh(req->RequestDataLength);
2843 	cm->cm_iovec[1].iov_base = response;
2844 	cm->cm_iovec[1].iov_len = ccb->smpio.smp_response_len;
2845 
2846 	cm->cm_uio.uio_resid = cm->cm_iovec[0].iov_len +
2847 			       cm->cm_iovec[1].iov_len;
2848 
2849 	/*
2850 	 * Trigger a warning message in mps_data_cb() for the user if we
2851 	 * wind up exceeding two S/G segments.  The chip expects one
2852 	 * segment for the request and another for the response.
2853 	 */
2854 	cm->cm_max_segs = 2;
2855 
2856 	cm->cm_desc.Default.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_DEFAULT_TYPE;
2857 	cm->cm_complete = mpssas_smpio_complete;
2858 	cm->cm_complete_data = ccb;
2859 
2860 	/*
2861 	 * Tell the mapping code that we're using a uio, and that this is
2862 	 * an SMP passthrough request.  There is a little special-case
2863 	 * logic there (in mps_data_cb()) to handle the bidirectional
2864 	 * transfer.
2865 	 */
2866 	cm->cm_flags |= MPS_CM_FLAGS_USE_UIO | MPS_CM_FLAGS_SMP_PASS |
2867 			MPS_CM_FLAGS_DATAIN | MPS_CM_FLAGS_DATAOUT;
2868 
2869 	/* The chip data format is little endian. */
2870 	req->SASAddress.High = htole32(sasaddr >> 32);
2871 	req->SASAddress.Low = htole32(sasaddr);
2872 
2873 	/*
2874 	 * XXX Note that we don't have a timeout/abort mechanism here.
2875 	 * From the manual, it looks like task management requests only
2876 	 * work for SCSI IO and SATA passthrough requests.  We may need to
2877 	 * have a mechanism to retry requests in the event of a chip reset
2878 	 * at least.  Hopefully the chip will insure that any errors short
2879 	 * of that are relayed back to the driver.
2880 	 */
2881 	error = mps_map_command(sc, cm);
2882 	if ((error != 0) && (error != EINPROGRESS)) {
2883 		mps_dprint(sc, MPS_ERROR,
2884 			   "%s: error %d returned from mps_map_command()\n",
2885 			   __func__, error);
2886 		goto bailout_error;
2887 	}
2888 
2889 	return;
2890 
2891 bailout_error:
2892 	mps_free_command(sc, cm);
2893 	mpssas_set_ccbstatus(ccb, CAM_RESRC_UNAVAIL);
2894 	xpt_done(ccb);
2895 	return;
2896 
2897 }
2898 
2899 static void
2900 mpssas_action_smpio(struct mpssas_softc *sassc, union ccb *ccb)
2901 {
2902 	struct mps_softc *sc;
2903 	struct mpssas_target *targ;
2904 	uint64_t sasaddr = 0;
2905 
2906 	sc = sassc->sc;
2907 
2908 	/*
2909 	 * Make sure the target exists.
2910 	 */
2911 	KASSERT(ccb->ccb_h.target_id < sassc->maxtargets,
2912 	    ("Target %d out of bounds in XPT_SMP_IO\n", ccb->ccb_h.target_id));
2913 	targ = &sassc->targets[ccb->ccb_h.target_id];
2914 	if (targ->handle == 0x0) {
2915 		mps_dprint(sc, MPS_ERROR,
2916 			   "%s: target %d does not exist!\n", __func__,
2917 			   ccb->ccb_h.target_id);
2918 		mpssas_set_ccbstatus(ccb, CAM_SEL_TIMEOUT);
2919 		xpt_done(ccb);
2920 		return;
2921 	}
2922 
2923 	/*
2924 	 * If this device has an embedded SMP target, we'll talk to it
2925 	 * directly.
2926 	 * figure out what the expander's address is.
2927 	 */
2928 	if ((targ->devinfo & MPI2_SAS_DEVICE_INFO_SMP_TARGET) != 0)
2929 		sasaddr = targ->sasaddr;
2930 
2931 	/*
2932 	 * If we don't have a SAS address for the expander yet, try
2933 	 * grabbing it from the page 0x83 information cached in the
2934 	 * transport layer for this target.  LSI expanders report the
2935 	 * expander SAS address as the port-associated SAS address in
2936 	 * Inquiry VPD page 0x83.  Maxim expanders don't report it in page
2937 	 * 0x83.
2938 	 *
2939 	 * XXX KDM disable this for now, but leave it commented out so that
2940 	 * it is obvious that this is another possible way to get the SAS
2941 	 * address.
2942 	 *
2943 	 * The parent handle method below is a little more reliable, and
2944 	 * the other benefit is that it works for devices other than SES
2945 	 * devices.  So you can send a SMP request to a da(4) device and it
2946 	 * will get routed to the expander that device is attached to.
2947 	 * (Assuming the da(4) device doesn't contain an SMP target...)
2948 	 */
2949 #if 0
2950 	if (sasaddr == 0)
2951 		sasaddr = xpt_path_sas_addr(ccb->ccb_h.path);
2952 #endif
2953 
2954 	/*
2955 	 * If we still don't have a SAS address for the expander, look for
2956 	 * the parent device of this device, which is probably the expander.
2957 	 */
2958 	if (sasaddr == 0) {
2959 #ifdef OLD_MPS_PROBE
2960 		struct mpssas_target *parent_target;
2961 #endif
2962 
2963 		if (targ->parent_handle == 0x0) {
2964 			mps_dprint(sc, MPS_ERROR,
2965 				   "%s: handle %d does not have a valid "
2966 				   "parent handle!\n", __func__, targ->handle);
2967 			mpssas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE);
2968 			goto bailout;
2969 		}
2970 #ifdef OLD_MPS_PROBE
2971 		parent_target = mpssas_find_target_by_handle(sassc, 0,
2972 			targ->parent_handle);
2973 
2974 		if (parent_target == NULL) {
2975 			mps_dprint(sc, MPS_ERROR,
2976 				   "%s: handle %d does not have a valid "
2977 				   "parent target!\n", __func__, targ->handle);
2978 			mpssas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE);
2979 			goto bailout;
2980 		}
2981 
2982 		if ((parent_target->devinfo &
2983 		     MPI2_SAS_DEVICE_INFO_SMP_TARGET) == 0) {
2984 			mps_dprint(sc, MPS_ERROR,
2985 				   "%s: handle %d parent %d does not "
2986 				   "have an SMP target!\n", __func__,
2987 				   targ->handle, parent_target->handle);
2988 			mpssas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE);
2989 			goto bailout;
2990 		}
2991 
2992 		sasaddr = parent_target->sasaddr;
2993 #else /* OLD_MPS_PROBE */
2994 		if ((targ->parent_devinfo &
2995 		     MPI2_SAS_DEVICE_INFO_SMP_TARGET) == 0) {
2996 			mps_dprint(sc, MPS_ERROR,
2997 				   "%s: handle %d parent %d does not "
2998 				   "have an SMP target!\n", __func__,
2999 				   targ->handle, targ->parent_handle);
3000 			mpssas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE);
3001 			goto bailout;
3002 		}
3003 		if (targ->parent_sasaddr == 0x0) {
3004 			mps_dprint(sc, MPS_ERROR,
3005 				   "%s: handle %d parent handle %d does "
3006 				   "not have a valid SAS address!\n",
3007 				   __func__, targ->handle, targ->parent_handle);
3008 			mpssas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE);
3009 			goto bailout;
3010 		}
3011 
3012 		sasaddr = targ->parent_sasaddr;
3013 #endif /* OLD_MPS_PROBE */
3014 	}
3015 
3016 	if (sasaddr == 0) {
3017 		mps_dprint(sc, MPS_INFO,
3018 			   "%s: unable to find SAS address for handle %d\n",
3019 			   __func__, targ->handle);
3020 		mpssas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE);
3021 		goto bailout;
3022 	}
3023 	mpssas_send_smpcmd(sassc, ccb, sasaddr);
3024 
3025 	return;
3026 
3027 bailout:
3028 	xpt_done(ccb);
3029 
3030 }
3031 
3032 static void
3033 mpssas_action_resetdev(struct mpssas_softc *sassc, union ccb *ccb)
3034 {
3035 	MPI2_SCSI_TASK_MANAGE_REQUEST *req;
3036 	struct mps_softc *sc;
3037 	struct mps_command *tm;
3038 	struct mpssas_target *targ;
3039 
3040 	MPS_FUNCTRACE(sassc->sc);
3041 	mtx_assert(&sassc->sc->mps_mtx, MA_OWNED);
3042 
3043 	KASSERT(ccb->ccb_h.target_id < sassc->maxtargets,
3044 	    ("Target %d out of bounds in XPT_RESET_DEV\n",
3045 	     ccb->ccb_h.target_id));
3046 	sc = sassc->sc;
3047 	tm = mpssas_alloc_tm(sc);
3048 	if (tm == NULL) {
3049 		mps_dprint(sc, MPS_ERROR,
3050 		    "command alloc failure in mpssas_action_resetdev\n");
3051 		mpssas_set_ccbstatus(ccb, CAM_RESRC_UNAVAIL);
3052 		xpt_done(ccb);
3053 		return;
3054 	}
3055 
3056 	targ = &sassc->targets[ccb->ccb_h.target_id];
3057 	req = (MPI2_SCSI_TASK_MANAGE_REQUEST *)tm->cm_req;
3058 	req->DevHandle = htole16(targ->handle);
3059 	req->TaskType = MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET;
3060 
3061 	/* SAS Hard Link Reset / SATA Link Reset */
3062 	req->MsgFlags = MPI2_SCSITASKMGMT_MSGFLAGS_LINK_RESET;
3063 
3064 	tm->cm_data = NULL;
3065 	tm->cm_complete = mpssas_resetdev_complete;
3066 	tm->cm_complete_data = ccb;
3067 	tm->cm_targ = targ;
3068 
3069 	mpssas_prepare_for_tm(sc, tm, targ, CAM_LUN_WILDCARD);
3070 	mps_map_command(sc, tm);
3071 }
3072 
3073 static void
3074 mpssas_resetdev_complete(struct mps_softc *sc, struct mps_command *tm)
3075 {
3076 	MPI2_SCSI_TASK_MANAGE_REPLY *resp;
3077 	union ccb *ccb;
3078 
3079 	MPS_FUNCTRACE(sc);
3080 	mtx_assert(&sc->mps_mtx, MA_OWNED);
3081 
3082 	resp = (MPI2_SCSI_TASK_MANAGE_REPLY *)tm->cm_reply;
3083 	ccb = tm->cm_complete_data;
3084 
3085 	/*
3086 	 * Currently there should be no way we can hit this case.  It only
3087 	 * happens when we have a failure to allocate chain frames, and
3088 	 * task management commands don't have S/G lists.
3089 	 */
3090 	if ((tm->cm_flags & MPS_CM_FLAGS_ERROR_MASK) != 0) {
3091 		MPI2_SCSI_TASK_MANAGE_REQUEST *req;
3092 
3093 		req = (MPI2_SCSI_TASK_MANAGE_REQUEST *)tm->cm_req;
3094 
3095 		mps_dprint(sc, MPS_ERROR,
3096 			   "%s: cm_flags = %#x for reset of handle %#04x! "
3097 			   "This should not happen!\n", __func__, tm->cm_flags,
3098 			   req->DevHandle);
3099 		mpssas_set_ccbstatus(ccb, CAM_REQ_CMP_ERR);
3100 		goto bailout;
3101 	}
3102 
3103 	mps_dprint(sc, MPS_XINFO,
3104 	    "%s: IOCStatus = 0x%x ResponseCode = 0x%x\n", __func__,
3105 	    le16toh(resp->IOCStatus), le32toh(resp->ResponseCode));
3106 
3107 	if (le32toh(resp->ResponseCode) == MPI2_SCSITASKMGMT_RSP_TM_COMPLETE) {
3108 		mpssas_set_ccbstatus(ccb, CAM_REQ_CMP);
3109 		mpssas_announce_reset(sc, AC_SENT_BDR, tm->cm_targ->tid,
3110 		    CAM_LUN_WILDCARD);
3111 	}
3112 	else
3113 		mpssas_set_ccbstatus(ccb, CAM_REQ_CMP_ERR);
3114 
3115 bailout:
3116 
3117 	mpssas_free_tm(sc, tm);
3118 	xpt_done(ccb);
3119 }
3120 
3121 static void
3122 mpssas_poll(struct cam_sim *sim)
3123 {
3124 	struct mpssas_softc *sassc;
3125 
3126 	sassc = cam_sim_softc(sim);
3127 
3128 	if (sassc->sc->mps_debug & MPS_TRACE) {
3129 		/* frequent debug messages during a panic just slow
3130 		 * everything down too much.
3131 		 */
3132 		mps_printf(sassc->sc, "%s clearing MPS_TRACE\n", __func__);
3133 		sassc->sc->mps_debug &= ~MPS_TRACE;
3134 	}
3135 
3136 	mps_intr_locked(sassc->sc);
3137 }
3138 
3139 static void
3140 mpssas_async(void *callback_arg, uint32_t code, struct cam_path *path,
3141 	     void *arg)
3142 {
3143 	struct mps_softc *sc;
3144 
3145 	sc = (struct mps_softc *)callback_arg;
3146 
3147 	mps_lock(sc);
3148 	switch (code) {
3149 	case AC_ADVINFO_CHANGED: {
3150 		struct mpssas_target *target;
3151 		struct mpssas_softc *sassc;
3152 		struct scsi_read_capacity_data_long rcap_buf;
3153 		struct ccb_dev_advinfo cdai;
3154 		struct mpssas_lun *lun;
3155 		lun_id_t lunid;
3156 		int found_lun;
3157 		uintptr_t buftype;
3158 
3159 		buftype = (uintptr_t)arg;
3160 
3161 		found_lun = 0;
3162 		sassc = sc->sassc;
3163 
3164 		/*
3165 		 * We're only interested in read capacity data changes.
3166 		 */
3167 		if (buftype != CDAI_TYPE_RCAPLONG)
3168 			break;
3169 
3170 		/*
3171 		 * We should have a handle for this, but check to make sure.
3172 		 */
3173 		KASSERT(xpt_path_target_id(path) < sassc->maxtargets,
3174 		    ("Target %d out of bounds in mpssas_async\n",
3175 		    xpt_path_target_id(path)));
3176 		target = &sassc->targets[xpt_path_target_id(path)];
3177 		if (target->handle == 0)
3178 			break;
3179 
3180 		lunid = xpt_path_lun_id(path);
3181 
3182 		SLIST_FOREACH(lun, &target->luns, lun_link) {
3183 			if (lun->lun_id == lunid) {
3184 				found_lun = 1;
3185 				break;
3186 			}
3187 		}
3188 
3189 		if (found_lun == 0) {
3190 			lun = malloc(sizeof(struct mpssas_lun), M_MPT2,
3191 				     M_NOWAIT | M_ZERO);
3192 			if (lun == NULL) {
3193 				mps_dprint(sc, MPS_ERROR, "Unable to alloc "
3194 					   "LUN for EEDP support.\n");
3195 				break;
3196 			}
3197 			lun->lun_id = lunid;
3198 			SLIST_INSERT_HEAD(&target->luns, lun, lun_link);
3199 		}
3200 
3201 		bzero(&rcap_buf, sizeof(rcap_buf));
3202 		bzero(&cdai, sizeof(cdai));
3203 		xpt_setup_ccb(&cdai.ccb_h, path, CAM_PRIORITY_NORMAL);
3204 		cdai.ccb_h.func_code = XPT_DEV_ADVINFO;
3205 		cdai.ccb_h.flags = CAM_DIR_IN;
3206 		cdai.buftype = CDAI_TYPE_RCAPLONG;
3207 		cdai.flags = CDAI_FLAG_NONE;
3208 		cdai.bufsiz = sizeof(rcap_buf);
3209 		cdai.buf = (uint8_t *)&rcap_buf;
3210 		xpt_action((union ccb *)&cdai);
3211 		if ((cdai.ccb_h.status & CAM_DEV_QFRZN) != 0)
3212 			cam_release_devq(cdai.ccb_h.path,
3213 					 0, 0, 0, FALSE);
3214 
3215 		if ((mpssas_get_ccbstatus((union ccb *)&cdai) == CAM_REQ_CMP)
3216 		 && (rcap_buf.prot & SRC16_PROT_EN)) {
3217 			switch (rcap_buf.prot & SRC16_P_TYPE) {
3218 			case SRC16_PTYPE_1:
3219 			case SRC16_PTYPE_3:
3220 				lun->eedp_formatted = TRUE;
3221 				lun->eedp_block_size =
3222 				    scsi_4btoul(rcap_buf.length);
3223 				break;
3224 			case SRC16_PTYPE_2:
3225 			default:
3226 				lun->eedp_formatted = FALSE;
3227 				lun->eedp_block_size = 0;
3228 				break;
3229 			}
3230 		} else {
3231 			lun->eedp_formatted = FALSE;
3232 			lun->eedp_block_size = 0;
3233 		}
3234 		break;
3235 	}
3236 	default:
3237 		break;
3238 	}
3239 	mps_unlock(sc);
3240 }
3241 
3242 /*
3243  * Freeze the devq and set the INRESET flag so that no I/O will be sent to
3244  * the target until the reset has completed.  The CCB holds the path which
3245  * is used to release the devq.  The devq is released and the CCB is freed
3246  * when the TM completes.
3247  * We only need to do this when we're entering reset, not at each time we
3248  * need to send an abort (which will happen if multiple commands timeout
3249  * while we're sending the abort). We do not release the queue for each
3250  * command we complete (just at the end when we free the tm), so freezing
3251  * it each time doesn't make sense.
3252  */
3253 void
3254 mpssas_prepare_for_tm(struct mps_softc *sc, struct mps_command *tm,
3255     struct mpssas_target *target, lun_id_t lun_id)
3256 {
3257 	union ccb *ccb;
3258 	path_id_t path_id;
3259 
3260 	ccb = xpt_alloc_ccb_nowait();
3261 	if (ccb) {
3262 		path_id = cam_sim_path(sc->sassc->sim);
3263 		if (xpt_create_path(&ccb->ccb_h.path, xpt_periph, path_id,
3264 		    target->tid, lun_id) != CAM_REQ_CMP) {
3265 			xpt_free_ccb(ccb);
3266 		} else {
3267 			tm->cm_ccb = ccb;
3268 			tm->cm_targ = target;
3269 			if ((target->flags & MPSSAS_TARGET_INRESET) == 0) {
3270 				mps_dprint(sc, MPS_XINFO | MPS_RECOVERY,
3271 				    "%s: Freezing devq for target ID %d\n",
3272 				    __func__, target->tid);
3273 				xpt_freeze_devq(ccb->ccb_h.path, 1);
3274 				target->flags |= MPSSAS_TARGET_INRESET;
3275 			}
3276 		}
3277 	}
3278 }
3279 
3280 int
3281 mpssas_startup(struct mps_softc *sc)
3282 {
3283 
3284 	/*
3285 	 * Send the port enable message and set the wait_for_port_enable flag.
3286 	 * This flag helps to keep the simq frozen until all discovery events
3287 	 * are processed.
3288 	 */
3289 	sc->wait_for_port_enable = 1;
3290 	mpssas_send_portenable(sc);
3291 	return (0);
3292 }
3293 
3294 static int
3295 mpssas_send_portenable(struct mps_softc *sc)
3296 {
3297 	MPI2_PORT_ENABLE_REQUEST *request;
3298 	struct mps_command *cm;
3299 
3300 	MPS_FUNCTRACE(sc);
3301 
3302 	if ((cm = mps_alloc_command(sc)) == NULL)
3303 		return (EBUSY);
3304 	request = (MPI2_PORT_ENABLE_REQUEST *)cm->cm_req;
3305 	request->Function = MPI2_FUNCTION_PORT_ENABLE;
3306 	request->MsgFlags = 0;
3307 	request->VP_ID = 0;
3308 	cm->cm_desc.Default.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_DEFAULT_TYPE;
3309 	cm->cm_complete = mpssas_portenable_complete;
3310 	cm->cm_data = NULL;
3311 	cm->cm_sge = NULL;
3312 
3313 	mps_map_command(sc, cm);
3314 	mps_dprint(sc, MPS_XINFO,
3315 	    "mps_send_portenable finished cm %p req %p complete %p\n",
3316 	    cm, cm->cm_req, cm->cm_complete);
3317 	return (0);
3318 }
3319 
3320 static void
3321 mpssas_portenable_complete(struct mps_softc *sc, struct mps_command *cm)
3322 {
3323 	MPI2_PORT_ENABLE_REPLY *reply;
3324 	struct mpssas_softc *sassc;
3325 
3326 	MPS_FUNCTRACE(sc);
3327 	sassc = sc->sassc;
3328 
3329 	/*
3330 	 * Currently there should be no way we can hit this case.  It only
3331 	 * happens when we have a failure to allocate chain frames, and
3332 	 * port enable commands don't have S/G lists.
3333 	 */
3334 	if ((cm->cm_flags & MPS_CM_FLAGS_ERROR_MASK) != 0) {
3335 		mps_dprint(sc, MPS_ERROR, "%s: cm_flags = %#x for port enable! "
3336 			   "This should not happen!\n", __func__, cm->cm_flags);
3337 	}
3338 
3339 	reply = (MPI2_PORT_ENABLE_REPLY *)cm->cm_reply;
3340 	if (reply == NULL)
3341 		mps_dprint(sc, MPS_FAULT, "Portenable NULL reply\n");
3342 	else if (le16toh(reply->IOCStatus & MPI2_IOCSTATUS_MASK) !=
3343 	    MPI2_IOCSTATUS_SUCCESS)
3344 		mps_dprint(sc, MPS_FAULT, "Portenable failed\n");
3345 
3346 	mps_free_command(sc, cm);
3347 
3348 	/*
3349 	 * Get WarpDrive info after discovery is complete but before the scan
3350 	 * starts.  At this point, all devices are ready to be exposed to the
3351 	 * OS.  If devices should be hidden instead, take them out of the
3352 	 * 'targets' array before the scan.  The devinfo for a disk will have
3353 	 * some info and a volume's will be 0.  Use that to remove disks.
3354 	 */
3355 	mps_wd_config_pages(sc);
3356 
3357 	/*
3358 	 * Done waiting for port enable to complete.  Decrement the refcount.
3359 	 * If refcount is 0, discovery is complete and a rescan of the bus can
3360 	 * take place.  Since the simq was explicitly frozen before port
3361 	 * enable, it must be explicitly released here to keep the
3362 	 * freeze/release count in sync.
3363 	 */
3364 	sc->wait_for_port_enable = 0;
3365 	sc->port_enable_complete = 1;
3366 	wakeup(&sc->port_enable_complete);
3367 	mpssas_startup_decrement(sassc);
3368 }
3369 
3370 int
3371 mpssas_check_id(struct mpssas_softc *sassc, int id)
3372 {
3373 	struct mps_softc *sc = sassc->sc;
3374 	char *ids;
3375 	char *name;
3376 
3377 	ids = &sc->exclude_ids[0];
3378 	while((name = strsep(&ids, ",")) != NULL) {
3379 		if (name[0] == '\0')
3380 			continue;
3381 		if (strtol(name, NULL, 0) == (long)id)
3382 			return (1);
3383 	}
3384 
3385 	return (0);
3386 }
3387 
3388 void
3389 mpssas_realloc_targets(struct mps_softc *sc, int maxtargets)
3390 {
3391 	struct mpssas_softc *sassc;
3392 	struct mpssas_lun *lun, *lun_tmp;
3393 	struct mpssas_target *targ;
3394 	int i;
3395 
3396 	sassc = sc->sassc;
3397 	/*
3398 	 * The number of targets is based on IOC Facts, so free all of
3399 	 * the allocated LUNs for each target and then the target buffer
3400 	 * itself.
3401 	 */
3402 	for (i=0; i< maxtargets; i++) {
3403 		targ = &sassc->targets[i];
3404 		SLIST_FOREACH_SAFE(lun, &targ->luns, lun_link, lun_tmp) {
3405 			free(lun, M_MPT2);
3406 		}
3407 	}
3408 	free(sassc->targets, M_MPT2);
3409 
3410 	sassc->targets = malloc(sizeof(struct mpssas_target) * maxtargets,
3411 	    M_MPT2, M_WAITOK|M_ZERO);
3412 }
3413