xref: /freebsd/sys/dev/mrsas/mrsas_cam.c (revision 7877fdebeeb35fad1cbbafce22598b1bdf97c786)
1 /*
2  * Copyright (c) 2015, AVAGO Tech. All rights reserved. Author: Marian Choy
3  * Copyright (c) 2014, LSI Corp. All rights reserved. Author: Marian Choy
4  * Support: freebsdraid@avagotech.com
5  *
6  * Redistribution and use in source and binary forms, with or without
7  * modification, are permitted provided that the following conditions are
8  * met:
9  *
10  * 1. Redistributions of source code must retain the above copyright notice,
11  * this list of conditions and the following disclaimer. 2. Redistributions
12  * in binary form must reproduce the above copyright notice, this list of
13  * conditions and the following disclaimer in the documentation and/or other
14  * materials provided with the distribution. 3. Neither the name of the
15  * <ORGANIZATION> nor the names of its contributors may be used to endorse or
16  * promote products derived from this software without specific prior written
17  * permission.
18  *
19  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
20  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22  * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
23  * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29  * POSSIBILITY OF SUCH DAMAGE.
30  *
31  */
32 
33 #include <sys/cdefs.h>
34 __FBSDID("$FreeBSD$");
35 
36 #include "dev/mrsas/mrsas.h"
37 
38 #include <cam/cam.h>
39 #include <cam/cam_ccb.h>
40 #include <cam/cam_sim.h>
41 #include <cam/cam_xpt_sim.h>
42 #include <cam/cam_debug.h>
43 #include <cam/cam_periph.h>
44 #include <cam/cam_xpt_periph.h>
45 
46 #include <cam/scsi/scsi_all.h>
47 #include <cam/scsi/scsi_message.h>
48 #include <sys/taskqueue.h>
49 #include <sys/kernel.h>
50 
51 #include <sys/time.h>			/* XXX for pcpu.h */
52 #include <sys/pcpu.h>			/* XXX for PCPU_GET */
53 
54 #define	smp_processor_id()  PCPU_GET(cpuid)
55 
56 /*
57  * Function prototypes
58  */
59 int	mrsas_cam_attach(struct mrsas_softc *sc);
60 int	mrsas_find_io_type(struct cam_sim *sim, union ccb *ccb);
61 int	mrsas_bus_scan(struct mrsas_softc *sc);
62 int	mrsas_bus_scan_sim(struct mrsas_softc *sc, struct cam_sim *sim);
63 int
64 mrsas_map_request(struct mrsas_softc *sc,
65     struct mrsas_mpt_cmd *cmd, union ccb *ccb);
66 int
67 mrsas_build_ldio_rw(struct mrsas_softc *sc, struct mrsas_mpt_cmd *cmd,
68     union ccb *ccb);
69 int
70 mrsas_build_ldio_nonrw(struct mrsas_softc *sc, struct mrsas_mpt_cmd *cmd,
71     union ccb *ccb);
72 int
73 mrsas_build_syspdio(struct mrsas_softc *sc, struct mrsas_mpt_cmd *cmd,
74     union ccb *ccb, struct cam_sim *sim, u_int8_t fp_possible);
75 int
76 mrsas_setup_io(struct mrsas_softc *sc, struct mrsas_mpt_cmd *cmd,
77     union ccb *ccb, u_int32_t device_id,
78     MRSAS_RAID_SCSI_IO_REQUEST * io_request);
79 void	mrsas_xpt_freeze(struct mrsas_softc *sc);
80 void	mrsas_xpt_release(struct mrsas_softc *sc);
81 void	mrsas_cam_detach(struct mrsas_softc *sc);
82 void	mrsas_release_mpt_cmd(struct mrsas_mpt_cmd *cmd);
83 void	mrsas_unmap_request(struct mrsas_softc *sc, struct mrsas_mpt_cmd *cmd);
84 void	mrsas_cmd_done(struct mrsas_softc *sc, struct mrsas_mpt_cmd *cmd);
85 void
86 mrsas_fire_cmd(struct mrsas_softc *sc, u_int32_t req_desc_lo,
87     u_int32_t req_desc_hi);
88 void
89 mrsas_set_pd_lba(MRSAS_RAID_SCSI_IO_REQUEST * io_request,
90     u_int8_t cdb_len, struct IO_REQUEST_INFO *io_info, union ccb *ccb,
91     MR_DRV_RAID_MAP_ALL * local_map_ptr, u_int32_t ref_tag,
92     u_int32_t ld_block_size);
93 static void mrsas_freeze_simq(struct mrsas_mpt_cmd *cmd, struct cam_sim *sim);
94 static void mrsas_cam_poll(struct cam_sim *sim);
95 static void mrsas_action(struct cam_sim *sim, union ccb *ccb);
96 static void mrsas_scsiio_timeout(void *data);
97 static int mrsas_track_scsiio(struct mrsas_softc *sc, target_id_t id, u_int32_t bus_id);
98 static void mrsas_tm_response_code(struct mrsas_softc *sc,
99     MPI2_SCSI_TASK_MANAGE_REPLY *mpi_reply);
100 static int mrsas_issue_tm(struct mrsas_softc *sc,
101     MRSAS_REQUEST_DESCRIPTOR_UNION *req_desc);
102 static void
103 mrsas_data_load_cb(void *arg, bus_dma_segment_t *segs,
104     int nseg, int error);
105 static int32_t
106 mrsas_startio(struct mrsas_softc *sc, struct cam_sim *sim,
107     union ccb *ccb);
108 
109 static boolean_t mrsas_is_prp_possible(struct mrsas_mpt_cmd *cmd,
110 	bus_dma_segment_t *segs, int nsegs);
111 static void mrsas_build_ieee_sgl(struct mrsas_mpt_cmd *cmd,
112 	bus_dma_segment_t *segs, int nseg);
113 static void mrsas_build_prp_nvme(struct mrsas_mpt_cmd *cmd,
114 	bus_dma_segment_t *segs, int nseg);
115 
116 struct mrsas_mpt_cmd *mrsas_get_mpt_cmd(struct mrsas_softc *sc);
117 MRSAS_REQUEST_DESCRIPTOR_UNION *
118 	mrsas_get_request_desc(struct mrsas_softc *sc, u_int16_t index);
119 
120 extern int mrsas_reset_targets(struct mrsas_softc *sc);
121 extern u_int16_t MR_TargetIdToLdGet(u_int32_t ldTgtId, MR_DRV_RAID_MAP_ALL * map);
122 extern u_int32_t
123 MR_LdBlockSizeGet(u_int32_t ldTgtId, MR_DRV_RAID_MAP_ALL * map);
124 extern void mrsas_isr(void *arg);
125 extern void mrsas_aen_handler(struct mrsas_softc *sc);
126 extern u_int8_t
127 MR_BuildRaidContext(struct mrsas_softc *sc,
128     struct IO_REQUEST_INFO *io_info, RAID_CONTEXT * pRAID_Context,
129     MR_DRV_RAID_MAP_ALL * map);
130 extern u_int16_t
131 MR_LdSpanArrayGet(u_int32_t ld, u_int32_t span,
132     MR_DRV_RAID_MAP_ALL * map);
133 extern u_int16_t
134 mrsas_get_updated_dev_handle(struct mrsas_softc *sc,
135     PLD_LOAD_BALANCE_INFO lbInfo, struct IO_REQUEST_INFO *io_info);
136 extern int mrsas_complete_cmd(struct mrsas_softc *sc, u_int32_t MSIxIndex);
137 extern MR_LD_RAID *MR_LdRaidGet(u_int32_t ld, MR_DRV_RAID_MAP_ALL * map);
138 extern void mrsas_disable_intr(struct mrsas_softc *sc);
139 extern void mrsas_enable_intr(struct mrsas_softc *sc);
140 void mrsas_prepare_secondRaid1_IO(struct mrsas_softc *sc,
141     struct mrsas_mpt_cmd *cmd);
142 
143 /*
144  * mrsas_cam_attach:	Main entry to CAM subsystem
145  * input:				Adapter instance soft state
146  *
147  * This function is called from mrsas_attach() during initialization to perform
148  * SIM allocations and XPT bus registration.  If the kernel version is 7.4 or
149  * earlier, it would also initiate a bus scan.
150  */
151 int
152 mrsas_cam_attach(struct mrsas_softc *sc)
153 {
154 	struct cam_devq *devq;
155 	int mrsas_cam_depth;
156 
157 	mrsas_cam_depth = sc->max_scsi_cmds;
158 
159 	if ((devq = cam_simq_alloc(mrsas_cam_depth)) == NULL) {
160 		device_printf(sc->mrsas_dev, "Cannot allocate SIM queue\n");
161 		return (ENOMEM);
162 	}
163 	/*
164 	 * Create SIM for bus 0 and register, also create path
165 	 */
166 	sc->sim_0 = cam_sim_alloc(mrsas_action, mrsas_cam_poll, "mrsas", sc,
167 	    device_get_unit(sc->mrsas_dev), &sc->sim_lock, mrsas_cam_depth,
168 	    mrsas_cam_depth, devq);
169 	if (sc->sim_0 == NULL) {
170 		cam_simq_free(devq);
171 		device_printf(sc->mrsas_dev, "Cannot register SIM\n");
172 		return (ENXIO);
173 	}
174 	/* Initialize taskqueue for Event Handling */
175 	TASK_INIT(&sc->ev_task, 0, (void *)mrsas_aen_handler, sc);
176 	sc->ev_tq = taskqueue_create("mrsas_taskq", M_NOWAIT | M_ZERO,
177 	    taskqueue_thread_enqueue, &sc->ev_tq);
178 
179 	/* Run the task queue with lowest priority */
180 	taskqueue_start_threads(&sc->ev_tq, 1, 255, "%s taskq",
181 	    device_get_nameunit(sc->mrsas_dev));
182 	mtx_lock(&sc->sim_lock);
183 	if (xpt_bus_register(sc->sim_0, sc->mrsas_dev, 0) != CAM_SUCCESS) {
184 		cam_sim_free(sc->sim_0, TRUE);	/* passing true frees the devq */
185 		mtx_unlock(&sc->sim_lock);
186 		return (ENXIO);
187 	}
188 	if (xpt_create_path(&sc->path_0, NULL, cam_sim_path(sc->sim_0),
189 	    CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD) != CAM_REQ_CMP) {
190 		xpt_bus_deregister(cam_sim_path(sc->sim_0));
191 		cam_sim_free(sc->sim_0, TRUE);	/* passing true will free the
192 						 * devq */
193 		mtx_unlock(&sc->sim_lock);
194 		return (ENXIO);
195 	}
196 	mtx_unlock(&sc->sim_lock);
197 
198 	/*
199 	 * Create SIM for bus 1 and register, also create path
200 	 */
201 	sc->sim_1 = cam_sim_alloc(mrsas_action, mrsas_cam_poll, "mrsas", sc,
202 	    device_get_unit(sc->mrsas_dev), &sc->sim_lock, mrsas_cam_depth,
203 	    mrsas_cam_depth, devq);
204 	if (sc->sim_1 == NULL) {
205 		cam_simq_free(devq);
206 		device_printf(sc->mrsas_dev, "Cannot register SIM\n");
207 		return (ENXIO);
208 	}
209 	mtx_lock(&sc->sim_lock);
210 	if (xpt_bus_register(sc->sim_1, sc->mrsas_dev, 1) != CAM_SUCCESS) {
211 		cam_sim_free(sc->sim_1, TRUE);	/* passing true frees the devq */
212 		mtx_unlock(&sc->sim_lock);
213 		return (ENXIO);
214 	}
215 	if (xpt_create_path(&sc->path_1, NULL, cam_sim_path(sc->sim_1),
216 	    CAM_TARGET_WILDCARD,
217 	    CAM_LUN_WILDCARD) != CAM_REQ_CMP) {
218 		xpt_bus_deregister(cam_sim_path(sc->sim_1));
219 		cam_sim_free(sc->sim_1, TRUE);
220 		mtx_unlock(&sc->sim_lock);
221 		return (ENXIO);
222 	}
223 	mtx_unlock(&sc->sim_lock);
224 
225 #if (__FreeBSD_version <= 704000)
226 	if (mrsas_bus_scan(sc)) {
227 		device_printf(sc->mrsas_dev, "Error in bus scan.\n");
228 		return (1);
229 	}
230 #endif
231 	return (0);
232 }
233 
234 /*
235  * mrsas_cam_detach:	De-allocates and teardown CAM
236  * input:				Adapter instance soft state
237  *
238  * De-registers and frees the paths and SIMs.
239  */
240 void
241 mrsas_cam_detach(struct mrsas_softc *sc)
242 {
243 	if (sc->ev_tq != NULL)
244 		taskqueue_free(sc->ev_tq);
245 	mtx_lock(&sc->sim_lock);
246 	if (sc->path_0)
247 		xpt_free_path(sc->path_0);
248 	if (sc->sim_0) {
249 		xpt_bus_deregister(cam_sim_path(sc->sim_0));
250 		cam_sim_free(sc->sim_0, FALSE);
251 	}
252 	if (sc->path_1)
253 		xpt_free_path(sc->path_1);
254 	if (sc->sim_1) {
255 		xpt_bus_deregister(cam_sim_path(sc->sim_1));
256 		cam_sim_free(sc->sim_1, TRUE);
257 	}
258 	mtx_unlock(&sc->sim_lock);
259 }
260 
261 /*
262  * mrsas_action:	SIM callback entry point
263  * input:			pointer to SIM pointer to CAM Control Block
264  *
265  * This function processes CAM subsystem requests. The type of request is stored
266  * in ccb->ccb_h.func_code.  The preprocessor #ifdef is necessary because
267  * ccb->cpi.maxio is not supported for FreeBSD version 7.4 or earlier.
268  */
269 static void
270 mrsas_action(struct cam_sim *sim, union ccb *ccb)
271 {
272 	struct mrsas_softc *sc = (struct mrsas_softc *)cam_sim_softc(sim);
273 	struct ccb_hdr *ccb_h = &(ccb->ccb_h);
274 	u_int32_t device_id;
275 
276 	/*
277      * Check if the system going down
278      * or the adapter is in unrecoverable critical error
279      */
280     if (sc->remove_in_progress ||
281         (sc->adprecovery == MRSAS_HW_CRITICAL_ERROR)) {
282         ccb->ccb_h.status |= CAM_DEV_NOT_THERE;
283         xpt_done(ccb);
284         return;
285     }
286 
287 	switch (ccb->ccb_h.func_code) {
288 	case XPT_SCSI_IO:
289 		{
290 			device_id = ccb_h->target_id;
291 
292 			/*
293 			 * bus 0 is LD, bus 1 is for system-PD
294 			 */
295 			if (cam_sim_bus(sim) == 1 &&
296 			    sc->pd_list[device_id].driveState != MR_PD_STATE_SYSTEM) {
297 				ccb->ccb_h.status |= CAM_DEV_NOT_THERE;
298 				xpt_done(ccb);
299 			} else {
300 				if (mrsas_startio(sc, sim, ccb)) {
301 					ccb->ccb_h.status |= CAM_REQ_INVALID;
302 					xpt_done(ccb);
303 				}
304 			}
305 			break;
306 		}
307 	case XPT_ABORT:
308 		{
309 			ccb->ccb_h.status = CAM_UA_ABORT;
310 			xpt_done(ccb);
311 			break;
312 		}
313 	case XPT_RESET_BUS:
314 		{
315 			xpt_done(ccb);
316 			break;
317 		}
318 	case XPT_GET_TRAN_SETTINGS:
319 		{
320 			ccb->cts.protocol = PROTO_SCSI;
321 			ccb->cts.protocol_version = SCSI_REV_2;
322 			ccb->cts.transport = XPORT_SPI;
323 			ccb->cts.transport_version = 2;
324 			ccb->cts.xport_specific.spi.valid = CTS_SPI_VALID_DISC;
325 			ccb->cts.xport_specific.spi.flags = CTS_SPI_FLAGS_DISC_ENB;
326 			ccb->cts.proto_specific.scsi.valid = CTS_SCSI_VALID_TQ;
327 			ccb->cts.proto_specific.scsi.flags = CTS_SCSI_FLAGS_TAG_ENB;
328 			ccb->ccb_h.status = CAM_REQ_CMP;
329 			xpt_done(ccb);
330 			break;
331 		}
332 	case XPT_SET_TRAN_SETTINGS:
333 		{
334 			ccb->ccb_h.status = CAM_FUNC_NOTAVAIL;
335 			xpt_done(ccb);
336 			break;
337 		}
338 	case XPT_CALC_GEOMETRY:
339 		{
340 			cam_calc_geometry(&ccb->ccg, 1);
341 			xpt_done(ccb);
342 			break;
343 		}
344 	case XPT_PATH_INQ:
345 		{
346 			ccb->cpi.version_num = 1;
347 			ccb->cpi.hba_inquiry = 0;
348 			ccb->cpi.target_sprt = 0;
349 #if (__FreeBSD_version >= 902001)
350 			ccb->cpi.hba_misc = PIM_UNMAPPED;
351 #else
352 			ccb->cpi.hba_misc = 0;
353 #endif
354 			ccb->cpi.hba_eng_cnt = 0;
355 			ccb->cpi.max_lun = MRSAS_SCSI_MAX_LUNS;
356 			ccb->cpi.unit_number = cam_sim_unit(sim);
357 			ccb->cpi.bus_id = cam_sim_bus(sim);
358 			ccb->cpi.initiator_id = MRSAS_SCSI_INITIATOR_ID;
359 			ccb->cpi.base_transfer_speed = 150000;
360 			strlcpy(ccb->cpi.sim_vid, "FreeBSD", SIM_IDLEN);
361 			strlcpy(ccb->cpi.hba_vid, "AVAGO", HBA_IDLEN);
362 			strlcpy(ccb->cpi.dev_name, cam_sim_name(sim), DEV_IDLEN);
363 			ccb->cpi.transport = XPORT_SPI;
364 			ccb->cpi.transport_version = 2;
365 			ccb->cpi.protocol = PROTO_SCSI;
366 			ccb->cpi.protocol_version = SCSI_REV_2;
367 			if (ccb->cpi.bus_id == 0)
368 				ccb->cpi.max_target = MRSAS_MAX_PD - 1;
369 			else
370 				ccb->cpi.max_target = MRSAS_MAX_LD_IDS - 1;
371 #if (__FreeBSD_version > 704000)
372 			ccb->cpi.maxio = sc->max_num_sge * MRSAS_PAGE_SIZE;
373 #endif
374 			ccb->ccb_h.status = CAM_REQ_CMP;
375 			xpt_done(ccb);
376 			break;
377 		}
378 	default:
379 		{
380 			ccb->ccb_h.status = CAM_REQ_INVALID;
381 			xpt_done(ccb);
382 			break;
383 		}
384 	}
385 }
386 
387 /*
388  * mrsas_scsiio_timeout:	Callback function for IO timed out
389  * input:					mpt command context
390  *
391  * This function will execute after timeout value provided by ccb header from
392  * CAM layer, if timer expires. Driver will run timer for all DCDM and LDIO
393  * coming from CAM layer. This function is callback function for IO timeout
394  * and it runs in no-sleep context. Set do_timedout_reset in Adapter context
395  * so that it will execute OCR/Kill adpter from ocr_thread context.
396  */
397 static void
398 mrsas_scsiio_timeout(void *data)
399 {
400 	struct mrsas_mpt_cmd *cmd;
401 	struct mrsas_softc *sc;
402 	u_int32_t target_id;
403 
404 	if (!data)
405 		return;
406 
407 	cmd = (struct mrsas_mpt_cmd *)data;
408 	sc = cmd->sc;
409 
410 	if (cmd->ccb_ptr == NULL) {
411 		printf("command timeout with NULL ccb\n");
412 		return;
413 	}
414 
415 	/*
416 	 * Below callout is dummy entry so that it will be cancelled from
417 	 * mrsas_cmd_done(). Now Controller will go to OCR/Kill Adapter based
418 	 * on OCR enable/disable property of Controller from ocr_thread
419 	 * context.
420 	 */
421 #if (__FreeBSD_version >= 1000510)
422 	callout_reset_sbt(&cmd->cm_callout, SBT_1S * 180, 0,
423 	    mrsas_scsiio_timeout, cmd, 0);
424 #else
425 	callout_reset(&cmd->cm_callout, (180000 * hz) / 1000,
426 	    mrsas_scsiio_timeout, cmd);
427 #endif
428 
429 	if (cmd->ccb_ptr->cpi.bus_id == 0)
430 		target_id = cmd->ccb_ptr->ccb_h.target_id;
431 	else
432 		target_id = (cmd->ccb_ptr->ccb_h.target_id + (MRSAS_MAX_PD - 1));
433 
434 	/* Save the cmd to be processed for TM, if it is not there in the array */
435 	if (sc->target_reset_pool[target_id] == NULL) {
436 		sc->target_reset_pool[target_id] = cmd;
437 		mrsas_atomic_inc(&sc->target_reset_outstanding);
438 	}
439 
440 	return;
441 }
442 
443 /*
444  * mrsas_startio:	SCSI IO entry point
445  * input:			Adapter instance soft state
446  * 					pointer to CAM Control Block
447  *
448  * This function is the SCSI IO entry point and it initiates IO processing. It
449  * copies the IO and depending if the IO is read/write or inquiry, it would
450  * call mrsas_build_ldio() or mrsas_build_dcdb(), respectively.  It returns 0
451  * if the command is sent to firmware successfully, otherwise it returns 1.
452  */
453 static int32_t
454 mrsas_startio(struct mrsas_softc *sc, struct cam_sim *sim,
455     union ccb *ccb)
456 {
457 	struct mrsas_mpt_cmd *cmd, *r1_cmd = NULL;
458 	struct ccb_hdr *ccb_h = &(ccb->ccb_h);
459 	struct ccb_scsiio *csio = &(ccb->csio);
460 	MRSAS_REQUEST_DESCRIPTOR_UNION *req_desc;
461 	u_int8_t cmd_type;
462 
463 	if ((csio->cdb_io.cdb_bytes[0]) == SYNCHRONIZE_CACHE &&
464 		(!sc->fw_sync_cache_support)) {
465 		ccb->ccb_h.status = CAM_REQ_CMP;
466 		xpt_done(ccb);
467 		return (0);
468 	}
469 	ccb_h->status |= CAM_SIM_QUEUED;
470 
471 	if (mrsas_atomic_inc_return(&sc->fw_outstanding) > sc->max_scsi_cmds) {
472 		ccb_h->status |= CAM_REQUEUE_REQ;
473 		xpt_done(ccb);
474 		mrsas_atomic_dec(&sc->fw_outstanding);
475 		return (0);
476 	}
477 
478 	cmd = mrsas_get_mpt_cmd(sc);
479 
480 	if (!cmd) {
481 		ccb_h->status |= CAM_REQUEUE_REQ;
482 		xpt_done(ccb);
483 		mrsas_atomic_dec(&sc->fw_outstanding);
484 		return (0);
485 	}
486 
487 	if ((ccb_h->flags & CAM_DIR_MASK) != CAM_DIR_NONE) {
488 		if (ccb_h->flags & CAM_DIR_IN)
489 			cmd->flags |= MRSAS_DIR_IN;
490 		if (ccb_h->flags & CAM_DIR_OUT)
491 			cmd->flags |= MRSAS_DIR_OUT;
492 	} else
493 		cmd->flags = MRSAS_DIR_NONE;	/* no data */
494 
495 /* For FreeBSD 9.2 and higher */
496 #if (__FreeBSD_version >= 902001)
497 	/*
498 	 * XXX We don't yet support physical addresses here.
499 	 */
500 	switch ((ccb->ccb_h.flags & CAM_DATA_MASK)) {
501 	case CAM_DATA_PADDR:
502 	case CAM_DATA_SG_PADDR:
503 		device_printf(sc->mrsas_dev, "%s: physical addresses not supported\n",
504 		    __func__);
505 		mrsas_release_mpt_cmd(cmd);
506 		ccb_h->status = CAM_REQ_INVALID;
507 		ccb_h->status &= ~CAM_SIM_QUEUED;
508 		goto done;
509 	case CAM_DATA_SG:
510 		device_printf(sc->mrsas_dev, "%s: scatter gather is not supported\n",
511 		    __func__);
512 		mrsas_release_mpt_cmd(cmd);
513 		ccb_h->status = CAM_REQ_INVALID;
514 		goto done;
515 	case CAM_DATA_VADDR:
516 		if (csio->dxfer_len > (sc->max_num_sge * MRSAS_PAGE_SIZE)) {
517 			mrsas_release_mpt_cmd(cmd);
518 			ccb_h->status = CAM_REQ_TOO_BIG;
519 			goto done;
520 		}
521 		cmd->length = csio->dxfer_len;
522 		if (cmd->length)
523 			cmd->data = csio->data_ptr;
524 		break;
525 	case CAM_DATA_BIO:
526 		if (csio->dxfer_len > (sc->max_num_sge * MRSAS_PAGE_SIZE)) {
527 			mrsas_release_mpt_cmd(cmd);
528 			ccb_h->status = CAM_REQ_TOO_BIG;
529 			goto done;
530 		}
531 		cmd->length = csio->dxfer_len;
532 		if (cmd->length)
533 			cmd->data = csio->data_ptr;
534 		break;
535 	default:
536 		ccb->ccb_h.status = CAM_REQ_INVALID;
537 		goto done;
538 	}
539 #else
540 	if (!(ccb_h->flags & CAM_DATA_PHYS)) {	/* Virtual data address */
541 		if (!(ccb_h->flags & CAM_SCATTER_VALID)) {
542 			if (csio->dxfer_len > (sc->max_num_sge * MRSAS_PAGE_SIZE)) {
543 				mrsas_release_mpt_cmd(cmd);
544 				ccb_h->status = CAM_REQ_TOO_BIG;
545 				goto done;
546 			}
547 			cmd->length = csio->dxfer_len;
548 			if (cmd->length)
549 				cmd->data = csio->data_ptr;
550 		} else {
551 			mrsas_release_mpt_cmd(cmd);
552 			ccb_h->status = CAM_REQ_INVALID;
553 			goto done;
554 		}
555 	} else {			/* Data addresses are physical. */
556 		mrsas_release_mpt_cmd(cmd);
557 		ccb_h->status = CAM_REQ_INVALID;
558 		ccb_h->status &= ~CAM_SIM_QUEUED;
559 		goto done;
560 	}
561 #endif
562 	/* save ccb ptr */
563 	cmd->ccb_ptr = ccb;
564 
565 	req_desc = mrsas_get_request_desc(sc, (cmd->index) - 1);
566 	if (!req_desc) {
567 		device_printf(sc->mrsas_dev, "Cannot get request_descriptor.\n");
568 		return (FAIL);
569 	}
570 	memset(req_desc, 0, sizeof(MRSAS_REQUEST_DESCRIPTOR_UNION));
571 	cmd->request_desc = req_desc;
572 
573 	if (ccb_h->flags & CAM_CDB_POINTER)
574 		bcopy(csio->cdb_io.cdb_ptr, cmd->io_request->CDB.CDB32, csio->cdb_len);
575 	else
576 		bcopy(csio->cdb_io.cdb_bytes, cmd->io_request->CDB.CDB32, csio->cdb_len);
577 	mtx_lock(&sc->raidmap_lock);
578 
579 	/* Check for IO type READ-WRITE targeted for Logical Volume */
580 	cmd_type = mrsas_find_io_type(sim, ccb);
581 	switch (cmd_type) {
582 	case READ_WRITE_LDIO:
583 		/* Build READ-WRITE IO for Logical Volume  */
584 		if (mrsas_build_ldio_rw(sc, cmd, ccb)) {
585 			device_printf(sc->mrsas_dev, "Build RW LDIO failed.\n");
586 			mtx_unlock(&sc->raidmap_lock);
587 			mrsas_release_mpt_cmd(cmd);
588 			return (1);
589 		}
590 		break;
591 	case NON_READ_WRITE_LDIO:
592 		/* Build NON READ-WRITE IO for Logical Volume  */
593 		if (mrsas_build_ldio_nonrw(sc, cmd, ccb)) {
594 			device_printf(sc->mrsas_dev, "Build NON-RW LDIO failed.\n");
595 			mtx_unlock(&sc->raidmap_lock);
596 			mrsas_release_mpt_cmd(cmd);
597 			return (1);
598 		}
599 		break;
600 	case READ_WRITE_SYSPDIO:
601 	case NON_READ_WRITE_SYSPDIO:
602 		if (sc->secure_jbod_support &&
603 		    (cmd_type == NON_READ_WRITE_SYSPDIO)) {
604 			/* Build NON-RW IO for JBOD */
605 			if (mrsas_build_syspdio(sc, cmd, ccb, sim, 0)) {
606 				device_printf(sc->mrsas_dev,
607 				    "Build SYSPDIO failed.\n");
608 				mtx_unlock(&sc->raidmap_lock);
609 				mrsas_release_mpt_cmd(cmd);
610 				return (1);
611 			}
612 		} else {
613 			/* Build RW IO for JBOD */
614 			if (mrsas_build_syspdio(sc, cmd, ccb, sim, 1)) {
615 				device_printf(sc->mrsas_dev,
616 				    "Build SYSPDIO failed.\n");
617 				mtx_unlock(&sc->raidmap_lock);
618 				mrsas_release_mpt_cmd(cmd);
619 				return (1);
620 			}
621 		}
622 	}
623 	mtx_unlock(&sc->raidmap_lock);
624 
625 	if (cmd->flags == MRSAS_DIR_IN)	/* from device */
626 		cmd->io_request->Control |= MPI2_SCSIIO_CONTROL_READ;
627 	else if (cmd->flags == MRSAS_DIR_OUT)	/* to device */
628 		cmd->io_request->Control |= MPI2_SCSIIO_CONTROL_WRITE;
629 
630 	cmd->io_request->SGLFlags = MPI2_SGE_FLAGS_64_BIT_ADDRESSING;
631 	cmd->io_request->SGLOffset0 = offsetof(MRSAS_RAID_SCSI_IO_REQUEST, SGL) / 4;
632 	cmd->io_request->SenseBufferLowAddress = cmd->sense_phys_addr;
633 	cmd->io_request->SenseBufferLength = MRSAS_SCSI_SENSE_BUFFERSIZE;
634 
635 	req_desc = cmd->request_desc;
636 	req_desc->SCSIIO.SMID = cmd->index;
637 
638 	/*
639 	 * Start timer for IO timeout. Default timeout value is 90 second.
640 	 */
641 	cmd->callout_owner = true;
642 #if (__FreeBSD_version >= 1000510)
643 	callout_reset_sbt(&cmd->cm_callout, SBT_1S * 180, 0,
644 	    mrsas_scsiio_timeout, cmd, 0);
645 #else
646 	callout_reset(&cmd->cm_callout, (180000 * hz) / 1000,
647 	    mrsas_scsiio_timeout, cmd);
648 #endif
649 
650 	if (mrsas_atomic_read(&sc->fw_outstanding) > sc->io_cmds_highwater)
651 		sc->io_cmds_highwater++;
652 
653 	/*
654 	 *  if it is raid 1/10 fp write capable.
655 	 *  try to get second command from pool and construct it.
656 	 *  From FW, it has confirmed that lba values of two PDs corresponds to
657 	 *  single R1/10 LD are always same
658 	 *
659 	 */
660 	/*
661 	 * driver side count always should be less than max_fw_cmds to get
662 	 * new command
663 	 */
664 	if (cmd->r1_alt_dev_handle != MR_DEVHANDLE_INVALID) {
665 		mrsas_prepare_secondRaid1_IO(sc, cmd);
666 		mrsas_fire_cmd(sc, req_desc->addr.u.low,
667 			req_desc->addr.u.high);
668 		r1_cmd = cmd->peer_cmd;
669 		mrsas_fire_cmd(sc, r1_cmd->request_desc->addr.u.low,
670 				r1_cmd->request_desc->addr.u.high);
671 	} else {
672 		mrsas_fire_cmd(sc, req_desc->addr.u.low,
673 			req_desc->addr.u.high);
674 	}
675 
676 	return (0);
677 
678 done:
679 	xpt_done(ccb);
680 	mrsas_atomic_dec(&sc->fw_outstanding);
681 	return (0);
682 }
683 
684 /*
685  * mrsas_find_io_type:	Determines if IO is read/write or inquiry
686  * input:			pointer to CAM Control Block
687  *
688  * This function determines if the IO is read/write or inquiry.  It returns a 1
689  * if the IO is read/write and 0 if it is inquiry.
690  */
691 int
692 mrsas_find_io_type(struct cam_sim *sim, union ccb *ccb)
693 {
694 	struct ccb_scsiio *csio = &(ccb->csio);
695 
696 	switch (csio->cdb_io.cdb_bytes[0]) {
697 	case READ_10:
698 	case WRITE_10:
699 	case READ_12:
700 	case WRITE_12:
701 	case READ_6:
702 	case WRITE_6:
703 	case READ_16:
704 	case WRITE_16:
705 		return (cam_sim_bus(sim) ?
706 		    READ_WRITE_SYSPDIO : READ_WRITE_LDIO);
707 	default:
708 		return (cam_sim_bus(sim) ?
709 		    NON_READ_WRITE_SYSPDIO : NON_READ_WRITE_LDIO);
710 	}
711 }
712 
713 /*
714  * mrsas_get_mpt_cmd:	Get a cmd from free command pool
715  * input:				Adapter instance soft state
716  *
717  * This function removes an MPT command from the command free list and
718  * initializes it.
719  */
720 struct mrsas_mpt_cmd *
721 mrsas_get_mpt_cmd(struct mrsas_softc *sc)
722 {
723 	struct mrsas_mpt_cmd *cmd = NULL;
724 
725 	mtx_lock(&sc->mpt_cmd_pool_lock);
726 	if (!TAILQ_EMPTY(&sc->mrsas_mpt_cmd_list_head)) {
727 		cmd = TAILQ_FIRST(&sc->mrsas_mpt_cmd_list_head);
728 		TAILQ_REMOVE(&sc->mrsas_mpt_cmd_list_head, cmd, next);
729 	} else {
730 		goto out;
731 	}
732 
733 	memset((uint8_t *)cmd->io_request, 0, MRSAS_MPI2_RAID_DEFAULT_IO_FRAME_SIZE);
734 	cmd->data = NULL;
735 	cmd->length = 0;
736 	cmd->flags = 0;
737 	cmd->error_code = 0;
738 	cmd->load_balance = 0;
739 	cmd->ccb_ptr = NULL;
740 out:
741 	mtx_unlock(&sc->mpt_cmd_pool_lock);
742 	return cmd;
743 }
744 
745 /*
746  * mrsas_release_mpt_cmd:	Return a cmd to free command pool
747  * input:					Command packet for return to free command pool
748  *
749  * This function returns an MPT command to the free command list.
750  */
751 void
752 mrsas_release_mpt_cmd(struct mrsas_mpt_cmd *cmd)
753 {
754 	struct mrsas_softc *sc = cmd->sc;
755 
756 	mtx_lock(&sc->mpt_cmd_pool_lock);
757 	cmd->r1_alt_dev_handle = MR_DEVHANDLE_INVALID;
758 	cmd->sync_cmd_idx = (u_int32_t)MRSAS_ULONG_MAX;
759 	cmd->peer_cmd = NULL;
760 	cmd->cmd_completed = 0;
761 	memset((uint8_t *)cmd->io_request, 0,
762 		sizeof(MRSAS_RAID_SCSI_IO_REQUEST));
763 	TAILQ_INSERT_HEAD(&(sc->mrsas_mpt_cmd_list_head), cmd, next);
764 	mtx_unlock(&sc->mpt_cmd_pool_lock);
765 
766 	return;
767 }
768 
769 /*
770  * mrsas_get_request_desc:	Get request descriptor from array
771  * input:					Adapter instance soft state
772  * 							SMID index
773  *
774  * This function returns a pointer to the request descriptor.
775  */
776 MRSAS_REQUEST_DESCRIPTOR_UNION *
777 mrsas_get_request_desc(struct mrsas_softc *sc, u_int16_t index)
778 {
779 	u_int8_t *p;
780 
781 	KASSERT(index < sc->max_fw_cmds, ("req_desc is out of range"));
782 	p = sc->req_desc + sizeof(MRSAS_REQUEST_DESCRIPTOR_UNION) * index;
783 
784 	return (MRSAS_REQUEST_DESCRIPTOR_UNION *) p;
785 }
786 
787 /* mrsas_prepare_secondRaid1_IO
788  * It prepares the raid 1 second IO
789  */
790 void
791 mrsas_prepare_secondRaid1_IO(struct mrsas_softc *sc,
792     struct mrsas_mpt_cmd *cmd)
793 {
794 	MRSAS_REQUEST_DESCRIPTOR_UNION *req_desc, *req_desc2 = NULL;
795 	struct mrsas_mpt_cmd *r1_cmd;
796 
797 	r1_cmd = cmd->peer_cmd;
798 	req_desc = cmd->request_desc;
799 
800 	/*
801 	 * copy the io request frame as well as 8 SGEs data for r1
802 	 * command
803 	 */
804 	memcpy(r1_cmd->io_request, cmd->io_request,
805 	    (sizeof(MRSAS_RAID_SCSI_IO_REQUEST)));
806 	memcpy(&r1_cmd->io_request->SGL, &cmd->io_request->SGL,
807 	    (sc->max_sge_in_main_msg * sizeof(MPI2_SGE_IO_UNION)));
808 
809 	/* sense buffer is different for r1 command */
810 	r1_cmd->io_request->SenseBufferLowAddress = r1_cmd->sense_phys_addr;
811 	r1_cmd->ccb_ptr = cmd->ccb_ptr;
812 
813 	req_desc2 = mrsas_get_request_desc(sc, r1_cmd->index - 1);
814 	req_desc2->addr.Words = 0;
815 	r1_cmd->request_desc = req_desc2;
816 	req_desc2->SCSIIO.SMID = r1_cmd->index;
817 	req_desc2->SCSIIO.RequestFlags = req_desc->SCSIIO.RequestFlags;
818 	r1_cmd->request_desc->SCSIIO.DevHandle = cmd->r1_alt_dev_handle;
819 	r1_cmd->r1_alt_dev_handle =  cmd->io_request->DevHandle;
820 	r1_cmd->io_request->DevHandle = cmd->r1_alt_dev_handle;
821 	cmd->io_request->RaidContext.raid_context_g35.smid.peerSMID =
822 	    r1_cmd->index;
823 	r1_cmd->io_request->RaidContext.raid_context_g35.smid.peerSMID =
824 		cmd->index;
825 	/*
826 	 * MSIxIndex of both commands request descriptors
827 	 * should be same
828 	 */
829 	r1_cmd->request_desc->SCSIIO.MSIxIndex = cmd->request_desc->SCSIIO.MSIxIndex;
830 	/* span arm is different for r1 cmd */
831 	r1_cmd->io_request->RaidContext.raid_context_g35.spanArm =
832 	    cmd->io_request->RaidContext.raid_context_g35.spanArm + 1;
833 
834 }
835 
836 /*
837  * mrsas_build_ldio_rw:	Builds an LDIO command
838  * input:				Adapter instance soft state
839  * 						Pointer to command packet
840  * 						Pointer to CCB
841  *
842  * This function builds the LDIO command packet.  It returns 0 if the command is
843  * built successfully, otherwise it returns a 1.
844  */
845 int
846 mrsas_build_ldio_rw(struct mrsas_softc *sc, struct mrsas_mpt_cmd *cmd,
847     union ccb *ccb)
848 {
849 	struct ccb_hdr *ccb_h = &(ccb->ccb_h);
850 	struct ccb_scsiio *csio = &(ccb->csio);
851 	u_int32_t device_id;
852 	MRSAS_RAID_SCSI_IO_REQUEST *io_request;
853 
854 	device_id = ccb_h->target_id;
855 
856 	io_request = cmd->io_request;
857 	io_request->RaidContext.raid_context.VirtualDiskTgtId = device_id;
858 	io_request->RaidContext.raid_context.status = 0;
859 	io_request->RaidContext.raid_context.exStatus = 0;
860 
861 	/* just the cdb len, other flags zero, and ORed-in later for FP */
862 	io_request->IoFlags = csio->cdb_len;
863 
864 	if (mrsas_setup_io(sc, cmd, ccb, device_id, io_request) != SUCCESS)
865 		device_printf(sc->mrsas_dev, "Build ldio or fpio error\n");
866 
867 	io_request->DataLength = cmd->length;
868 
869 	if (mrsas_map_request(sc, cmd, ccb) == SUCCESS) {
870 		if (cmd->sge_count > sc->max_num_sge) {
871 			device_printf(sc->mrsas_dev, "Error: sge_count (0x%x) exceeds"
872 			    "max (0x%x) allowed\n", cmd->sge_count, sc->max_num_sge);
873 			return (FAIL);
874 		}
875 		if (sc->is_ventura || sc->is_aero)
876 			io_request->RaidContext.raid_context_g35.numSGE = cmd->sge_count;
877 		else {
878 			/*
879 			 * numSGE store lower 8 bit of sge_count. numSGEExt store
880 			 * higher 8 bit of sge_count
881 			 */
882 			io_request->RaidContext.raid_context.numSGE = cmd->sge_count;
883 			io_request->RaidContext.raid_context.numSGEExt = (uint8_t)(cmd->sge_count >> 8);
884 		}
885 
886 	} else {
887 		device_printf(sc->mrsas_dev, "Data map/load failed.\n");
888 		return (FAIL);
889 	}
890 	return (0);
891 }
892 
893 /* stream detection on read and and write IOs */
894 static void
895 mrsas_stream_detect(struct mrsas_softc *sc, struct mrsas_mpt_cmd *cmd,
896     struct IO_REQUEST_INFO *io_info)
897 {
898 	u_int32_t device_id = io_info->ldTgtId;
899 	LD_STREAM_DETECT *current_ld_SD = sc->streamDetectByLD[device_id];
900 	u_int32_t *track_stream = &current_ld_SD->mruBitMap;
901 	u_int32_t streamNum, shiftedValues, unshiftedValues;
902 	u_int32_t indexValueMask, shiftedValuesMask;
903 	int i;
904 	boolean_t isReadAhead = false;
905 	STREAM_DETECT *current_SD;
906 
907 	/* find possible stream */
908 	for (i = 0; i < MAX_STREAMS_TRACKED; ++i) {
909 		streamNum = (*track_stream >> (i * BITS_PER_INDEX_STREAM)) &
910 				STREAM_MASK;
911 		current_SD = &current_ld_SD->streamTrack[streamNum];
912 		/*
913 		 * if we found a stream, update the raid context and
914 		 * also update the mruBitMap
915 		 */
916 		if (current_SD->nextSeqLBA &&
917 		    io_info->ldStartBlock >= current_SD->nextSeqLBA &&
918 		    (io_info->ldStartBlock <= (current_SD->nextSeqLBA+32)) &&
919 		    (current_SD->isRead == io_info->isRead)) {
920 			if (io_info->ldStartBlock != current_SD->nextSeqLBA &&
921 			    (!io_info->isRead || !isReadAhead)) {
922 				/*
923 				 * Once the API availible we need to change this.
924 				 * At this point we are not allowing any gap
925 				 */
926 				continue;
927 			}
928 			cmd->io_request->RaidContext.raid_context_g35.streamDetected = TRUE;
929 			current_SD->nextSeqLBA = io_info->ldStartBlock + io_info->numBlocks;
930 			/*
931 			 * update the mruBitMap LRU
932 			 */
933 			shiftedValuesMask = (1 << i * BITS_PER_INDEX_STREAM) - 1 ;
934 			shiftedValues = ((*track_stream & shiftedValuesMask) <<
935 			    BITS_PER_INDEX_STREAM);
936 			indexValueMask = STREAM_MASK << i * BITS_PER_INDEX_STREAM;
937 			unshiftedValues = (*track_stream) &
938 			    (~(shiftedValuesMask | indexValueMask));
939 			*track_stream =
940 			    (unshiftedValues | shiftedValues | streamNum);
941 			return;
942 		}
943 	}
944 	/*
945 	 * if we did not find any stream, create a new one from the least recently used
946 	 */
947 	streamNum = (*track_stream >>
948 	    ((MAX_STREAMS_TRACKED - 1) * BITS_PER_INDEX_STREAM)) & STREAM_MASK;
949 	current_SD = &current_ld_SD->streamTrack[streamNum];
950 	current_SD->isRead = io_info->isRead;
951 	current_SD->nextSeqLBA = io_info->ldStartBlock + io_info->numBlocks;
952 	*track_stream = (((*track_stream & ZERO_LAST_STREAM) << 4) | streamNum);
953 	return;
954 }
955 
956 /*
957  * mrsas_setup_io:	Set up data including Fast Path I/O
958  * input:			Adapter instance soft state
959  * 					Pointer to command packet
960  * 					Pointer to CCB
961  *
962  * This function builds the DCDB inquiry command.  It returns 0 if the command
963  * is built successfully, otherwise it returns a 1.
964  */
965 int
966 mrsas_setup_io(struct mrsas_softc *sc, struct mrsas_mpt_cmd *cmd,
967     union ccb *ccb, u_int32_t device_id,
968     MRSAS_RAID_SCSI_IO_REQUEST * io_request)
969 {
970 	struct ccb_hdr *ccb_h = &(ccb->ccb_h);
971 	struct ccb_scsiio *csio = &(ccb->csio);
972 	struct IO_REQUEST_INFO io_info;
973 	MR_DRV_RAID_MAP_ALL *map_ptr;
974 	struct mrsas_mpt_cmd *r1_cmd = NULL;
975 
976 	MR_LD_RAID *raid;
977 	u_int8_t fp_possible;
978 	u_int32_t start_lba_hi, start_lba_lo, ld_block_size, ld;
979 	u_int32_t datalength = 0;
980 
981 	io_request->RaidContext.raid_context.VirtualDiskTgtId = device_id;
982 
983 	start_lba_lo = 0;
984 	start_lba_hi = 0;
985 	fp_possible = 0;
986 
987 	/*
988 	 * READ_6 (0x08) or WRITE_6 (0x0A) cdb
989 	 */
990 	if (csio->cdb_len == 6) {
991 		datalength = (u_int32_t)csio->cdb_io.cdb_bytes[4];
992 		start_lba_lo = ((u_int32_t)csio->cdb_io.cdb_bytes[1] << 16) |
993 		    ((u_int32_t)csio->cdb_io.cdb_bytes[2] << 8) |
994 		    (u_int32_t)csio->cdb_io.cdb_bytes[3];
995 		start_lba_lo &= 0x1FFFFF;
996 	}
997 	/*
998 	 * READ_10 (0x28) or WRITE_6 (0x2A) cdb
999 	 */
1000 	else if (csio->cdb_len == 10) {
1001 		datalength = (u_int32_t)csio->cdb_io.cdb_bytes[8] |
1002 		    ((u_int32_t)csio->cdb_io.cdb_bytes[7] << 8);
1003 		start_lba_lo = ((u_int32_t)csio->cdb_io.cdb_bytes[2] << 24) |
1004 		    ((u_int32_t)csio->cdb_io.cdb_bytes[3] << 16) |
1005 		    (u_int32_t)csio->cdb_io.cdb_bytes[4] << 8 |
1006 		    ((u_int32_t)csio->cdb_io.cdb_bytes[5]);
1007 	}
1008 	/*
1009 	 * READ_12 (0xA8) or WRITE_12 (0xAA) cdb
1010 	 */
1011 	else if (csio->cdb_len == 12) {
1012 		datalength = (u_int32_t)csio->cdb_io.cdb_bytes[6] << 24 |
1013 		    ((u_int32_t)csio->cdb_io.cdb_bytes[7] << 16) |
1014 		    ((u_int32_t)csio->cdb_io.cdb_bytes[8] << 8) |
1015 		    ((u_int32_t)csio->cdb_io.cdb_bytes[9]);
1016 		start_lba_lo = ((u_int32_t)csio->cdb_io.cdb_bytes[2] << 24) |
1017 		    ((u_int32_t)csio->cdb_io.cdb_bytes[3] << 16) |
1018 		    (u_int32_t)csio->cdb_io.cdb_bytes[4] << 8 |
1019 		    ((u_int32_t)csio->cdb_io.cdb_bytes[5]);
1020 	}
1021 	/*
1022 	 * READ_16 (0x88) or WRITE_16 (0xx8A) cdb
1023 	 */
1024 	else if (csio->cdb_len == 16) {
1025 		datalength = (u_int32_t)csio->cdb_io.cdb_bytes[10] << 24 |
1026 		    ((u_int32_t)csio->cdb_io.cdb_bytes[11] << 16) |
1027 		    ((u_int32_t)csio->cdb_io.cdb_bytes[12] << 8) |
1028 		    ((u_int32_t)csio->cdb_io.cdb_bytes[13]);
1029 		start_lba_lo = ((u_int32_t)csio->cdb_io.cdb_bytes[6] << 24) |
1030 		    ((u_int32_t)csio->cdb_io.cdb_bytes[7] << 16) |
1031 		    (u_int32_t)csio->cdb_io.cdb_bytes[8] << 8 |
1032 		    ((u_int32_t)csio->cdb_io.cdb_bytes[9]);
1033 		start_lba_hi = ((u_int32_t)csio->cdb_io.cdb_bytes[2] << 24) |
1034 		    ((u_int32_t)csio->cdb_io.cdb_bytes[3] << 16) |
1035 		    (u_int32_t)csio->cdb_io.cdb_bytes[4] << 8 |
1036 		    ((u_int32_t)csio->cdb_io.cdb_bytes[5]);
1037 	}
1038 	memset(&io_info, 0, sizeof(struct IO_REQUEST_INFO));
1039 	io_info.ldStartBlock = ((u_int64_t)start_lba_hi << 32) | start_lba_lo;
1040 	io_info.numBlocks = datalength;
1041 	io_info.ldTgtId = device_id;
1042 	io_info.r1_alt_dev_handle = MR_DEVHANDLE_INVALID;
1043 
1044 	io_request->DataLength = cmd->length;
1045 
1046 	switch (ccb_h->flags & CAM_DIR_MASK) {
1047 	case CAM_DIR_IN:
1048 		io_info.isRead = 1;
1049 		break;
1050 	case CAM_DIR_OUT:
1051 		io_info.isRead = 0;
1052 		break;
1053 	case CAM_DIR_NONE:
1054 	default:
1055 		mrsas_dprint(sc, MRSAS_TRACE, "From %s : DMA Flag is %d \n", __func__, ccb_h->flags & CAM_DIR_MASK);
1056 		break;
1057 	}
1058 
1059 	map_ptr = sc->ld_drv_map[(sc->map_id & 1)];
1060 	ld_block_size = MR_LdBlockSizeGet(device_id, map_ptr);
1061 
1062 	ld = MR_TargetIdToLdGet(device_id, map_ptr);
1063 	if ((ld >= MAX_LOGICAL_DRIVES_EXT) || (!sc->fast_path_io)) {
1064 		io_request->RaidContext.raid_context.regLockFlags = 0;
1065 		fp_possible = 0;
1066 	} else {
1067 		if (MR_BuildRaidContext(sc, &io_info, &io_request->RaidContext.raid_context, map_ptr))
1068 			fp_possible = io_info.fpOkForIo;
1069 	}
1070 
1071 	raid = MR_LdRaidGet(ld, map_ptr);
1072 	/* Store the TM capability value in cmd */
1073 	cmd->tmCapable = raid->capability.tmCapable;
1074 
1075 	cmd->request_desc->SCSIIO.MSIxIndex =
1076 	    sc->msix_vectors ? smp_processor_id() % sc->msix_vectors : 0;
1077 
1078 	if (sc->is_ventura || sc->is_aero) {
1079 		if (sc->streamDetectByLD) {
1080 			mtx_lock(&sc->stream_lock);
1081 			mrsas_stream_detect(sc, cmd, &io_info);
1082 			mtx_unlock(&sc->stream_lock);
1083 			/* In ventura if stream detected for a read and
1084 			 * it is read ahead capable make this IO as LDIO */
1085 			if (io_request->RaidContext.raid_context_g35.streamDetected &&
1086 					io_info.isRead && io_info.raCapable)
1087 				fp_possible = FALSE;
1088 		}
1089 
1090 		/* Set raid 1/10 fast path write capable bit in io_info.
1091 		 * Note - reset peer_cmd and r1_alt_dev_handle if fp_possible
1092 		 * disabled after this point. Try not to add more check for
1093 		 * fp_possible toggle after this.
1094 		 */
1095 		if (fp_possible &&
1096 				(io_info.r1_alt_dev_handle != MR_DEVHANDLE_INVALID) &&
1097 				(raid->level == 1) && !io_info.isRead) {
1098 			r1_cmd = mrsas_get_mpt_cmd(sc);
1099 			if (mrsas_atomic_inc_return(&sc->fw_outstanding) > sc->max_scsi_cmds) {
1100 				fp_possible = FALSE;
1101 				mrsas_atomic_dec(&sc->fw_outstanding);
1102 			} else {
1103 				r1_cmd = mrsas_get_mpt_cmd(sc);
1104 				if (!r1_cmd) {
1105 					fp_possible = FALSE;
1106 					mrsas_atomic_dec(&sc->fw_outstanding);
1107 				}
1108 				else {
1109 					cmd->peer_cmd = r1_cmd;
1110 					r1_cmd->peer_cmd = cmd;
1111 				}
1112  			}
1113 		}
1114 	}
1115 
1116 	if (fp_possible) {
1117 		mrsas_set_pd_lba(io_request, csio->cdb_len, &io_info, ccb, map_ptr,
1118 		    start_lba_lo, ld_block_size);
1119 		io_request->Function = MPI2_FUNCTION_SCSI_IO_REQUEST;
1120 		cmd->request_desc->SCSIIO.RequestFlags =
1121 		    (MPI2_REQ_DESCRIPT_FLAGS_FP_IO <<
1122 		    MRSAS_REQ_DESCRIPT_FLAGS_TYPE_SHIFT);
1123 		if (sc->mrsas_gen3_ctrl) {
1124 			if (io_request->RaidContext.raid_context.regLockFlags == REGION_TYPE_UNUSED)
1125 				cmd->request_desc->SCSIIO.RequestFlags =
1126 				    (MRSAS_REQ_DESCRIPT_FLAGS_NO_LOCK <<
1127 				    MRSAS_REQ_DESCRIPT_FLAGS_TYPE_SHIFT);
1128 			io_request->RaidContext.raid_context.Type = MPI2_TYPE_CUDA;
1129 			io_request->RaidContext.raid_context.nseg = 0x1;
1130 			io_request->IoFlags |= MPI25_SAS_DEVICE0_FLAGS_ENABLED_FAST_PATH;
1131 			io_request->RaidContext.raid_context.regLockFlags |=
1132 			    (MR_RL_FLAGS_GRANT_DESTINATION_CUDA |
1133 			    MR_RL_FLAGS_SEQ_NUM_ENABLE);
1134 		} else if (sc->is_ventura || sc->is_aero) {
1135 			io_request->RaidContext.raid_context_g35.Type = MPI2_TYPE_CUDA;
1136 			io_request->RaidContext.raid_context_g35.nseg = 0x1;
1137 			io_request->RaidContext.raid_context_g35.routingFlags.bits.sqn = 1;
1138 			io_request->IoFlags |= MPI25_SAS_DEVICE0_FLAGS_ENABLED_FAST_PATH;
1139 			if (io_request->RaidContext.raid_context_g35.routingFlags.bits.sld) {
1140 					io_request->RaidContext.raid_context_g35.RAIDFlags =
1141 					(MR_RAID_FLAGS_IO_SUB_TYPE_CACHE_BYPASS
1142 					<< MR_RAID_CTX_RAID_FLAGS_IO_SUB_TYPE_SHIFT);
1143 			}
1144 		}
1145 		if ((sc->load_balance_info[device_id].loadBalanceFlag) &&
1146 		    (io_info.isRead)) {
1147 			io_info.devHandle =
1148 			    mrsas_get_updated_dev_handle(sc,
1149 			    &sc->load_balance_info[device_id], &io_info);
1150 			cmd->load_balance = MRSAS_LOAD_BALANCE_FLAG;
1151 			cmd->pd_r1_lb = io_info.pd_after_lb;
1152 			if (sc->is_ventura || sc->is_aero)
1153 				io_request->RaidContext.raid_context_g35.spanArm = io_info.span_arm;
1154 			else
1155 				io_request->RaidContext.raid_context.spanArm = io_info.span_arm;
1156 		} else
1157 			cmd->load_balance = 0;
1158 
1159 		if (sc->is_ventura || sc->is_aero)
1160 				cmd->r1_alt_dev_handle = io_info.r1_alt_dev_handle;
1161 		else
1162 				cmd->r1_alt_dev_handle = MR_DEVHANDLE_INVALID;
1163 
1164 		cmd->request_desc->SCSIIO.DevHandle = io_info.devHandle;
1165 		io_request->DevHandle = io_info.devHandle;
1166 		cmd->pdInterface = io_info.pdInterface;
1167 	} else {
1168 		/* Not FP IO */
1169 		io_request->RaidContext.raid_context.timeoutValue = map_ptr->raidMap.fpPdIoTimeoutSec;
1170 		cmd->request_desc->SCSIIO.RequestFlags =
1171 		    (MRSAS_REQ_DESCRIPT_FLAGS_LD_IO <<
1172 		    MRSAS_REQ_DESCRIPT_FLAGS_TYPE_SHIFT);
1173 		if (sc->mrsas_gen3_ctrl) {
1174 			if (io_request->RaidContext.raid_context.regLockFlags == REGION_TYPE_UNUSED)
1175 				cmd->request_desc->SCSIIO.RequestFlags =
1176 				    (MRSAS_REQ_DESCRIPT_FLAGS_NO_LOCK <<
1177 				    MRSAS_REQ_DESCRIPT_FLAGS_TYPE_SHIFT);
1178 			io_request->RaidContext.raid_context.Type = MPI2_TYPE_CUDA;
1179 			io_request->RaidContext.raid_context.regLockFlags |=
1180 			    (MR_RL_FLAGS_GRANT_DESTINATION_CPU0 |
1181 			    MR_RL_FLAGS_SEQ_NUM_ENABLE);
1182 			io_request->RaidContext.raid_context.nseg = 0x1;
1183 		} else if (sc->is_ventura || sc->is_aero) {
1184 			io_request->RaidContext.raid_context_g35.Type = MPI2_TYPE_CUDA;
1185 			io_request->RaidContext.raid_context_g35.routingFlags.bits.sqn = 1;
1186 			io_request->RaidContext.raid_context_g35.nseg = 0x1;
1187 		}
1188 		io_request->Function = MRSAS_MPI2_FUNCTION_LD_IO_REQUEST;
1189 		io_request->DevHandle = device_id;
1190 	}
1191 	return (0);
1192 }
1193 
1194 /*
1195  * mrsas_build_ldio_nonrw:	Builds an LDIO command
1196  * input:				Adapter instance soft state
1197  * 						Pointer to command packet
1198  * 						Pointer to CCB
1199  *
1200  * This function builds the LDIO command packet.  It returns 0 if the command is
1201  * built successfully, otherwise it returns a 1.
1202  */
1203 int
1204 mrsas_build_ldio_nonrw(struct mrsas_softc *sc, struct mrsas_mpt_cmd *cmd,
1205     union ccb *ccb)
1206 {
1207 	struct ccb_hdr *ccb_h = &(ccb->ccb_h);
1208 	u_int32_t device_id, ld;
1209 	MR_DRV_RAID_MAP_ALL *map_ptr;
1210 	MR_LD_RAID *raid;
1211 	RAID_CONTEXT *pRAID_Context;
1212 	MRSAS_RAID_SCSI_IO_REQUEST *io_request;
1213 
1214 	io_request = cmd->io_request;
1215 	device_id = ccb_h->target_id;
1216 
1217 	map_ptr = sc->ld_drv_map[(sc->map_id & 1)];
1218 	ld = MR_TargetIdToLdGet(device_id, map_ptr);
1219 	raid = MR_LdRaidGet(ld, map_ptr);
1220 	/* get RAID_Context pointer */
1221 	pRAID_Context = &io_request->RaidContext.raid_context;
1222 	/* Store the TM capability value in cmd */
1223 	cmd->tmCapable = raid->capability.tmCapable;
1224 
1225 	/* FW path for LD Non-RW (SCSI management commands) */
1226 	io_request->Function = MRSAS_MPI2_FUNCTION_LD_IO_REQUEST;
1227 	io_request->DevHandle = device_id;
1228 	cmd->request_desc->SCSIIO.RequestFlags =
1229 	    (MPI2_REQ_DESCRIPT_FLAGS_SCSI_IO <<
1230 	    MRSAS_REQ_DESCRIPT_FLAGS_TYPE_SHIFT);
1231 
1232 	io_request->RaidContext.raid_context.VirtualDiskTgtId = device_id;
1233 	io_request->LUN[1] = ccb_h->target_lun & 0xF;
1234 	io_request->DataLength = cmd->length;
1235 
1236 	if (mrsas_map_request(sc, cmd, ccb) == SUCCESS) {
1237 		if (cmd->sge_count > sc->max_num_sge) {
1238 			device_printf(sc->mrsas_dev, "Error: sge_count (0x%x) exceeds"
1239 			    "max (0x%x) allowed\n", cmd->sge_count, sc->max_num_sge);
1240 			return (1);
1241 		}
1242 		if (sc->is_ventura || sc->is_aero)
1243 			io_request->RaidContext.raid_context_g35.numSGE = cmd->sge_count;
1244 		else {
1245 			/*
1246 			 * numSGE store lower 8 bit of sge_count. numSGEExt store
1247 			 * higher 8 bit of sge_count
1248 			 */
1249 			io_request->RaidContext.raid_context.numSGE = cmd->sge_count;
1250 			io_request->RaidContext.raid_context.numSGEExt = (uint8_t)(cmd->sge_count >> 8);
1251 		}
1252 	} else {
1253 		device_printf(sc->mrsas_dev, "Data map/load failed.\n");
1254 		return (1);
1255 	}
1256 	return (0);
1257 }
1258 
1259 /*
1260  * mrsas_build_syspdio:	Builds an DCDB command
1261  * input:				Adapter instance soft state
1262  * 						Pointer to command packet
1263  * 						Pointer to CCB
1264  *
1265  * This function builds the DCDB inquiry command.  It returns 0 if the command
1266  * is built successfully, otherwise it returns a 1.
1267  */
1268 int
1269 mrsas_build_syspdio(struct mrsas_softc *sc, struct mrsas_mpt_cmd *cmd,
1270     union ccb *ccb, struct cam_sim *sim, u_int8_t fp_possible)
1271 {
1272 	struct ccb_hdr *ccb_h = &(ccb->ccb_h);
1273 	u_int32_t device_id;
1274 	MR_DRV_RAID_MAP_ALL *local_map_ptr;
1275 	MRSAS_RAID_SCSI_IO_REQUEST *io_request;
1276 	RAID_CONTEXT *pRAID_Context;
1277 	struct MR_PD_CFG_SEQ_NUM_SYNC *pd_sync;
1278 
1279 	io_request = cmd->io_request;
1280 	/* get RAID_Context pointer */
1281 	pRAID_Context = &io_request->RaidContext.raid_context;
1282 	device_id = ccb_h->target_id;
1283 	local_map_ptr = sc->ld_drv_map[(sc->map_id & 1)];
1284 	io_request->RaidContext.raid_context.RAIDFlags = MR_RAID_FLAGS_IO_SUB_TYPE_SYSTEM_PD
1285 	    << MR_RAID_CTX_RAID_FLAGS_IO_SUB_TYPE_SHIFT;
1286 	io_request->RaidContext.raid_context.regLockFlags = 0;
1287 	io_request->RaidContext.raid_context.regLockRowLBA = 0;
1288 	io_request->RaidContext.raid_context.regLockLength = 0;
1289 
1290 	cmd->pdInterface = sc->target_list[device_id].interface_type;
1291 
1292 	/* If FW supports PD sequence number */
1293 	if (sc->use_seqnum_jbod_fp &&
1294 	    sc->pd_list[device_id].driveType == 0x00) {
1295 		//printf("Using Drv seq num\n");
1296 		pd_sync = (void *)sc->jbodmap_mem[(sc->pd_seq_map_id - 1) & 1];
1297 		cmd->tmCapable = pd_sync->seq[device_id].capability.tmCapable;
1298 		/* More than 256 PD/JBOD support for Ventura */
1299 		if (sc->support_morethan256jbod)
1300 			io_request->RaidContext.raid_context.VirtualDiskTgtId =
1301 				pd_sync->seq[device_id].pdTargetId;
1302 		else
1303 			io_request->RaidContext.raid_context.VirtualDiskTgtId =
1304 				device_id + 255;
1305 		io_request->RaidContext.raid_context.configSeqNum = pd_sync->seq[device_id].seqNum;
1306 		io_request->DevHandle = pd_sync->seq[device_id].devHandle;
1307 		if (sc->is_ventura || sc->is_aero)
1308 			io_request->RaidContext.raid_context_g35.routingFlags.bits.sqn = 1;
1309 		else
1310 			io_request->RaidContext.raid_context.regLockFlags |=
1311 			    (MR_RL_FLAGS_SEQ_NUM_ENABLE | MR_RL_FLAGS_GRANT_DESTINATION_CUDA);
1312 		/* raid_context.Type = MPI2_TYPE_CUDA is valid only,
1313 		 * if FW support Jbod Sequence number
1314 		 */
1315 		io_request->RaidContext.raid_context.Type = MPI2_TYPE_CUDA;
1316 		io_request->RaidContext.raid_context.nseg = 0x1;
1317 	} else if (sc->fast_path_io) {
1318 		//printf("Using LD RAID map\n");
1319 		io_request->RaidContext.raid_context.VirtualDiskTgtId = device_id;
1320 		io_request->RaidContext.raid_context.configSeqNum = 0;
1321 		local_map_ptr = sc->ld_drv_map[(sc->map_id & 1)];
1322 		io_request->DevHandle =
1323 		    local_map_ptr->raidMap.devHndlInfo[device_id].curDevHdl;
1324 	} else {
1325 		//printf("Using FW PATH\n");
1326 		/* Want to send all IO via FW path */
1327 		io_request->RaidContext.raid_context.VirtualDiskTgtId = device_id;
1328 		io_request->RaidContext.raid_context.configSeqNum = 0;
1329 		io_request->DevHandle = MR_DEVHANDLE_INVALID;
1330 	}
1331 
1332 	cmd->request_desc->SCSIIO.DevHandle = io_request->DevHandle;
1333 	cmd->request_desc->SCSIIO.MSIxIndex =
1334 	    sc->msix_vectors ? smp_processor_id() % sc->msix_vectors : 0;
1335 
1336 	if (!fp_possible) {
1337 		/* system pd firmware path */
1338 		io_request->Function = MRSAS_MPI2_FUNCTION_LD_IO_REQUEST;
1339 		cmd->request_desc->SCSIIO.RequestFlags =
1340 		    (MPI2_REQ_DESCRIPT_FLAGS_SCSI_IO <<
1341 		    MRSAS_REQ_DESCRIPT_FLAGS_TYPE_SHIFT);
1342 		io_request->RaidContext.raid_context.timeoutValue =
1343 		    local_map_ptr->raidMap.fpPdIoTimeoutSec;
1344 		io_request->RaidContext.raid_context.VirtualDiskTgtId = device_id;
1345 	} else {
1346 		/* system pd fast path */
1347 		io_request->Function = MPI2_FUNCTION_SCSI_IO_REQUEST;
1348 		io_request->RaidContext.raid_context.timeoutValue = local_map_ptr->raidMap.fpPdIoTimeoutSec;
1349 
1350 		/*
1351 		 * NOTE - For system pd RW cmds only IoFlags will be FAST_PATH
1352 		 * Because the NON RW cmds will now go via FW Queue
1353 		 * and not the Exception queue
1354 		 */
1355 		if (sc->mrsas_gen3_ctrl || sc->is_ventura || sc->is_aero)
1356 			io_request->IoFlags |= MPI25_SAS_DEVICE0_FLAGS_ENABLED_FAST_PATH;
1357 
1358 		cmd->request_desc->SCSIIO.RequestFlags =
1359 		    (MPI2_REQ_DESCRIPT_FLAGS_FP_IO <<
1360 		    MRSAS_REQ_DESCRIPT_FLAGS_TYPE_SHIFT);
1361 	}
1362 
1363 	io_request->LUN[1] = ccb_h->target_lun & 0xF;
1364 	io_request->DataLength = cmd->length;
1365 
1366 	if (mrsas_map_request(sc, cmd, ccb) == SUCCESS) {
1367 		if (cmd->sge_count > sc->max_num_sge) {
1368 			device_printf(sc->mrsas_dev, "Error: sge_count (0x%x) exceeds"
1369 			    "max (0x%x) allowed\n", cmd->sge_count, sc->max_num_sge);
1370 			return (1);
1371 		}
1372 		if (sc->is_ventura || sc->is_aero)
1373 			io_request->RaidContext.raid_context_g35.numSGE = cmd->sge_count;
1374 		else {
1375 			/*
1376 			 * numSGE store lower 8 bit of sge_count. numSGEExt store
1377 			 * higher 8 bit of sge_count
1378 			 */
1379 			io_request->RaidContext.raid_context.numSGE = cmd->sge_count;
1380 			io_request->RaidContext.raid_context.numSGEExt = (uint8_t)(cmd->sge_count >> 8);
1381 		}
1382 	} else {
1383 		device_printf(sc->mrsas_dev, "Data map/load failed.\n");
1384 		return (1);
1385 	}
1386 	return (0);
1387 }
1388 
1389 /*
1390  * mrsas_is_prp_possible:	This function will tell whether PRPs should be built or not
1391  * sc:						Adapter instance soft state
1392  * cmd:						MPT command frame pointer
1393  * nsesg:					Number of OS SGEs
1394  *
1395  * This function will check whether IO is qualified to build PRPs
1396  * return:				true: if PRP should be built
1397  *						false: if IEEE SGLs should be built
1398  */
1399 static boolean_t mrsas_is_prp_possible(struct mrsas_mpt_cmd *cmd,
1400 	bus_dma_segment_t *segs, int nsegs)
1401 {
1402 	struct mrsas_softc *sc = cmd->sc;
1403 	int i;
1404 	u_int32_t data_length = 0;
1405 	bool build_prp = false;
1406 	u_int32_t mr_nvme_pg_size;
1407 
1408 	mr_nvme_pg_size = max(sc->nvme_page_size, MR_DEFAULT_NVME_PAGE_SIZE);
1409 	data_length = cmd->length;
1410 
1411 	if (data_length > (mr_nvme_pg_size * 5))
1412 		build_prp = true;
1413 	else if ((data_length > (mr_nvme_pg_size * 4)) &&
1414 		(data_length <= (mr_nvme_pg_size * 5)))  {
1415 		/* check if 1st SG entry size is < residual beyond 4 pages */
1416 		if ((segs[0].ds_len) < (data_length - (mr_nvme_pg_size * 4)))
1417 			build_prp = true;
1418 	}
1419 
1420 	/*check for SGE holes here*/
1421 	for (i = 0; i < nsegs; i++) {
1422 		/* check for mid SGEs */
1423 		if ((i != 0) && (i != (nsegs - 1))) {
1424 				if ((segs[i].ds_addr % mr_nvme_pg_size) ||
1425 					(segs[i].ds_len % mr_nvme_pg_size)) {
1426 					build_prp = false;
1427 					mrsas_atomic_inc(&sc->sge_holes);
1428 					break;
1429 				}
1430 		}
1431 
1432 		/* check for first SGE*/
1433 		if ((nsegs > 1) && (i == 0)) {
1434 				if ((segs[i].ds_addr + segs[i].ds_len) % mr_nvme_pg_size) {
1435 					build_prp = false;
1436 					mrsas_atomic_inc(&sc->sge_holes);
1437 					break;
1438 				}
1439 		}
1440 
1441 		/* check for Last SGE*/
1442 		if ((nsegs > 1) && (i == (nsegs - 1))) {
1443 				if (segs[i].ds_addr % mr_nvme_pg_size) {
1444 					build_prp = false;
1445 					mrsas_atomic_inc(&sc->sge_holes);
1446 					break;
1447 				}
1448 		}
1449 	}
1450 
1451 	return build_prp;
1452 }
1453 
1454 /*
1455  * mrsas_map_request:	Map and load data
1456  * input:				Adapter instance soft state
1457  * 						Pointer to command packet
1458  *
1459  * For data from OS, map and load the data buffer into bus space.  The SG list
1460  * is built in the callback.  If the  bus dmamap load is not successful,
1461  * cmd->error_code will contain the  error code and a 1 is returned.
1462  */
1463 int
1464 mrsas_map_request(struct mrsas_softc *sc,
1465     struct mrsas_mpt_cmd *cmd, union ccb *ccb)
1466 {
1467 	u_int32_t retcode = 0;
1468 	struct cam_sim *sim;
1469 
1470 	sim = xpt_path_sim(cmd->ccb_ptr->ccb_h.path);
1471 
1472 	if (cmd->data != NULL) {
1473 		/* Map data buffer into bus space */
1474 		mtx_lock(&sc->io_lock);
1475 #if (__FreeBSD_version >= 902001)
1476 		retcode = bus_dmamap_load_ccb(sc->data_tag, cmd->data_dmamap, ccb,
1477 		    mrsas_data_load_cb, cmd, 0);
1478 #else
1479 		retcode = bus_dmamap_load(sc->data_tag, cmd->data_dmamap, cmd->data,
1480 		    cmd->length, mrsas_data_load_cb, cmd, BUS_DMA_NOWAIT);
1481 #endif
1482 		mtx_unlock(&sc->io_lock);
1483 		if (retcode)
1484 			device_printf(sc->mrsas_dev, "bus_dmamap_load(): retcode = %d\n", retcode);
1485 		if (retcode == EINPROGRESS) {
1486 			device_printf(sc->mrsas_dev, "request load in progress\n");
1487 			mrsas_freeze_simq(cmd, sim);
1488 		}
1489 	}
1490 	if (cmd->error_code)
1491 		return (1);
1492 	return (retcode);
1493 }
1494 
1495 /*
1496  * mrsas_unmap_request:	Unmap and unload data
1497  * input:				Adapter instance soft state
1498  * 						Pointer to command packet
1499  *
1500  * This function unmaps and unloads data from OS.
1501  */
1502 void
1503 mrsas_unmap_request(struct mrsas_softc *sc, struct mrsas_mpt_cmd *cmd)
1504 {
1505 	if (cmd->data != NULL) {
1506 		if (cmd->flags & MRSAS_DIR_IN)
1507 			bus_dmamap_sync(sc->data_tag, cmd->data_dmamap, BUS_DMASYNC_POSTREAD);
1508 		if (cmd->flags & MRSAS_DIR_OUT)
1509 			bus_dmamap_sync(sc->data_tag, cmd->data_dmamap, BUS_DMASYNC_POSTWRITE);
1510 		mtx_lock(&sc->io_lock);
1511 		bus_dmamap_unload(sc->data_tag, cmd->data_dmamap);
1512 		mtx_unlock(&sc->io_lock);
1513 	}
1514 }
1515 
1516 /**
1517  * mrsas_build_ieee_sgl -	Prepare IEEE SGLs
1518  * @sc:						Adapter soft state
1519  * @segs:					OS SGEs pointers
1520  * @nseg:					Number of OS SGEs
1521  * @cmd:					Fusion command frame
1522  * return:					void
1523  */
1524 static void mrsas_build_ieee_sgl(struct mrsas_mpt_cmd *cmd, bus_dma_segment_t *segs, int nseg)
1525 {
1526 	struct mrsas_softc *sc = cmd->sc;
1527 	MRSAS_RAID_SCSI_IO_REQUEST *io_request;
1528 	pMpi25IeeeSgeChain64_t sgl_ptr;
1529 	int i = 0, sg_processed = 0;
1530 
1531 	io_request = cmd->io_request;
1532 	sgl_ptr = (pMpi25IeeeSgeChain64_t)&io_request->SGL;
1533 
1534 	if (sc->mrsas_gen3_ctrl || sc->is_ventura || sc->is_aero) {
1535 		pMpi25IeeeSgeChain64_t sgl_ptr_end = sgl_ptr;
1536 
1537 		sgl_ptr_end += sc->max_sge_in_main_msg - 1;
1538 		sgl_ptr_end->Flags = 0;
1539 	}
1540 	if (nseg != 0) {
1541 		for (i = 0; i < nseg; i++) {
1542 			sgl_ptr->Address = segs[i].ds_addr;
1543 			sgl_ptr->Length = segs[i].ds_len;
1544 			sgl_ptr->Flags = 0;
1545 			if (sc->mrsas_gen3_ctrl || sc->is_ventura || sc->is_aero) {
1546 				if (i == nseg - 1)
1547 					sgl_ptr->Flags = IEEE_SGE_FLAGS_END_OF_LIST;
1548 			}
1549 			sgl_ptr++;
1550 			sg_processed = i + 1;
1551 			if ((sg_processed == (sc->max_sge_in_main_msg - 1)) &&
1552 				(nseg > sc->max_sge_in_main_msg)) {
1553 				pMpi25IeeeSgeChain64_t sg_chain;
1554 
1555 				if (sc->mrsas_gen3_ctrl || sc->is_ventura || sc->is_aero) {
1556 					if ((cmd->io_request->IoFlags & MPI25_SAS_DEVICE0_FLAGS_ENABLED_FAST_PATH)
1557 						!= MPI25_SAS_DEVICE0_FLAGS_ENABLED_FAST_PATH)
1558 						cmd->io_request->ChainOffset = sc->chain_offset_io_request;
1559 					else
1560 						cmd->io_request->ChainOffset = 0;
1561 				} else
1562 					cmd->io_request->ChainOffset = sc->chain_offset_io_request;
1563 				sg_chain = sgl_ptr;
1564 				if (sc->mrsas_gen3_ctrl || sc->is_ventura || sc->is_aero)
1565 					sg_chain->Flags = IEEE_SGE_FLAGS_CHAIN_ELEMENT;
1566 				else
1567 					sg_chain->Flags = (IEEE_SGE_FLAGS_CHAIN_ELEMENT | MPI2_IEEE_SGE_FLAGS_IOCPLBNTA_ADDR);
1568 				sg_chain->Length = (sizeof(MPI2_SGE_IO_UNION) * (nseg - sg_processed));
1569 				sg_chain->Address = cmd->chain_frame_phys_addr;
1570 				sgl_ptr = (pMpi25IeeeSgeChain64_t)cmd->chain_frame;
1571 			}
1572 		}
1573 	}
1574 }
1575 
1576 /**
1577  * mrsas_build_prp_nvme - Prepare PRPs(Physical Region Page)- SGLs specific to NVMe drives only
1578  * @sc:						Adapter soft state
1579  * @segs:					OS SGEs pointers
1580  * @nseg:					Number of OS SGEs
1581  * @cmd:					Fusion command frame
1582  * return:					void
1583  */
1584 static void mrsas_build_prp_nvme(struct mrsas_mpt_cmd *cmd, bus_dma_segment_t *segs, int nseg)
1585 {
1586 	struct mrsas_softc *sc = cmd->sc;
1587 	int sge_len, offset, num_prp_in_chain = 0;
1588 	pMpi25IeeeSgeChain64_t main_chain_element, ptr_first_sgl, sgl_ptr;
1589 	u_int64_t *ptr_sgl;
1590 	bus_addr_t ptr_sgl_phys;
1591 	u_int64_t sge_addr;
1592 	u_int32_t page_mask, page_mask_result, i = 0;
1593 	u_int32_t first_prp_len;
1594 	int data_len = cmd->length;
1595 	u_int32_t mr_nvme_pg_size = max(sc->nvme_page_size,
1596 					MR_DEFAULT_NVME_PAGE_SIZE);
1597 
1598 	sgl_ptr = (pMpi25IeeeSgeChain64_t) &cmd->io_request->SGL;
1599 	/*
1600 	 * NVMe has a very convoluted PRP format.  One PRP is required
1601 	 * for each page or partial page.  We need to split up OS SG
1602 	 * entries if they are longer than one page or cross a page
1603 	 * boundary.  We also have to insert a PRP list pointer entry as
1604 	 * the last entry in each physical page of the PRP list.
1605 	 *
1606 	 * NOTE: The first PRP "entry" is actually placed in the first
1607 	 * SGL entry in the main message in IEEE 64 format.  The 2nd
1608 	 * entry in the main message is the chain element, and the rest
1609 	 * of the PRP entries are built in the contiguous PCIe buffer.
1610 	 */
1611 	page_mask = mr_nvme_pg_size - 1;
1612 	ptr_sgl = (u_int64_t *) cmd->chain_frame;
1613 	ptr_sgl_phys = cmd->chain_frame_phys_addr;
1614 	memset(ptr_sgl, 0, sc->max_chain_frame_sz);
1615 
1616 	/* Build chain frame element which holds all PRPs except first*/
1617 	main_chain_element = (pMpi25IeeeSgeChain64_t)
1618 	    ((u_int8_t *)sgl_ptr + sizeof(MPI25_IEEE_SGE_CHAIN64));
1619 
1620 	main_chain_element->Address = cmd->chain_frame_phys_addr;
1621 	main_chain_element->NextChainOffset = 0;
1622 	main_chain_element->Flags = IEEE_SGE_FLAGS_CHAIN_ELEMENT |
1623 					IEEE_SGE_FLAGS_SYSTEM_ADDR |
1624 					MPI26_IEEE_SGE_FLAGS_NSF_NVME_PRP;
1625 
1626 	/* Build first PRP, SGE need not to be PAGE aligned*/
1627 	ptr_first_sgl = sgl_ptr;
1628 	sge_addr = segs[i].ds_addr;
1629 	sge_len = segs[i].ds_len;
1630 	i++;
1631 
1632 	offset = (u_int32_t) (sge_addr & page_mask);
1633 	first_prp_len = mr_nvme_pg_size - offset;
1634 
1635 	ptr_first_sgl->Address = sge_addr;
1636 	ptr_first_sgl->Length = first_prp_len;
1637 
1638 	data_len -= first_prp_len;
1639 
1640 	if (sge_len > first_prp_len) {
1641 		sge_addr += first_prp_len;
1642 		sge_len -= first_prp_len;
1643 	} else if (sge_len == first_prp_len) {
1644 		sge_addr = segs[i].ds_addr;
1645 		sge_len = segs[i].ds_len;
1646 		i++;
1647 	}
1648 
1649 	for (;;) {
1650 		offset = (u_int32_t) (sge_addr & page_mask);
1651 
1652 		/* Put PRP pointer due to page boundary*/
1653 		page_mask_result = (uintptr_t)(ptr_sgl + 1) & page_mask;
1654 		if (!page_mask_result) {
1655 			device_printf(sc->mrsas_dev, "BRCM: Put prp pointer as we are at page boundary"
1656 					" ptr_sgl: 0x%p\n", ptr_sgl);
1657 			ptr_sgl_phys++;
1658 			*ptr_sgl = (uintptr_t)ptr_sgl_phys;
1659 			ptr_sgl++;
1660 			num_prp_in_chain++;
1661 		}
1662 
1663 		*ptr_sgl = sge_addr;
1664 		ptr_sgl++;
1665 		ptr_sgl_phys++;
1666 		num_prp_in_chain++;
1667 
1668 		sge_addr += mr_nvme_pg_size;
1669 		sge_len -= mr_nvme_pg_size;
1670 		data_len -= mr_nvme_pg_size;
1671 
1672 		if (data_len <= 0)
1673 			break;
1674 
1675 		if (sge_len > 0)
1676 			continue;
1677 
1678 		sge_addr = segs[i].ds_addr;
1679 		sge_len = segs[i].ds_len;
1680 		i++;
1681 	}
1682 
1683 	main_chain_element->Length = num_prp_in_chain * sizeof(u_int64_t);
1684 	mrsas_atomic_inc(&sc->prp_count);
1685 
1686 }
1687 
1688 /*
1689  * mrsas_data_load_cb:	Callback entry point to build SGLs
1690  * input:				Pointer to command packet as argument
1691  *						Pointer to segment
1692  *						Number of segments Error
1693  *
1694  * This is the callback function of the bus dma map load.  It builds SG list
1695  */
1696 static void
1697 mrsas_data_load_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
1698 {
1699 	struct mrsas_mpt_cmd *cmd = (struct mrsas_mpt_cmd *)arg;
1700 	struct mrsas_softc *sc = cmd->sc;
1701 	boolean_t build_prp = false;
1702 
1703 	if (error) {
1704 		cmd->error_code = error;
1705 		device_printf(sc->mrsas_dev, "mrsas_data_load_cb_prp: error=%d\n", error);
1706 		if (error == EFBIG) {
1707 			cmd->ccb_ptr->ccb_h.status = CAM_REQ_TOO_BIG;
1708 			return;
1709 		}
1710 	}
1711 	if (cmd->flags & MRSAS_DIR_IN)
1712 		bus_dmamap_sync(cmd->sc->data_tag, cmd->data_dmamap,
1713 		    BUS_DMASYNC_PREREAD);
1714 	if (cmd->flags & MRSAS_DIR_OUT)
1715 		bus_dmamap_sync(cmd->sc->data_tag, cmd->data_dmamap,
1716 		    BUS_DMASYNC_PREWRITE);
1717 	if (nseg > sc->max_num_sge) {
1718 		device_printf(sc->mrsas_dev, "SGE count is too large or 0.\n");
1719 		return;
1720 	}
1721 
1722 	/* Check for whether PRPs should be built or IEEE SGLs*/
1723 	if ((cmd->io_request->IoFlags & MPI25_SAS_DEVICE0_FLAGS_ENABLED_FAST_PATH) &&
1724 			(cmd->pdInterface == NVME_PD))
1725 		build_prp = mrsas_is_prp_possible(cmd, segs, nseg);
1726 
1727 	if (build_prp == true)
1728 		mrsas_build_prp_nvme(cmd, segs, nseg);
1729 	else
1730 		mrsas_build_ieee_sgl(cmd, segs, nseg);
1731 
1732 	cmd->sge_count = nseg;
1733 }
1734 
1735 /*
1736  * mrsas_freeze_simq:	Freeze SIM queue
1737  * input:				Pointer to command packet
1738  * 						Pointer to SIM
1739  *
1740  * This function freezes the sim queue.
1741  */
1742 static void
1743 mrsas_freeze_simq(struct mrsas_mpt_cmd *cmd, struct cam_sim *sim)
1744 {
1745 	union ccb *ccb = (union ccb *)(cmd->ccb_ptr);
1746 
1747 	xpt_freeze_simq(sim, 1);
1748 	ccb->ccb_h.status |= CAM_RELEASE_SIMQ;
1749 	ccb->ccb_h.status |= CAM_REQUEUE_REQ;
1750 }
1751 
1752 void
1753 mrsas_xpt_freeze(struct mrsas_softc *sc)
1754 {
1755 	xpt_freeze_simq(sc->sim_0, 1);
1756 	xpt_freeze_simq(sc->sim_1, 1);
1757 }
1758 
1759 void
1760 mrsas_xpt_release(struct mrsas_softc *sc)
1761 {
1762 	xpt_release_simq(sc->sim_0, 1);
1763 	xpt_release_simq(sc->sim_1, 1);
1764 }
1765 
1766 /*
1767  * mrsas_cmd_done:	Perform remaining command completion
1768  * input:			Adapter instance soft state  Pointer to command packet
1769  *
1770  * This function calls ummap request and releases the MPT command.
1771  */
1772 void
1773 mrsas_cmd_done(struct mrsas_softc *sc, struct mrsas_mpt_cmd *cmd)
1774 {
1775 	mrsas_unmap_request(sc, cmd);
1776 
1777 	mtx_lock(&sc->sim_lock);
1778 	if (cmd->callout_owner) {
1779 		callout_stop(&cmd->cm_callout);
1780 		cmd->callout_owner  = false;
1781 	}
1782 	xpt_done(cmd->ccb_ptr);
1783 	cmd->ccb_ptr = NULL;
1784 	mtx_unlock(&sc->sim_lock);
1785 	mrsas_release_mpt_cmd(cmd);
1786 }
1787 
1788 /*
1789  * mrsas_cam_poll:	Polling entry point
1790  * input:			Pointer to SIM
1791  *
1792  * This is currently a stub function.
1793  */
1794 static void
1795 mrsas_cam_poll(struct cam_sim *sim)
1796 {
1797 	int i;
1798 	struct mrsas_softc *sc = (struct mrsas_softc *)cam_sim_softc(sim);
1799 
1800 	if (sc->msix_vectors != 0){
1801 		for (i=0; i<sc->msix_vectors; i++){
1802 			mrsas_complete_cmd(sc, i);
1803 		}
1804 	} else {
1805 		mrsas_complete_cmd(sc, 0);
1806 	}
1807 }
1808 
1809 /*
1810  * mrsas_bus_scan:	Perform bus scan
1811  * input:			Adapter instance soft state
1812  *
1813  * This mrsas_bus_scan function is needed for FreeBSD 7.x.  Also, it should not
1814  * be called in FreeBSD 8.x and later versions, where the bus scan is
1815  * automatic.
1816  */
1817 int
1818 mrsas_bus_scan(struct mrsas_softc *sc)
1819 {
1820 	union ccb *ccb_0;
1821 	union ccb *ccb_1;
1822 
1823 	if ((ccb_0 = xpt_alloc_ccb()) == NULL) {
1824 		return (ENOMEM);
1825 	}
1826 	if ((ccb_1 = xpt_alloc_ccb()) == NULL) {
1827 		xpt_free_ccb(ccb_0);
1828 		return (ENOMEM);
1829 	}
1830 	mtx_lock(&sc->sim_lock);
1831 	if (xpt_create_path(&ccb_0->ccb_h.path, xpt_periph, cam_sim_path(sc->sim_0),
1832 	    CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD) != CAM_REQ_CMP) {
1833 		xpt_free_ccb(ccb_0);
1834 		xpt_free_ccb(ccb_1);
1835 		mtx_unlock(&sc->sim_lock);
1836 		return (EIO);
1837 	}
1838 	if (xpt_create_path(&ccb_1->ccb_h.path, xpt_periph, cam_sim_path(sc->sim_1),
1839 	    CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD) != CAM_REQ_CMP) {
1840 		xpt_free_ccb(ccb_0);
1841 		xpt_free_ccb(ccb_1);
1842 		mtx_unlock(&sc->sim_lock);
1843 		return (EIO);
1844 	}
1845 	mtx_unlock(&sc->sim_lock);
1846 	xpt_rescan(ccb_0);
1847 	xpt_rescan(ccb_1);
1848 
1849 	return (0);
1850 }
1851 
1852 /*
1853  * mrsas_bus_scan_sim:	Perform bus scan per SIM
1854  * input:				adapter instance soft state
1855  *
1856  * This function will be called from Event handler on LD creation/deletion,
1857  * JBOD on/off.
1858  */
1859 int
1860 mrsas_bus_scan_sim(struct mrsas_softc *sc, struct cam_sim *sim)
1861 {
1862 	union ccb *ccb;
1863 
1864 	if ((ccb = xpt_alloc_ccb()) == NULL) {
1865 		return (ENOMEM);
1866 	}
1867 	mtx_lock(&sc->sim_lock);
1868 	if (xpt_create_path(&ccb->ccb_h.path, xpt_periph, cam_sim_path(sim),
1869 	    CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD) != CAM_REQ_CMP) {
1870 		xpt_free_ccb(ccb);
1871 		mtx_unlock(&sc->sim_lock);
1872 		return (EIO);
1873 	}
1874 	mtx_unlock(&sc->sim_lock);
1875 	xpt_rescan(ccb);
1876 
1877 	return (0);
1878 }
1879 
1880 /*
1881  * mrsas_track_scsiio:  Track IOs for a given target in the mpt_cmd_list
1882  * input:           Adapter instance soft state
1883  *                  Target ID of target
1884  *                  Bus ID of the target
1885  *
1886  * This function checks for any pending IO in the whole mpt_cmd_list pool
1887  * with the bus_id and target_id passed in arguments. If some IO is found
1888  * that means target reset is not successfully completed.
1889  *
1890  * Returns FAIL if IOs pending to the target device, else return SUCCESS
1891  */
1892 static int
1893 mrsas_track_scsiio(struct mrsas_softc *sc, target_id_t tgt_id, u_int32_t bus_id)
1894 {
1895 	int i;
1896 	struct mrsas_mpt_cmd *mpt_cmd = NULL;
1897 
1898 	for (i = 0 ; i < sc->max_fw_cmds; i++) {
1899 		mpt_cmd = sc->mpt_cmd_list[i];
1900 
1901 		/*
1902 		 * Check if the target_id and bus_id is same as the timeout IO
1903 		 */
1904 		if (mpt_cmd->ccb_ptr) {
1905 			/* bus_id = 1 denotes a VD */
1906 			if (bus_id == 1)
1907 				tgt_id =
1908 				    (mpt_cmd->ccb_ptr->ccb_h.target_id - (MRSAS_MAX_PD - 1));
1909 
1910 			if (mpt_cmd->ccb_ptr->cpi.bus_id == bus_id &&
1911 			    mpt_cmd->ccb_ptr->ccb_h.target_id == tgt_id) {
1912 				device_printf(sc->mrsas_dev,
1913 				    "IO commands pending to target id %d\n", tgt_id);
1914 				return FAIL;
1915 			}
1916 		}
1917 	}
1918 
1919 	return SUCCESS;
1920 }
1921 
1922 #if TM_DEBUG
1923 /*
1924  * mrsas_tm_response_code: Prints TM response code received from FW
1925  * input:           Adapter instance soft state
1926  *                  MPI reply returned from firmware
1927  *
1928  * Returns nothing.
1929  */
1930 static void
1931 mrsas_tm_response_code(struct mrsas_softc *sc,
1932 	MPI2_SCSI_TASK_MANAGE_REPLY *mpi_reply)
1933 {
1934 	char *desc;
1935 
1936 	switch (mpi_reply->ResponseCode) {
1937 	case MPI2_SCSITASKMGMT_RSP_TM_COMPLETE:
1938 		desc = "task management request completed";
1939 		break;
1940 	case MPI2_SCSITASKMGMT_RSP_INVALID_FRAME:
1941 		desc = "invalid frame";
1942 		break;
1943 	case MPI2_SCSITASKMGMT_RSP_TM_NOT_SUPPORTED:
1944 		desc = "task management request not supported";
1945 		break;
1946 	case MPI2_SCSITASKMGMT_RSP_TM_FAILED:
1947 		desc = "task management request failed";
1948 		break;
1949 	case MPI2_SCSITASKMGMT_RSP_TM_SUCCEEDED:
1950 		desc = "task management request succeeded";
1951 		break;
1952 	case MPI2_SCSITASKMGMT_RSP_TM_INVALID_LUN:
1953 		desc = "invalid lun";
1954 		break;
1955 	case 0xA:
1956 		desc = "overlapped tag attempted";
1957 		break;
1958 	case MPI2_SCSITASKMGMT_RSP_IO_QUEUED_ON_IOC:
1959 		desc = "task queued, however not sent to target";
1960 		break;
1961 	default:
1962 		desc = "unknown";
1963 		break;
1964 	}
1965 	device_printf(sc->mrsas_dev, "response_code(%01x): %s\n",
1966 	    mpi_reply->ResponseCode, desc);
1967 	device_printf(sc->mrsas_dev,
1968 	    "TerminationCount/DevHandle/Function/TaskType/IOCStat/IOCLoginfo\n"
1969 	    "0x%x/0x%x/0x%x/0x%x/0x%x/0x%x\n",
1970 	    mpi_reply->TerminationCount, mpi_reply->DevHandle,
1971 	    mpi_reply->Function, mpi_reply->TaskType,
1972 	    mpi_reply->IOCStatus, mpi_reply->IOCLogInfo);
1973 }
1974 #endif
1975 
1976 /*
1977  * mrsas_issue_tm:  Fires the TM command to FW and waits for completion
1978  * input:           Adapter instance soft state
1979  *                  reqest descriptor compiled by mrsas_reset_targets
1980  *
1981  * Returns FAIL if TM command TIMEDOUT from FW else SUCCESS.
1982  */
1983 static int
1984 mrsas_issue_tm(struct mrsas_softc *sc,
1985 	MRSAS_REQUEST_DESCRIPTOR_UNION *req_desc)
1986 {
1987 	int sleep_stat;
1988 
1989 	mrsas_fire_cmd(sc, req_desc->addr.u.low, req_desc->addr.u.high);
1990 	sleep_stat = msleep(&sc->ocr_chan, &sc->sim_lock, PRIBIO, "tm_sleep", 50*hz);
1991 
1992 	if (sleep_stat == EWOULDBLOCK) {
1993 		device_printf(sc->mrsas_dev, "tm cmd TIMEDOUT\n");
1994 		return FAIL;
1995 	}
1996 
1997 	return SUCCESS;
1998 }
1999 
2000 /*
2001  * mrsas_reset_targets : Gathers info to fire a target reset command
2002  * input:           Adapter instance soft state
2003  *
2004  * This function compiles data for a target reset command to be fired to the FW
2005  * and then traverse the target_reset_pool to see targets with TIMEDOUT IOs.
2006  *
2007  * Returns SUCCESS or FAIL
2008  */
2009 int mrsas_reset_targets(struct mrsas_softc *sc)
2010 {
2011 	struct mrsas_mpt_cmd *tm_mpt_cmd = NULL;
2012 	struct mrsas_mpt_cmd *tgt_mpt_cmd = NULL;
2013 	MR_TASK_MANAGE_REQUEST *mr_request;
2014 	MPI2_SCSI_TASK_MANAGE_REQUEST *tm_mpi_request;
2015 	MRSAS_REQUEST_DESCRIPTOR_UNION *req_desc;
2016 	int retCode = FAIL, count, i, outstanding;
2017 	u_int32_t MSIxIndex, bus_id;
2018 	target_id_t tgt_id;
2019 #if TM_DEBUG
2020 	MPI2_SCSI_TASK_MANAGE_REPLY *mpi_reply;
2021 #endif
2022 
2023 	outstanding = mrsas_atomic_read(&sc->fw_outstanding);
2024 
2025 	if (!outstanding) {
2026 		device_printf(sc->mrsas_dev, "NO IOs pending...\n");
2027 		mrsas_atomic_set(&sc->target_reset_outstanding, 0);
2028 		retCode = SUCCESS;
2029 		goto return_status;
2030 	} else if (sc->adprecovery != MRSAS_HBA_OPERATIONAL) {
2031 		device_printf(sc->mrsas_dev, "Controller is not operational\n");
2032 		goto return_status;
2033 	} else {
2034 		/* Some more error checks will be added in future */
2035 	}
2036 
2037 	/* Get an mpt frame and an index to fire the TM cmd */
2038 	tm_mpt_cmd = mrsas_get_mpt_cmd(sc);
2039 	if (!tm_mpt_cmd) {
2040 		retCode = FAIL;
2041 		goto return_status;
2042 	}
2043 
2044 	req_desc = mrsas_get_request_desc(sc, (tm_mpt_cmd->index) - 1);
2045 	if (!req_desc) {
2046 		device_printf(sc->mrsas_dev, "Cannot get request_descriptor for tm.\n");
2047 		retCode = FAIL;
2048 		goto release_mpt;
2049 	}
2050 	memset(req_desc, 0, sizeof(MRSAS_REQUEST_DESCRIPTOR_UNION));
2051 
2052 	req_desc->HighPriority.SMID = tm_mpt_cmd->index;
2053 	req_desc->HighPriority.RequestFlags =
2054 	    (MPI2_REQ_DESCRIPT_FLAGS_HIGH_PRIORITY <<
2055 	    MRSAS_REQ_DESCRIPT_FLAGS_TYPE_SHIFT);
2056 	req_desc->HighPriority.MSIxIndex =  0;
2057 	req_desc->HighPriority.LMID = 0;
2058 	req_desc->HighPriority.Reserved1 = 0;
2059 	tm_mpt_cmd->request_desc = req_desc;
2060 
2061 	mr_request = (MR_TASK_MANAGE_REQUEST *) tm_mpt_cmd->io_request;
2062 	memset(mr_request, 0, sizeof(MR_TASK_MANAGE_REQUEST));
2063 
2064 	tm_mpi_request = (MPI2_SCSI_TASK_MANAGE_REQUEST *) &mr_request->TmRequest;
2065 	tm_mpi_request->Function = MPI2_FUNCTION_SCSI_TASK_MGMT;
2066 	tm_mpi_request->TaskType = MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET;
2067 	tm_mpi_request->TaskMID = 0; /* smid task */
2068 	tm_mpi_request->LUN[1] = 0;
2069 
2070 	/* Traverse the tm_mpt pool to get valid entries */
2071 	for (i = 0 ; i < MRSAS_MAX_TM_TARGETS; i++) {
2072 		if(!sc->target_reset_pool[i]) {
2073 			continue;
2074 		} else {
2075 			tgt_mpt_cmd = sc->target_reset_pool[i];
2076 		}
2077 
2078 		tgt_id = i;
2079 
2080 		/* See if the target is tm capable or NOT */
2081 		if (!tgt_mpt_cmd->tmCapable) {
2082 			device_printf(sc->mrsas_dev, "Task management NOT SUPPORTED for "
2083 			    "CAM target:%d\n", tgt_id);
2084 
2085 			retCode = FAIL;
2086 			goto release_mpt;
2087 		}
2088 
2089 		tm_mpi_request->DevHandle = tgt_mpt_cmd->io_request->DevHandle;
2090 
2091 		if (i < (MRSAS_MAX_PD - 1)) {
2092 			mr_request->uTmReqReply.tmReqFlags.isTMForPD = 1;
2093 			bus_id = 0;
2094 		} else {
2095 			mr_request->uTmReqReply.tmReqFlags.isTMForLD = 1;
2096 			bus_id = 1;
2097 		}
2098 
2099 		device_printf(sc->mrsas_dev, "TM will be fired for "
2100 		    "CAM target:%d and bus_id %d\n", tgt_id, bus_id);
2101 
2102 		sc->ocr_chan = (void *)&tm_mpt_cmd;
2103 		retCode = mrsas_issue_tm(sc, req_desc);
2104 		if (retCode == FAIL)
2105 			goto release_mpt;
2106 
2107 #if TM_DEBUG
2108 		mpi_reply =
2109 		    (MPI2_SCSI_TASK_MANAGE_REPLY *) &mr_request->uTmReqReply.TMReply;
2110 		mrsas_tm_response_code(sc, mpi_reply);
2111 #endif
2112 		mrsas_atomic_dec(&sc->target_reset_outstanding);
2113 		sc->target_reset_pool[i] = NULL;
2114 
2115 		/* Check for pending cmds in the mpt_cmd_pool with the tgt_id */
2116 		mrsas_disable_intr(sc);
2117 		/* Wait for 1 second to complete parallel ISR calling same
2118 		 * mrsas_complete_cmd()
2119 		 */
2120 		msleep(&sc->ocr_chan, &sc->sim_lock, PRIBIO, "mrsas_reset_wakeup",
2121 		   1 * hz);
2122 		count = sc->msix_vectors > 0 ? sc->msix_vectors : 1;
2123 		mtx_unlock(&sc->sim_lock);
2124 		for (MSIxIndex = 0; MSIxIndex < count; MSIxIndex++)
2125 		    mrsas_complete_cmd(sc, MSIxIndex);
2126 		mtx_lock(&sc->sim_lock);
2127 		retCode = mrsas_track_scsiio(sc, tgt_id, bus_id);
2128 		mrsas_enable_intr(sc);
2129 
2130 		if (retCode == FAIL)
2131 			goto release_mpt;
2132 	}
2133 
2134 	device_printf(sc->mrsas_dev, "Number of targets outstanding "
2135 	    "after reset: %d\n", mrsas_atomic_read(&sc->target_reset_outstanding));
2136 
2137 release_mpt:
2138 	mrsas_release_mpt_cmd(tm_mpt_cmd);
2139 return_status:
2140 	device_printf(sc->mrsas_dev, "target reset %s!!\n",
2141 		(retCode == SUCCESS) ? "SUCCESS" : "FAIL");
2142 
2143 	return retCode;
2144 }
2145