79 static void mpi3mr_repost_reply_buf(struct mpi3mr_softc *sc,
81 static int mpi3mr_complete_admin_cmd(struct mpi3mr_softc *sc);
82 static void mpi3mr_port_enable_complete(struct mpi3mr_softc *sc,
84 static void mpi3mr_flush_io(struct mpi3mr_softc *sc);
85 static int mpi3mr_issue_reset(struct mpi3mr_softc *sc, U16 reset_type,
87 static void mpi3mr_dev_rmhs_send_tm(struct mpi3mr_softc *sc, U16 handle,
89 static void mpi3mr_dev_rmhs_complete_iou(struct mpi3mr_softc *sc,
91 static void mpi3mr_dev_rmhs_complete_tm(struct mpi3mr_softc *sc,
93 static void mpi3mr_send_evt_ack(struct mpi3mr_softc *sc, U8 event,
95 static void mpi3mr_print_fault_info(struct mpi3mr_softc *sc);
96 static inline void mpi3mr_set_diagsave(struct mpi3mr_softc *sc);
145 	    U32 timeout, struct mpi3mr_softc *sc)  in wait_for_completion_timeout_tm()  argument
150 		msleep(&sc->tm_chan, &sc->mpi3mr_mtx, PRIBIO,  in wait_for_completion_timeout_tm()
163 poll_for_command_completion(struct mpi3mr_softc *sc,  in poll_for_command_completion()  argument
168                mpi3mr_complete_admin_cmd(sc);  in poll_for_command_completion()
189 mpi3mr_trigger_snapdump(struct mpi3mr_softc *sc, U16 reason_code)  in mpi3mr_trigger_snapdump()  argument
193 	mpi3mr_dprint(sc, MPI3MR_INFO, "snapdump triggered: reason code: %s\n",  in mpi3mr_trigger_snapdump()
196 	mpi3mr_set_diagsave(sc);  in mpi3mr_trigger_snapdump()
197 	mpi3mr_issue_reset(sc, MPI3_SYSIF_HOST_DIAG_RESET_ACTION_DIAG_FAULT,  in mpi3mr_trigger_snapdump()
201 		host_diagnostic = mpi3mr_regread(sc, MPI3_SYSIF_HOST_DIAG_OFFSET);  in mpi3mr_trigger_snapdump()
224 static void mpi3mr_check_rh_fault_ioc(struct mpi3mr_softc *sc, U16 reason_code)  in mpi3mr_check_rh_fault_ioc()  argument
228 	if (sc->unrecoverable) {  in mpi3mr_check_rh_fault_ioc()
229 		mpi3mr_dprint(sc, MPI3MR_ERROR, "controller is unrecoverable\n");  in mpi3mr_check_rh_fault_ioc()
233 	ioc_status = mpi3mr_regread(sc, MPI3_SYSIF_IOC_STATUS_OFFSET);  in mpi3mr_check_rh_fault_ioc()
236 		mpi3mr_print_fault_info(sc);  in mpi3mr_check_rh_fault_ioc()
240 	mpi3mr_trigger_snapdump(sc, reason_code);  in mpi3mr_check_rh_fault_ioc()
245 static void * mpi3mr_get_reply_virt_addr(struct mpi3mr_softc *sc,  in mpi3mr_get_reply_virt_addr()  argument
250 	if ((phys_addr < sc->reply_buf_dma_min_address) ||  in mpi3mr_get_reply_virt_addr()
251 	    (phys_addr > sc->reply_buf_dma_max_address))  in mpi3mr_get_reply_virt_addr()
254 	return sc->reply_buf + (phys_addr - sc->reply_buf_phys);  in mpi3mr_get_reply_virt_addr()
257 static void * mpi3mr_get_sensebuf_virt_addr(struct mpi3mr_softc *sc,  in mpi3mr_get_sensebuf_virt_addr()  argument
262 	return sc->sense_buf + (phys_addr - sc->sense_buf_phys);  in mpi3mr_get_sensebuf_virt_addr()
265 static void mpi3mr_repost_reply_buf(struct mpi3mr_softc *sc,  in mpi3mr_repost_reply_buf()  argument
270 	mtx_lock_spin(&sc->reply_free_q_lock);  in mpi3mr_repost_reply_buf()
271 	old_idx  =  sc->reply_free_q_host_index;  in mpi3mr_repost_reply_buf()
272 	sc->reply_free_q_host_index = ((sc->reply_free_q_host_index ==  in mpi3mr_repost_reply_buf()
273 	    (sc->reply_free_q_sz - 1)) ? 0 :  in mpi3mr_repost_reply_buf()
274 	    (sc->reply_free_q_host_index + 1));  in mpi3mr_repost_reply_buf()
275 	sc->reply_free_q[old_idx] = reply_dma;  in mpi3mr_repost_reply_buf()
276 	mpi3mr_regwrite(sc, MPI3_SYSIF_REPLY_FREE_HOST_INDEX_OFFSET,  in mpi3mr_repost_reply_buf()
277 		sc->reply_free_q_host_index);  in mpi3mr_repost_reply_buf()
278 	mtx_unlock_spin(&sc->reply_free_q_lock);  in mpi3mr_repost_reply_buf()
281 static void mpi3mr_repost_sense_buf(struct mpi3mr_softc *sc,  in mpi3mr_repost_sense_buf()  argument
286 	mtx_lock_spin(&sc->sense_buf_q_lock);  in mpi3mr_repost_sense_buf()
287 	old_idx  =  sc->sense_buf_q_host_index;  in mpi3mr_repost_sense_buf()
288 	sc->sense_buf_q_host_index = ((sc->sense_buf_q_host_index ==  in mpi3mr_repost_sense_buf()
289 	    (sc->sense_buf_q_sz - 1)) ? 0 :  in mpi3mr_repost_sense_buf()
290 	    (sc->sense_buf_q_host_index + 1));  in mpi3mr_repost_sense_buf()
291 	sc->sense_buf_q[old_idx] = sense_buf_phys;  in mpi3mr_repost_sense_buf()
292 	mpi3mr_regwrite(sc, MPI3_SYSIF_SENSE_BUF_FREE_HOST_INDEX_OFFSET,  in mpi3mr_repost_sense_buf()
293 		sc->sense_buf_q_host_index);  in mpi3mr_repost_sense_buf()
294 	mtx_unlock_spin(&sc->sense_buf_q_lock);  in mpi3mr_repost_sense_buf()
298 void mpi3mr_set_io_divert_for_all_vd_in_tg(struct mpi3mr_softc *sc,  in mpi3mr_set_io_divert_for_all_vd_in_tg()  argument
303 	mtx_lock_spin(&sc->target_lock);  in mpi3mr_set_io_divert_for_all_vd_in_tg()
304 	TAILQ_FOREACH(target, &sc->cam_sc->tgt_list, tgt_next) {  in mpi3mr_set_io_divert_for_all_vd_in_tg()
308 	mtx_unlock_spin(&sc->target_lock);  in mpi3mr_set_io_divert_for_all_vd_in_tg()
323 int mpi3mr_submit_admin_cmd(struct mpi3mr_softc *sc, void *admin_req,  in mpi3mr_submit_admin_cmd()  argument
330 	mtx_lock_spin(&sc->admin_req_lock);  in mpi3mr_submit_admin_cmd()
331 	areq_pi = sc->admin_req_pi;  in mpi3mr_submit_admin_cmd()
332 	areq_ci = sc->admin_req_ci;  in mpi3mr_submit_admin_cmd()
333 	max_entries = sc->num_admin_reqs;  in mpi3mr_submit_admin_cmd()
335 	if (sc->unrecoverable)  in mpi3mr_submit_admin_cmd()
341 		    sc->name);  in mpi3mr_submit_admin_cmd()
345 	areq_entry = (U8 *)sc->admin_req + (areq_pi *  in mpi3mr_submit_admin_cmd()
352 	sc->admin_req_pi = areq_pi;  in mpi3mr_submit_admin_cmd()
354 	mpi3mr_regwrite(sc, MPI3_SYSIF_ADMIN_REQ_Q_PI_OFFSET, sc->admin_req_pi);  in mpi3mr_submit_admin_cmd()
357 	mtx_unlock_spin(&sc->admin_req_lock);  in mpi3mr_submit_admin_cmd()
393 int mpi3mr_submit_io(struct mpi3mr_softc *sc,  in mpi3mr_submit_io()  argument
399 	U16 req_sz = sc->facts.op_req_sz;  in mpi3mr_submit_io()
407 		irq_ctx = &sc->irq_ctx[op_req_q->reply_qid - 1];  in mpi3mr_submit_io()
408 		mpi3mr_complete_io_cmd(sc, irq_ctx);  in mpi3mr_submit_io()
412 				sc->name);  in mpi3mr_submit_io()
425 	mpi3mr_atomic_inc(&sc->op_reply_q[op_req_q->reply_qid - 1].pend_ios);  in mpi3mr_submit_io()
427 	mpi3mr_regwrite(sc, MPI3_SYSIF_OPER_REQ_Q_N_PI_OFFSET(op_req_q->qid), op_req_q->pi);  in mpi3mr_submit_io()
428 	if (sc->mpi3mr_debug & MPI3MR_TRACE) {  in mpi3mr_submit_io()
429 		device_printf(sc->mpi3mr_dev, "IO submission: QID:%d PI:0x%x\n", op_req_q->qid, op_req_q->pi);  in mpi3mr_submit_io()
458 void mpi3mr_enable_interrupts(struct mpi3mr_softc *sc)  in mpi3mr_enable_interrupts()  argument
460 	sc->intr_enabled = 1;  in mpi3mr_enable_interrupts()
463 void mpi3mr_disable_interrupts(struct mpi3mr_softc *sc)  in mpi3mr_disable_interrupts()  argument
465 	sc->intr_enabled = 0;  in mpi3mr_disable_interrupts()
477 static int mpi3mr_delete_op_reply_queue(struct mpi3mr_softc *sc, U16 qid)  in mpi3mr_delete_op_reply_queue()  argument
484 	op_reply_q = &sc->op_reply_q[qid - 1];  in mpi3mr_delete_op_reply_queue()
490 		    sc->name);  in mpi3mr_delete_op_reply_queue()
496 	mtx_lock(&sc->init_cmds.completion.lock);  in mpi3mr_delete_op_reply_queue()
497 	if (sc->init_cmds.state & MPI3MR_CMD_PENDING) {  in mpi3mr_delete_op_reply_queue()
500 		    sc->name);  in mpi3mr_delete_op_reply_queue()
501 		mtx_unlock(&sc->init_cmds.completion.lock);  in mpi3mr_delete_op_reply_queue()
505 	if (sc->init_cmds.state & MPI3MR_CMD_PENDING) {  in mpi3mr_delete_op_reply_queue()
508 		    sc->name);  in mpi3mr_delete_op_reply_queue()
511 	sc->init_cmds.state = MPI3MR_CMD_PENDING;  in mpi3mr_delete_op_reply_queue()
512 	sc->init_cmds.is_waiting = 1;  in mpi3mr_delete_op_reply_queue()
513 	sc->init_cmds.callback = NULL;  in mpi3mr_delete_op_reply_queue()
518 	init_completion(&sc->init_cmds.completion);  in mpi3mr_delete_op_reply_queue()
519 	retval = mpi3mr_submit_admin_cmd(sc, &delq_req, sizeof(delq_req));  in mpi3mr_delete_op_reply_queue()
522 		    sc->name);  in mpi3mr_delete_op_reply_queue()
525 	wait_for_completion_timeout(&sc->init_cmds.completion,  in mpi3mr_delete_op_reply_queue()
527 	if (!(sc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {  in mpi3mr_delete_op_reply_queue()
529 		    sc->name);  in mpi3mr_delete_op_reply_queue()
530 		mpi3mr_check_rh_fault_ioc(sc,  in mpi3mr_delete_op_reply_queue()
532 		sc->unrecoverable = 1;  in mpi3mr_delete_op_reply_queue()
537 	if ((sc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)  in mpi3mr_delete_op_reply_queue()
540 		    " Loginfo(0x%08x) \n" , sc->name,  in mpi3mr_delete_op_reply_queue()
541 		    (sc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),  in mpi3mr_delete_op_reply_queue()
542 		    sc->init_cmds.ioc_loginfo);  in mpi3mr_delete_op_reply_queue()
546 	sc->irq_ctx[qid - 1].op_reply_q = NULL;  in mpi3mr_delete_op_reply_queue()
548 	if (sc->op_reply_q[qid - 1].q_base_phys != 0)  in mpi3mr_delete_op_reply_queue()
549 		bus_dmamap_unload(sc->op_reply_q[qid - 1].q_base_tag, sc->op_reply_q[qid - 1].q_base_dmamap);  in mpi3mr_delete_op_reply_queue()
550 	if (sc->op_reply_q[qid - 1].q_base != NULL)  in mpi3mr_delete_op_reply_queue()
551 …bus_dmamem_free(sc->op_reply_q[qid - 1].q_base_tag, sc->op_reply_q[qid - 1].q_base, sc->op_reply_q…  in mpi3mr_delete_op_reply_queue()
552 	if (sc->op_reply_q[qid - 1].q_base_tag != NULL)  in mpi3mr_delete_op_reply_queue()
553 		bus_dma_tag_destroy(sc->op_reply_q[qid - 1].q_base_tag);  in mpi3mr_delete_op_reply_queue()
555 	sc->op_reply_q[qid - 1].q_base = NULL;  in mpi3mr_delete_op_reply_queue()
556 	sc->op_reply_q[qid - 1].qid = 0;  in mpi3mr_delete_op_reply_queue()
558 	sc->init_cmds.state = MPI3MR_CMD_NOTUSED;  in mpi3mr_delete_op_reply_queue()
559 	mtx_unlock(&sc->init_cmds.completion.lock);  in mpi3mr_delete_op_reply_queue()
574 static int mpi3mr_create_op_reply_queue(struct mpi3mr_softc *sc, U16 qid)  in mpi3mr_create_op_reply_queue()  argument
581 	op_reply_q = &sc->op_reply_q[qid - 1];  in mpi3mr_create_op_reply_queue()
587 		    sc->name, op_reply_q->qid);  in mpi3mr_create_op_reply_queue()
592 	if (pci_get_revid(sc->mpi3mr_dev) == SAS4116_CHIP_REV_A0)  in mpi3mr_create_op_reply_queue()
597 	op_reply_q->qsz = op_reply_q->num_replies * sc->op_reply_sz;  in mpi3mr_create_op_reply_queue()
604 		if (bus_dma_tag_create(sc->mpi3mr_parent_dmat,    /* parent */  in mpi3mr_create_op_reply_queue()
606 					sc->dma_loaddr,		/* lowaddr */  in mpi3mr_create_op_reply_queue()
615 			mpi3mr_dprint(sc, MPI3MR_ERROR, "Cannot allocate Operational reply DMA tag\n");  in mpi3mr_create_op_reply_queue()
621 			mpi3mr_dprint(sc, MPI3MR_ERROR, "%s: Cannot allocate replies memory\n", __func__);  in mpi3mr_create_op_reply_queue()
627 …mpi3mr_dprint(sc, MPI3MR_XINFO, "Operational Reply queue ID: %d phys addr= %#016jx virt_addr: %pa …  in mpi3mr_create_op_reply_queue()
634 			    sc->name, qid);  in mpi3mr_create_op_reply_queue()
641 	mtx_lock(&sc->init_cmds.completion.lock);  in mpi3mr_create_op_reply_queue()
642 	if (sc->init_cmds.state & MPI3MR_CMD_PENDING) {  in mpi3mr_create_op_reply_queue()
645 		    sc->name);  in mpi3mr_create_op_reply_queue()
646 		mtx_unlock(&sc->init_cmds.completion.lock);  in mpi3mr_create_op_reply_queue()
650 	sc->init_cmds.state = MPI3MR_CMD_PENDING;  in mpi3mr_create_op_reply_queue()
651 	sc->init_cmds.is_waiting = 1;  in mpi3mr_create_op_reply_queue()
652 	sc->init_cmds.callback = NULL;  in mpi3mr_create_op_reply_queue()
657 	create_req.MSIxIndex = sc->irq_ctx[qid - 1].msix_index;  in mpi3mr_create_op_reply_queue()
661 	init_completion(&sc->init_cmds.completion);  in mpi3mr_create_op_reply_queue()
662 	retval = mpi3mr_submit_admin_cmd(sc, &create_req,  in mpi3mr_create_op_reply_queue()
666 		    sc->name);  in mpi3mr_create_op_reply_queue()
670 	wait_for_completion_timeout(&sc->init_cmds.completion,  in mpi3mr_create_op_reply_queue()
672 	if (!(sc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {  in mpi3mr_create_op_reply_queue()
674 		    sc->name);  in mpi3mr_create_op_reply_queue()
675 		mpi3mr_check_rh_fault_ioc(sc,  in mpi3mr_create_op_reply_queue()
677 		sc->unrecoverable = 1;  in mpi3mr_create_op_reply_queue()
682 	if ((sc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)  in mpi3mr_create_op_reply_queue()
685 		    " Loginfo(0x%08x) \n" , sc->name,  in mpi3mr_create_op_reply_queue()
686 		    (sc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),  in mpi3mr_create_op_reply_queue()
687 		    sc->init_cmds.ioc_loginfo);  in mpi3mr_create_op_reply_queue()
692 	sc->irq_ctx[qid - 1].op_reply_q = op_reply_q;  in mpi3mr_create_op_reply_queue()
695 	sc->init_cmds.state = MPI3MR_CMD_NOTUSED;  in mpi3mr_create_op_reply_queue()
696 	mtx_unlock(&sc->init_cmds.completion.lock);  in mpi3mr_create_op_reply_queue()
723 static int mpi3mr_create_op_req_queue(struct mpi3mr_softc *sc, U16 req_qid, U8 reply_qid)  in mpi3mr_create_op_req_queue()  argument
730 	op_req_q = &sc->op_req_q[req_qid - 1];  in mpi3mr_create_op_req_queue()
736 		    sc->name, op_req_q->qid);  in mpi3mr_create_op_req_queue()
743 	op_req_q->qsz = op_req_q->num_reqs * sc->facts.op_req_sz;  in mpi3mr_create_op_req_queue()
750 		if (bus_dma_tag_create(sc->mpi3mr_parent_dmat,    /* parent */  in mpi3mr_create_op_req_queue()
752 					sc->dma_loaddr,		/* lowaddr */  in mpi3mr_create_op_req_queue()
761 			mpi3mr_dprint(sc, MPI3MR_ERROR, "Cannot allocate request DMA tag\n");  in mpi3mr_create_op_req_queue()
767 			mpi3mr_dprint(sc, MPI3MR_ERROR, "%s: Cannot allocate replies memory\n", __func__);  in mpi3mr_create_op_req_queue()
776 …mpi3mr_dprint(sc, MPI3MR_XINFO, "Operational Request QID: %d phys addr= %#016jx virt addr= %pa siz…  in mpi3mr_create_op_req_queue()
782 			    sc->name, req_qid);  in mpi3mr_create_op_req_queue()
789 	mtx_lock(&sc->init_cmds.completion.lock);  in mpi3mr_create_op_req_queue()
790 	if (sc->init_cmds.state & MPI3MR_CMD_PENDING) {  in mpi3mr_create_op_req_queue()
793 		    sc->name);  in mpi3mr_create_op_req_queue()
794 		mtx_unlock(&sc->init_cmds.completion.lock);  in mpi3mr_create_op_req_queue()
798 	sc->init_cmds.state = MPI3MR_CMD_PENDING;  in mpi3mr_create_op_req_queue()
799 	sc->init_cmds.is_waiting = 1;  in mpi3mr_create_op_req_queue()
800 	sc->init_cmds.callback = NULL;  in mpi3mr_create_op_req_queue()
809 	init_completion(&sc->init_cmds.completion);  in mpi3mr_create_op_req_queue()
810 	retval = mpi3mr_submit_admin_cmd(sc, &create_req,  in mpi3mr_create_op_req_queue()
814 		    sc->name);  in mpi3mr_create_op_req_queue()
818 	wait_for_completion_timeout(&sc->init_cmds.completion,  in mpi3mr_create_op_req_queue()
821 	if (!(sc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {  in mpi3mr_create_op_req_queue()
823 		    sc->name);  in mpi3mr_create_op_req_queue()
824 		mpi3mr_check_rh_fault_ioc(sc,  in mpi3mr_create_op_req_queue()
826 		sc->unrecoverable = 1;  in mpi3mr_create_op_req_queue()
831 	if ((sc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)  in mpi3mr_create_op_req_queue()
834 		    " Loginfo(0x%08x) \n" , sc->name,  in mpi3mr_create_op_req_queue()
835 		    (sc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),  in mpi3mr_create_op_req_queue()
836 		    sc->init_cmds.ioc_loginfo);  in mpi3mr_create_op_req_queue()
843 	sc->init_cmds.state = MPI3MR_CMD_NOTUSED;  in mpi3mr_create_op_req_queue()
844 	mtx_unlock(&sc->init_cmds.completion.lock);  in mpi3mr_create_op_req_queue()
866 static int mpi3mr_create_op_queues(struct mpi3mr_softc *sc)  in mpi3mr_create_op_queues()  argument
871 	num_queues = min(sc->facts.max_op_reply_q,  in mpi3mr_create_op_queues()
872 	    sc->facts.max_op_req_q);  in mpi3mr_create_op_queues()
873 	num_queues = min(num_queues, sc->msix_count);  in mpi3mr_create_op_queues()
879 	if (sc->num_queues)  in mpi3mr_create_op_queues()
880 		num_queues = sc->num_queues;  in mpi3mr_create_op_queues()
882 	mpi3mr_dprint(sc, MPI3MR_XINFO, "Trying to create %d Operational Q pairs\n",  in mpi3mr_create_op_queues()
885 	if (!sc->op_req_q) {  in mpi3mr_create_op_queues()
886 		sc->op_req_q = malloc(sizeof(struct mpi3mr_op_req_queue) *  in mpi3mr_create_op_queues()
889 		if (!sc->op_req_q) {  in mpi3mr_create_op_queues()
890 			mpi3mr_dprint(sc, MPI3MR_ERROR, "Failed to alloc memory for Request queue info\n");  in mpi3mr_create_op_queues()
896 	if (!sc->op_reply_q) {  in mpi3mr_create_op_queues()
897 		sc->op_reply_q = malloc(sizeof(struct mpi3mr_op_reply_queue) * num_queues,  in mpi3mr_create_op_queues()
900 		if (!sc->op_reply_q) {  in mpi3mr_create_op_queues()
901 			mpi3mr_dprint(sc, MPI3MR_ERROR, "Failed to alloc memory for Reply queue info\n");  in mpi3mr_create_op_queues()
907 	sc->num_hosttag_op_req_q = (sc->max_host_ios + 1) / num_queues;  in mpi3mr_create_op_queues()
912 		if (mpi3mr_create_op_reply_queue(sc, qid)) {  in mpi3mr_create_op_queues()
913 			mpi3mr_dprint(sc, MPI3MR_ERROR, "Failed to create Reply queue %d\n",  in mpi3mr_create_op_queues()
917 		if (mpi3mr_create_op_req_queue(sc, qid,  in mpi3mr_create_op_queues()
918 		    sc->op_reply_q[qid - 1].qid)) {  in mpi3mr_create_op_queues()
919 			mpi3mr_delete_op_reply_queue(sc, qid);  in mpi3mr_create_op_queues()
920 			mpi3mr_dprint(sc, MPI3MR_ERROR, "Failed to create Request queue %d\n",  in mpi3mr_create_op_queues()
933 	if (!sc->num_queues) {  in mpi3mr_create_op_queues()
934 		sc->num_queues = i;  in mpi3mr_create_op_queues()
937 			mpi3mr_dprint(sc, MPI3MR_ERROR, "Number of queues (%d) post reset are not same as"   in mpi3mr_create_op_queues()
943 	mpi3mr_dprint(sc, MPI3MR_INFO, "Successfully created %d Operational Queue pairs\n",  in mpi3mr_create_op_queues()
944 	    sc->num_queues);  in mpi3mr_create_op_queues()
945 	mpi3mr_dprint(sc, MPI3MR_INFO, "Request Queue QD: %d Reply queue QD: %d\n",  in mpi3mr_create_op_queues()
946 	    sc->op_req_q[0].num_reqs, sc->op_reply_q[0].num_replies);  in mpi3mr_create_op_queues()
950 	if (sc->op_req_q) {  in mpi3mr_create_op_queues()
951 		free(sc->op_req_q, M_MPI3MR);  in mpi3mr_create_op_queues()
952 		sc->op_req_q = NULL;  in mpi3mr_create_op_queues()
954 	if (sc->op_reply_q) {  in mpi3mr_create_op_queues()
955 		free(sc->op_reply_q, M_MPI3MR);  in mpi3mr_create_op_queues()
956 		sc->op_reply_q = NULL;  in mpi3mr_create_op_queues()
968 static int mpi3mr_setup_admin_qpair(struct mpi3mr_softc *sc)  in mpi3mr_setup_admin_qpair()  argument
973 	sc->admin_req_q_sz = MPI3MR_AREQQ_SIZE;  in mpi3mr_setup_admin_qpair()
974 	sc->num_admin_reqs = sc->admin_req_q_sz / MPI3MR_AREQ_FRAME_SZ;  in mpi3mr_setup_admin_qpair()
975 	sc->admin_req_ci = sc->admin_req_pi = 0;  in mpi3mr_setup_admin_qpair()
977 	sc->admin_reply_q_sz = MPI3MR_AREPQ_SIZE;  in mpi3mr_setup_admin_qpair()
978 	sc->num_admin_replies = sc->admin_reply_q_sz/ MPI3MR_AREP_FRAME_SZ;  in mpi3mr_setup_admin_qpair()
979 	sc->admin_reply_ci = 0;  in mpi3mr_setup_admin_qpair()
980 	sc->admin_reply_ephase = 1;  in mpi3mr_setup_admin_qpair()
982 	if (!sc->admin_req) {  in mpi3mr_setup_admin_qpair()
988 		if (bus_dma_tag_create(sc->mpi3mr_parent_dmat,    /* parent */  in mpi3mr_setup_admin_qpair()
993 					sc->admin_req_q_sz,	/* maxsize */  in mpi3mr_setup_admin_qpair()
995 					sc->admin_req_q_sz,	/* maxsegsize */  in mpi3mr_setup_admin_qpair()
998 					&sc->admin_req_tag)) {  in mpi3mr_setup_admin_qpair()
999 			mpi3mr_dprint(sc, MPI3MR_ERROR, "Cannot allocate request DMA tag\n");  in mpi3mr_setup_admin_qpair()
1003 		if (bus_dmamem_alloc(sc->admin_req_tag, (void **)&sc->admin_req,  in mpi3mr_setup_admin_qpair()
1004 		    BUS_DMA_NOWAIT, &sc->admin_req_dmamap)) {  in mpi3mr_setup_admin_qpair()
1005 			mpi3mr_dprint(sc, MPI3MR_ERROR, "%s: Cannot allocate replies memory\n", __func__);  in mpi3mr_setup_admin_qpair()
1008 		bzero(sc->admin_req, sc->admin_req_q_sz);  in mpi3mr_setup_admin_qpair()
1009 		bus_dmamap_load(sc->admin_req_tag, sc->admin_req_dmamap, sc->admin_req, sc->admin_req_q_sz,  in mpi3mr_setup_admin_qpair()
1010 		    mpi3mr_memaddr_cb, &sc->admin_req_phys, BUS_DMA_NOWAIT);  in mpi3mr_setup_admin_qpair()
1011 		mpi3mr_dprint(sc, MPI3MR_XINFO, "Admin Req queue phys addr= %#016jx size= %d\n",  in mpi3mr_setup_admin_qpair()
1012 		    (uintmax_t)sc->admin_req_phys, sc->admin_req_q_sz);  in mpi3mr_setup_admin_qpair()
1014 		if (!sc->admin_req)  in mpi3mr_setup_admin_qpair()
1018 			    sc->name);  in mpi3mr_setup_admin_qpair()
1023 	if (!sc->admin_reply) {  in mpi3mr_setup_admin_qpair()
1024 		mtx_init(&sc->admin_reply_lock, "Admin Reply Queue Lock", NULL, MTX_SPIN);  in mpi3mr_setup_admin_qpair()
1026 		if (bus_dma_tag_create(sc->mpi3mr_parent_dmat,    /* parent */  in mpi3mr_setup_admin_qpair()
1031 					sc->admin_reply_q_sz,	/* maxsize */  in mpi3mr_setup_admin_qpair()
1033 					sc->admin_reply_q_sz,	/* maxsegsize */  in mpi3mr_setup_admin_qpair()
1036 					&sc->admin_reply_tag)) {  in mpi3mr_setup_admin_qpair()
1037 			mpi3mr_dprint(sc, MPI3MR_ERROR, "Cannot allocate reply DMA tag\n");  in mpi3mr_setup_admin_qpair()
1041 		if (bus_dmamem_alloc(sc->admin_reply_tag, (void **)&sc->admin_reply,  in mpi3mr_setup_admin_qpair()
1042 		    BUS_DMA_NOWAIT, &sc->admin_reply_dmamap)) {  in mpi3mr_setup_admin_qpair()
1043 			mpi3mr_dprint(sc, MPI3MR_ERROR, "%s: Cannot allocate replies memory\n", __func__);  in mpi3mr_setup_admin_qpair()
1046 		bzero(sc->admin_reply, sc->admin_reply_q_sz);  in mpi3mr_setup_admin_qpair()
1047 …bus_dmamap_load(sc->admin_reply_tag, sc->admin_reply_dmamap, sc->admin_reply, sc->admin_reply_q_sz,  in mpi3mr_setup_admin_qpair()
1048 		    mpi3mr_memaddr_cb, &sc->admin_reply_phys, BUS_DMA_NOWAIT);  in mpi3mr_setup_admin_qpair()
1049 		mpi3mr_dprint(sc, MPI3MR_XINFO, "Admin Reply queue phys addr= %#016jx size= %d\n",  in mpi3mr_setup_admin_qpair()
1050 		    (uintmax_t)sc->admin_reply_phys, sc->admin_req_q_sz);  in mpi3mr_setup_admin_qpair()
1053 		if (!sc->admin_reply)  in mpi3mr_setup_admin_qpair()
1057 			    sc->name);  in mpi3mr_setup_admin_qpair()
1062 	num_adm_entries = (sc->num_admin_replies << 16) |  in mpi3mr_setup_admin_qpair()
1063 				(sc->num_admin_reqs);  in mpi3mr_setup_admin_qpair()
1064 	mpi3mr_regwrite(sc, MPI3_SYSIF_ADMIN_Q_NUM_ENTRIES_OFFSET, num_adm_entries);  in mpi3mr_setup_admin_qpair()
1065 	mpi3mr_regwrite64(sc, MPI3_SYSIF_ADMIN_REQ_Q_ADDR_LOW_OFFSET, sc->admin_req_phys);  in mpi3mr_setup_admin_qpair()
1066 	mpi3mr_regwrite64(sc, MPI3_SYSIF_ADMIN_REPLY_Q_ADDR_LOW_OFFSET, sc->admin_reply_phys);  in mpi3mr_setup_admin_qpair()
1067 	mpi3mr_regwrite(sc, MPI3_SYSIF_ADMIN_REQ_Q_PI_OFFSET, sc->admin_req_pi);  in mpi3mr_setup_admin_qpair()
1068 	mpi3mr_regwrite(sc, MPI3_SYSIF_ADMIN_REPLY_Q_CI_OFFSET, sc->admin_reply_ci);  in mpi3mr_setup_admin_qpair()
1074 	if (sc->admin_reply_phys)  in mpi3mr_setup_admin_qpair()
1075 		bus_dmamap_unload(sc->admin_reply_tag, sc->admin_reply_dmamap);  in mpi3mr_setup_admin_qpair()
1077 	if (sc->admin_reply != NULL)  in mpi3mr_setup_admin_qpair()
1078 		bus_dmamem_free(sc->admin_reply_tag, sc->admin_reply,  in mpi3mr_setup_admin_qpair()
1079 		    sc->admin_reply_dmamap);  in mpi3mr_setup_admin_qpair()
1081 	if (sc->admin_reply_tag != NULL)  in mpi3mr_setup_admin_qpair()
1082 		bus_dma_tag_destroy(sc->admin_reply_tag);  in mpi3mr_setup_admin_qpair()
1085 	if (sc->admin_req_phys)  in mpi3mr_setup_admin_qpair()
1086 		bus_dmamap_unload(sc->admin_req_tag, sc->admin_req_dmamap);  in mpi3mr_setup_admin_qpair()
1088 	if (sc->admin_req != NULL)  in mpi3mr_setup_admin_qpair()
1089 		bus_dmamem_free(sc->admin_req_tag, sc->admin_req,  in mpi3mr_setup_admin_qpair()
1090 		    sc->admin_req_dmamap);  in mpi3mr_setup_admin_qpair()
1092 	if (sc->admin_req_tag != NULL)  in mpi3mr_setup_admin_qpair()
1093 		bus_dma_tag_destroy(sc->admin_req_tag);  in mpi3mr_setup_admin_qpair()
1107 static void mpi3mr_print_fault_info(struct mpi3mr_softc *sc)  in mpi3mr_print_fault_info()  argument
1111 	ioc_status = mpi3mr_regread(sc, MPI3_SYSIF_IOC_STATUS_OFFSET);  in mpi3mr_print_fault_info()
1114 		code = mpi3mr_regread(sc, MPI3_SYSIF_FAULT_OFFSET) &  in mpi3mr_print_fault_info()
1116 		code1 = mpi3mr_regread(sc, MPI3_SYSIF_FAULT_INFO0_OFFSET);  in mpi3mr_print_fault_info()
1117 		code2 = mpi3mr_regread(sc, MPI3_SYSIF_FAULT_INFO1_OFFSET);  in mpi3mr_print_fault_info()
1118 		code3 = mpi3mr_regread(sc, MPI3_SYSIF_FAULT_INFO2_OFFSET);  in mpi3mr_print_fault_info()
1120 		    sc->name, code, code1, code2, code3);  in mpi3mr_print_fault_info()
1124 enum mpi3mr_iocstate mpi3mr_get_iocstate(struct mpi3mr_softc *sc)  in mpi3mr_get_iocstate()  argument
1129 	ioc_status = mpi3mr_regread(sc, MPI3_SYSIF_IOC_STATUS_OFFSET);  in mpi3mr_get_iocstate()
1130 	ioc_control = mpi3mr_regread(sc, MPI3_SYSIF_IOC_CONFIG_OFFSET);  in mpi3mr_get_iocstate()
1132 	if(sc->unrecoverable)  in mpi3mr_get_iocstate()
1150 static inline void mpi3mr_clear_resethistory(struct mpi3mr_softc *sc)  in mpi3mr_clear_resethistory()  argument
1154 	ioc_status = mpi3mr_regread(sc, MPI3_SYSIF_IOC_STATUS_OFFSET);  in mpi3mr_clear_resethistory()
1156 		mpi3mr_regwrite(sc, MPI3_SYSIF_IOC_STATUS_OFFSET, ioc_status);  in mpi3mr_clear_resethistory()
1170 static int mpi3mr_mur_ioc(struct mpi3mr_softc *sc, U16 reset_reason)  in mpi3mr_mur_ioc()  argument
1175         mpi3mr_dprint(sc, MPI3MR_INFO, "Issuing Message Unit Reset(MUR)\n");  in mpi3mr_mur_ioc()
1176         if (sc->unrecoverable) {  in mpi3mr_mur_ioc()
1177                 mpi3mr_dprint(sc, MPI3MR_ERROR, "IOC is unrecoverable MUR not issued\n");  in mpi3mr_mur_ioc()
1180         mpi3mr_clear_resethistory(sc);  in mpi3mr_mur_ioc()
1184 			(sc->facts.ioc_num <<  in mpi3mr_mur_ioc()
1186 	mpi3mr_regwrite(sc, MPI3_SYSIF_SCRATCHPAD0_OFFSET, scratch_pad0);  in mpi3mr_mur_ioc()
1187 	ioc_config = mpi3mr_regread(sc, MPI3_SYSIF_IOC_CONFIG_OFFSET);  in mpi3mr_mur_ioc()
1189 	mpi3mr_regwrite(sc, MPI3_SYSIF_IOC_CONFIG_OFFSET, ioc_config);  in mpi3mr_mur_ioc()
1193 		ioc_status = mpi3mr_regread(sc, MPI3_SYSIF_IOC_STATUS_OFFSET);  in mpi3mr_mur_ioc()
1195                         mpi3mr_clear_resethistory(sc);  in mpi3mr_mur_ioc()
1197 				mpi3mr_regread(sc, MPI3_SYSIF_IOC_CONFIG_OFFSET);  in mpi3mr_mur_ioc()
1208 	ioc_status = mpi3mr_regread(sc, MPI3_SYSIF_IOC_STATUS_OFFSET);  in mpi3mr_mur_ioc()
1209 	ioc_config = mpi3mr_regread(sc, MPI3_SYSIF_IOC_CONFIG_OFFSET);  in mpi3mr_mur_ioc()
1211         mpi3mr_dprint(sc, MPI3MR_INFO, "IOC Status/Config after %s MUR is (0x%x)/(0x%x)\n",  in mpi3mr_mur_ioc()
1225 static int mpi3mr_bring_ioc_ready(struct mpi3mr_softc *sc)  in mpi3mr_bring_ioc_ready()  argument
1230 	ioc_config = mpi3mr_regread(sc, MPI3_SYSIF_IOC_CONFIG_OFFSET);  in mpi3mr_bring_ioc_ready()
1232 	mpi3mr_regwrite(sc, MPI3_SYSIF_IOC_CONFIG_OFFSET, ioc_config);  in mpi3mr_bring_ioc_ready()
1234         timeout = sc->ready_timeout * 10;  in mpi3mr_bring_ioc_ready()
1236                 current_state = mpi3mr_get_iocstate(sc);  in mpi3mr_bring_ioc_ready()
1408 static inline bool mpi3mr_diagfault_success(struct mpi3mr_softc *sc,  in mpi3mr_diagfault_success()  argument
1413 	mpi3mr_print_fault_info(sc);  in mpi3mr_diagfault_success()
1427 static int mpi3mr_issue_iocfacts(struct mpi3mr_softc *sc,  in mpi3mr_issue_iocfacts()  argument
1449         if (bus_dma_tag_create(sc->mpi3mr_parent_dmat,    /* parent */  in mpi3mr_issue_iocfacts()
1460 		mpi3mr_dprint(sc, MPI3MR_ERROR, "Cannot allocate request DMA tag\n");  in mpi3mr_issue_iocfacts()
1466 		mpi3mr_dprint(sc, MPI3MR_ERROR, "Func: %s line: %d Data  DMA mem alloc failed\n",  in mpi3mr_issue_iocfacts()
1474 	mpi3mr_dprint(sc, MPI3MR_XINFO, "Func: %s line: %d IOCfacts data phys addr= %#016jx size= %d\n",  in mpi3mr_issue_iocfacts()
1481 		    sc->name);  in mpi3mr_issue_iocfacts()
1485 	mtx_lock(&sc->init_cmds.completion.lock);  in mpi3mr_issue_iocfacts()
1488 	if (sc->init_cmds.state & MPI3MR_CMD_PENDING) {  in mpi3mr_issue_iocfacts()
1491 		    sc->name);  in mpi3mr_issue_iocfacts()
1492 		mtx_unlock(&sc->init_cmds.completion.lock);  in mpi3mr_issue_iocfacts()
1496 	sc->init_cmds.state = MPI3MR_CMD_PENDING;  in mpi3mr_issue_iocfacts()
1497 	sc->init_cmds.is_waiting = 1;  in mpi3mr_issue_iocfacts()
1498 	sc->init_cmds.callback = NULL;  in mpi3mr_issue_iocfacts()
1505 	init_completion(&sc->init_cmds.completion);  in mpi3mr_issue_iocfacts()
1507 	retval = mpi3mr_submit_admin_cmd(sc, &iocfacts_req,  in mpi3mr_issue_iocfacts()
1512 		    sc->name);  in mpi3mr_issue_iocfacts()
1516 	wait_for_completion_timeout(&sc->init_cmds.completion,  in mpi3mr_issue_iocfacts()
1518 	if (!(sc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {  in mpi3mr_issue_iocfacts()
1520 		    sc->name);  in mpi3mr_issue_iocfacts()
1521 		mpi3mr_check_rh_fault_ioc(sc,  in mpi3mr_issue_iocfacts()
1523 		sc->unrecoverable = 1;  in mpi3mr_issue_iocfacts()
1528 	if ((sc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)  in mpi3mr_issue_iocfacts()
1531 		    " Loginfo(0x%08x) \n" , sc->name,  in mpi3mr_issue_iocfacts()
1532 		    (sc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),  in mpi3mr_issue_iocfacts()
1533 		    sc->init_cmds.ioc_loginfo);  in mpi3mr_issue_iocfacts()
1540 	sc->init_cmds.state = MPI3MR_CMD_NOTUSED;  in mpi3mr_issue_iocfacts()
1541 	mtx_unlock(&sc->init_cmds.completion.lock);  in mpi3mr_issue_iocfacts()
1563 static int mpi3mr_process_factsdata(struct mpi3mr_softc *sc,  in mpi3mr_process_factsdata()  argument
1572 		mpi3mr_dprint(sc, MPI3MR_INFO, "IOCFacts data length mismatch "  in mpi3mr_process_factsdata()
1578 	ioc_config = mpi3mr_regread(sc, MPI3_SYSIF_IOC_CONFIG_OFFSET);  in mpi3mr_process_factsdata()
1583 		 mpi3mr_dprint(sc, MPI3MR_INFO, "IOCFacts data reqFrameSize mismatch "  in mpi3mr_process_factsdata()
1588 	memset(&sc->facts, 0, sizeof(sc->facts));  in mpi3mr_process_factsdata()
1591 	sc->facts.op_req_sz = req_sz;  in mpi3mr_process_factsdata()
1592 	sc->op_reply_sz = 1 << ((ioc_config &  in mpi3mr_process_factsdata()
1596 	sc->facts.ioc_num = facts_data->IOCNumber;  in mpi3mr_process_factsdata()
1597         sc->facts.who_init = facts_data->WhoInit;  in mpi3mr_process_factsdata()
1598         sc->facts.max_msix_vectors = facts_data->MaxMSIxVectors;  in mpi3mr_process_factsdata()
1599 	sc->facts.personality = (facts_flags &  in mpi3mr_process_factsdata()
1601 	sc->facts.dma_mask = (facts_flags &  in mpi3mr_process_factsdata()
1604         sc->facts.protocol_flags = facts_data->ProtocolFlags;  in mpi3mr_process_factsdata()
1605         sc->facts.mpi_version = (facts_data->MPIVersion.Word);  in mpi3mr_process_factsdata()
1606         sc->facts.max_reqs = (facts_data->MaxOutstandingRequests);  in mpi3mr_process_factsdata()
1607         sc->facts.product_id = (facts_data->ProductID);  in mpi3mr_process_factsdata()
1608 	sc->facts.reply_sz = (facts_data->ReplyFrameSize) * 4;  in mpi3mr_process_factsdata()
1609         sc->facts.exceptions = (facts_data->IOCExceptions);  in mpi3mr_process_factsdata()
1610         sc->facts.max_perids = (facts_data->MaxPersistentID);  in mpi3mr_process_factsdata()
1611         sc->facts.max_vds = (facts_data->MaxVDs);  in mpi3mr_process_factsdata()
1612         sc->facts.max_hpds = (facts_data->MaxHostPDs);  in mpi3mr_process_factsdata()
1613         sc->facts.max_advhpds = (facts_data->MaxAdvHostPDs);  in mpi3mr_process_factsdata()
1614         sc->facts.max_raidpds = (facts_data->MaxRAIDPDs);  in mpi3mr_process_factsdata()
1615         sc->facts.max_nvme = (facts_data->MaxNVMe);  in mpi3mr_process_factsdata()
1616         sc->facts.max_pcieswitches =  in mpi3mr_process_factsdata()
1618         sc->facts.max_sasexpanders =  in mpi3mr_process_factsdata()
1620         sc->facts.max_sasinitiators =  in mpi3mr_process_factsdata()
1622         sc->facts.max_enclosures = (facts_data->MaxEnclosures);  in mpi3mr_process_factsdata()
1623         sc->facts.min_devhandle = (facts_data->MinDevHandle);  in mpi3mr_process_factsdata()
1624         sc->facts.max_devhandle = (facts_data->MaxDevHandle);  in mpi3mr_process_factsdata()
1625 	sc->facts.max_op_req_q =  in mpi3mr_process_factsdata()
1627 	sc->facts.max_op_reply_q =  in mpi3mr_process_factsdata()
1629         sc->facts.ioc_capabilities =  in mpi3mr_process_factsdata()
1631         sc->facts.fw_ver.build_num =  in mpi3mr_process_factsdata()
1633         sc->facts.fw_ver.cust_id =  in mpi3mr_process_factsdata()
1635         sc->facts.fw_ver.ph_minor = facts_data->FWVersion.PhaseMinor;  in mpi3mr_process_factsdata()
1636         sc->facts.fw_ver.ph_major = facts_data->FWVersion.PhaseMajor;  in mpi3mr_process_factsdata()
1637         sc->facts.fw_ver.gen_minor = facts_data->FWVersion.GenMinor;  in mpi3mr_process_factsdata()
1638         sc->facts.fw_ver.gen_major = facts_data->FWVersion.GenMajor;  in mpi3mr_process_factsdata()
1639         sc->max_msix_vectors = min(sc->max_msix_vectors,  in mpi3mr_process_factsdata()
1640             sc->facts.max_msix_vectors);  in mpi3mr_process_factsdata()
1641         sc->facts.sge_mod_mask = facts_data->SGEModifierMask;  in mpi3mr_process_factsdata()
1642         sc->facts.sge_mod_value = facts_data->SGEModifierValue;  in mpi3mr_process_factsdata()
1643         sc->facts.sge_mod_shift = facts_data->SGEModifierShift;  in mpi3mr_process_factsdata()
1644         sc->facts.shutdown_timeout =  in mpi3mr_process_factsdata()
1646 	sc->facts.max_dev_per_tg = facts_data->MaxDevicesPerThrottleGroup;  in mpi3mr_process_factsdata()
1647 	sc->facts.io_throttle_data_length =  in mpi3mr_process_factsdata()
1649 	sc->facts.max_io_throttle_group =  in mpi3mr_process_factsdata()
1651 	sc->facts.io_throttle_low = facts_data->IOThrottleLow;  in mpi3mr_process_factsdata()
1652 	sc->facts.io_throttle_high = facts_data->IOThrottleHigh;  in mpi3mr_process_factsdata()
1655 	if (sc->facts.io_throttle_data_length)  in mpi3mr_process_factsdata()
1656 		sc->io_throttle_data_length =  in mpi3mr_process_factsdata()
1657 		    (sc->facts.io_throttle_data_length * 2 * 4);  in mpi3mr_process_factsdata()
1660 		sc->io_throttle_data_length = MPI3MR_MAX_SECTORS + 2;  in mpi3mr_process_factsdata()
1662 	sc->io_throttle_high = (sc->facts.io_throttle_high * 2 * 1024);  in mpi3mr_process_factsdata()
1663 	sc->io_throttle_low = (sc->facts.io_throttle_low * 2 * 1024);  in mpi3mr_process_factsdata()
1665 	fwver = &sc->facts.fw_ver;  in mpi3mr_process_factsdata()
1666 	snprintf(sc->fw_version, sizeof(sc->fw_version),  in mpi3mr_process_factsdata()
1671 	mpi3mr_dprint(sc, MPI3MR_INFO, "ioc_num(%d), maxopQ(%d), maxopRepQ(%d), maxdh(%d),"  in mpi3mr_process_factsdata()
1673 	    sc->facts.ioc_num, sc->facts.max_op_req_q,  in mpi3mr_process_factsdata()
1674 	    sc->facts.max_op_reply_q, sc->facts.max_devhandle,  in mpi3mr_process_factsdata()
1675             sc->facts.max_reqs, sc->facts.min_devhandle,  in mpi3mr_process_factsdata()
1676             sc->facts.max_pds, sc->facts.max_msix_vectors,  in mpi3mr_process_factsdata()
1677             sc->facts.max_perids);  in mpi3mr_process_factsdata()
1678         mpi3mr_dprint(sc, MPI3MR_INFO, "SGEModMask 0x%x SGEModVal 0x%x SGEModShift 0x%x\n",  in mpi3mr_process_factsdata()
1679             sc->facts.sge_mod_mask, sc->facts.sge_mod_value,  in mpi3mr_process_factsdata()
1680             sc->facts.sge_mod_shift);  in mpi3mr_process_factsdata()
1681 	mpi3mr_dprint(sc, MPI3MR_INFO,  in mpi3mr_process_factsdata()
1683 	    sc->facts.max_dev_per_tg, sc->facts.max_io_throttle_group,  in mpi3mr_process_factsdata()
1684 	    sc->facts.io_throttle_data_length * 4,  in mpi3mr_process_factsdata()
1685 	    sc->facts.io_throttle_high, sc->facts.io_throttle_low);  in mpi3mr_process_factsdata()
1687 	sc->max_host_ios = sc->facts.max_reqs -  in mpi3mr_process_factsdata()
1696 	if (sc->facts.dma_mask == 0 ||  in mpi3mr_process_factsdata()
1697 	    (sc->facts.dma_mask >= sizeof(bus_addr_t) * 8))  in mpi3mr_process_factsdata()
1698 		sc->dma_loaddr = BUS_SPACE_MAXADDR;  in mpi3mr_process_factsdata()
1700 		sc->dma_loaddr = ~((1ull << sc->facts.dma_mask) - 1);  in mpi3mr_process_factsdata()
1701 	mpi3mr_dprint(sc, MPI3MR_INFO,  in mpi3mr_process_factsdata()
1703 	    sc->facts.dma_mask, sc->dma_loaddr);  in mpi3mr_process_factsdata()
1708 static inline void mpi3mr_setup_reply_free_queues(struct mpi3mr_softc *sc)  in mpi3mr_setup_reply_free_queues()  argument
1714 	for (i = 0, phys_addr = sc->reply_buf_phys;  in mpi3mr_setup_reply_free_queues()
1715 	    i < sc->num_reply_bufs; i++, phys_addr += sc->reply_sz)  in mpi3mr_setup_reply_free_queues()
1716 		sc->reply_free_q[i] = phys_addr;  in mpi3mr_setup_reply_free_queues()
1717 	sc->reply_free_q[i] = (0);  in mpi3mr_setup_reply_free_queues()
1720 	for (i = 0, phys_addr = sc->sense_buf_phys;  in mpi3mr_setup_reply_free_queues()
1721 	    i < sc->num_sense_bufs; i++, phys_addr += MPI3MR_SENSEBUF_SZ)  in mpi3mr_setup_reply_free_queues()
1722 		sc->sense_buf_q[i] = phys_addr;  in mpi3mr_setup_reply_free_queues()
1723 	sc->sense_buf_q[i] = (0);  in mpi3mr_setup_reply_free_queues()
1727 static int mpi3mr_reply_dma_alloc(struct mpi3mr_softc *sc)  in mpi3mr_reply_dma_alloc()  argument
1731 	sc->num_reply_bufs = sc->facts.max_reqs + MPI3MR_NUM_EVTREPLIES;  in mpi3mr_reply_dma_alloc()
1732 	sc->reply_free_q_sz = sc->num_reply_bufs + 1;  in mpi3mr_reply_dma_alloc()
1733 	sc->num_sense_bufs = sc->facts.max_reqs / MPI3MR_SENSEBUF_FACTOR;  in mpi3mr_reply_dma_alloc()
1734 	sc->sense_buf_q_sz = sc->num_sense_bufs + 1;  in mpi3mr_reply_dma_alloc()
1736 	sz = sc->num_reply_bufs * sc->reply_sz;  in mpi3mr_reply_dma_alloc()
1738 	if (bus_dma_tag_create(sc->mpi3mr_parent_dmat,  /* parent */  in mpi3mr_reply_dma_alloc()
1740 				sc->dma_loaddr,		/* lowaddr */  in mpi3mr_reply_dma_alloc()
1748                                 &sc->reply_buf_tag)) {  in mpi3mr_reply_dma_alloc()
1749 		mpi3mr_dprint(sc, MPI3MR_ERROR, "Cannot allocate request DMA tag\n");  in mpi3mr_reply_dma_alloc()
1753 	if (bus_dmamem_alloc(sc->reply_buf_tag, (void **)&sc->reply_buf,  in mpi3mr_reply_dma_alloc()
1754 	    BUS_DMA_NOWAIT, &sc->reply_buf_dmamap)) {  in mpi3mr_reply_dma_alloc()
1755 		mpi3mr_dprint(sc, MPI3MR_ERROR, "Func: %s line: %d  DMA mem alloc failed\n",  in mpi3mr_reply_dma_alloc()
1760 	bzero(sc->reply_buf, sz);  in mpi3mr_reply_dma_alloc()
1761         bus_dmamap_load(sc->reply_buf_tag, sc->reply_buf_dmamap, sc->reply_buf, sz,  in mpi3mr_reply_dma_alloc()
1762 	    mpi3mr_memaddr_cb, &sc->reply_buf_phys, BUS_DMA_NOWAIT);  in mpi3mr_reply_dma_alloc()
1764 	sc->reply_buf_dma_min_address = sc->reply_buf_phys;  in mpi3mr_reply_dma_alloc()
1765 	sc->reply_buf_dma_max_address = sc->reply_buf_phys + sz;  in mpi3mr_reply_dma_alloc()
1766 	mpi3mr_dprint(sc, MPI3MR_XINFO, "reply buf (0x%p): depth(%d), frame_size(%d), "  in mpi3mr_reply_dma_alloc()
1768 	    sc->reply_buf, sc->num_reply_bufs, sc->reply_sz,  in mpi3mr_reply_dma_alloc()
1769 	    (sz / 1024), (unsigned long long)sc->reply_buf_phys);  in mpi3mr_reply_dma_alloc()
1772 	sz = sc->reply_free_q_sz * 8;  in mpi3mr_reply_dma_alloc()
1774         if (bus_dma_tag_create(sc->mpi3mr_parent_dmat,    /* parent */  in mpi3mr_reply_dma_alloc()
1776 				sc->dma_loaddr,		/* lowaddr */  in mpi3mr_reply_dma_alloc()
1784                                 &sc->reply_free_q_tag)) {  in mpi3mr_reply_dma_alloc()
1785 		mpi3mr_dprint(sc, MPI3MR_ERROR, "Cannot allocate reply free queue DMA tag\n");  in mpi3mr_reply_dma_alloc()
1789         if (bus_dmamem_alloc(sc->reply_free_q_tag, (void **)&sc->reply_free_q,  in mpi3mr_reply_dma_alloc()
1790 	    BUS_DMA_NOWAIT, &sc->reply_free_q_dmamap)) {  in mpi3mr_reply_dma_alloc()
1791 		mpi3mr_dprint(sc, MPI3MR_ERROR, "Func: %s line: %d  DMA mem alloc failed\n",  in mpi3mr_reply_dma_alloc()
1796 	bzero(sc->reply_free_q, sz);  in mpi3mr_reply_dma_alloc()
1797         bus_dmamap_load(sc->reply_free_q_tag, sc->reply_free_q_dmamap, sc->reply_free_q, sz,  in mpi3mr_reply_dma_alloc()
1798 	    mpi3mr_memaddr_cb, &sc->reply_free_q_phys, BUS_DMA_NOWAIT);  in mpi3mr_reply_dma_alloc()
1800 	mpi3mr_dprint(sc, MPI3MR_XINFO, "reply_free_q (0x%p): depth(%d), frame_size(%d), "  in mpi3mr_reply_dma_alloc()
1802 	    sc->reply_free_q, sc->reply_free_q_sz, 8, (sz / 1024),  in mpi3mr_reply_dma_alloc()
1803 	    (unsigned long long)sc->reply_free_q_phys);  in mpi3mr_reply_dma_alloc()
1806 	sz = sc->num_sense_bufs * MPI3MR_SENSEBUF_SZ;  in mpi3mr_reply_dma_alloc()
1808         if (bus_dma_tag_create(sc->mpi3mr_parent_dmat,    /* parent */  in mpi3mr_reply_dma_alloc()
1810 				sc->dma_loaddr,		/* lowaddr */  in mpi3mr_reply_dma_alloc()
1818                                 &sc->sense_buf_tag)) {  in mpi3mr_reply_dma_alloc()
1819 		mpi3mr_dprint(sc, MPI3MR_ERROR, "Cannot allocate Sense buffer DMA tag\n");  in mpi3mr_reply_dma_alloc()
1823 	if (bus_dmamem_alloc(sc->sense_buf_tag, (void **)&sc->sense_buf,  in mpi3mr_reply_dma_alloc()
1824 	    BUS_DMA_NOWAIT, &sc->sense_buf_dmamap)) {  in mpi3mr_reply_dma_alloc()
1825 		mpi3mr_dprint(sc, MPI3MR_ERROR, "Func: %s line: %d  DMA mem alloc failed\n",  in mpi3mr_reply_dma_alloc()
1830 	bzero(sc->sense_buf, sz);  in mpi3mr_reply_dma_alloc()
1831         bus_dmamap_load(sc->sense_buf_tag, sc->sense_buf_dmamap, sc->sense_buf, sz,  in mpi3mr_reply_dma_alloc()
1832 	    mpi3mr_memaddr_cb, &sc->sense_buf_phys, BUS_DMA_NOWAIT);  in mpi3mr_reply_dma_alloc()
1834 	mpi3mr_dprint(sc, MPI3MR_XINFO, "sense_buf (0x%p): depth(%d), frame_size(%d), "  in mpi3mr_reply_dma_alloc()
1836 	    sc->sense_buf, sc->num_sense_bufs, MPI3MR_SENSEBUF_SZ,  in mpi3mr_reply_dma_alloc()
1837 	    (sz / 1024), (unsigned long long)sc->sense_buf_phys);  in mpi3mr_reply_dma_alloc()
1840 	sz = sc->sense_buf_q_sz * 8;  in mpi3mr_reply_dma_alloc()
1842         if (bus_dma_tag_create(sc->mpi3mr_parent_dmat,    /* parent */  in mpi3mr_reply_dma_alloc()
1844 				sc->dma_loaddr,		/* lowaddr */  in mpi3mr_reply_dma_alloc()
1852                                 &sc->sense_buf_q_tag)) {  in mpi3mr_reply_dma_alloc()
1853 		mpi3mr_dprint(sc, MPI3MR_ERROR, "Cannot allocate Sense buffer Queue DMA tag\n");  in mpi3mr_reply_dma_alloc()
1857 	if (bus_dmamem_alloc(sc->sense_buf_q_tag, (void **)&sc->sense_buf_q,  in mpi3mr_reply_dma_alloc()
1858 	    BUS_DMA_NOWAIT, &sc->sense_buf_q_dmamap)) {  in mpi3mr_reply_dma_alloc()
1859 		mpi3mr_dprint(sc, MPI3MR_ERROR, "Func: %s line: %d  DMA mem alloc failed\n",  in mpi3mr_reply_dma_alloc()
1864 	bzero(sc->sense_buf_q, sz);  in mpi3mr_reply_dma_alloc()
1865         bus_dmamap_load(sc->sense_buf_q_tag, sc->sense_buf_q_dmamap, sc->sense_buf_q, sz,  in mpi3mr_reply_dma_alloc()
1866 	    mpi3mr_memaddr_cb, &sc->sense_buf_q_phys, BUS_DMA_NOWAIT);  in mpi3mr_reply_dma_alloc()
1868 	mpi3mr_dprint(sc, MPI3MR_XINFO, "sense_buf_q (0x%p): depth(%d), frame_size(%d), "  in mpi3mr_reply_dma_alloc()
1870 	    sc->sense_buf_q, sc->sense_buf_q_sz, 8, (sz / 1024),  in mpi3mr_reply_dma_alloc()
1871 	    (unsigned long long)sc->sense_buf_q_phys);  in mpi3mr_reply_dma_alloc()
1876 static int mpi3mr_reply_alloc(struct mpi3mr_softc *sc)  in mpi3mr_reply_alloc()  argument
1881 	if (sc->init_cmds.reply)  in mpi3mr_reply_alloc()
1884 	sc->init_cmds.reply = malloc(sc->reply_sz,  in mpi3mr_reply_alloc()
1887 	if (!sc->init_cmds.reply) {  in mpi3mr_reply_alloc()
1889 		    sc->name);  in mpi3mr_reply_alloc()
1893 	sc->ioctl_cmds.reply = malloc(sc->reply_sz, M_MPI3MR, M_NOWAIT | M_ZERO);  in mpi3mr_reply_alloc()
1894 	if (!sc->ioctl_cmds.reply) {  in mpi3mr_reply_alloc()
1896 		    sc->name);  in mpi3mr_reply_alloc()
1900 	sc->host_tm_cmds.reply = malloc(sc->reply_sz, M_MPI3MR, M_NOWAIT | M_ZERO);  in mpi3mr_reply_alloc()
1901 	if (!sc->host_tm_cmds.reply) {  in mpi3mr_reply_alloc()
1903 		    sc->name);  in mpi3mr_reply_alloc()
1907 		sc->dev_rmhs_cmds[i].reply = malloc(sc->reply_sz,  in mpi3mr_reply_alloc()
1909 		if (!sc->dev_rmhs_cmds[i].reply) {  in mpi3mr_reply_alloc()
1912 			    sc->name, i);  in mpi3mr_reply_alloc()
1918 		sc->evtack_cmds[i].reply = malloc(sc->reply_sz,  in mpi3mr_reply_alloc()
1920 		if (!sc->evtack_cmds[i].reply)  in mpi3mr_reply_alloc()
1924 	sc->dev_handle_bitmap_sz = MPI3MR_DIV_ROUND_UP(sc->facts.max_devhandle, 8);  in mpi3mr_reply_alloc()
1926 	sc->removepend_bitmap = malloc(sc->dev_handle_bitmap_sz,  in mpi3mr_reply_alloc()
1928 	if (!sc->removepend_bitmap) {  in mpi3mr_reply_alloc()
1930 		    sc->name);  in mpi3mr_reply_alloc()
1934 	sc->devrem_bitmap_sz = MPI3MR_DIV_ROUND_UP(MPI3MR_NUM_DEVRMCMD, 8);  in mpi3mr_reply_alloc()
1935 	sc->devrem_bitmap = malloc(sc->devrem_bitmap_sz,  in mpi3mr_reply_alloc()
1937 	if (!sc->devrem_bitmap) {  in mpi3mr_reply_alloc()
1939 		    sc->name);  in mpi3mr_reply_alloc()
1943 	sc->evtack_cmds_bitmap_sz = MPI3MR_DIV_ROUND_UP(MPI3MR_NUM_EVTACKCMD, 8);  in mpi3mr_reply_alloc()
1945 	sc->evtack_cmds_bitmap = malloc(sc->evtack_cmds_bitmap_sz,  in mpi3mr_reply_alloc()
1947 	if (!sc->evtack_cmds_bitmap)  in mpi3mr_reply_alloc()
1950 	if (mpi3mr_reply_dma_alloc(sc)) {  in mpi3mr_reply_alloc()
1952 		    sc->name, __func__, __LINE__);  in mpi3mr_reply_alloc()
1957 	mpi3mr_setup_reply_free_queues(sc);  in mpi3mr_reply_alloc()
1960 	mpi3mr_cleanup_interrupts(sc);  in mpi3mr_reply_alloc()
1961 	mpi3mr_free_mem(sc);  in mpi3mr_reply_alloc()
1967 mpi3mr_print_fw_pkg_ver(struct mpi3mr_softc *sc)  in mpi3mr_print_fw_pkg_ver()  argument
1979 	if (bus_dma_tag_create(sc->mpi3mr_parent_dmat,  /* parent */  in mpi3mr_print_fw_pkg_ver()
1981 				sc->dma_loaddr,		/* lowaddr */  in mpi3mr_print_fw_pkg_ver()
1990 		mpi3mr_dprint(sc, MPI3MR_ERROR, "Cannot allocate fw package version request DMA tag\n");  in mpi3mr_print_fw_pkg_ver()
1995 		mpi3mr_dprint(sc, MPI3MR_ERROR, "Func: %s line: %d fw package version DMA mem alloc failed\n",  in mpi3mr_print_fw_pkg_ver()
2005 …mpi3mr_dprint(sc, MPI3MR_XINFO, "Func: %s line: %d fw package version phys addr= %#016jx size= %d\…  in mpi3mr_print_fw_pkg_ver()
2009 		mpi3mr_dprint(sc, MPI3MR_ERROR, "Memory alloc for fw package version failed\n");  in mpi3mr_print_fw_pkg_ver()
2014 	mtx_lock(&sc->init_cmds.completion.lock);  in mpi3mr_print_fw_pkg_ver()
2015 	if (sc->init_cmds.state & MPI3MR_CMD_PENDING) {  in mpi3mr_print_fw_pkg_ver()
2016 		mpi3mr_dprint(sc, MPI3MR_INFO,"Issue CI Header Upload: command is in use\n");  in mpi3mr_print_fw_pkg_ver()
2017 		mtx_unlock(&sc->init_cmds.completion.lock);  in mpi3mr_print_fw_pkg_ver()
2020 	sc->init_cmds.state = MPI3MR_CMD_PENDING;  in mpi3mr_print_fw_pkg_ver()
2021 	sc->init_cmds.is_waiting = 1;  in mpi3mr_print_fw_pkg_ver()
2022 	sc->init_cmds.callback = NULL;  in mpi3mr_print_fw_pkg_ver()
2032 	init_completion(&sc->init_cmds.completion);  in mpi3mr_print_fw_pkg_ver()
2033 	if ((retval = mpi3mr_submit_admin_cmd(sc, &ci_upload, sizeof(ci_upload)))) {  in mpi3mr_print_fw_pkg_ver()
2034 		mpi3mr_dprint(sc, MPI3MR_ERROR, "Issue CI Header Upload: Admin Post failed\n");  in mpi3mr_print_fw_pkg_ver()
2037 	wait_for_completion_timeout(&sc->init_cmds.completion,  in mpi3mr_print_fw_pkg_ver()
2039 	if (!(sc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {  in mpi3mr_print_fw_pkg_ver()
2040 		mpi3mr_dprint(sc, MPI3MR_ERROR, "Issue CI Header Upload: command timed out\n");  in mpi3mr_print_fw_pkg_ver()
2041 		sc->init_cmds.is_waiting = 0;  in mpi3mr_print_fw_pkg_ver()
2042 		if (!(sc->init_cmds.state & MPI3MR_CMD_RESET))  in mpi3mr_print_fw_pkg_ver()
2043 			mpi3mr_check_rh_fault_ioc(sc,  in mpi3mr_print_fw_pkg_ver()
2047 	if ((GET_IOC_STATUS(sc->init_cmds.ioc_status)) != MPI3_IOCSTATUS_SUCCESS) {  in mpi3mr_print_fw_pkg_ver()
2048 		mpi3mr_dprint(sc, MPI3MR_ERROR,  in mpi3mr_print_fw_pkg_ver()
2050 			      GET_IOC_STATUS(sc->init_cmds.ioc_status), sc->init_cmds.ioc_loginfo);  in mpi3mr_print_fw_pkg_ver()
2055 	mpi3mr_dprint(sc, MPI3MR_XINFO,  in mpi3mr_print_fw_pkg_ver()
2061 	mpi3mr_dprint(sc, MPI3MR_INFO, "FW Package Version: %02d.%02d.%02d.%02d\n",  in mpi3mr_print_fw_pkg_ver()
2067 	sc->init_cmds.state = MPI3MR_CMD_NOTUSED;  in mpi3mr_print_fw_pkg_ver()
2068 	mtx_unlock(&sc->init_cmds.completion.lock);  in mpi3mr_print_fw_pkg_ver()
2089 static int mpi3mr_issue_iocinit(struct mpi3mr_softc *sc)  in mpi3mr_issue_iocinit()  argument
2101 	if (bus_dma_tag_create(sc->mpi3mr_parent_dmat,  /* parent */  in mpi3mr_issue_iocinit()
2103 				sc->dma_loaddr,		/* lowaddr */  in mpi3mr_issue_iocinit()
2112 		mpi3mr_dprint(sc, MPI3MR_ERROR, "Cannot allocate request DMA tag\n");  in mpi3mr_issue_iocinit()
2118 		mpi3mr_dprint(sc, MPI3MR_ERROR, "Func: %s line: %d Data  DMA mem alloc failed\n",  in mpi3mr_issue_iocinit()
2126 …mpi3mr_dprint(sc, MPI3MR_XINFO, "Func: %s line: %d IOCfacts drvr_info phys addr= %#016jx size= %d\…  in mpi3mr_issue_iocinit()
2133 		    sc->name);  in mpi3mr_issue_iocinit()
2144 	memcpy((U8 *)&sc->driver_info, (U8 *)drvr_info, sizeof(sc->driver_info));  in mpi3mr_issue_iocinit()
2147 	mtx_lock(&sc->init_cmds.completion.lock);  in mpi3mr_issue_iocinit()
2148 	if (sc->init_cmds.state & MPI3MR_CMD_PENDING) {  in mpi3mr_issue_iocinit()
2151 		    sc->name);  in mpi3mr_issue_iocinit()
2152 		mtx_unlock(&sc->init_cmds.completion.lock);  in mpi3mr_issue_iocinit()
2155 	sc->init_cmds.state = MPI3MR_CMD_PENDING;  in mpi3mr_issue_iocinit()
2156 	sc->init_cmds.is_waiting = 1;  in mpi3mr_issue_iocinit()
2157 	sc->init_cmds.callback = NULL;  in mpi3mr_issue_iocinit()
2165         iocinit_req.ReplyFreeQueueDepth = sc->reply_free_q_sz;  in mpi3mr_issue_iocinit()
2167                 sc->reply_free_q_phys;  in mpi3mr_issue_iocinit()
2170                 sc->sense_buf_q_sz;  in mpi3mr_issue_iocinit()
2172                 sc->sense_buf_q_phys;  in mpi3mr_issue_iocinit()
2181 	init_completion(&sc->init_cmds.completion);  in mpi3mr_issue_iocinit()
2182 	retval = mpi3mr_submit_admin_cmd(sc, &iocinit_req,  in mpi3mr_issue_iocinit()
2187 		    sc->name);  in mpi3mr_issue_iocinit()
2191 	wait_for_completion_timeout(&sc->init_cmds.completion,  in mpi3mr_issue_iocinit()
2193 	if (!(sc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {  in mpi3mr_issue_iocinit()
2195 		    sc->name);  in mpi3mr_issue_iocinit()
2196 		mpi3mr_check_rh_fault_ioc(sc,  in mpi3mr_issue_iocinit()
2198 		sc->unrecoverable = 1;  in mpi3mr_issue_iocinit()
2203 	if ((sc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)  in mpi3mr_issue_iocinit()
2206 		    " Loginfo(0x%08x) \n" , sc->name,  in mpi3mr_issue_iocinit()
2207 		    (sc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),  in mpi3mr_issue_iocinit()
2208 		    sc->init_cmds.ioc_loginfo);  in mpi3mr_issue_iocinit()
2214 	sc->init_cmds.state = MPI3MR_CMD_NOTUSED;  in mpi3mr_issue_iocinit()
2215 	mtx_unlock(&sc->init_cmds.completion.lock);  in mpi3mr_issue_iocinit()
2228 mpi3mr_display_ioc_info(struct mpi3mr_softc *sc)  in mpi3mr_display_ioc_info()  argument
2233         switch (sc->facts.personality) {  in mpi3mr_display_ioc_info()
2245 	mpi3mr_dprint(sc, MPI3MR_INFO, "Current Personality: %s\n", personality);  in mpi3mr_display_ioc_info()
2247 	mpi3mr_dprint(sc, MPI3MR_INFO, "%s\n", sc->fw_version);  in mpi3mr_display_ioc_info()
2249         mpi3mr_dprint(sc, MPI3MR_INFO, "Protocol=(");  in mpi3mr_display_ioc_info()
2251         if (sc->facts.protocol_flags &  in mpi3mr_display_ioc_info()
2257         if (sc->facts.protocol_flags &  in mpi3mr_display_ioc_info()
2263         if (sc->facts.protocol_flags &  in mpi3mr_display_ioc_info()
2272         if (sc->facts.ioc_capabilities &  in mpi3mr_display_ioc_info()
2291 static void mpi3mr_unmask_events(struct mpi3mr_softc *sc, U16 event)  in mpi3mr_unmask_events()  argument
2301 		sc->event_masks[0] &= ~desired_event;  in mpi3mr_unmask_events()
2303 		sc->event_masks[1] &= ~desired_event;  in mpi3mr_unmask_events()
2305 		sc->event_masks[2] &= ~desired_event;  in mpi3mr_unmask_events()
2307 		sc->event_masks[3] &= ~desired_event;  in mpi3mr_unmask_events()
2310 static void mpi3mr_set_events_mask(struct mpi3mr_softc *sc)  in mpi3mr_set_events_mask()  argument
2314 		sc->event_masks[i] = -1;  in mpi3mr_set_events_mask()
2316         mpi3mr_unmask_events(sc, MPI3_EVENT_DEVICE_ADDED);  in mpi3mr_set_events_mask()
2317         mpi3mr_unmask_events(sc, MPI3_EVENT_DEVICE_INFO_CHANGED);  in mpi3mr_set_events_mask()
2318         mpi3mr_unmask_events(sc, MPI3_EVENT_DEVICE_STATUS_CHANGE);  in mpi3mr_set_events_mask()
2320         mpi3mr_unmask_events(sc, MPI3_EVENT_ENCL_DEVICE_STATUS_CHANGE);  in mpi3mr_set_events_mask()
2322         mpi3mr_unmask_events(sc, MPI3_EVENT_SAS_TOPOLOGY_CHANGE_LIST);  in mpi3mr_set_events_mask()
2323         mpi3mr_unmask_events(sc, MPI3_EVENT_SAS_DISCOVERY);  in mpi3mr_set_events_mask()
2324         mpi3mr_unmask_events(sc, MPI3_EVENT_SAS_DEVICE_DISCOVERY_ERROR);  in mpi3mr_set_events_mask()
2325         mpi3mr_unmask_events(sc, MPI3_EVENT_SAS_BROADCAST_PRIMITIVE);  in mpi3mr_set_events_mask()
2327         mpi3mr_unmask_events(sc, MPI3_EVENT_PCIE_TOPOLOGY_CHANGE_LIST);  in mpi3mr_set_events_mask()
2328         mpi3mr_unmask_events(sc, MPI3_EVENT_PCIE_ENUMERATION);  in mpi3mr_set_events_mask()
2330         mpi3mr_unmask_events(sc, MPI3_EVENT_PREPARE_FOR_RESET);  in mpi3mr_set_events_mask()
2331         mpi3mr_unmask_events(sc, MPI3_EVENT_CABLE_MGMT);  in mpi3mr_set_events_mask()
2332         mpi3mr_unmask_events(sc, MPI3_EVENT_ENERGY_PACK_CHANGE);  in mpi3mr_set_events_mask()
2344 int mpi3mr_issue_event_notification(struct mpi3mr_softc *sc)  in mpi3mr_issue_event_notification()  argument
2351 	mtx_lock(&sc->init_cmds.completion.lock);  in mpi3mr_issue_event_notification()
2352 	if (sc->init_cmds.state & MPI3MR_CMD_PENDING) {  in mpi3mr_issue_event_notification()
2355 		    sc->name);  in mpi3mr_issue_event_notification()
2356 		mtx_unlock(&sc->init_cmds.completion.lock);  in mpi3mr_issue_event_notification()
2359 	sc->init_cmds.state = MPI3MR_CMD_PENDING;  in mpi3mr_issue_event_notification()
2360 	sc->init_cmds.is_waiting = 1;  in mpi3mr_issue_event_notification()
2361 	sc->init_cmds.callback = NULL;  in mpi3mr_issue_event_notification()
2366 		    (sc->event_masks[i]);  in mpi3mr_issue_event_notification()
2367 	init_completion(&sc->init_cmds.completion);  in mpi3mr_issue_event_notification()
2368 	retval = mpi3mr_submit_admin_cmd(sc, &evtnotify_req,  in mpi3mr_issue_event_notification()
2372 		    sc->name);  in mpi3mr_issue_event_notification()
2376 	poll_for_command_completion(sc,  in mpi3mr_issue_event_notification()
2377 				    &sc->init_cmds,  in mpi3mr_issue_event_notification()
2379 	if (!(sc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {  in mpi3mr_issue_event_notification()
2381 		    sc->name);  in mpi3mr_issue_event_notification()
2382 		mpi3mr_check_rh_fault_ioc(sc,  in mpi3mr_issue_event_notification()
2388 	if ((sc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)  in mpi3mr_issue_event_notification()
2391 		    " Loginfo(0x%08x) \n" , sc->name,  in mpi3mr_issue_event_notification()
2392 		    (sc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),  in mpi3mr_issue_event_notification()
2393 		    sc->init_cmds.ioc_loginfo);  in mpi3mr_issue_event_notification()
2399 	sc->init_cmds.state = MPI3MR_CMD_NOTUSED;  in mpi3mr_issue_event_notification()
2400 	mtx_unlock(&sc->init_cmds.completion.lock);  in mpi3mr_issue_event_notification()
2407 mpi3mr_register_events(struct mpi3mr_softc *sc)  in mpi3mr_register_events()  argument
2411 	mpi3mr_set_events_mask(sc);  in mpi3mr_register_events()
2413 	error = mpi3mr_issue_event_notification(sc);  in mpi3mr_register_events()
2417 		    sc->name, error);  in mpi3mr_register_events()
2434 int mpi3mr_process_event_ack(struct mpi3mr_softc *sc, U8 event,  in mpi3mr_process_event_ack()  argument
2441 	mtx_lock(&sc->init_cmds.completion.lock);  in mpi3mr_process_event_ack()
2442 	if (sc->init_cmds.state & MPI3MR_CMD_PENDING) {  in mpi3mr_process_event_ack()
2445 		    sc->name);  in mpi3mr_process_event_ack()
2446 		mtx_unlock(&sc->init_cmds.completion.lock);  in mpi3mr_process_event_ack()
2449 	sc->init_cmds.state = MPI3MR_CMD_PENDING;  in mpi3mr_process_event_ack()
2450 	sc->init_cmds.is_waiting = 1;  in mpi3mr_process_event_ack()
2451 	sc->init_cmds.callback = NULL;  in mpi3mr_process_event_ack()
2457 	init_completion(&sc->init_cmds.completion);  in mpi3mr_process_event_ack()
2458 	retval = mpi3mr_submit_admin_cmd(sc, &evtack_req,  in mpi3mr_process_event_ack()
2462 		    sc->name);  in mpi3mr_process_event_ack()
2466 	wait_for_completion_timeout(&sc->init_cmds.completion,  in mpi3mr_process_event_ack()
2468 	if (!(sc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {  in mpi3mr_process_event_ack()
2470 		    sc->name);  in mpi3mr_process_event_ack()
2475 	if ((sc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)  in mpi3mr_process_event_ack()
2478 		    " Loginfo(0x%08x) \n" , sc->name,  in mpi3mr_process_event_ack()
2479 		    (sc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),  in mpi3mr_process_event_ack()
2480 		    sc->init_cmds.ioc_loginfo);  in mpi3mr_process_event_ack()
2486 	sc->init_cmds.state = MPI3MR_CMD_NOTUSED;  in mpi3mr_process_event_ack()
2487 	mtx_unlock(&sc->init_cmds.completion.lock);  in mpi3mr_process_event_ack()
2494 static int mpi3mr_alloc_chain_bufs(struct mpi3mr_softc *sc)  in mpi3mr_alloc_chain_bufs()  argument
2500 	num_chains = sc->max_host_ios;  in mpi3mr_alloc_chain_bufs()
2502 	sc->chain_buf_count = num_chains;  in mpi3mr_alloc_chain_bufs()
2505 	sc->chain_sgl_list = malloc(sz, M_MPI3MR, M_NOWAIT | M_ZERO);  in mpi3mr_alloc_chain_bufs()
2507 	if (!sc->chain_sgl_list) {  in mpi3mr_alloc_chain_bufs()
2509 		    sc->name);  in mpi3mr_alloc_chain_bufs()
2516         if (bus_dma_tag_create(sc->mpi3mr_parent_dmat,  /* parent */  in mpi3mr_alloc_chain_bufs()
2518 				sc->dma_loaddr,		/* lowaddr */  in mpi3mr_alloc_chain_bufs()
2526                                 &sc->chain_sgl_list_tag)) {  in mpi3mr_alloc_chain_bufs()
2527 		mpi3mr_dprint(sc, MPI3MR_ERROR, "Cannot allocate Chain buffer DMA tag\n");  in mpi3mr_alloc_chain_bufs()
2532 		if (bus_dmamem_alloc(sc->chain_sgl_list_tag, (void **)&sc->chain_sgl_list[i].buf,  in mpi3mr_alloc_chain_bufs()
2533 		    BUS_DMA_NOWAIT, &sc->chain_sgl_list[i].buf_dmamap)) {  in mpi3mr_alloc_chain_bufs()
2534 			mpi3mr_dprint(sc, MPI3MR_ERROR, "Func: %s line: %d  DMA mem alloc failed\n",  in mpi3mr_alloc_chain_bufs()
2539 		bzero(sc->chain_sgl_list[i].buf, sz);  in mpi3mr_alloc_chain_bufs()
2540 …bus_dmamap_load(sc->chain_sgl_list_tag, sc->chain_sgl_list[i].buf_dmamap, sc->chain_sgl_list[i].bu…  in mpi3mr_alloc_chain_bufs()
2541 		    mpi3mr_memaddr_cb, &sc->chain_sgl_list[i].buf_phys, BUS_DMA_NOWAIT);  in mpi3mr_alloc_chain_bufs()
2542 		mpi3mr_dprint(sc, MPI3MR_XINFO, "Func: %s line: %d phys addr= %#016jx size= %d\n",  in mpi3mr_alloc_chain_bufs()
2543 		    __func__, __LINE__, (uintmax_t)sc->chain_sgl_list[i].buf_phys, sz);  in mpi3mr_alloc_chain_bufs()
2546 	sc->chain_bitmap_sz = MPI3MR_DIV_ROUND_UP(num_chains, 8);  in mpi3mr_alloc_chain_bufs()
2548 	sc->chain_bitmap = malloc(sc->chain_bitmap_sz, M_MPI3MR, M_NOWAIT | M_ZERO);  in mpi3mr_alloc_chain_bufs()
2549 	if (!sc->chain_bitmap) {  in mpi3mr_alloc_chain_bufs()
2550 		mpi3mr_dprint(sc, MPI3MR_INFO, "Cannot alloc memory for chain bitmap\n");  in mpi3mr_alloc_chain_bufs()
2558 		if (sc->chain_sgl_list[i].buf_phys != 0)  in mpi3mr_alloc_chain_bufs()
2559 			bus_dmamap_unload(sc->chain_sgl_list_tag, sc->chain_sgl_list[i].buf_dmamap);  in mpi3mr_alloc_chain_bufs()
2560 		if (sc->chain_sgl_list[i].buf != NULL)  in mpi3mr_alloc_chain_bufs()
2561 …bus_dmamem_free(sc->chain_sgl_list_tag, sc->chain_sgl_list[i].buf, sc->chain_sgl_list[i].buf_dmama…  in mpi3mr_alloc_chain_bufs()
2563 	if (sc->chain_sgl_list_tag != NULL)  in mpi3mr_alloc_chain_bufs()
2564 		bus_dma_tag_destroy(sc->chain_sgl_list_tag);  in mpi3mr_alloc_chain_bufs()
2568 static int mpi3mr_pel_alloc(struct mpi3mr_softc *sc)  in mpi3mr_pel_alloc()  argument
2572 	if (!sc->pel_cmds.reply) {  in mpi3mr_pel_alloc()
2573 		sc->pel_cmds.reply = malloc(sc->reply_sz, M_MPI3MR, M_NOWAIT | M_ZERO);  in mpi3mr_pel_alloc()
2574 		if (!sc->pel_cmds.reply) {  in mpi3mr_pel_alloc()
2576 			    sc->name);  in mpi3mr_pel_alloc()
2581 	if (!sc->pel_abort_cmd.reply) {  in mpi3mr_pel_alloc()
2582 		sc->pel_abort_cmd.reply = malloc(sc->reply_sz, M_MPI3MR, M_NOWAIT | M_ZERO);  in mpi3mr_pel_alloc()
2583 		if (!sc->pel_abort_cmd.reply) {  in mpi3mr_pel_alloc()
2585 			    sc->name);  in mpi3mr_pel_alloc()
2590 	if (!sc->pel_seq_number) {  in mpi3mr_pel_alloc()
2591 		sc->pel_seq_number_sz = sizeof(Mpi3PELSeq_t);  in mpi3mr_pel_alloc()
2592 		if (bus_dma_tag_create(sc->mpi3mr_parent_dmat,   /* parent */  in mpi3mr_pel_alloc()
2594 				 sc->dma_loaddr,	         /* lowaddr */  in mpi3mr_pel_alloc()
2597 				 sc->pel_seq_number_sz,		 /* maxsize */  in mpi3mr_pel_alloc()
2599 				 sc->pel_seq_number_sz,          /* maxsegsize */  in mpi3mr_pel_alloc()
2602 				 &sc->pel_seq_num_dmatag)) {  in mpi3mr_pel_alloc()
2603 			 mpi3mr_dprint(sc, MPI3MR_ERROR, "Cannot create PEL seq number dma memory tag\n");  in mpi3mr_pel_alloc()
2608 		if (bus_dmamem_alloc(sc->pel_seq_num_dmatag, (void **)&sc->pel_seq_number,  in mpi3mr_pel_alloc()
2609 		    BUS_DMA_NOWAIT, &sc->pel_seq_num_dmamap)) {  in mpi3mr_pel_alloc()
2610 			mpi3mr_dprint(sc, MPI3MR_ERROR, "Cannot allocate PEL seq number kernel buffer dma memory\n");  in mpi3mr_pel_alloc()
2615 		bzero(sc->pel_seq_number, sc->pel_seq_number_sz);  in mpi3mr_pel_alloc()
2617 		bus_dmamap_load(sc->pel_seq_num_dmatag, sc->pel_seq_num_dmamap, sc->pel_seq_number,  in mpi3mr_pel_alloc()
2618 		    sc->pel_seq_number_sz, mpi3mr_memaddr_cb, &sc->pel_seq_number_dma, BUS_DMA_NOWAIT);  in mpi3mr_pel_alloc()
2620 		if (!sc->pel_seq_number) {  in mpi3mr_pel_alloc()
2621 			printf(IOCNAME "%s:%d Cannot load PEL seq number dma memory for size: %d\n", sc->name,  in mpi3mr_pel_alloc()
2622 				__func__, __LINE__, sc->pel_seq_number_sz);  in mpi3mr_pel_alloc()
2640 mpi3mr_validate_fw_update(struct mpi3mr_softc *sc)  in mpi3mr_validate_fw_update()  argument
2645 	if (sc->facts.reply_sz > sc->reply_sz) {  in mpi3mr_validate_fw_update()
2646 		mpi3mr_dprint(sc, MPI3MR_ERROR,  in mpi3mr_validate_fw_update()
2648 		    sc->reply_sz, sc->reply_sz);  in mpi3mr_validate_fw_update()
2652 	if (sc->num_io_throttle_group != sc->facts.max_io_throttle_group) {  in mpi3mr_validate_fw_update()
2653 		mpi3mr_dprint(sc, MPI3MR_ERROR,  in mpi3mr_validate_fw_update()
2655 		    sc->num_io_throttle_group,  in mpi3mr_validate_fw_update()
2656 		    sc->facts.max_io_throttle_group);  in mpi3mr_validate_fw_update()
2660 	if (sc->facts.max_op_reply_q < sc->num_queues) {  in mpi3mr_validate_fw_update()
2661 		mpi3mr_dprint(sc, MPI3MR_ERROR,  in mpi3mr_validate_fw_update()
2663 		    sc->num_queues,  in mpi3mr_validate_fw_update()
2664 		    sc->facts.max_op_reply_q);  in mpi3mr_validate_fw_update()
2668 	if (sc->facts.max_op_req_q < sc->num_queues) {  in mpi3mr_validate_fw_update()
2669 		mpi3mr_dprint(sc, MPI3MR_ERROR,  in mpi3mr_validate_fw_update()
2671 		    sc->num_queues, sc->facts.max_op_req_q);  in mpi3mr_validate_fw_update()
2675 	dev_handle_bitmap_sz = MPI3MR_DIV_ROUND_UP(sc->facts.max_devhandle, 8);  in mpi3mr_validate_fw_update()
2677 	if (dev_handle_bitmap_sz > sc->dev_handle_bitmap_sz) {  in mpi3mr_validate_fw_update()
2678 		removepend_bitmap = realloc(sc->removepend_bitmap,  in mpi3mr_validate_fw_update()
2682 			mpi3mr_dprint(sc, MPI3MR_ERROR,  in mpi3mr_validate_fw_update()
2684 			    sc->dev_handle_bitmap_sz, dev_handle_bitmap_sz);  in mpi3mr_validate_fw_update()
2688 		memset(removepend_bitmap + sc->dev_handle_bitmap_sz, 0,  in mpi3mr_validate_fw_update()
2689 		    dev_handle_bitmap_sz - sc->dev_handle_bitmap_sz);  in mpi3mr_validate_fw_update()
2690 		sc->removepend_bitmap = removepend_bitmap;  in mpi3mr_validate_fw_update()
2691 		mpi3mr_dprint(sc, MPI3MR_INFO,  in mpi3mr_validate_fw_update()
2693 		    sc->dev_handle_bitmap_sz, dev_handle_bitmap_sz);  in mpi3mr_validate_fw_update()
2694 		sc->dev_handle_bitmap_sz = dev_handle_bitmap_sz;  in mpi3mr_validate_fw_update()
2709 int mpi3mr_initialize_ioc(struct mpi3mr_softc *sc, U8 init_type)  in mpi3mr_initialize_ioc()  argument
2719 	sc->cpu_count = mp_ncpus;  in mpi3mr_initialize_ioc()
2721 	ioc_status = mpi3mr_regread(sc, MPI3_SYSIF_IOC_STATUS_OFFSET);  in mpi3mr_initialize_ioc()
2722 	ioc_control = mpi3mr_regread(sc, MPI3_SYSIF_IOC_CONFIG_OFFSET);  in mpi3mr_initialize_ioc()
2723 	ioc_info = mpi3mr_regread64(sc, MPI3_SYSIF_IOC_INFO_LOW_OFFSET);  in mpi3mr_initialize_ioc()
2725 	mpi3mr_dprint(sc, MPI3MR_INFO, "SOD ioc_status: 0x%x ioc_control: 0x%x "  in mpi3mr_initialize_ioc()
2729 	sc->ready_timeout =  in mpi3mr_initialize_ioc()
2733 	ioc_state = mpi3mr_get_iocstate(sc);  in mpi3mr_initialize_ioc()
2735 	mpi3mr_dprint(sc, MPI3MR_INFO, "IOC state: %s   IOC ready timeout: %d\n",  in mpi3mr_initialize_ioc()
2736 	    mpi3mr_iocstate_name(ioc_state), sc->ready_timeout);  in mpi3mr_initialize_ioc()
2740 		timeout = sc->ready_timeout * 10;  in mpi3mr_initialize_ioc()
2745 		ioc_state = mpi3mr_get_iocstate(sc);  in mpi3mr_initialize_ioc()
2746 		mpi3mr_dprint(sc, MPI3MR_INFO,  in mpi3mr_initialize_ioc()
2752                 retval = mpi3mr_mur_ioc(sc, MPI3MR_RESET_FROM_BRINGUP);  in mpi3mr_initialize_ioc()
2754                         mpi3mr_dprint(sc, MPI3MR_ERROR, "Failed to MU reset IOC, error 0x%x\n",  in mpi3mr_initialize_ioc()
2757                 ioc_state = mpi3mr_get_iocstate(sc);  in mpi3mr_initialize_ioc()
2761                 mpi3mr_print_fault_info(sc);  in mpi3mr_initialize_ioc()
2762 		 mpi3mr_dprint(sc, MPI3MR_ERROR, "issuing soft reset to bring to reset state\n");  in mpi3mr_initialize_ioc()
2763                  retval = mpi3mr_issue_reset(sc,  in mpi3mr_initialize_ioc()
2767                         mpi3mr_dprint(sc, MPI3MR_ERROR,  in mpi3mr_initialize_ioc()
2774 	ioc_state = mpi3mr_get_iocstate(sc);  in mpi3mr_initialize_ioc()
2777 		mpi3mr_dprint(sc, MPI3MR_ERROR, "Cannot bring IOC to reset state\n");  in mpi3mr_initialize_ioc()
2781 	retval = mpi3mr_setup_admin_qpair(sc);  in mpi3mr_initialize_ioc()
2783 		mpi3mr_dprint(sc, MPI3MR_ERROR, "Failed to setup Admin queues, error 0x%x\n",  in mpi3mr_initialize_ioc()
2788 	retval = mpi3mr_bring_ioc_ready(sc);  in mpi3mr_initialize_ioc()
2790 		mpi3mr_dprint(sc, MPI3MR_ERROR, "Failed to bring IOC ready, error 0x%x\n",  in mpi3mr_initialize_ioc()
2796 		retval = mpi3mr_alloc_interrupts(sc, 1);  in mpi3mr_initialize_ioc()
2798 			mpi3mr_dprint(sc, MPI3MR_ERROR, "Failed to allocate interrupts, error 0x%x\n",  in mpi3mr_initialize_ioc()
2803 		retval = mpi3mr_setup_irqs(sc);  in mpi3mr_initialize_ioc()
2805 			mpi3mr_dprint(sc, MPI3MR_ERROR, "Failed to setup ISR, error 0x%x\n",  in mpi3mr_initialize_ioc()
2811 	mpi3mr_enable_interrupts(sc);  in mpi3mr_initialize_ioc()
2814 		mtx_init(&sc->mpi3mr_mtx, "SIM lock", NULL, MTX_DEF);  in mpi3mr_initialize_ioc()
2815 		mtx_init(&sc->io_lock, "IO lock", NULL, MTX_DEF);  in mpi3mr_initialize_ioc()
2816 		mtx_init(&sc->admin_req_lock, "Admin Request Queue lock", NULL, MTX_SPIN);  in mpi3mr_initialize_ioc()
2817 		mtx_init(&sc->reply_free_q_lock, "Reply free Queue lock", NULL, MTX_SPIN);  in mpi3mr_initialize_ioc()
2818 		mtx_init(&sc->sense_buf_q_lock, "Sense buffer Queue lock", NULL, MTX_SPIN);  in mpi3mr_initialize_ioc()
2819 		mtx_init(&sc->chain_buf_lock, "Chain buffer lock", NULL, MTX_SPIN);  in mpi3mr_initialize_ioc()
2820 		mtx_init(&sc->cmd_pool_lock, "Command pool lock", NULL, MTX_DEF);  in mpi3mr_initialize_ioc()
2821 		mtx_init(&sc->fwevt_lock, "Firmware Event lock", NULL, MTX_DEF);  in mpi3mr_initialize_ioc()
2822 		mtx_init(&sc->target_lock, "Target lock", NULL, MTX_SPIN);  in mpi3mr_initialize_ioc()
2823 		mtx_init(&sc->reset_mutex, "Reset lock", NULL, MTX_DEF);  in mpi3mr_initialize_ioc()
2825 		mtx_init(&sc->init_cmds.completion.lock, "Init commands lock", NULL, MTX_DEF);  in mpi3mr_initialize_ioc()
2826 		sc->init_cmds.reply = NULL;  in mpi3mr_initialize_ioc()
2827 		sc->init_cmds.state = MPI3MR_CMD_NOTUSED;  in mpi3mr_initialize_ioc()
2828 		sc->init_cmds.dev_handle = MPI3MR_INVALID_DEV_HANDLE;  in mpi3mr_initialize_ioc()
2829 		sc->init_cmds.host_tag = MPI3MR_HOSTTAG_INITCMDS;  in mpi3mr_initialize_ioc()
2831 		mtx_init(&sc->ioctl_cmds.completion.lock, "IOCTL commands lock", NULL, MTX_DEF);  in mpi3mr_initialize_ioc()
2832 		sc->ioctl_cmds.reply = NULL;  in mpi3mr_initialize_ioc()
2833 		sc->ioctl_cmds.state = MPI3MR_CMD_NOTUSED;  in mpi3mr_initialize_ioc()
2834 		sc->ioctl_cmds.dev_handle = MPI3MR_INVALID_DEV_HANDLE;  in mpi3mr_initialize_ioc()
2835 		sc->ioctl_cmds.host_tag = MPI3MR_HOSTTAG_IOCTLCMDS;  in mpi3mr_initialize_ioc()
2837 		mtx_init(&sc->pel_abort_cmd.completion.lock, "PEL Abort command lock", NULL, MTX_DEF);  in mpi3mr_initialize_ioc()
2838 		sc->pel_abort_cmd.reply = NULL;  in mpi3mr_initialize_ioc()
2839 		sc->pel_abort_cmd.state = MPI3MR_CMD_NOTUSED;  in mpi3mr_initialize_ioc()
2840 		sc->pel_abort_cmd.dev_handle = MPI3MR_INVALID_DEV_HANDLE;  in mpi3mr_initialize_ioc()
2841 		sc->pel_abort_cmd.host_tag = MPI3MR_HOSTTAG_PELABORT;  in mpi3mr_initialize_ioc()
2843 		mtx_init(&sc->host_tm_cmds.completion.lock, "TM commands lock", NULL, MTX_DEF);  in mpi3mr_initialize_ioc()
2844 		sc->host_tm_cmds.reply = NULL;  in mpi3mr_initialize_ioc()
2845 		sc->host_tm_cmds.state = MPI3MR_CMD_NOTUSED;  in mpi3mr_initialize_ioc()
2846 		sc->host_tm_cmds.dev_handle = MPI3MR_INVALID_DEV_HANDLE;  in mpi3mr_initialize_ioc()
2847 		sc->host_tm_cmds.host_tag = MPI3MR_HOSTTAG_TMS;  in mpi3mr_initialize_ioc()
2849 		TAILQ_INIT(&sc->cmd_list_head);  in mpi3mr_initialize_ioc()
2850 		TAILQ_INIT(&sc->event_list);  in mpi3mr_initialize_ioc()
2851 		TAILQ_INIT(&sc->delayed_rmhs_list);  in mpi3mr_initialize_ioc()
2852 		TAILQ_INIT(&sc->delayed_evtack_cmds_list);  in mpi3mr_initialize_ioc()
2856 			mtx_init(&sc->dev_rmhs_cmds[i].completion.lock, str, NULL, MTX_DEF);  in mpi3mr_initialize_ioc()
2857 			sc->dev_rmhs_cmds[i].reply = NULL;  in mpi3mr_initialize_ioc()
2858 			sc->dev_rmhs_cmds[i].state = MPI3MR_CMD_NOTUSED;  in mpi3mr_initialize_ioc()
2859 			sc->dev_rmhs_cmds[i].dev_handle = MPI3MR_INVALID_DEV_HANDLE;  in mpi3mr_initialize_ioc()
2860 			sc->dev_rmhs_cmds[i].host_tag = MPI3MR_HOSTTAG_DEVRMCMD_MIN  in mpi3mr_initialize_ioc()
2865 	retval = mpi3mr_issue_iocfacts(sc, &facts_data);  in mpi3mr_initialize_ioc()
2867 		mpi3mr_dprint(sc, MPI3MR_ERROR, "Failed to Issue IOC Facts, retval: 0x%x\n",  in mpi3mr_initialize_ioc()
2872 	retval = mpi3mr_process_factsdata(sc, &facts_data);  in mpi3mr_initialize_ioc()
2874 		mpi3mr_dprint(sc, MPI3MR_ERROR, "IOC Facts data processing failedi, retval: 0x%x\n",  in mpi3mr_initialize_ioc()
2879 	sc->num_io_throttle_group = sc->facts.max_io_throttle_group;  in mpi3mr_initialize_ioc()
2880 	mpi3mr_atomic_set(&sc->pend_large_data_sz, 0);  in mpi3mr_initialize_ioc()
2883 		retval = mpi3mr_validate_fw_update(sc);  in mpi3mr_initialize_ioc()
2887 		sc->reply_sz = sc->facts.reply_sz;  in mpi3mr_initialize_ioc()
2890 	mpi3mr_display_ioc_info(sc);  in mpi3mr_initialize_ioc()
2892 	retval = mpi3mr_reply_alloc(sc);  in mpi3mr_initialize_ioc()
2894 		mpi3mr_dprint(sc, MPI3MR_ERROR, "Failed to allocated reply and sense buffers, retval: 0x%x\n",  in mpi3mr_initialize_ioc()
2900 		retval = mpi3mr_alloc_chain_bufs(sc);  in mpi3mr_initialize_ioc()
2902 			mpi3mr_dprint(sc, MPI3MR_ERROR, "Failed to allocated chain buffers, retval: 0x%x\n",  in mpi3mr_initialize_ioc()
2908 	retval = mpi3mr_issue_iocinit(sc);  in mpi3mr_initialize_ioc()
2910 		mpi3mr_dprint(sc, MPI3MR_ERROR, "Failed to Issue IOC Init, retval: 0x%x\n",  in mpi3mr_initialize_ioc()
2915 	mpi3mr_print_fw_pkg_ver(sc);  in mpi3mr_initialize_ioc()
2917 	sc->reply_free_q_host_index = sc->num_reply_bufs;  in mpi3mr_initialize_ioc()
2918 	mpi3mr_regwrite(sc, MPI3_SYSIF_REPLY_FREE_HOST_INDEX_OFFSET,  in mpi3mr_initialize_ioc()
2919 		sc->reply_free_q_host_index);  in mpi3mr_initialize_ioc()
2921 	sc->sense_buf_q_host_index = sc->num_sense_bufs;  in mpi3mr_initialize_ioc()
2923 	mpi3mr_regwrite(sc, MPI3_SYSIF_SENSE_BUF_FREE_HOST_INDEX_OFFSET,  in mpi3mr_initialize_ioc()
2924 		sc->sense_buf_q_host_index);  in mpi3mr_initialize_ioc()
2927 		retval = mpi3mr_alloc_interrupts(sc, 0);  in mpi3mr_initialize_ioc()
2929 			mpi3mr_dprint(sc, MPI3MR_ERROR, "Failed to allocate interrupts, retval: 0x%x\n",  in mpi3mr_initialize_ioc()
2934 		retval = mpi3mr_setup_irqs(sc);  in mpi3mr_initialize_ioc()
2937 			    sc->name, retval);  in mpi3mr_initialize_ioc()
2941 		mpi3mr_enable_interrupts(sc);  in mpi3mr_initialize_ioc()
2944 		mpi3mr_enable_interrupts(sc);  in mpi3mr_initialize_ioc()
2946 	retval = mpi3mr_create_op_queues(sc);  in mpi3mr_initialize_ioc()
2949 		mpi3mr_dprint(sc, MPI3MR_ERROR, "Failed to create operational queues, error: %d\n",  in mpi3mr_initialize_ioc()
2954 	if (!sc->throttle_groups && sc->num_io_throttle_group) {  in mpi3mr_initialize_ioc()
2955 		mpi3mr_dprint(sc, MPI3MR_ERROR, "allocating memory for throttle groups\n");  in mpi3mr_initialize_ioc()
2957 		sc->throttle_groups = (struct mpi3mr_throttle_group_info *)  in mpi3mr_initialize_ioc()
2958 					  malloc(sc->num_io_throttle_group *  in mpi3mr_initialize_ioc()
2960 		if (!sc->throttle_groups)  in mpi3mr_initialize_ioc()
2965 		mpi3mr_dprint(sc, MPI3MR_INFO, "Re-register events\n");  in mpi3mr_initialize_ioc()
2966 		retval = mpi3mr_register_events(sc);  in mpi3mr_initialize_ioc()
2968 			mpi3mr_dprint(sc, MPI3MR_INFO, "Failed to re-register events, retval: 0x%x\n",  in mpi3mr_initialize_ioc()
2973 		mpi3mr_dprint(sc, MPI3MR_INFO, "Issuing Port Enable\n");  in mpi3mr_initialize_ioc()
2974 		retval = mpi3mr_issue_port_enable(sc, 0);  in mpi3mr_initialize_ioc()
2976 			mpi3mr_dprint(sc, MPI3MR_INFO, "Failed to issue port enable, retval: 0x%x\n",  in mpi3mr_initialize_ioc()
2981 	retval = mpi3mr_pel_alloc(sc);  in mpi3mr_initialize_ioc()
2983 		mpi3mr_dprint(sc, MPI3MR_ERROR, "Failed to allocate memory for PEL, retval: 0x%x\n",  in mpi3mr_initialize_ioc()
2995 static void mpi3mr_port_enable_complete(struct mpi3mr_softc *sc,  in mpi3mr_port_enable_complete()  argument
3000 	printf(IOCNAME "Completing Port Enable Request\n", sc->name);  in mpi3mr_port_enable_complete()
3001 	sc->mpi3mr_flags |= MPI3MR_FLAGS_PORT_ENABLE_DONE;  in mpi3mr_port_enable_complete()
3002 	mpi3mr_startup_decrement(sc->cam_sc);  in mpi3mr_port_enable_complete()
3005 int mpi3mr_issue_port_enable(struct mpi3mr_softc *sc, U8 async)  in mpi3mr_issue_port_enable()  argument
3011 	mtx_lock(&sc->init_cmds.completion.lock);  in mpi3mr_issue_port_enable()
3012 	if (sc->init_cmds.state & MPI3MR_CMD_PENDING) {  in mpi3mr_issue_port_enable()
3014 		printf(IOCNAME "Issue PortEnable: Init command is in use\n", sc->name);  in mpi3mr_issue_port_enable()
3015 		mtx_unlock(&sc->init_cmds.completion.lock);  in mpi3mr_issue_port_enable()
3019 	sc->init_cmds.state = MPI3MR_CMD_PENDING;  in mpi3mr_issue_port_enable()
3022 		sc->init_cmds.is_waiting = 0;  in mpi3mr_issue_port_enable()
3023 		sc->init_cmds.callback = mpi3mr_port_enable_complete;  in mpi3mr_issue_port_enable()
3025 		sc->init_cmds.is_waiting = 1;  in mpi3mr_issue_port_enable()
3026 		sc->init_cmds.callback = NULL;  in mpi3mr_issue_port_enable()
3027 		init_completion(&sc->init_cmds.completion);  in mpi3mr_issue_port_enable()
3032 	printf(IOCNAME "Sending Port Enable Request\n", sc->name);  in mpi3mr_issue_port_enable()
3033 	retval = mpi3mr_submit_admin_cmd(sc, &pe_req, sizeof(pe_req));  in mpi3mr_issue_port_enable()
3036 		    sc->name);  in mpi3mr_issue_port_enable()
3041 		wait_for_completion_timeout(&sc->init_cmds.completion,  in mpi3mr_issue_port_enable()
3043 		if (!(sc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {  in mpi3mr_issue_port_enable()
3045 			    sc->name);  in mpi3mr_issue_port_enable()
3047 			mpi3mr_check_rh_fault_ioc(sc, MPI3MR_RESET_FROM_PE_TIMEOUT);  in mpi3mr_issue_port_enable()
3050 		mpi3mr_port_enable_complete(sc, &sc->init_cmds);  in mpi3mr_issue_port_enable()
3053 	mtx_unlock(&sc->init_cmds.completion.lock);  in mpi3mr_issue_port_enable()
3062 	struct mpi3mr_softc *sc;  in mpi3mr_watchdog_thread()  local
3066 	sc = (struct mpi3mr_softc *)arg;  in mpi3mr_watchdog_thread()
3068 	mpi3mr_dprint(sc, MPI3MR_XINFO, "%s\n", __func__);  in mpi3mr_watchdog_thread()
3070 	sc->watchdog_thread_active = 1;  in mpi3mr_watchdog_thread()
3071 	mtx_lock(&sc->reset_mutex);  in mpi3mr_watchdog_thread()
3073 		if (sc->mpi3mr_flags & MPI3MR_FLAGS_SHUTDOWN ||   in mpi3mr_watchdog_thread()
3074 		    (sc->unrecoverable == 1)) {  in mpi3mr_watchdog_thread()
3075 			mpi3mr_dprint(sc, MPI3MR_INFO,  in mpi3mr_watchdog_thread()
3077 			   sc->mpi3mr_flags & MPI3MR_FLAGS_SHUTDOWN ? "Shutdown" :  in mpi3mr_watchdog_thread()
3081 		mtx_unlock(&sc->reset_mutex);  in mpi3mr_watchdog_thread()
3083 		if ((sc->prepare_for_reset) &&  in mpi3mr_watchdog_thread()
3084 		    ((sc->prepare_for_reset_timeout_counter++) >=  in mpi3mr_watchdog_thread()
3086 			mpi3mr_soft_reset_handler(sc,  in mpi3mr_watchdog_thread()
3091 		ioc_status = mpi3mr_regread(sc, MPI3_SYSIF_IOC_STATUS_OFFSET);  in mpi3mr_watchdog_thread()
3094 			mpi3mr_soft_reset_handler(sc, MPI3MR_RESET_FROM_FIRMWARE, 0);  in mpi3mr_watchdog_thread()
3098 		ioc_state = mpi3mr_get_iocstate(sc);  in mpi3mr_watchdog_thread()
3100 			fault = mpi3mr_regread(sc, MPI3_SYSIF_FAULT_OFFSET) &  in mpi3mr_watchdog_thread()
3103 			host_diagnostic = mpi3mr_regread(sc, MPI3_SYSIF_HOST_DIAG_OFFSET);  in mpi3mr_watchdog_thread()
3105 				if (!sc->diagsave_timeout) {  in mpi3mr_watchdog_thread()
3106 					mpi3mr_print_fault_info(sc);  in mpi3mr_watchdog_thread()
3107 					mpi3mr_dprint(sc, MPI3MR_INFO,  in mpi3mr_watchdog_thread()
3110 				if ((sc->diagsave_timeout++) <= MPI3_SYSIF_DIAG_SAVE_TIMEOUT)  in mpi3mr_watchdog_thread()
3113 			mpi3mr_print_fault_info(sc);  in mpi3mr_watchdog_thread()
3114 			sc->diagsave_timeout = 0;  in mpi3mr_watchdog_thread()
3118 				mpi3mr_dprint(sc, MPI3MR_INFO,  in mpi3mr_watchdog_thread()
3121 				sc->unrecoverable = 1;  in mpi3mr_watchdog_thread()
3126 			    || (sc->reset_in_progress))  in mpi3mr_watchdog_thread()
3129 				mpi3mr_soft_reset_handler(sc,  in mpi3mr_watchdog_thread()
3132 				mpi3mr_soft_reset_handler(sc,  in mpi3mr_watchdog_thread()
3137 		if (sc->reset.type == MPI3MR_TRIGGER_SOFT_RESET) {  in mpi3mr_watchdog_thread()
3138 			mpi3mr_print_fault_info(sc);  in mpi3mr_watchdog_thread()
3139 			mpi3mr_soft_reset_handler(sc, sc->reset.reason, 1);  in mpi3mr_watchdog_thread()
3142 		mtx_lock(&sc->reset_mutex);  in mpi3mr_watchdog_thread()
3147 		if ((sc->mpi3mr_flags & MPI3MR_FLAGS_SHUTDOWN) == 0 &&  in mpi3mr_watchdog_thread()
3148 		    !sc->unrecoverable) {  in mpi3mr_watchdog_thread()
3149 			msleep(&sc->watchdog_chan, &sc->reset_mutex, PRIBIO,  in mpi3mr_watchdog_thread()
3153 	mtx_unlock(&sc->reset_mutex);  in mpi3mr_watchdog_thread()
3154 	sc->watchdog_thread_active = 0;  in mpi3mr_watchdog_thread()
3158 static void mpi3mr_display_event_data(struct mpi3mr_softc *sc,  in mpi3mr_display_event_data()  argument
3186 		mpi3mr_dprint(sc, MPI3MR_EVENT, "Device Added: Dev=0x%04x Form=0x%x Perst id: 0x%x\n",  in mpi3mr_display_event_data()
3194 		mpi3mr_dprint(sc, MPI3MR_EVENT, "Device Info Changed: Dev=0x%04x Form=0x%x\n",  in mpi3mr_display_event_data()
3202 		mpi3mr_dprint(sc, MPI3MR_EVENT, "Device Status Change: Dev=0x%04x RC=0x%x\n",  in mpi3mr_display_event_data()
3210 		mpi3mr_dprint(sc, MPI3MR_EVENT, "SAS Discovery: (%s)",  in mpi3mr_display_event_data()
3214 		    (sc->mpi3mr_debug & MPI3MR_EVENT)) {  in mpi3mr_display_event_data()
3220 		if (sc->mpi3mr_debug & MPI3MR_EVENT)  in mpi3mr_display_event_data()
3258 		mpi3mr_dprint(sc, MPI3MR_EVENT, "PCIE Enumeration: (%s)",  in mpi3mr_display_event_data()
3263 			mpi3mr_dprint(sc, MPI3MR_EVENT, "enumeration_status(0x%08x)",  in mpi3mr_display_event_data()
3265 		if (sc->mpi3mr_debug & MPI3MR_EVENT)  in mpi3mr_display_event_data()
3277 	mpi3mr_dprint(sc, MPI3MR_EVENT, "%s\n", desc);  in mpi3mr_display_event_data()
3286 	mtx_lock_spin(&cam_sc->sc->target_lock);  in mpi3mr_find_target_by_per_id()
3292 	mtx_unlock_spin(&cam_sc->sc->target_lock);  in mpi3mr_find_target_by_per_id()
3302 	mtx_lock_spin(&cam_sc->sc->target_lock);  in mpi3mr_find_target_by_dev_handle()
3308 	mtx_unlock_spin(&cam_sc->sc->target_lock);  in mpi3mr_find_target_by_dev_handle()
3312 void mpi3mr_update_device(struct mpi3mr_softc *sc,  in mpi3mr_update_device()  argument
3419 		if (vdinf->IOThrottleGroup < sc->num_io_throttle_group) {  in mpi3mr_update_device()
3420 			tg = sc->throttle_groups + vdinf->IOThrottleGroup;  in mpi3mr_update_device()
3440 int mpi3mr_create_device(struct mpi3mr_softc *sc,  in mpi3mr_create_device()  argument
3449 	mtx_lock_spin(&sc->target_lock);  in mpi3mr_create_device()
3450 	TAILQ_FOREACH(target, &sc->cam_sc->tgt_list, tgt_next) {  in mpi3mr_create_device()
3456 	mtx_unlock_spin(&sc->target_lock);  in mpi3mr_create_device()
3459 			mpi3mr_update_device(sc, target, dev_pg0, true);  in mpi3mr_create_device()
3470 			mpi3mr_update_device(sc, target, dev_pg0, true);  in mpi3mr_create_device()
3471 			mtx_lock_spin(&sc->target_lock);  in mpi3mr_create_device()
3472 			TAILQ_INSERT_TAIL(&sc->cam_sc->tgt_list, target, tgt_next);  in mpi3mr_create_device()
3474 			mtx_unlock_spin(&sc->target_lock);  in mpi3mr_create_device()
3491 static void mpi3mr_dev_rmhs_complete_iou(struct mpi3mr_softc *sc,  in mpi3mr_dev_rmhs_complete_iou()  argument
3498 	mpi3mr_dprint(sc, MPI3MR_EVENT,  in mpi3mr_dev_rmhs_complete_iou()
3505 			mpi3mr_dprint(sc, MPI3MR_EVENT,  in mpi3mr_dev_rmhs_complete_iou()
3509 			mpi3mr_dev_rmhs_send_tm(sc, drv_cmd->dev_handle,  in mpi3mr_dev_rmhs_complete_iou()
3513 		mpi3mr_dprint(sc, MPI3MR_ERROR,  in mpi3mr_dev_rmhs_complete_iou()
3517 		mtx_lock_spin(&sc->target_lock);  in mpi3mr_dev_rmhs_complete_iou()
3518 		TAILQ_FOREACH(tgtdev, &sc->cam_sc->tgt_list, tgt_next) {  in mpi3mr_dev_rmhs_complete_iou()
3522 		mtx_unlock_spin(&sc->target_lock);  in mpi3mr_dev_rmhs_complete_iou()
3524 		mpi3mr_dprint(sc, MPI3MR_INFO,  in mpi3mr_dev_rmhs_complete_iou()
3527 		mpi3mr_clear_bit(drv_cmd->dev_handle, sc->removepend_bitmap);  in mpi3mr_dev_rmhs_complete_iou()
3530 	if (!TAILQ_EMPTY(&sc->delayed_rmhs_list)) {  in mpi3mr_dev_rmhs_complete_iou()
3531 		delayed_dev_rmhs = TAILQ_FIRST(&sc->delayed_rmhs_list);  in mpi3mr_dev_rmhs_complete_iou()
3535 		mpi3mr_dprint(sc, MPI3MR_EVENT,  in mpi3mr_dev_rmhs_complete_iou()
3538 		mpi3mr_dev_rmhs_send_tm(sc, drv_cmd->dev_handle, drv_cmd,  in mpi3mr_dev_rmhs_complete_iou()
3540 		TAILQ_REMOVE(&sc->delayed_rmhs_list, delayed_dev_rmhs, list);  in mpi3mr_dev_rmhs_complete_iou()
3548 	mpi3mr_clear_bit(cmd_idx, sc->devrem_bitmap);  in mpi3mr_dev_rmhs_complete_iou()
3562 static void mpi3mr_dev_rmhs_complete_tm(struct mpi3mr_softc *sc,  in mpi3mr_dev_rmhs_complete_tm()  argument
3576 		    sc->name, drv_cmd->dev_handle, drv_cmd->ioc_status,  in mpi3mr_dev_rmhs_complete_tm()
3581 	    sc->name, drv_cmd->dev_handle, cmd_idx);  in mpi3mr_dev_rmhs_complete_tm()
3593 	retval = mpi3mr_submit_admin_cmd(sc, &iou_ctrl, sizeof(iou_ctrl));  in mpi3mr_dev_rmhs_complete_tm()
3596 		    sc->name);  in mpi3mr_dev_rmhs_complete_tm()
3606 	mpi3mr_clear_bit(cmd_idx, sc->devrem_bitmap);  in mpi3mr_dev_rmhs_complete_tm()
3622 static void mpi3mr_dev_rmhs_send_tm(struct mpi3mr_softc *sc, U16 handle,  in mpi3mr_dev_rmhs_send_tm()  argument
3635 		cmd_idx = mpi3mr_find_first_zero_bit(sc->devrem_bitmap,  in mpi3mr_dev_rmhs_send_tm()
3638 			if (!mpi3mr_test_and_set_bit(cmd_idx, sc->devrem_bitmap))  in mpi3mr_dev_rmhs_send_tm()
3652 		TAILQ_INSERT_TAIL(&(sc->delayed_rmhs_list), delayed_dev_rmhs, list);  in mpi3mr_dev_rmhs_send_tm()
3653 		mpi3mr_dprint(sc, MPI3MR_EVENT, "%s :DevRmHs: tr:handle(0x%04x) is postponed\n",  in mpi3mr_dev_rmhs_send_tm()
3659 	drv_cmd = &sc->dev_rmhs_cmds[cmd_idx];  in mpi3mr_dev_rmhs_send_tm()
3663 	mpi3mr_dprint(sc, MPI3MR_EVENT,  in mpi3mr_dev_rmhs_send_tm()
3669 		mpi3mr_dprint(sc, MPI3MR_EVENT, "%s :Issue TM: Command is in use\n", __func__);  in mpi3mr_dev_rmhs_send_tm()
3683 	mpi3mr_set_bit(handle, sc->removepend_bitmap);  in mpi3mr_dev_rmhs_send_tm()
3684 	retval = mpi3mr_submit_admin_cmd(sc, &tm_req, sizeof(tm_req));  in mpi3mr_dev_rmhs_send_tm()
3686 		mpi3mr_dprint(sc, MPI3MR_ERROR, "%s :Issue DevRmHsTM: Admin Post failed\n",  in mpi3mr_dev_rmhs_send_tm()
3697 	mpi3mr_clear_bit(cmd_idx, sc->devrem_bitmap);  in mpi3mr_dev_rmhs_send_tm()
3711 static void mpi3mr_complete_evt_ack(struct mpi3mr_softc *sc,  in mpi3mr_complete_evt_ack()  argument
3718 		mpi3mr_dprint(sc, MPI3MR_EVENT,  in mpi3mr_complete_evt_ack()
3724 	if (!TAILQ_EMPTY(&sc->delayed_evtack_cmds_list)) {  in mpi3mr_complete_evt_ack()
3725 		delayed_evtack = TAILQ_FIRST(&sc->delayed_evtack_cmds_list);  in mpi3mr_complete_evt_ack()
3726 		mpi3mr_dprint(sc, MPI3MR_EVENT,  in mpi3mr_complete_evt_ack()
3729 		mpi3mr_send_evt_ack(sc, delayed_evtack->event, drv_cmd,  in mpi3mr_complete_evt_ack()
3731 		TAILQ_REMOVE(&sc->delayed_evtack_cmds_list, delayed_evtack, list);  in mpi3mr_complete_evt_ack()
3737 	mpi3mr_clear_bit(cmd_idx, sc->evtack_cmds_bitmap);  in mpi3mr_complete_evt_ack()
3754 static void mpi3mr_send_evt_ack(struct mpi3mr_softc *sc, U8 event,  in mpi3mr_send_evt_ack()  argument
3767 		cmd_idx = mpi3mr_find_first_zero_bit(sc->evtack_cmds_bitmap,  in mpi3mr_send_evt_ack()
3771 			    sc->evtack_cmds_bitmap))  in mpi3mr_send_evt_ack()
3784 		TAILQ_INSERT_TAIL(&(sc->delayed_evtack_cmds_list), delayed_evtack, list);  in mpi3mr_send_evt_ack()
3785 		mpi3mr_dprint(sc, MPI3MR_EVENT, "%s : Event ack for event:%d is postponed\n",  in mpi3mr_send_evt_ack()
3789 	drv_cmd = &sc->evtack_cmds[cmd_idx];  in mpi3mr_send_evt_ack()
3796 		mpi3mr_dprint(sc, MPI3MR_EVENT, "%s: Command is in use\n", __func__);  in mpi3mr_send_evt_ack()
3806 	retval = mpi3mr_submit_admin_cmd(sc, &evtack_req,  in mpi3mr_send_evt_ack()
3810 		mpi3mr_dprint(sc, MPI3MR_ERROR, "%s: Admin Post failed\n", __func__);  in mpi3mr_send_evt_ack()
3818 	mpi3mr_clear_bit(cmd_idx, sc->evtack_cmds_bitmap);  in mpi3mr_send_evt_ack()
3833 static void mpi3mr_pcietopochg_evt_th(struct mpi3mr_softc *sc,  in mpi3mr_pcietopochg_evt_th()  argument
3848 		tgtdev = mpi3mr_find_target_by_dev_handle(sc->cam_sc, handle);  in mpi3mr_pcietopochg_evt_th()
3856 			mpi3mr_dev_rmhs_send_tm(sc, handle, NULL,  in mpi3mr_pcietopochg_evt_th()
3892 static void mpi3mr_sastopochg_evt_th(struct mpi3mr_softc *sc,  in mpi3mr_sastopochg_evt_th()  argument
3908 		tgtdev = mpi3mr_find_target_by_dev_handle(sc->cam_sc, handle);  in mpi3mr_sastopochg_evt_th()
3916 			mpi3mr_dev_rmhs_send_tm(sc, handle, NULL,  in mpi3mr_sastopochg_evt_th()
3951 static void mpi3mr_devstatuschg_evt_th(struct mpi3mr_softc *sc,  in mpi3mr_devstatuschg_evt_th()  argument
3986 	tgtdev = mpi3mr_find_target_by_dev_handle(sc->cam_sc, dev_handle);  in mpi3mr_devstatuschg_evt_th()
3989 		mpi3mr_dprint(sc, MPI3MR_ERROR, "%s :target with dev_handle:0x%x not found\n",  in mpi3mr_devstatuschg_evt_th()
4014 		mpi3mr_dev_rmhs_send_tm(sc, dev_handle, NULL,  in mpi3mr_devstatuschg_evt_th()
4018 		mpi3mr_dev_rmhs_send_tm(sc, dev_handle, NULL,  in mpi3mr_devstatuschg_evt_th()
4031 static void mpi3mr_preparereset_evt_th(struct mpi3mr_softc *sc,  in mpi3mr_preparereset_evt_th()  argument
4038 		mpi3mr_dprint(sc, MPI3MR_EVENT, "%s :Recieved PrepForReset Event with RC=START\n",  in mpi3mr_preparereset_evt_th()
4040 		if (sc->prepare_for_reset)  in mpi3mr_preparereset_evt_th()
4042 		sc->prepare_for_reset = 1;  in mpi3mr_preparereset_evt_th()
4043 		sc->prepare_for_reset_timeout_counter = 0;  in mpi3mr_preparereset_evt_th()
4045 		mpi3mr_dprint(sc, MPI3MR_EVENT, "%s :Recieved PrepForReset Event with RC=ABORT\n",  in mpi3mr_preparereset_evt_th()
4047 		sc->prepare_for_reset = 0;  in mpi3mr_preparereset_evt_th()
4048 		sc->prepare_for_reset_timeout_counter = 0;  in mpi3mr_preparereset_evt_th()
4052 		mpi3mr_send_evt_ack(sc, event_reply->Event, NULL,  in mpi3mr_preparereset_evt_th()
4065 static void mpi3mr_energypackchg_evt_th(struct mpi3mr_softc *sc,  in mpi3mr_energypackchg_evt_th()  argument
4073 		mpi3mr_dprint(sc, MPI3MR_ERROR,  in mpi3mr_energypackchg_evt_th()
4079 	mpi3mr_dprint(sc, MPI3MR_EVENT,  in mpi3mr_energypackchg_evt_th()
4081 	    __func__, sc->facts.shutdown_timeout, shutdown_timeout);  in mpi3mr_energypackchg_evt_th()
4082 	sc->facts.shutdown_timeout = shutdown_timeout;  in mpi3mr_energypackchg_evt_th()
4094 static void mpi3mr_cablemgmt_evt_th(struct mpi3mr_softc *sc,  in mpi3mr_cablemgmt_evt_th()  argument
4103 		mpi3mr_dprint(sc, MPI3MR_INFO, "An active cable with ReceptacleID %d cannot be powered.\n"  in mpi3mr_cablemgmt_evt_th()
4112 		mpi3mr_dprint(sc, MPI3MR_INFO, "A cable with ReceptacleID %d is not running at optimal speed\n",  in mpi3mr_cablemgmt_evt_th()
4130 static void mpi3mr_process_events(struct mpi3mr_softc *sc,  in mpi3mr_process_events()  argument
4138 	if (sc->mpi3mr_flags & MPI3MR_FLAGS_SHUTDOWN)  in mpi3mr_process_events()
4152 		if (mpi3mr_create_device(sc, dev_pg0))  in mpi3mr_process_events()
4153 			mpi3mr_dprint(sc, MPI3MR_ERROR,  in mpi3mr_process_events()
4164 		mpi3mr_devstatuschg_evt_th(sc, event_reply);  in mpi3mr_process_events()
4170 		mpi3mr_sastopochg_evt_th(sc, event_reply);  in mpi3mr_process_events()
4176 		mpi3mr_pcietopochg_evt_th(sc, event_reply);  in mpi3mr_process_events()
4181 		mpi3mr_preparereset_evt_th(sc, event_reply);  in mpi3mr_process_events()
4193 		mpi3mr_energypackchg_evt_th(sc, event_reply);  in mpi3mr_process_events()
4198 		mpi3mr_cablemgmt_evt_th(sc, event_reply);  in mpi3mr_process_events()
4209 		mpi3mr_dprint(sc, MPI3MR_INFO, "%s :Event 0x%02x is not handled by driver\n",  in mpi3mr_process_events()
4237 		    sc->track_mapping_events)  in mpi3mr_process_events()
4238 			sc->pending_map_events++;  in mpi3mr_process_events()
4245 		    !(sc->mpi3mr_flags & MPI3MR_FLAGS_PORT_ENABLE_DONE))  in mpi3mr_process_events()
4246 			mpi3mr_startup_increment(sc->cam_sc);  in mpi3mr_process_events()
4253 		mtx_lock(&sc->fwevt_lock);  in mpi3mr_process_events()
4254 		TAILQ_INSERT_TAIL(&sc->cam_sc->ev_queue, fw_event, ev_link);  in mpi3mr_process_events()
4255 		taskqueue_enqueue(sc->cam_sc->ev_tq, &sc->cam_sc->ev_task);  in mpi3mr_process_events()
4256 		mtx_unlock(&sc->fwevt_lock);  in mpi3mr_process_events()
4263 static void mpi3mr_handle_events(struct mpi3mr_softc *sc, uintptr_t data,  in mpi3mr_handle_events()  argument
4269 	sc->change_count = event_reply->IOCChangeCount;  in mpi3mr_handle_events()
4270 	mpi3mr_display_event_data(sc, event_reply);  in mpi3mr_handle_events()
4272 	mpi3mr_process_events(sc, data, event_reply);  in mpi3mr_handle_events()
4275 static void mpi3mr_process_admin_reply_desc(struct mpi3mr_softc *sc,  in mpi3mr_process_admin_reply_desc()  argument
4304 		def_reply = mpi3mr_get_reply_virt_addr(sc, *reply_dma);  in mpi3mr_process_admin_reply_desc()
4314 			sense_buf = mpi3mr_get_sensebuf_virt_addr(sc,  in mpi3mr_process_admin_reply_desc()
4327 		cmdptr = &sc->init_cmds;  in mpi3mr_process_admin_reply_desc()
4330 		cmdptr = &sc->ioctl_cmds;  in mpi3mr_process_admin_reply_desc()
4333 		cmdptr = &sc->host_tm_cmds;  in mpi3mr_process_admin_reply_desc()
4334 		wakeup((void *)&sc->tm_chan);  in mpi3mr_process_admin_reply_desc()
4337 		cmdptr = &sc->pel_abort_cmd;  in mpi3mr_process_admin_reply_desc()
4340 		cmdptr = &sc->pel_cmds;  in mpi3mr_process_admin_reply_desc()
4345 			mpi3mr_handle_events(sc, *reply_dma ,def_reply);  in mpi3mr_process_admin_reply_desc()
4353 		cmdptr = &sc->dev_rmhs_cmds[idx];  in mpi3mr_process_admin_reply_desc()
4359 		cmdptr = &sc->evtack_cmds[idx];  in mpi3mr_process_admin_reply_desc()
4371 				    sc->reply_sz);  in mpi3mr_process_admin_reply_desc()
4382 				cmdptr->callback(sc, cmdptr);  in mpi3mr_process_admin_reply_desc()
4387 		mpi3mr_repost_sense_buf(sc,  in mpi3mr_process_admin_reply_desc()
4398 static int mpi3mr_complete_admin_cmd(struct mpi3mr_softc *sc)  in mpi3mr_complete_admin_cmd()  argument
4400 	U32 exp_phase = sc->admin_reply_ephase;  in mpi3mr_complete_admin_cmd()
4401 	U32 adm_reply_ci = sc->admin_reply_ci;  in mpi3mr_complete_admin_cmd()
4407 	mtx_lock_spin(&sc->admin_reply_lock);  in mpi3mr_complete_admin_cmd()
4408 	if (sc->admin_in_use == false) {  in mpi3mr_complete_admin_cmd()
4409 		sc->admin_in_use = true;  in mpi3mr_complete_admin_cmd()
4410 		mtx_unlock_spin(&sc->admin_reply_lock);  in mpi3mr_complete_admin_cmd()
4412 		mtx_unlock_spin(&sc->admin_reply_lock);  in mpi3mr_complete_admin_cmd()
4416 	reply_desc = (Mpi3DefaultReplyDescriptor_t *)sc->admin_reply +  in mpi3mr_complete_admin_cmd()
4421 		mtx_lock_spin(&sc->admin_reply_lock);  in mpi3mr_complete_admin_cmd()
4422 		sc->admin_in_use = false;  in mpi3mr_complete_admin_cmd()
4423 		mtx_unlock_spin(&sc->admin_reply_lock);  in mpi3mr_complete_admin_cmd()
4428 		sc->admin_req_ci = reply_desc->RequestQueueCI;  in mpi3mr_complete_admin_cmd()
4429 		mpi3mr_process_admin_reply_desc(sc, reply_desc, &reply_dma);  in mpi3mr_complete_admin_cmd()
4431 			mpi3mr_repost_reply_buf(sc, reply_dma);  in mpi3mr_complete_admin_cmd()
4433 		if (++adm_reply_ci == sc->num_admin_replies) {  in mpi3mr_complete_admin_cmd()
4438 			(Mpi3DefaultReplyDescriptor_t *)sc->admin_reply +  in mpi3mr_complete_admin_cmd()
4445 			mpi3mr_regwrite(sc, MPI3_SYSIF_ADMIN_REPLY_Q_CI_OFFSET, adm_reply_ci);  in mpi3mr_complete_admin_cmd()
4450 	mpi3mr_regwrite(sc, MPI3_SYSIF_ADMIN_REPLY_Q_CI_OFFSET, adm_reply_ci);  in mpi3mr_complete_admin_cmd()
4451 	sc->admin_reply_ci = adm_reply_ci;  in mpi3mr_complete_admin_cmd()
4452 	sc->admin_reply_ephase = exp_phase;  in mpi3mr_complete_admin_cmd()
4453 	mtx_lock_spin(&sc->admin_reply_lock);  in mpi3mr_complete_admin_cmd()
4454 	sc->admin_in_use = false;  in mpi3mr_complete_admin_cmd()
4455 	mtx_unlock_spin(&sc->admin_reply_lock);  in mpi3mr_complete_admin_cmd()
4460 mpi3mr_cmd_done(struct mpi3mr_softc *sc, struct mpi3mr_cmd *cmd)  in mpi3mr_cmd_done()  argument
4462 	mpi3mr_unmap_request(sc, cmd);  in mpi3mr_cmd_done()
4464 	mtx_lock(&sc->mpi3mr_mtx);  in mpi3mr_cmd_done()
4470 	if (sc->unrecoverable)  in mpi3mr_cmd_done()
4475 	mtx_unlock(&sc->mpi3mr_mtx);  in mpi3mr_cmd_done()
4479 void mpi3mr_process_op_reply_desc(struct mpi3mr_softc *sc,  in mpi3mr_process_op_reply_desc()  argument
4519 		scsi_reply = mpi3mr_get_reply_virt_addr(sc,  in mpi3mr_process_op_reply_desc()
4522 			mpi3mr_dprint(sc, MPI3MR_ERROR, "scsi_reply is NULL, "  in mpi3mr_process_op_reply_desc()
4536 		sense_buf = mpi3mr_get_sensebuf_virt_addr(sc,  in mpi3mr_process_op_reply_desc()
4542 			mpi3mr_dprint(sc, MPI3MR_ERROR, "Ran out of sense buffers\n");  in mpi3mr_process_op_reply_desc()
4553 	cm = sc->cmd_list[host_tag];  in mpi3mr_process_op_reply_desc()
4558 	cam_sc = sc->cam_sc;  in mpi3mr_process_op_reply_desc()
4566 	if (sc->iot_enable) {  in mpi3mr_process_op_reply_desc()
4575 		if ((data_len_blks >= sc->io_throttle_data_length) &&  in mpi3mr_process_op_reply_desc()
4577 			mpi3mr_atomic_sub(&sc->pend_large_data_sz, data_len_blks);  in mpi3mr_process_op_reply_desc()
4579 			    &sc->pend_large_data_sz);  in mpi3mr_process_op_reply_desc()
4585 					mpi3mr_dprint(sc, MPI3MR_IOT,  in mpi3mr_process_op_reply_desc()
4592 						    sc->io_throttle_low,  in mpi3mr_process_op_reply_desc()
4597 				    sc->io_throttle_low) &&  in mpi3mr_process_op_reply_desc()
4600 					mpi3mr_dprint(sc, MPI3MR_IOT,  in mpi3mr_process_op_reply_desc()
4604 					    sc, tg, 0);  in mpi3mr_process_op_reply_desc()
4608 					mpi3mr_dprint(sc, MPI3MR_IOT,  in mpi3mr_process_op_reply_desc()
4613 					    sc->io_throttle_low);  in mpi3mr_process_op_reply_desc()
4617 				if (ioc_pend_data_len <= sc->io_throttle_low) {  in mpi3mr_process_op_reply_desc()
4619 					mpi3mr_dprint(sc, MPI3MR_IOT,  in mpi3mr_process_op_reply_desc()
4626 			ioc_pend_data_len = mpi3mr_atomic_read(&sc->pend_large_data_sz);  in mpi3mr_process_op_reply_desc()
4629 					mpi3mr_dprint(sc, MPI3MR_IOT,  in mpi3mr_process_op_reply_desc()
4634 					    sc->io_throttle_low);  in mpi3mr_process_op_reply_desc()
4638 				if ( ioc_pend_data_len <= sc->io_throttle_low) {  in mpi3mr_process_op_reply_desc()
4639 					mpi3mr_dprint(sc, MPI3MR_IOT,  in mpi3mr_process_op_reply_desc()
4645 			} else if (ioc_pend_data_len <= sc->io_throttle_low) {  in mpi3mr_process_op_reply_desc()
4648 					mpi3mr_dprint(sc, MPI3MR_IOT,  in mpi3mr_process_op_reply_desc()
4655 						    sc->io_throttle_low,  in mpi3mr_process_op_reply_desc()
4661 					mpi3mr_dprint(sc, MPI3MR_IOT,  in mpi3mr_process_op_reply_desc()
4665 					    sc, tg, 0);  in mpi3mr_process_op_reply_desc()
4723 		mpi3mr_dprint(sc, MPI3MR_TRACE,  in mpi3mr_process_op_reply_desc()
4728 		mpi3mr_dprint(sc, MPI3MR_TRACE,  in mpi3mr_process_op_reply_desc()
4744 			mpi3mr_dprint(sc, MPI3MR_XINFO, "func: %s line: %d recovered error\n",  __func__, __LINE__);  in mpi3mr_process_op_reply_desc()
4828 	mpi3mr_cmd_done(sc, cm);  in mpi3mr_process_op_reply_desc()
4829 	mpi3mr_dprint(sc, MPI3MR_TRACE, "Completion IO path :"  in mpi3mr_process_op_reply_desc()
4833 	mpi3mr_atomic_dec(&sc->fw_outstanding);  in mpi3mr_process_op_reply_desc()
4837 		mpi3mr_repost_sense_buf(sc,  in mpi3mr_process_op_reply_desc()
4849 int mpi3mr_complete_io_cmd(struct mpi3mr_softc *sc,  in mpi3mr_complete_io_cmd()  argument
4870 	mpi3mr_dprint(sc, MPI3MR_TRACE, "[QID:%d]:reply_desc: (%pa) reply_ci: %x"  in mpi3mr_complete_io_cmd()
4886 		sc->op_req_q[req_qid - 1].ci =  in mpi3mr_complete_io_cmd()
4889 		mpi3mr_process_op_reply_desc(sc, reply_desc, &reply_dma);  in mpi3mr_complete_io_cmd()
4892 			mpi3mr_repost_reply_buf(sc, reply_dma);  in mpi3mr_complete_io_cmd()
4905 			mpi3mr_regwrite(sc, MPI3_SYSIF_OPER_REPLY_Q_N_CI_OFFSET(op_reply_q->qid), reply_ci);  in mpi3mr_complete_io_cmd()
4912 	mpi3mr_regwrite(sc, MPI3_SYSIF_OPER_REPLY_Q_N_CI_OFFSET(op_reply_q->qid), reply_ci);  in mpi3mr_complete_io_cmd()
4931 	struct mpi3mr_softc *sc = irq_ctx->sc;  in mpi3mr_isr()  local
4939 	if (!sc->intr_enabled)  in mpi3mr_isr()
4943 		mpi3mr_complete_admin_cmd(sc);  in mpi3mr_isr()
4946 		mpi3mr_complete_io_cmd(sc, irq_ctx);  in mpi3mr_isr()
4959 mpi3mr_alloc_requests(struct mpi3mr_softc *sc)  in mpi3mr_alloc_requests()  argument
4965 	ret = bus_dma_tag_create( sc->mpi3mr_parent_dmat,    /* parent */  in mpi3mr_alloc_requests()
4967 				sc->dma_loaddr,		/* lowaddr */  in mpi3mr_alloc_requests()
4975 				&sc->io_lock,	/* lockarg */  in mpi3mr_alloc_requests()
4976 				&sc->buffer_dmat);  in mpi3mr_alloc_requests()
4978 		mpi3mr_dprint(sc, MPI3MR_ERROR, "Cannot allocate buffer DMA tag ret: %d\n", ret);  in mpi3mr_alloc_requests()
4987 	sc->cmd_list = malloc(sizeof(struct mpi3mr_cmd *) * sc->max_host_ios,  in mpi3mr_alloc_requests()
4990 	if (!sc->cmd_list) {  in mpi3mr_alloc_requests()
4991 		device_printf(sc->mpi3mr_dev, "Cannot alloc memory for mpt_cmd_list.\n");  in mpi3mr_alloc_requests()
4995 	for (i = 0; i < sc->max_host_ios; i++) {  in mpi3mr_alloc_requests()
4996 		sc->cmd_list[i] = malloc(sizeof(struct mpi3mr_cmd),  in mpi3mr_alloc_requests()
4998 		if (!sc->cmd_list[i]) {  in mpi3mr_alloc_requests()
5000 				free(sc->cmd_list[j], M_MPI3MR);  in mpi3mr_alloc_requests()
5001 			free(sc->cmd_list, M_MPI3MR);  in mpi3mr_alloc_requests()
5002 			sc->cmd_list = NULL;  in mpi3mr_alloc_requests()
5007 	for (i = 1; i < sc->max_host_ios; i++) {  in mpi3mr_alloc_requests()
5008 		cmd = sc->cmd_list[i];  in mpi3mr_alloc_requests()
5010 		cmd->sc = sc;  in mpi3mr_alloc_requests()
5012 		callout_init_mtx(&cmd->callout, &sc->mpi3mr_mtx, 0);  in mpi3mr_alloc_requests()
5014 		TAILQ_INSERT_TAIL(&(sc->cmd_list_head), cmd, next);  in mpi3mr_alloc_requests()
5015 		if (bus_dmamap_create(sc->buffer_dmat, 0, &cmd->dmamap))  in mpi3mr_alloc_requests()
5029 mpi3mr_get_command(struct mpi3mr_softc *sc)  in mpi3mr_get_command()  argument
5033 	mtx_lock(&sc->cmd_pool_lock);  in mpi3mr_get_command()
5034 	if (!TAILQ_EMPTY(&sc->cmd_list_head)) {  in mpi3mr_get_command()
5035 		cmd = TAILQ_FIRST(&sc->cmd_list_head);  in mpi3mr_get_command()
5036 		TAILQ_REMOVE(&sc->cmd_list_head, cmd, next);  in mpi3mr_get_command()
5041 	mpi3mr_dprint(sc, MPI3MR_TRACE, "Get command SMID: 0x%x\n", cmd->hosttag);  in mpi3mr_get_command()
5051 	mtx_unlock(&sc->cmd_pool_lock);  in mpi3mr_get_command()
5064 	struct mpi3mr_softc *sc = cmd->sc;  in mpi3mr_release_command()  local
5066 	mtx_lock(&sc->cmd_pool_lock);  in mpi3mr_release_command()
5067 	TAILQ_INSERT_HEAD(&(sc->cmd_list_head), cmd, next);  in mpi3mr_release_command()
5070 	mpi3mr_dprint(sc, MPI3MR_TRACE, "Release command SMID: 0x%x\n", cmd->hosttag);  in mpi3mr_release_command()
5071 	mtx_unlock(&sc->cmd_pool_lock);  in mpi3mr_release_command()
5084 static void mpi3mr_free_ioctl_dma_memory(struct mpi3mr_softc *sc)  in mpi3mr_free_ioctl_dma_memory()  argument
5090 		mem_desc = &sc->ioctl_sge[i];  in mpi3mr_free_ioctl_dma_memory()
5100 	mem_desc = &sc->ioctl_chain_sge;  in mpi3mr_free_ioctl_dma_memory()
5109 	mem_desc = &sc->ioctl_resp_sge;  in mpi3mr_free_ioctl_dma_memory()
5118 	sc->ioctl_sges_allocated = false;  in mpi3mr_free_ioctl_dma_memory()
5130 void mpi3mr_alloc_ioctl_dma_memory(struct mpi3mr_softc *sc)  in mpi3mr_alloc_ioctl_dma_memory()  argument
5136 		mem_desc = &sc->ioctl_sge[i];  in mpi3mr_alloc_ioctl_dma_memory()
5139 		if (bus_dma_tag_create(sc->mpi3mr_parent_dmat,    /* parent */  in mpi3mr_alloc_ioctl_dma_memory()
5141 					sc->dma_loaddr,		/* lowaddr */  in mpi3mr_alloc_ioctl_dma_memory()
5150 			mpi3mr_dprint(sc, MPI3MR_ERROR, "Cannot allocate request DMA tag\n");  in mpi3mr_alloc_ioctl_dma_memory()
5156 			mpi3mr_dprint(sc, MPI3MR_ERROR, "%s: Cannot allocate replies memory\n", __func__);  in mpi3mr_alloc_ioctl_dma_memory()
5167 	mem_desc = &sc->ioctl_chain_sge;  in mpi3mr_alloc_ioctl_dma_memory()
5169 	if (bus_dma_tag_create(sc->mpi3mr_parent_dmat,    /* parent */  in mpi3mr_alloc_ioctl_dma_memory()
5171 				sc->dma_loaddr,		/* lowaddr */  in mpi3mr_alloc_ioctl_dma_memory()
5180 		mpi3mr_dprint(sc, MPI3MR_ERROR, "Cannot allocate request DMA tag\n");  in mpi3mr_alloc_ioctl_dma_memory()
5186 		mpi3mr_dprint(sc, MPI3MR_ERROR, "%s: Cannot allocate replies memory\n", __func__);  in mpi3mr_alloc_ioctl_dma_memory()
5196 	mem_desc = &sc->ioctl_resp_sge;  in mpi3mr_alloc_ioctl_dma_memory()
5198 	if (bus_dma_tag_create(sc->mpi3mr_parent_dmat,    /* parent */  in mpi3mr_alloc_ioctl_dma_memory()
5200 				sc->dma_loaddr,		/* lowaddr */  in mpi3mr_alloc_ioctl_dma_memory()
5209 		mpi3mr_dprint(sc, MPI3MR_ERROR, "Cannot allocate request DMA tag\n");  in mpi3mr_alloc_ioctl_dma_memory()
5215 		mpi3mr_dprint(sc, MPI3MR_ERROR, "Cannot allocate replies memory\n");  in mpi3mr_alloc_ioctl_dma_memory()
5225 	sc->ioctl_sges_allocated = true;  in mpi3mr_alloc_ioctl_dma_memory()
5231 	mpi3mr_free_ioctl_dma_memory(sc);  in mpi3mr_alloc_ioctl_dma_memory()
5235 mpi3mr_destory_mtx(struct mpi3mr_softc *sc)  in mpi3mr_destory_mtx()  argument
5241 	if (sc->admin_reply) {  in mpi3mr_destory_mtx()
5242 		if (mtx_initialized(&sc->admin_reply_lock))  in mpi3mr_destory_mtx()
5243 			mtx_destroy(&sc->admin_reply_lock);  in mpi3mr_destory_mtx()
5246 	if (sc->op_reply_q) {  in mpi3mr_destory_mtx()
5247 		for(i = 0; i < sc->num_queues; i++) {  in mpi3mr_destory_mtx()
5248 			op_reply_q = sc->op_reply_q + i;  in mpi3mr_destory_mtx()
5254 	if (sc->op_req_q) {  in mpi3mr_destory_mtx()
5255 		for(i = 0; i < sc->num_queues; i++) {  in mpi3mr_destory_mtx()
5256 			op_req_q = sc->op_req_q + i;  in mpi3mr_destory_mtx()
5262 	if (mtx_initialized(&sc->init_cmds.completion.lock))  in mpi3mr_destory_mtx()
5263 		mtx_destroy(&sc->init_cmds.completion.lock);  in mpi3mr_destory_mtx()
5265 	if (mtx_initialized(&sc->ioctl_cmds.completion.lock))  in mpi3mr_destory_mtx()
5266 		mtx_destroy(&sc->ioctl_cmds.completion.lock);  in mpi3mr_destory_mtx()
5268 	if (mtx_initialized(&sc->host_tm_cmds.completion.lock))  in mpi3mr_destory_mtx()
5269 		mtx_destroy(&sc->host_tm_cmds.completion.lock);  in mpi3mr_destory_mtx()
5272 		if (mtx_initialized(&sc->dev_rmhs_cmds[i].completion.lock))  in mpi3mr_destory_mtx()
5273 			mtx_destroy(&sc->dev_rmhs_cmds[i].completion.lock);  in mpi3mr_destory_mtx()
5276 	if (mtx_initialized(&sc->reset_mutex))  in mpi3mr_destory_mtx()
5277 		mtx_destroy(&sc->reset_mutex);  in mpi3mr_destory_mtx()
5279 	if (mtx_initialized(&sc->target_lock))  in mpi3mr_destory_mtx()
5280 		mtx_destroy(&sc->target_lock);  in mpi3mr_destory_mtx()
5282 	if (mtx_initialized(&sc->fwevt_lock))  in mpi3mr_destory_mtx()
5283 		mtx_destroy(&sc->fwevt_lock);  in mpi3mr_destory_mtx()
5285 	if (mtx_initialized(&sc->cmd_pool_lock))  in mpi3mr_destory_mtx()
5286 		mtx_destroy(&sc->cmd_pool_lock);  in mpi3mr_destory_mtx()
5288 	if (mtx_initialized(&sc->reply_free_q_lock))  in mpi3mr_destory_mtx()
5289 		mtx_destroy(&sc->reply_free_q_lock);  in mpi3mr_destory_mtx()
5291 	if (mtx_initialized(&sc->sense_buf_q_lock))  in mpi3mr_destory_mtx()
5292 		mtx_destroy(&sc->sense_buf_q_lock);  in mpi3mr_destory_mtx()
5294 	if (mtx_initialized(&sc->chain_buf_lock))  in mpi3mr_destory_mtx()
5295 		mtx_destroy(&sc->chain_buf_lock);  in mpi3mr_destory_mtx()
5297 	if (mtx_initialized(&sc->admin_req_lock))  in mpi3mr_destory_mtx()
5298 		mtx_destroy(&sc->admin_req_lock);  in mpi3mr_destory_mtx()
5300 	if (mtx_initialized(&sc->mpi3mr_mtx))  in mpi3mr_destory_mtx()
5301 		mtx_destroy(&sc->mpi3mr_mtx);  in mpi3mr_destory_mtx()
5311 mpi3mr_free_mem(struct mpi3mr_softc *sc)  in mpi3mr_free_mem()  argument
5318 	if (sc->cmd_list) {  in mpi3mr_free_mem()
5319 		for (i = 0; i < sc->max_host_ios; i++) {  in mpi3mr_free_mem()
5320 			free(sc->cmd_list[i], M_MPI3MR);  in mpi3mr_free_mem()
5322 		free(sc->cmd_list, M_MPI3MR);  in mpi3mr_free_mem()
5323 		sc->cmd_list = NULL;  in mpi3mr_free_mem()
5326 	if (sc->pel_seq_number && sc->pel_seq_number_dma) {  in mpi3mr_free_mem()
5327 		bus_dmamap_unload(sc->pel_seq_num_dmatag, sc->pel_seq_num_dmamap);  in mpi3mr_free_mem()
5328 		bus_dmamem_free(sc->pel_seq_num_dmatag, sc->pel_seq_number, sc->pel_seq_num_dmamap);  in mpi3mr_free_mem()
5329 		sc->pel_seq_number = NULL;  in mpi3mr_free_mem()
5330 		if (sc->pel_seq_num_dmatag != NULL)  in mpi3mr_free_mem()
5331 			bus_dma_tag_destroy(sc->pel_seq_num_dmatag);  in mpi3mr_free_mem()
5334 	if (sc->throttle_groups) {  in mpi3mr_free_mem()
5335 		free(sc->throttle_groups, M_MPI3MR);  in mpi3mr_free_mem()
5336 		sc->throttle_groups = NULL;  in mpi3mr_free_mem()
5340 	if (sc->op_req_q) {  in mpi3mr_free_mem()
5341 		for (i = 0; i < sc->num_queues; i++) {  in mpi3mr_free_mem()
5342 			op_req_q = sc->op_req_q + i;  in mpi3mr_free_mem()
5351 		free(sc->op_req_q, M_MPI3MR);  in mpi3mr_free_mem()
5352 		sc->op_req_q = NULL;  in mpi3mr_free_mem()
5355 	if (sc->op_reply_q) {  in mpi3mr_free_mem()
5356 		for (i = 0; i < sc->num_queues; i++) {  in mpi3mr_free_mem()
5357 			op_reply_q = sc->op_reply_q + i;  in mpi3mr_free_mem()
5366 		free(sc->op_reply_q, M_MPI3MR);  in mpi3mr_free_mem()
5367 		sc->op_reply_q = NULL;  in mpi3mr_free_mem()
5371 	if (sc->chain_sgl_list) {  in mpi3mr_free_mem()
5372 		for (i = 0; i < sc->chain_buf_count; i++) {  in mpi3mr_free_mem()
5373 			if (sc->chain_sgl_list[i].buf && sc->chain_sgl_list[i].buf_phys) {  in mpi3mr_free_mem()
5374 				bus_dmamap_unload(sc->chain_sgl_list_tag, sc->chain_sgl_list[i].buf_dmamap);  in mpi3mr_free_mem()
5375 				bus_dmamem_free(sc->chain_sgl_list_tag, sc->chain_sgl_list[i].buf,  in mpi3mr_free_mem()
5376 						sc->chain_sgl_list[i].buf_dmamap);  in mpi3mr_free_mem()
5377 				sc->chain_sgl_list[i].buf = NULL;  in mpi3mr_free_mem()
5380 		if (sc->chain_sgl_list_tag != NULL)  in mpi3mr_free_mem()
5381 			bus_dma_tag_destroy(sc->chain_sgl_list_tag);  in mpi3mr_free_mem()
5382 		free(sc->chain_sgl_list, M_MPI3MR);  in mpi3mr_free_mem()
5383 		sc->chain_sgl_list = NULL;  in mpi3mr_free_mem()
5386 	if (sc->chain_bitmap) {  in mpi3mr_free_mem()
5387 		free(sc->chain_bitmap, M_MPI3MR);  in mpi3mr_free_mem()
5388 		sc->chain_bitmap = NULL;  in mpi3mr_free_mem()
5391 	for (i = 0; i < sc->msix_count; i++) {  in mpi3mr_free_mem()
5392 		irq_ctx = sc->irq_ctx + i;  in mpi3mr_free_mem()
5398 	if (sc->reply_buf && sc->reply_buf_phys) {   in mpi3mr_free_mem()
5399 		bus_dmamap_unload(sc->reply_buf_tag, sc->reply_buf_dmamap);  in mpi3mr_free_mem()
5400 		bus_dmamem_free(sc->reply_buf_tag, sc->reply_buf,  in mpi3mr_free_mem()
5401 				sc->reply_buf_dmamap);  in mpi3mr_free_mem()
5402 		sc->reply_buf = NULL;  in mpi3mr_free_mem()
5403 		if (sc->reply_buf_tag != NULL)  in mpi3mr_free_mem()
5404 			bus_dma_tag_destroy(sc->reply_buf_tag);  in mpi3mr_free_mem()
5408 	if (sc->reply_free_q && sc->reply_free_q_phys) {  in mpi3mr_free_mem()
5409 		bus_dmamap_unload(sc->reply_free_q_tag, sc->reply_free_q_dmamap);  in mpi3mr_free_mem()
5410 		bus_dmamem_free(sc->reply_free_q_tag, sc->reply_free_q,  in mpi3mr_free_mem()
5411 				sc->reply_free_q_dmamap);  in mpi3mr_free_mem()
5412 		sc->reply_free_q = NULL;  in mpi3mr_free_mem()
5413 		if (sc->reply_free_q_tag != NULL)  in mpi3mr_free_mem()
5414 			bus_dma_tag_destroy(sc->reply_free_q_tag);  in mpi3mr_free_mem()
5418 	if (sc->sense_buf && sc->sense_buf_phys) {  in mpi3mr_free_mem()
5419 		bus_dmamap_unload(sc->sense_buf_tag, sc->sense_buf_dmamap);  in mpi3mr_free_mem()
5420 		bus_dmamem_free(sc->sense_buf_tag, sc->sense_buf,  in mpi3mr_free_mem()
5421 				sc->sense_buf_dmamap);  in mpi3mr_free_mem()
5422 		sc->sense_buf = NULL;  in mpi3mr_free_mem()
5423 		if (sc->sense_buf_tag != NULL)  in mpi3mr_free_mem()
5424 			bus_dma_tag_destroy(sc->sense_buf_tag);  in mpi3mr_free_mem()
5428 	if (sc->sense_buf_q && sc->sense_buf_q_phys) {  in mpi3mr_free_mem()
5429 		bus_dmamap_unload(sc->sense_buf_q_tag, sc->sense_buf_q_dmamap);  in mpi3mr_free_mem()
5430 		bus_dmamem_free(sc->sense_buf_q_tag, sc->sense_buf_q,  in mpi3mr_free_mem()
5431 				sc->sense_buf_q_dmamap);  in mpi3mr_free_mem()
5432 		sc->sense_buf_q = NULL;  in mpi3mr_free_mem()
5433 		if (sc->sense_buf_q_tag != NULL)  in mpi3mr_free_mem()
5434 			bus_dma_tag_destroy(sc->sense_buf_q_tag);  in mpi3mr_free_mem()
5438 	if (sc->init_cmds.reply) {  in mpi3mr_free_mem()
5439 		free(sc->init_cmds.reply, M_MPI3MR);  in mpi3mr_free_mem()
5440 		sc->init_cmds.reply = NULL;  in mpi3mr_free_mem()
5443 	if (sc->ioctl_cmds.reply) {  in mpi3mr_free_mem()
5444 		free(sc->ioctl_cmds.reply, M_MPI3MR);  in mpi3mr_free_mem()
5445 		sc->ioctl_cmds.reply = NULL;  in mpi3mr_free_mem()
5448 	if (sc->pel_cmds.reply) {  in mpi3mr_free_mem()
5449 		free(sc->pel_cmds.reply, M_MPI3MR);  in mpi3mr_free_mem()
5450 		sc->pel_cmds.reply = NULL;  in mpi3mr_free_mem()
5453 	if (sc->pel_abort_cmd.reply) {  in mpi3mr_free_mem()
5454 		free(sc->pel_abort_cmd.reply, M_MPI3MR);  in mpi3mr_free_mem()
5455 		sc->pel_abort_cmd.reply = NULL;  in mpi3mr_free_mem()
5458 	if (sc->host_tm_cmds.reply) {  in mpi3mr_free_mem()
5459 		free(sc->host_tm_cmds.reply, M_MPI3MR);  in mpi3mr_free_mem()
5460 		sc->host_tm_cmds.reply = NULL;  in mpi3mr_free_mem()
5463 	if (sc->log_data_buffer) {  in mpi3mr_free_mem()
5464 		free(sc->log_data_buffer, M_MPI3MR);  in mpi3mr_free_mem()
5465 		sc->log_data_buffer = NULL;  in mpi3mr_free_mem()
5469 		if (sc->dev_rmhs_cmds[i].reply) {  in mpi3mr_free_mem()
5470 			free(sc->dev_rmhs_cmds[i].reply, M_MPI3MR);  in mpi3mr_free_mem()
5471 			sc->dev_rmhs_cmds[i].reply = NULL;  in mpi3mr_free_mem()
5476 		if (sc->evtack_cmds[i].reply) {  in mpi3mr_free_mem()
5477 			free(sc->evtack_cmds[i].reply, M_MPI3MR);  in mpi3mr_free_mem()
5478 			sc->evtack_cmds[i].reply = NULL;  in mpi3mr_free_mem()
5482 	if (sc->removepend_bitmap) {  in mpi3mr_free_mem()
5483 		free(sc->removepend_bitmap, M_MPI3MR);  in mpi3mr_free_mem()
5484 		sc->removepend_bitmap = NULL;  in mpi3mr_free_mem()
5487 	if (sc->devrem_bitmap) {  in mpi3mr_free_mem()
5488 		free(sc->devrem_bitmap, M_MPI3MR);  in mpi3mr_free_mem()
5489 		sc->devrem_bitmap = NULL;  in mpi3mr_free_mem()
5492 	if (sc->evtack_cmds_bitmap) {  in mpi3mr_free_mem()
5493 		free(sc->evtack_cmds_bitmap, M_MPI3MR);  in mpi3mr_free_mem()
5494 		sc->evtack_cmds_bitmap = NULL;  in mpi3mr_free_mem()
5498 	if (sc->admin_reply && sc->admin_reply_phys) {  in mpi3mr_free_mem()
5499 		bus_dmamap_unload(sc->admin_reply_tag, sc->admin_reply_dmamap);  in mpi3mr_free_mem()
5500 		bus_dmamem_free(sc->admin_reply_tag, sc->admin_reply,  in mpi3mr_free_mem()
5501 				sc->admin_reply_dmamap);  in mpi3mr_free_mem()
5502 		sc->admin_reply = NULL;  in mpi3mr_free_mem()
5503 		if (sc->admin_reply_tag != NULL)  in mpi3mr_free_mem()
5504 			bus_dma_tag_destroy(sc->admin_reply_tag);  in mpi3mr_free_mem()
5508 	if (sc->admin_req && sc->admin_req_phys) {  in mpi3mr_free_mem()
5509 		bus_dmamap_unload(sc->admin_req_tag, sc->admin_req_dmamap);  in mpi3mr_free_mem()
5510 		bus_dmamem_free(sc->admin_req_tag, sc->admin_req,  in mpi3mr_free_mem()
5511 				sc->admin_req_dmamap);  in mpi3mr_free_mem()
5512 		sc->admin_req = NULL;  in mpi3mr_free_mem()
5513 		if (sc->admin_req_tag != NULL)  in mpi3mr_free_mem()
5514 			bus_dma_tag_destroy(sc->admin_req_tag);  in mpi3mr_free_mem()
5516 	mpi3mr_free_ioctl_dma_memory(sc);  in mpi3mr_free_mem()
5530 static inline void mpi3mr_drv_cmd_comp_reset(struct mpi3mr_softc *sc,  in mpi3mr_drv_cmd_comp_reset()  argument
5540 			cmdptr->callback(sc, cmdptr);  in mpi3mr_drv_cmd_comp_reset()
5552 static void mpi3mr_flush_drv_cmds(struct mpi3mr_softc *sc)  in mpi3mr_flush_drv_cmds()  argument
5557 	cmdptr = &sc->init_cmds;  in mpi3mr_flush_drv_cmds()
5558 	mpi3mr_drv_cmd_comp_reset(sc, cmdptr);  in mpi3mr_flush_drv_cmds()
5560 	cmdptr = &sc->ioctl_cmds;  in mpi3mr_flush_drv_cmds()
5561 	mpi3mr_drv_cmd_comp_reset(sc, cmdptr);  in mpi3mr_flush_drv_cmds()
5563 	cmdptr = &sc->host_tm_cmds;  in mpi3mr_flush_drv_cmds()
5564 	mpi3mr_drv_cmd_comp_reset(sc, cmdptr);  in mpi3mr_flush_drv_cmds()
5567 		cmdptr = &sc->dev_rmhs_cmds[i];  in mpi3mr_flush_drv_cmds()
5568 		mpi3mr_drv_cmd_comp_reset(sc, cmdptr);  in mpi3mr_flush_drv_cmds()
5572 		cmdptr = &sc->evtack_cmds[i];  in mpi3mr_flush_drv_cmds()
5573 		mpi3mr_drv_cmd_comp_reset(sc, cmdptr);  in mpi3mr_flush_drv_cmds()
5576 	cmdptr = &sc->pel_cmds;  in mpi3mr_flush_drv_cmds()
5577 	mpi3mr_drv_cmd_comp_reset(sc, cmdptr);  in mpi3mr_flush_drv_cmds()
5579 	cmdptr = &sc->pel_abort_cmd;  in mpi3mr_flush_drv_cmds()
5580 	mpi3mr_drv_cmd_comp_reset(sc, cmdptr);  in mpi3mr_flush_drv_cmds()
5594 static void mpi3mr_memset_buffers(struct mpi3mr_softc *sc)  in mpi3mr_memset_buffers()  argument
5599 	memset(sc->admin_req, 0, sc->admin_req_q_sz);  in mpi3mr_memset_buffers()
5600 	memset(sc->admin_reply, 0, sc->admin_reply_q_sz);  in mpi3mr_memset_buffers()
5602 	memset(sc->init_cmds.reply, 0, sc->reply_sz);  in mpi3mr_memset_buffers()
5603 	memset(sc->ioctl_cmds.reply, 0, sc->reply_sz);  in mpi3mr_memset_buffers()
5604 	memset(sc->host_tm_cmds.reply, 0, sc->reply_sz);  in mpi3mr_memset_buffers()
5605 	memset(sc->pel_cmds.reply, 0, sc->reply_sz);  in mpi3mr_memset_buffers()
5606 	memset(sc->pel_abort_cmd.reply, 0, sc->reply_sz);  in mpi3mr_memset_buffers()
5608 		memset(sc->dev_rmhs_cmds[i].reply, 0, sc->reply_sz);  in mpi3mr_memset_buffers()
5610 		memset(sc->evtack_cmds[i].reply, 0, sc->reply_sz);  in mpi3mr_memset_buffers()
5611 	memset(sc->removepend_bitmap, 0, sc->dev_handle_bitmap_sz);  in mpi3mr_memset_buffers()
5612 	memset(sc->devrem_bitmap, 0, sc->devrem_bitmap_sz);  in mpi3mr_memset_buffers()
5613 	memset(sc->evtack_cmds_bitmap, 0, sc->evtack_cmds_bitmap_sz);  in mpi3mr_memset_buffers()
5615 	for (i = 0; i < sc->num_queues; i++) {  in mpi3mr_memset_buffers()
5616 		sc->op_reply_q[i].qid = 0;  in mpi3mr_memset_buffers()
5617 		sc->op_reply_q[i].ci = 0;  in mpi3mr_memset_buffers()
5618 		sc->op_reply_q[i].num_replies = 0;  in mpi3mr_memset_buffers()
5619 		sc->op_reply_q[i].ephase = 0;  in mpi3mr_memset_buffers()
5620 		mpi3mr_atomic_set(&sc->op_reply_q[i].pend_ios, 0);  in mpi3mr_memset_buffers()
5621 		memset(sc->op_reply_q[i].q_base, 0, sc->op_reply_q[i].qsz);  in mpi3mr_memset_buffers()
5623 		sc->op_req_q[i].ci = 0;  in mpi3mr_memset_buffers()
5624 		sc->op_req_q[i].pi = 0;  in mpi3mr_memset_buffers()
5625 		sc->op_req_q[i].num_reqs = 0;  in mpi3mr_memset_buffers()
5626 		sc->op_req_q[i].qid = 0;  in mpi3mr_memset_buffers()
5627 		sc->op_req_q[i].reply_qid = 0;  in mpi3mr_memset_buffers()
5628 		memset(sc->op_req_q[i].q_base, 0, sc->op_req_q[i].qsz);  in mpi3mr_memset_buffers()
5631 	mpi3mr_atomic_set(&sc->pend_large_data_sz, 0);  in mpi3mr_memset_buffers()
5632 	if (sc->throttle_groups) {  in mpi3mr_memset_buffers()
5633 		tg = sc->throttle_groups;  in mpi3mr_memset_buffers()
5634 		for (i = 0; i < sc->num_io_throttle_group; i++, tg++) {  in mpi3mr_memset_buffers()
5655 static void mpi3mr_invalidate_devhandles(struct mpi3mr_softc *sc)  in mpi3mr_invalidate_devhandles()  argument
5659 	mtx_lock_spin(&sc->target_lock);  in mpi3mr_invalidate_devhandles()
5660 	TAILQ_FOREACH(target, &sc->cam_sc->tgt_list, tgt_next) {  in mpi3mr_invalidate_devhandles()
5669 	mtx_unlock_spin(&sc->target_lock);  in mpi3mr_invalidate_devhandles()
5683 static void mpi3mr_rfresh_tgtdevs(struct mpi3mr_softc *sc)  in mpi3mr_rfresh_tgtdevs()  argument
5688 	TAILQ_FOREACH_SAFE(target, &sc->cam_sc->tgt_list, tgt_next, target_temp) {  in mpi3mr_rfresh_tgtdevs()
5691 				mpi3mr_remove_device_from_os(sc, target->dev_handle);  in mpi3mr_rfresh_tgtdevs()
5692 			mpi3mr_remove_device_from_list(sc, target, true);  in mpi3mr_rfresh_tgtdevs()
5694 				mpi3mr_remove_device_from_os(sc, target->dev_handle);  in mpi3mr_rfresh_tgtdevs()
5698 	TAILQ_FOREACH(target, &sc->cam_sc->tgt_list, tgt_next) {  in mpi3mr_rfresh_tgtdevs()
5701 			mpi3mr_add_device(sc, target->per_id);  in mpi3mr_rfresh_tgtdevs()
5707 static void mpi3mr_flush_io(struct mpi3mr_softc *sc)  in mpi3mr_flush_io()  argument
5713 	for (i = 0; i < sc->max_host_ios; i++) {  in mpi3mr_flush_io()
5714 		cmd = sc->cmd_list[i];  in mpi3mr_flush_io()
5720 				mpi3mr_atomic_dec(&sc->fw_outstanding);  in mpi3mr_flush_io()
5722 				mpi3mr_cmd_done(sc, cmd);  in mpi3mr_flush_io()
5739 static inline void mpi3mr_clear_reset_history(struct mpi3mr_softc *sc)  in mpi3mr_clear_reset_history()  argument
5743 	ioc_status = mpi3mr_regread(sc, MPI3_SYSIF_IOC_STATUS_OFFSET);  in mpi3mr_clear_reset_history()
5745 		mpi3mr_regwrite(sc, MPI3_SYSIF_IOC_STATUS_OFFSET, ioc_status);  in mpi3mr_clear_reset_history()
5757 static inline void mpi3mr_set_diagsave(struct mpi3mr_softc *sc)  in mpi3mr_set_diagsave()  argument
5762 	    mpi3mr_regread(sc, MPI3_SYSIF_IOC_CONFIG_OFFSET);  in mpi3mr_set_diagsave()
5764 	mpi3mr_regwrite(sc, MPI3_SYSIF_IOC_CONFIG_OFFSET, ioc_config);  in mpi3mr_set_diagsave()
5780 static int mpi3mr_issue_reset(struct mpi3mr_softc *sc, U16 reset_type,  in mpi3mr_issue_reset()  argument
5791 	if (sc->unrecoverable)  in mpi3mr_issue_reset()
5799 	mpi3mr_dprint(sc, MPI3MR_INFO, "%s reset due to %s(0x%x)\n",  in mpi3mr_issue_reset()
5803 	mpi3mr_clear_reset_history(sc);  in mpi3mr_issue_reset()
5805 		mpi3mr_dprint(sc, MPI3MR_INFO,  in mpi3mr_issue_reset()
5809 			mpi3mr_dprint(sc, MPI3MR_ERROR,  in mpi3mr_issue_reset()
5813 			sc->unrecoverable = 1;  in mpi3mr_issue_reset()
5817 		mpi3mr_regwrite(sc, MPI3_SYSIF_WRITE_SEQUENCE_OFFSET,  in mpi3mr_issue_reset()
5819 		mpi3mr_regwrite(sc, MPI3_SYSIF_WRITE_SEQUENCE_OFFSET,  in mpi3mr_issue_reset()
5821 		mpi3mr_regwrite(sc, MPI3_SYSIF_WRITE_SEQUENCE_OFFSET,  in mpi3mr_issue_reset()
5823 		mpi3mr_regwrite(sc, MPI3_SYSIF_WRITE_SEQUENCE_OFFSET,  in mpi3mr_issue_reset()
5825 		mpi3mr_regwrite(sc, MPI3_SYSIF_WRITE_SEQUENCE_OFFSET,  in mpi3mr_issue_reset()
5827 		mpi3mr_regwrite(sc, MPI3_SYSIF_WRITE_SEQUENCE_OFFSET,  in mpi3mr_issue_reset()
5829 		mpi3mr_regwrite(sc, MPI3_SYSIF_WRITE_SEQUENCE_OFFSET,  in mpi3mr_issue_reset()
5833 		host_diagnostic = mpi3mr_regread(sc, MPI3_SYSIF_HOST_DIAG_OFFSET);  in mpi3mr_issue_reset()
5834 		mpi3mr_dprint(sc, MPI3MR_INFO,  in mpi3mr_issue_reset()
5841 			(sc->facts.ioc_num <<  in mpi3mr_issue_reset()
5843 	mpi3mr_regwrite(sc, MPI3_SYSIF_SCRATCHPAD0_OFFSET, scratch_pad0);  in mpi3mr_issue_reset()
5844 	mpi3mr_regwrite(sc, MPI3_SYSIF_HOST_DIAG_OFFSET, host_diagnostic | reset_type);  in mpi3mr_issue_reset()
5848 			ioc_status = mpi3mr_regread(sc, MPI3_SYSIF_IOC_STATUS_OFFSET);  in mpi3mr_issue_reset()
5852 				    mpi3mr_regread(sc, MPI3_SYSIF_IOC_CONFIG_OFFSET);  in mpi3mr_issue_reset()
5855 					mpi3mr_clear_reset_history(sc);  in mpi3mr_issue_reset()
5864 			ioc_status = mpi3mr_regread(sc, MPI3_SYSIF_IOC_STATUS_OFFSET);  in mpi3mr_issue_reset()
5865 			if (mpi3mr_diagfault_success(sc, ioc_status)) {  in mpi3mr_issue_reset()
5873 	mpi3mr_regwrite(sc, MPI3_SYSIF_WRITE_SEQUENCE_OFFSET,   in mpi3mr_issue_reset()
5876 	ioc_status = mpi3mr_regread(sc, MPI3_SYSIF_IOC_STATUS_OFFSET);  in mpi3mr_issue_reset()
5877 	ioc_config = mpi3mr_regread(sc, MPI3_SYSIF_IOC_CONFIG_OFFSET);  in mpi3mr_issue_reset()
5879 	mpi3mr_dprint(sc, MPI3MR_INFO,  in mpi3mr_issue_reset()
5885 		sc->unrecoverable = 1;  in mpi3mr_issue_reset()
5890 inline void mpi3mr_cleanup_event_taskq(struct mpi3mr_softc *sc)  in mpi3mr_cleanup_event_taskq()  argument
5899 	taskqueue_block(sc->cam_sc->ev_tq);  in mpi3mr_cleanup_event_taskq()
5900 	while (taskqueue_cancel(sc->cam_sc->ev_tq, &sc->cam_sc->ev_task, NULL) != 0) {  in mpi3mr_cleanup_event_taskq()
5901 		taskqueue_drain(sc->cam_sc->ev_tq, &sc->cam_sc->ev_task);  in mpi3mr_cleanup_event_taskq()
5921 int mpi3mr_soft_reset_handler(struct mpi3mr_softc *sc,  in mpi3mr_soft_reset_handler()  argument
5927 	mpi3mr_dprint(sc, MPI3MR_INFO, "soft reset invoked: reason code: %s\n",  in mpi3mr_soft_reset_handler()
5931 	     (sc->reset.ioctl_reset_snapdump != true))  in mpi3mr_soft_reset_handler()
5934 	mpi3mr_dprint(sc, MPI3MR_INFO,  in mpi3mr_soft_reset_handler()
5936 	while (sc->diagsave_timeout)  in mpi3mr_soft_reset_handler()
5939 	ioc_state = mpi3mr_get_iocstate(sc);  in mpi3mr_soft_reset_handler()
5941 		mpi3mr_dprint(sc, MPI3MR_ERROR, "controller is in unrecoverable state, exit\n");  in mpi3mr_soft_reset_handler()
5942 		sc->reset.type = MPI3MR_NO_RESET;  in mpi3mr_soft_reset_handler()
5943 		sc->reset.reason = MPI3MR_DEFAULT_RESET_REASON;  in mpi3mr_soft_reset_handler()
5944 		sc->reset.status = -1;   in mpi3mr_soft_reset_handler()
5945 		sc->reset.ioctl_reset_snapdump = false;	  in mpi3mr_soft_reset_handler()
5949 	if (sc->reset_in_progress) {  in mpi3mr_soft_reset_handler()
5950 		mpi3mr_dprint(sc, MPI3MR_INFO, "reset is already in progress, exit\n");  in mpi3mr_soft_reset_handler()
5955 	xpt_freeze_simq(sc->cam_sc->sim, 1);  in mpi3mr_soft_reset_handler()
5957 	mpi3mr_cleanup_event_taskq(sc);  in mpi3mr_soft_reset_handler()
5959 	sc->reset_in_progress = 1;  in mpi3mr_soft_reset_handler()
5960 	sc->block_ioctls = 1;  in mpi3mr_soft_reset_handler()
5962 	while (mpi3mr_atomic_read(&sc->pend_ioctls) && (i < PEND_IOCTLS_COMP_WAIT_TIME)) {  in mpi3mr_soft_reset_handler()
5963 		ioc_state = mpi3mr_get_iocstate(sc);  in mpi3mr_soft_reset_handler()
5968 			mpi3mr_dprint(sc, MPI3MR_INFO,  in mpi3mr_soft_reset_handler()
5978 		mpi3mr_dprint(sc, MPI3MR_INFO, "Turn off events prior to reset\n");  in mpi3mr_soft_reset_handler()
5981 			sc->event_masks[i] = -1;  in mpi3mr_soft_reset_handler()
5982 		mpi3mr_issue_event_notification(sc);  in mpi3mr_soft_reset_handler()
5985 	mpi3mr_disable_interrupts(sc);  in mpi3mr_soft_reset_handler()
5988 		mpi3mr_trigger_snapdump(sc, reset_reason);  in mpi3mr_soft_reset_handler()
5990 	retval = mpi3mr_issue_reset(sc,  in mpi3mr_soft_reset_handler()
5993 		mpi3mr_dprint(sc, MPI3MR_ERROR, "Failed to issue soft reset to the ioc\n");  in mpi3mr_soft_reset_handler()
5997 	mpi3mr_flush_drv_cmds(sc);  in mpi3mr_soft_reset_handler()
5998 	mpi3mr_flush_io(sc);  in mpi3mr_soft_reset_handler()
5999 	mpi3mr_invalidate_devhandles(sc);  in mpi3mr_soft_reset_handler()
6000 	mpi3mr_memset_buffers(sc);  in mpi3mr_soft_reset_handler()
6002 	if (sc->prepare_for_reset) {  in mpi3mr_soft_reset_handler()
6003 		sc->prepare_for_reset = 0;  in mpi3mr_soft_reset_handler()
6004 		sc->prepare_for_reset_timeout_counter = 0;  in mpi3mr_soft_reset_handler()
6007 	retval = mpi3mr_initialize_ioc(sc, MPI3MR_INIT_TYPE_RESET);  in mpi3mr_soft_reset_handler()
6009 		mpi3mr_dprint(sc, MPI3MR_ERROR, "reinit after soft reset failed: reason %d\n",  in mpi3mr_soft_reset_handler()
6017 		sc->diagsave_timeout = 0;  in mpi3mr_soft_reset_handler()
6018 		sc->reset_in_progress = 0;  in mpi3mr_soft_reset_handler()
6019 		mpi3mr_rfresh_tgtdevs(sc);  in mpi3mr_soft_reset_handler()
6020 		sc->ts_update_counter = 0;  in mpi3mr_soft_reset_handler()
6021 		sc->block_ioctls = 0;  in mpi3mr_soft_reset_handler()
6022 		sc->pel_abort_requested = 0;  in mpi3mr_soft_reset_handler()
6023 		if (sc->pel_wait_pend) {  in mpi3mr_soft_reset_handler()
6024 			sc->pel_cmds.retry_count = 0;  in mpi3mr_soft_reset_handler()
6025 			mpi3mr_issue_pel_wait(sc, &sc->pel_cmds);  in mpi3mr_soft_reset_handler()
6026 			mpi3mr_app_send_aen(sc);  in mpi3mr_soft_reset_handler()
6029 		mpi3mr_issue_reset(sc,  in mpi3mr_soft_reset_handler()
6031 		sc->unrecoverable = 1;  in mpi3mr_soft_reset_handler()
6032 		sc->reset_in_progress = 0;  in mpi3mr_soft_reset_handler()
6035 	mpi3mr_dprint(sc, MPI3MR_INFO, "Soft Reset: %s\n", ((retval == 0) ? "SUCCESS" : "FAILED"));  in mpi3mr_soft_reset_handler()
6037 	taskqueue_unblock(sc->cam_sc->ev_tq);  in mpi3mr_soft_reset_handler()
6038 	xpt_release_simq(sc->cam_sc->sim, 1);  in mpi3mr_soft_reset_handler()
6040 	sc->reset.type = MPI3MR_NO_RESET;  in mpi3mr_soft_reset_handler()
6041 	sc->reset.reason = MPI3MR_DEFAULT_RESET_REASON;  in mpi3mr_soft_reset_handler()
6042 	sc->reset.status = retval;   in mpi3mr_soft_reset_handler()
6043 	sc->reset.ioctl_reset_snapdump = false;	  in mpi3mr_soft_reset_handler()
6057 static void mpi3mr_issue_ioc_shutdown(struct mpi3mr_softc *sc)  in mpi3mr_issue_ioc_shutdown()  argument
6063 	mpi3mr_dprint(sc, MPI3MR_INFO, "sending shutdown notification\n");  in mpi3mr_issue_ioc_shutdown()
6064 	if (sc->unrecoverable) {  in mpi3mr_issue_ioc_shutdown()
6065 		mpi3mr_dprint(sc, MPI3MR_ERROR,   in mpi3mr_issue_ioc_shutdown()
6069 	ioc_status = mpi3mr_regread(sc, MPI3_SYSIF_IOC_STATUS_OFFSET);  in mpi3mr_issue_ioc_shutdown()
6072 		mpi3mr_dprint(sc, MPI3MR_ERROR, "shutdown already in progress\n");  in mpi3mr_issue_ioc_shutdown()
6076 	ioc_config = mpi3mr_regread(sc, MPI3_SYSIF_IOC_CONFIG_OFFSET);  in mpi3mr_issue_ioc_shutdown()
6080 	mpi3mr_regwrite(sc, MPI3_SYSIF_IOC_CONFIG_OFFSET, ioc_config);  in mpi3mr_issue_ioc_shutdown()
6082 	if (sc->facts.shutdown_timeout)  in mpi3mr_issue_ioc_shutdown()
6083 		timeout = sc->facts.shutdown_timeout * 10;  in mpi3mr_issue_ioc_shutdown()
6086 		ioc_status = mpi3mr_regread(sc, MPI3_SYSIF_IOC_STATUS_OFFSET);  in mpi3mr_issue_ioc_shutdown()
6093 		if (sc->unrecoverable)  in mpi3mr_issue_ioc_shutdown()
6097 			mpi3mr_print_fault_info(sc);  in mpi3mr_issue_ioc_shutdown()
6102 			if (mpi3mr_issue_reset(sc,  in mpi3mr_issue_ioc_shutdown()
6107 			ioc_config = mpi3mr_regread(sc, MPI3_SYSIF_IOC_CONFIG_OFFSET);  in mpi3mr_issue_ioc_shutdown()
6111 			mpi3mr_regwrite(sc, MPI3_SYSIF_IOC_CONFIG_OFFSET, ioc_config);  in mpi3mr_issue_ioc_shutdown()
6113 			if (sc->facts.shutdown_timeout)  in mpi3mr_issue_ioc_shutdown()
6114 				timeout = sc->facts.shutdown_timeout * 10;  in mpi3mr_issue_ioc_shutdown()
6123 	ioc_status = mpi3mr_regread(sc, MPI3_SYSIF_IOC_STATUS_OFFSET);  in mpi3mr_issue_ioc_shutdown()
6124 	ioc_config = mpi3mr_regread(sc, MPI3_SYSIF_IOC_CONFIG_OFFSET);  in mpi3mr_issue_ioc_shutdown()
6129 			mpi3mr_dprint(sc, MPI3MR_ERROR,  in mpi3mr_issue_ioc_shutdown()
6133 	mpi3mr_dprint(sc, MPI3MR_INFO,  in mpi3mr_issue_ioc_shutdown()
6148 void mpi3mr_cleanup_ioc(struct mpi3mr_softc *sc)  in mpi3mr_cleanup_ioc()  argument
6152 	mpi3mr_dprint(sc, MPI3MR_INFO, "cleaning up the controller\n");  in mpi3mr_cleanup_ioc()
6153 	mpi3mr_disable_interrupts(sc);  in mpi3mr_cleanup_ioc()
6155 	ioc_state = mpi3mr_get_iocstate(sc);  in mpi3mr_cleanup_ioc()
6157 	if ((!sc->unrecoverable) && (!sc->reset_in_progress) &&  in mpi3mr_cleanup_ioc()
6159 		if (mpi3mr_mur_ioc(sc,  in mpi3mr_cleanup_ioc()
6161 			mpi3mr_issue_reset(sc,  in mpi3mr_cleanup_ioc()
6164 		mpi3mr_issue_ioc_shutdown(sc);  in mpi3mr_cleanup_ioc()
6167 	mpi3mr_dprint(sc, MPI3MR_INFO, "controller cleanup completed\n");  in mpi3mr_cleanup_ioc()