1 // SPDX-License-Identifier: GPL-2.0-or-later
5  * Copyright (C) 2017-2023 Broadcom Inc.
6  *  (mailto: mpi3mr-linuxdrv.pdl@broadcom.com)
11 #include <linux/io-64-nonatomic-lo-hi.h>
23 MODULE_PARM_DESC(poll_queues, "Number of queues for io_uring poll mode. (Range 1 - 126)");
46 	pi = op_req_q->pi;  in mpi3mr_check_req_qfull()
47 	ci = READ_ONCE(op_req_q->ci);  in mpi3mr_check_req_qfull()
48 	max_entries = op_req_q->num_requests;  in mpi3mr_check_req_qfull()
50 	if ((ci == (pi + 1)) || ((!ci) && (pi == (max_entries - 1))))  in mpi3mr_check_req_qfull()
60 	max_vectors = mrioc->intr_info_count;  in mpi3mr_sync_irqs()
63 		synchronize_irq(pci_irq_vector(mrioc->pdev, i));  in mpi3mr_sync_irqs()
68 	mrioc->intr_enabled = 0;  in mpi3mr_ioc_disable_intr()
74 	mrioc->intr_enabled = 1;  in mpi3mr_ioc_enable_intr()
83 	if (!mrioc->intr_info)  in mpi3mr_cleanup_isr()
86 	for (i = 0; i < mrioc->intr_info_count; i++)  in mpi3mr_cleanup_isr()
87 		free_irq(pci_irq_vector(mrioc->pdev, i),  in mpi3mr_cleanup_isr()
88 		    (mrioc->intr_info + i));  in mpi3mr_cleanup_isr()
90 	kfree(mrioc->intr_info);  in mpi3mr_cleanup_isr()
91 	mrioc->intr_info = NULL;  in mpi3mr_cleanup_isr()
92 	mrioc->intr_info_count = 0;  in mpi3mr_cleanup_isr()
93 	mrioc->is_intr_info_set = false;  in mpi3mr_cleanup_isr()
94 	pci_free_irq_vectors(mrioc->pdev);  in mpi3mr_cleanup_isr()
102 	sgel->flags = flags;  in mpi3mr_add_sg_single()
103 	sgel->length = cpu_to_le32(length);  in mpi3mr_add_sg_single()
104 	sgel->address = cpu_to_le64(dma_addr);  in mpi3mr_add_sg_single()
111 	mpi3mr_add_sg_single(paddr, sgl_flags, 0, -1);  in mpi3mr_build_zero_len_sge()
120 	if ((phys_addr < mrioc->reply_buf_dma) ||  in mpi3mr_get_reply_virt_addr()
121 	    (phys_addr > mrioc->reply_buf_dma_max_address))  in mpi3mr_get_reply_virt_addr()
124 	return mrioc->reply_buf + (phys_addr - mrioc->reply_buf_dma);  in mpi3mr_get_reply_virt_addr()
133 	return mrioc->sense_buf + (phys_addr - mrioc->sense_buf_dma);  in mpi3mr_get_sensebuf_virt_addr()
142 	spin_lock_irqsave(&mrioc->reply_free_queue_lock, flags);  in mpi3mr_repost_reply_buf()
143 	old_idx  =  mrioc->reply_free_queue_host_index;  in mpi3mr_repost_reply_buf()
144 	mrioc->reply_free_queue_host_index = (  in mpi3mr_repost_reply_buf()
145 	    (mrioc->reply_free_queue_host_index ==  in mpi3mr_repost_reply_buf()
146 	    (mrioc->reply_free_qsz - 1)) ? 0 :  in mpi3mr_repost_reply_buf()
147 	    (mrioc->reply_free_queue_host_index + 1));  in mpi3mr_repost_reply_buf()
148 	mrioc->reply_free_q[old_idx] = cpu_to_le64(reply_dma);  in mpi3mr_repost_reply_buf()
149 	writel(mrioc->reply_free_queue_host_index,  in mpi3mr_repost_reply_buf()
150 	    &mrioc->sysif_regs->reply_free_host_index);  in mpi3mr_repost_reply_buf()
151 	spin_unlock_irqrestore(&mrioc->reply_free_queue_lock, flags);  in mpi3mr_repost_reply_buf()
160 	spin_lock_irqsave(&mrioc->sbq_lock, flags);  in mpi3mr_repost_sense_buf()
161 	old_idx  =  mrioc->sbq_host_index;  in mpi3mr_repost_sense_buf()
162 	mrioc->sbq_host_index = ((mrioc->sbq_host_index ==  in mpi3mr_repost_sense_buf()
163 	    (mrioc->sense_buf_q_sz - 1)) ? 0 :  in mpi3mr_repost_sense_buf()
164 	    (mrioc->sbq_host_index + 1));  in mpi3mr_repost_sense_buf()
165 	mrioc->sense_buf_q[old_idx] = cpu_to_le64(sense_buf_dma);  in mpi3mr_repost_sense_buf()
166 	writel(mrioc->sbq_host_index,  in mpi3mr_repost_sense_buf()
167 	    &mrioc->sysif_regs->sense_buffer_free_host_index);  in mpi3mr_repost_sense_buf()
168 	spin_unlock_irqrestore(&mrioc->sbq_lock, flags);  in mpi3mr_repost_sense_buf()
177 	if (!(mrioc->logging_level & MPI3_DEBUG_EVENT))  in mpi3mr_print_event_data()
180 	event = event_reply->event;  in mpi3mr_print_event_data()
201 		    (struct mpi3_device_page0 *)event_reply->event_data;  in mpi3mr_print_event_data()
203 		    event_data->dev_handle, event_data->device_form);  in mpi3mr_print_event_data()
209 		    (struct mpi3_device_page0 *)event_reply->event_data;  in mpi3mr_print_event_data()
211 		    event_data->dev_handle, event_data->device_form);  in mpi3mr_print_event_data()
217 		    (struct mpi3_event_data_device_status_change *)event_reply->event_data;  in mpi3mr_print_event_data()
219 		    event_data->dev_handle, event_data->reason_code);  in mpi3mr_print_event_data()
225 		    (struct mpi3_event_data_sas_discovery *)event_reply->event_data;  in mpi3mr_print_event_data()
227 		    (event_data->reason_code == MPI3_EVENT_SAS_DISC_RC_STARTED) ?  in mpi3mr_print_event_data()
229 		    le32_to_cpu(event_data->discovery_status));  in mpi3mr_print_event_data()
268 		    (struct mpi3_event_data_pcie_enumeration *)event_reply->event_data;  in mpi3mr_print_event_data()
270 		    (event_data->reason_code ==  in mpi3mr_print_event_data()
272 		if (event_data->enumeration_status)  in mpi3mr_print_event_data()
274 			    le32_to_cpu(event_data->enumeration_status));  in mpi3mr_print_event_data()
297 	mrioc->change_count = le16_to_cpu(event_reply->ioc_change_count);  in mpi3mr_handle_events()
310 		return &mrioc->init_cmds;  in mpi3mr_get_drv_cmd()
312 		return &mrioc->cfg_cmds;  in mpi3mr_get_drv_cmd()
314 		return &mrioc->bsg_cmds;  in mpi3mr_get_drv_cmd()
316 		return &mrioc->host_tm_cmds;  in mpi3mr_get_drv_cmd()
318 		return &mrioc->pel_abort_cmd;  in mpi3mr_get_drv_cmd()
320 		return &mrioc->pel_cmds;  in mpi3mr_get_drv_cmd()
322 		return &mrioc->transport_cmds;  in mpi3mr_get_drv_cmd()
324 		if (def_reply && def_reply->function ==  in mpi3mr_get_drv_cmd()
333 		idx = host_tag - MPI3MR_HOSTTAG_DEVRMCMD_MIN;  in mpi3mr_get_drv_cmd()
334 		return &mrioc->dev_rmhs_cmds[idx];  in mpi3mr_get_drv_cmd()
339 		idx = host_tag - MPI3MR_HOSTTAG_EVTACKCMD_MIN;  in mpi3mr_get_drv_cmd()
340 		return &mrioc->evtack_cmds[idx];  in mpi3mr_get_drv_cmd()
363 	reply_desc_type = le16_to_cpu(reply_desc->reply_flags) &  in mpi3mr_process_admin_reply_desc()
368 		host_tag = le16_to_cpu(status_desc->host_tag);  in mpi3mr_process_admin_reply_desc()
369 		ioc_status = le16_to_cpu(status_desc->ioc_status);  in mpi3mr_process_admin_reply_desc()
372 			ioc_loginfo = le32_to_cpu(status_desc->ioc_log_info);  in mpi3mr_process_admin_reply_desc()
378 		*reply_dma = le64_to_cpu(addr_desc->reply_frame_address);  in mpi3mr_process_admin_reply_desc()
382 		host_tag = le16_to_cpu(def_reply->host_tag);  in mpi3mr_process_admin_reply_desc()
383 		ioc_status = le16_to_cpu(def_reply->ioc_status);  in mpi3mr_process_admin_reply_desc()
386 			ioc_loginfo = le32_to_cpu(def_reply->ioc_log_info);  in mpi3mr_process_admin_reply_desc()
388 		if (def_reply->function == MPI3_FUNCTION_SCSI_IO) {  in mpi3mr_process_admin_reply_desc()
391 			    le64_to_cpu(scsi_reply->sense_data_buffer_address));  in mpi3mr_process_admin_reply_desc()
392 			sense_count = le32_to_cpu(scsi_reply->sense_count);  in mpi3mr_process_admin_reply_desc()
404 		host_tag = le16_to_cpu(success_desc->host_tag);  in mpi3mr_process_admin_reply_desc()
412 		if (cmdptr->state & MPI3MR_CMD_PENDING) {  in mpi3mr_process_admin_reply_desc()
413 			cmdptr->state |= MPI3MR_CMD_COMPLETE;  in mpi3mr_process_admin_reply_desc()
414 			cmdptr->ioc_loginfo = ioc_loginfo;  in mpi3mr_process_admin_reply_desc()
416 				cmdptr->ioc_status = ioc_status;  in mpi3mr_process_admin_reply_desc()
418 				cmdptr->ioc_status = masked_ioc_status;  in mpi3mr_process_admin_reply_desc()
419 			cmdptr->state &= ~MPI3MR_CMD_PENDING;  in mpi3mr_process_admin_reply_desc()
421 				cmdptr->state |= MPI3MR_CMD_REPLY_VALID;  in mpi3mr_process_admin_reply_desc()
422 				memcpy((u8 *)cmdptr->reply, (u8 *)def_reply,  in mpi3mr_process_admin_reply_desc()
423 				    mrioc->reply_sz);  in mpi3mr_process_admin_reply_desc()
425 			if (sense_buf && cmdptr->sensebuf) {  in mpi3mr_process_admin_reply_desc()
426 				cmdptr->is_sense = 1;  in mpi3mr_process_admin_reply_desc()
427 				memcpy(cmdptr->sensebuf, sense_buf,  in mpi3mr_process_admin_reply_desc()
430 			if (cmdptr->is_waiting) {  in mpi3mr_process_admin_reply_desc()
431 				complete(&cmdptr->done);  in mpi3mr_process_admin_reply_desc()
432 				cmdptr->is_waiting = 0;  in mpi3mr_process_admin_reply_desc()
433 			} else if (cmdptr->callback)  in mpi3mr_process_admin_reply_desc()
434 				cmdptr->callback(mrioc, cmdptr);  in mpi3mr_process_admin_reply_desc()
440 		    le64_to_cpu(scsi_reply->sense_data_buffer_address));  in mpi3mr_process_admin_reply_desc()
445 	u32 exp_phase = mrioc->admin_reply_ephase;  in mpi3mr_process_admin_reply_q()
446 	u32 admin_reply_ci = mrioc->admin_reply_ci;  in mpi3mr_process_admin_reply_q()
452 	if (!atomic_add_unless(&mrioc->admin_reply_q_in_use, 1, 1)) {  in mpi3mr_process_admin_reply_q()
453 		atomic_inc(&mrioc->admin_pend_isr);  in mpi3mr_process_admin_reply_q()
457 	atomic_set(&mrioc->admin_pend_isr, 0);  in mpi3mr_process_admin_reply_q()
458 	reply_desc = (struct mpi3_default_reply_descriptor *)mrioc->admin_reply_base +  in mpi3mr_process_admin_reply_q()
461 	if ((le16_to_cpu(reply_desc->reply_flags) &  in mpi3mr_process_admin_reply_q()
463 		atomic_dec(&mrioc->admin_reply_q_in_use);  in mpi3mr_process_admin_reply_q()
468 		if (mrioc->unrecoverable || mrioc->io_admin_reset_sync)  in mpi3mr_process_admin_reply_q()
471 		mrioc->admin_req_ci = le16_to_cpu(reply_desc->request_queue_ci);  in mpi3mr_process_admin_reply_q()
477 		if (++admin_reply_ci == mrioc->num_admin_replies) {  in mpi3mr_process_admin_reply_q()
482 		    (struct mpi3_default_reply_descriptor *)mrioc->admin_reply_base +  in mpi3mr_process_admin_reply_q()
484 		if ((le16_to_cpu(reply_desc->reply_flags) &  in mpi3mr_process_admin_reply_q()
489 			    &mrioc->sysif_regs->admin_reply_queue_ci);  in mpi3mr_process_admin_reply_q()
494 	writel(admin_reply_ci, &mrioc->sysif_regs->admin_reply_queue_ci);  in mpi3mr_process_admin_reply_q()
495 	mrioc->admin_reply_ci = admin_reply_ci;  in mpi3mr_process_admin_reply_q()
496 	mrioc->admin_reply_ephase = exp_phase;  in mpi3mr_process_admin_reply_q()
497 	atomic_dec(&mrioc->admin_reply_q_in_use);  in mpi3mr_process_admin_reply_q()
503  * mpi3mr_get_reply_desc - get reply descriptor frame corresponding to
514 	struct segments *segments = op_reply_q->q_segments;  in mpi3mr_get_reply_desc()
518 	    segments[reply_ci / op_reply_q->segment_qd].segment;  in mpi3mr_get_reply_desc()
520 	    (reply_ci % op_reply_q->segment_qd);  in mpi3mr_get_reply_desc()
525  * mpi3mr_process_op_reply_q - Operational reply queue handler
547 	reply_qidx = op_reply_q->qid - 1;  in mpi3mr_process_op_reply_q()
549 	if (!atomic_add_unless(&op_reply_q->in_use, 1, 1))  in mpi3mr_process_op_reply_q()
552 	exp_phase = op_reply_q->ephase;  in mpi3mr_process_op_reply_q()
553 	reply_ci = op_reply_q->ci;  in mpi3mr_process_op_reply_q()
556 	if ((le16_to_cpu(reply_desc->reply_flags) &  in mpi3mr_process_op_reply_q()
558 		atomic_dec(&op_reply_q->in_use);  in mpi3mr_process_op_reply_q()
563 		if (mrioc->unrecoverable || mrioc->io_admin_reset_sync)  in mpi3mr_process_op_reply_q()
566 		req_q_idx = le16_to_cpu(reply_desc->request_queue_id) - 1;  in mpi3mr_process_op_reply_q()
567 		op_req_q = &mrioc->req_qinfo[req_q_idx];  in mpi3mr_process_op_reply_q()
569 		WRITE_ONCE(op_req_q->ci, le16_to_cpu(reply_desc->request_queue_ci));  in mpi3mr_process_op_reply_q()
578 		if (++reply_ci == op_reply_q->num_replies) {  in mpi3mr_process_op_reply_q()
585 		if ((le16_to_cpu(reply_desc->reply_flags) &  in mpi3mr_process_op_reply_q()
593 		if (num_op_reply > mrioc->max_host_ios) {  in mpi3mr_process_op_reply_q()
594 			op_reply_q->enable_irq_poll = true;  in mpi3mr_process_op_reply_q()
600 			    &mrioc->sysif_regs->oper_queue_indexes[reply_qidx].consumer_index);  in mpi3mr_process_op_reply_q()
601 			atomic_sub(threshold_comps, &op_reply_q->pend_ios);  in mpi3mr_process_op_reply_q()
607 	    &mrioc->sysif_regs->oper_queue_indexes[reply_qidx].consumer_index);  in mpi3mr_process_op_reply_q()
608 	op_reply_q->ci = reply_ci;  in mpi3mr_process_op_reply_q()
609 	op_reply_q->ephase = exp_phase;  in mpi3mr_process_op_reply_q()
610 	atomic_sub(threshold_comps, &op_reply_q->pend_ios);  in mpi3mr_process_op_reply_q()
611 	atomic_dec(&op_reply_q->in_use);  in mpi3mr_process_op_reply_q()
616  * mpi3mr_blk_mq_poll - Operational reply queue handler
632 	mrioc = (struct mpi3mr_ioc *)shost->hostdata;  in mpi3mr_blk_mq_poll()
634 	if ((mrioc->reset_in_progress || mrioc->prepare_for_reset ||  in mpi3mr_blk_mq_poll()
635 	    mrioc->unrecoverable || mrioc->pci_err_recovery))  in mpi3mr_blk_mq_poll()
639 			&mrioc->op_reply_qinfo[queue_num]);  in mpi3mr_blk_mq_poll()
654 	mrioc = intr_info->mrioc;  in mpi3mr_isr_primary()
656 	if (!mrioc->intr_enabled)  in mpi3mr_isr_primary()
659 	midx = intr_info->msix_index;  in mpi3mr_isr_primary()
663 	if (intr_info->op_reply_q)  in mpi3mr_isr_primary()
665 		    intr_info->op_reply_q);  in mpi3mr_isr_primary()
690 	if (!intr_info->op_reply_q)  in mpi3mr_isr()
693 	if (!intr_info->op_reply_q->enable_irq_poll ||  in mpi3mr_isr()
694 	    !atomic_read(&intr_info->op_reply_q->pend_ios))  in mpi3mr_isr()
697 	disable_irq_nosync(intr_info->os_irq);  in mpi3mr_isr()
703  * mpi3mr_isr_poll - Reply queue polling routine
719 	if (!intr_info || !intr_info->op_reply_q)  in mpi3mr_isr_poll()
722 	mrioc = intr_info->mrioc;  in mpi3mr_isr_poll()
723 	midx = intr_info->msix_index;  in mpi3mr_isr_poll()
727 		if (!mrioc->intr_enabled || mrioc->unrecoverable)  in mpi3mr_isr_poll()
732 		if (intr_info->op_reply_q)  in mpi3mr_isr_poll()
735 				intr_info->op_reply_q);  in mpi3mr_isr_poll()
739 	} while (atomic_read(&intr_info->op_reply_q->pend_ios) &&  in mpi3mr_isr_poll()
740 	    (num_op_reply < mrioc->max_host_ios));  in mpi3mr_isr_poll()
742 	intr_info->op_reply_q->enable_irq_poll = false;  in mpi3mr_isr_poll()
743 	enable_irq(intr_info->os_irq);  in mpi3mr_isr_poll()
751  * mpi3mr_request_irq - Request IRQ and register ISR
761 	struct pci_dev *pdev = mrioc->pdev;  in mpi3mr_request_irq()
762 	struct mpi3mr_intr_info *intr_info = mrioc->intr_info + index;  in mpi3mr_request_irq()
765 	intr_info->mrioc = mrioc;  in mpi3mr_request_irq()
766 	intr_info->msix_index = index;  in mpi3mr_request_irq()
767 	intr_info->op_reply_q = NULL;  in mpi3mr_request_irq()
769 	snprintf(intr_info->name, MPI3MR_NAME_LENGTH, "%s%d-msix%d",  in mpi3mr_request_irq()
770 	    mrioc->driver_name, mrioc->id, index);  in mpi3mr_request_irq()
774 	    mpi3mr_isr_poll, IRQF_SHARED, intr_info->name, intr_info);  in mpi3mr_request_irq()
777 	    NULL, IRQF_SHARED, intr_info->name, intr_info);  in mpi3mr_request_irq()
781 		    intr_info->name, pci_irq_vector(pdev, index));  in mpi3mr_request_irq()
785 	intr_info->os_irq = pci_irq_vector(pdev, index);  in mpi3mr_request_irq()
791 	if (!mrioc->requested_poll_qcount)  in mpi3mr_calc_poll_queues()
796 		(mrioc->requested_poll_qcount < max_vectors - 2)) {  in mpi3mr_calc_poll_queues()
799 		    mrioc->requested_poll_qcount, max_vectors);  in mpi3mr_calc_poll_queues()
803 		    mrioc->requested_poll_qcount, max_vectors);  in mpi3mr_calc_poll_queues()
804 		mrioc->requested_poll_qcount = 0;  in mpi3mr_calc_poll_queues()
809  * mpi3mr_setup_isr - Setup ISR for the controller
825 	if (mrioc->is_intr_info_set)  in mpi3mr_setup_isr()
832 		retval = pci_alloc_irq_vectors(mrioc->pdev,  in mpi3mr_setup_isr()
841 		    min_t(int, mrioc->cpu_count + 1 +  in mpi3mr_setup_isr()
842 			mrioc->requested_poll_qcount, mrioc->msix_count);  in mpi3mr_setup_isr()
847 		    "MSI-X vectors supported: %d, no of cores: %d,",  in mpi3mr_setup_isr()
848 		    mrioc->msix_count, mrioc->cpu_count);  in mpi3mr_setup_isr()
850 		    "MSI-x vectors requested: %d poll_queues %d\n",  in mpi3mr_setup_isr()
851 		    max_vectors, mrioc->requested_poll_qcount);  in mpi3mr_setup_isr()
853 		desc.post_vectors = mrioc->requested_poll_qcount;  in mpi3mr_setup_isr()
857 		retval = pci_alloc_irq_vectors_affinity(mrioc->pdev,  in mpi3mr_setup_isr()
868 		 * If only one MSI-x is allocated, then MSI-x 0 will be shared  in mpi3mr_setup_isr()
872 			mrioc->op_reply_q_offset = 0;  in mpi3mr_setup_isr()
880 		mrioc->op_reply_q_offset = (max_vectors > 1) ? 1 : 0;  in mpi3mr_setup_isr()
886 	mrioc->intr_info = kzalloc(sizeof(struct mpi3mr_intr_info) * max_vectors,  in mpi3mr_setup_isr()
888 	if (!mrioc->intr_info) {  in mpi3mr_setup_isr()
889 		retval = -ENOMEM;  in mpi3mr_setup_isr()
890 		pci_free_irq_vectors(mrioc->pdev);  in mpi3mr_setup_isr()
896 			mrioc->intr_info_count = i;  in mpi3mr_setup_isr()
901 		mrioc->is_intr_info_set = true;  in mpi3mr_setup_isr()
902 	mrioc->intr_info_count = max_vectors;  in mpi3mr_setup_isr()
992  * mpi3mr_reset_rc_name - get reset reason code name
1023  * mpi3mr_reset_type_name - get reset type name
1045  * mpi3mr_is_fault_recoverable - Read fault code and decide
1054 	fault = (readl(&mrioc->sysif_regs->fault) &  in mpi3mr_is_fault_recoverable()
1075  * mpi3mr_print_fault_info - Display fault information
1087 	ioc_status = readl(&mrioc->sysif_regs->ioc_status);  in mpi3mr_print_fault_info()
1090 		code = readl(&mrioc->sysif_regs->fault);  in mpi3mr_print_fault_info()
1091 		code1 = readl(&mrioc->sysif_regs->fault_info[0]);  in mpi3mr_print_fault_info()
1092 		code2 = readl(&mrioc->sysif_regs->fault_info[1]);  in mpi3mr_print_fault_info()
1093 		code3 = readl(&mrioc->sysif_regs->fault_info[2]);  in mpi3mr_print_fault_info()
1102  * mpi3mr_get_iocstate - Get IOC State
1115 	ioc_status = readl(&mrioc->sysif_regs->ioc_status);  in mpi3mr_get_iocstate()
1116 	ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);  in mpi3mr_get_iocstate()
1118 	if (mrioc->unrecoverable)  in mpi3mr_get_iocstate()
1137  * mpi3mr_free_ioctl_dma_memory - free memory for ioctl dma
1149 	if (!mrioc->ioctl_dma_pool)  in mpi3mr_free_ioctl_dma_memory()
1153 		mem_desc = &mrioc->ioctl_sge[i];  in mpi3mr_free_ioctl_dma_memory()
1154 		if (mem_desc->addr) {  in mpi3mr_free_ioctl_dma_memory()
1155 			dma_pool_free(mrioc->ioctl_dma_pool,  in mpi3mr_free_ioctl_dma_memory()
1156 				      mem_desc->addr,  in mpi3mr_free_ioctl_dma_memory()
1157 				      mem_desc->dma_addr);  in mpi3mr_free_ioctl_dma_memory()
1158 			mem_desc->addr = NULL;  in mpi3mr_free_ioctl_dma_memory()
1161 	dma_pool_destroy(mrioc->ioctl_dma_pool);  in mpi3mr_free_ioctl_dma_memory()
1162 	mrioc->ioctl_dma_pool = NULL;  in mpi3mr_free_ioctl_dma_memory()
1163 	mem_desc = &mrioc->ioctl_chain_sge;  in mpi3mr_free_ioctl_dma_memory()
1165 	if (mem_desc->addr) {  in mpi3mr_free_ioctl_dma_memory()
1166 		dma_free_coherent(&mrioc->pdev->dev, mem_desc->size,  in mpi3mr_free_ioctl_dma_memory()
1167 				  mem_desc->addr, mem_desc->dma_addr);  in mpi3mr_free_ioctl_dma_memory()
1168 		mem_desc->addr = NULL;  in mpi3mr_free_ioctl_dma_memory()
1170 	mem_desc = &mrioc->ioctl_resp_sge;  in mpi3mr_free_ioctl_dma_memory()
1171 	if (mem_desc->addr) {  in mpi3mr_free_ioctl_dma_memory()
1172 		dma_free_coherent(&mrioc->pdev->dev, mem_desc->size,  in mpi3mr_free_ioctl_dma_memory()
1173 				  mem_desc->addr, mem_desc->dma_addr);  in mpi3mr_free_ioctl_dma_memory()
1174 		mem_desc->addr = NULL;  in mpi3mr_free_ioctl_dma_memory()
1177 	mrioc->ioctl_sges_allocated = false;  in mpi3mr_free_ioctl_dma_memory()
1181  * mpi3mr_alloc_ioctl_dma_memory - Alloc memory for ioctl dma
1195 	mrioc->ioctl_dma_pool = dma_pool_create("ioctl dma pool",  in mpi3mr_alloc_ioctl_dma_memory()
1196 						&mrioc->pdev->dev,  in mpi3mr_alloc_ioctl_dma_memory()
1200 	if (!mrioc->ioctl_dma_pool) {  in mpi3mr_alloc_ioctl_dma_memory()
1206 		mem_desc = &mrioc->ioctl_sge[i];  in mpi3mr_alloc_ioctl_dma_memory()
1207 		mem_desc->size = MPI3MR_IOCTL_SGE_SIZE;  in mpi3mr_alloc_ioctl_dma_memory()
1208 		mem_desc->addr = dma_pool_zalloc(mrioc->ioctl_dma_pool,  in mpi3mr_alloc_ioctl_dma_memory()
1210 						 &mem_desc->dma_addr);  in mpi3mr_alloc_ioctl_dma_memory()
1211 		if (!mem_desc->addr)  in mpi3mr_alloc_ioctl_dma_memory()
1215 	mem_desc = &mrioc->ioctl_chain_sge;  in mpi3mr_alloc_ioctl_dma_memory()
1216 	mem_desc->size = MPI3MR_PAGE_SIZE_4K;  in mpi3mr_alloc_ioctl_dma_memory()
1217 	mem_desc->addr = dma_alloc_coherent(&mrioc->pdev->dev,  in mpi3mr_alloc_ioctl_dma_memory()
1218 					    mem_desc->size,  in mpi3mr_alloc_ioctl_dma_memory()
1219 					    &mem_desc->dma_addr,  in mpi3mr_alloc_ioctl_dma_memory()
1221 	if (!mem_desc->addr)  in mpi3mr_alloc_ioctl_dma_memory()
1224 	mem_desc = &mrioc->ioctl_resp_sge;  in mpi3mr_alloc_ioctl_dma_memory()
1225 	mem_desc->size = MPI3MR_PAGE_SIZE_4K;  in mpi3mr_alloc_ioctl_dma_memory()
1226 	mem_desc->addr = dma_alloc_coherent(&mrioc->pdev->dev,  in mpi3mr_alloc_ioctl_dma_memory()
1227 					    mem_desc->size,  in mpi3mr_alloc_ioctl_dma_memory()
1228 					    &mem_desc->dma_addr,  in mpi3mr_alloc_ioctl_dma_memory()
1230 	if (!mem_desc->addr)  in mpi3mr_alloc_ioctl_dma_memory()
1233 	mrioc->ioctl_sges_allocated = true;  in mpi3mr_alloc_ioctl_dma_memory()
1243  * mpi3mr_clear_reset_history - clear reset history
1255 	ioc_status = readl(&mrioc->sysif_regs->ioc_status);  in mpi3mr_clear_reset_history()
1257 		writel(ioc_status, &mrioc->sysif_regs->ioc_status);  in mpi3mr_clear_reset_history()
1261  * mpi3mr_issue_and_process_mur - Message unit Reset handler
1268  * Return: 0 on success, -1 on failure.
1274 	int retval = -1;  in mpi3mr_issue_and_process_mur()
1277 	if (mrioc->unrecoverable) {  in mpi3mr_issue_and_process_mur()
1284 			(mrioc->facts.ioc_num <<  in mpi3mr_issue_and_process_mur()
1286 	writel(scratch_pad0, &mrioc->sysif_regs->scratchpad[0]);  in mpi3mr_issue_and_process_mur()
1287 	ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);  in mpi3mr_issue_and_process_mur()
1289 	writel(ioc_config, &mrioc->sysif_regs->ioc_configuration);  in mpi3mr_issue_and_process_mur()
1293 		ioc_status = readl(&mrioc->sysif_regs->ioc_status);  in mpi3mr_issue_and_process_mur()
1303 	} while (--timeout);  in mpi3mr_issue_and_process_mur()
1305 	ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);  in mpi3mr_issue_and_process_mur()
1317  * mpi3mr_revalidate_factsdata - validate IOCFacts parameters
1322  * older values else return -EPERM
1329 	if (mrioc->facts.reply_sz > mrioc->reply_sz) {  in mpi3mr_revalidate_factsdata()
1331 		    "cannot increase reply size from %d to %d\n",  in mpi3mr_revalidate_factsdata()
1332 		    mrioc->reply_sz, mrioc->facts.reply_sz);  in mpi3mr_revalidate_factsdata()
1333 		return -EPERM;  in mpi3mr_revalidate_factsdata()
1336 	if (mrioc->facts.max_op_reply_q < mrioc->num_op_reply_q) {  in mpi3mr_revalidate_factsdata()
1339 		    mrioc->num_op_reply_q,  in mpi3mr_revalidate_factsdata()
1340 		    mrioc->facts.max_op_reply_q);  in mpi3mr_revalidate_factsdata()
1341 		return -EPERM;  in mpi3mr_revalidate_factsdata()
1344 	if (mrioc->facts.max_op_req_q < mrioc->num_op_req_q) {  in mpi3mr_revalidate_factsdata()
1347 		    mrioc->num_op_req_q, mrioc->facts.max_op_req_q);  in mpi3mr_revalidate_factsdata()
1348 		return -EPERM;  in mpi3mr_revalidate_factsdata()
1351 	if (mrioc->shost->max_sectors != (mrioc->facts.max_data_length / 512))  in mpi3mr_revalidate_factsdata()
1355 			    mrioc->shost->max_sectors * 512, mrioc->facts.max_data_length);  in mpi3mr_revalidate_factsdata()
1357 	if ((mrioc->sas_transport_enabled) && (mrioc->facts.ioc_capabilities &  in mpi3mr_revalidate_factsdata()
1364 	if (mrioc->seg_tb_support) {  in mpi3mr_revalidate_factsdata()
1365 		if (!(mrioc->facts.ioc_capabilities &  in mpi3mr_revalidate_factsdata()
1372 			mrioc->diag_buffers[0].disabled_after_reset = true;  in mpi3mr_revalidate_factsdata()
1374 			mrioc->diag_buffers[0].disabled_after_reset = false;  in mpi3mr_revalidate_factsdata()
1377 	if (mrioc->facts.max_devhandle > mrioc->dev_handle_bitmap_bits) {  in mpi3mr_revalidate_factsdata()
1378 		removepend_bitmap = bitmap_zalloc(mrioc->facts.max_devhandle,  in mpi3mr_revalidate_factsdata()
1383 				mrioc->dev_handle_bitmap_bits,  in mpi3mr_revalidate_factsdata()
1384 				mrioc->facts.max_devhandle);  in mpi3mr_revalidate_factsdata()
1385 			return -EPERM;  in mpi3mr_revalidate_factsdata()
1387 		bitmap_free(mrioc->removepend_bitmap);  in mpi3mr_revalidate_factsdata()
1388 		mrioc->removepend_bitmap = removepend_bitmap;  in mpi3mr_revalidate_factsdata()
1391 			 mrioc->dev_handle_bitmap_bits,  in mpi3mr_revalidate_factsdata()
1392 			 mrioc->facts.max_devhandle);  in mpi3mr_revalidate_factsdata()
1393 		mrioc->dev_handle_bitmap_bits = mrioc->facts.max_devhandle;  in mpi3mr_revalidate_factsdata()
1400  * mpi3mr_bring_ioc_ready - Bring controller to ready state
1419 	ioc_status = readl(&mrioc->sysif_regs->ioc_status);  in mpi3mr_bring_ioc_ready()
1420 	ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);  in mpi3mr_bring_ioc_ready()
1421 	base_info = lo_hi_readq(&mrioc->sysif_regs->ioc_information);  in mpi3mr_bring_ioc_ready()
1426 		mrioc->unrecoverable = 1;  in mpi3mr_bring_ioc_ready()
1431 	mrioc->ready_timeout =  in mpi3mr_bring_ioc_ready()
1435 	ioc_info(mrioc, "ready timeout: %d seconds\n", mrioc->ready_timeout);  in mpi3mr_bring_ioc_ready()
1441 	timeout = mrioc->ready_timeout * 10;  in mpi3mr_bring_ioc_ready()
1450 		if (!pci_device_is_present(mrioc->pdev)) {  in mpi3mr_bring_ioc_ready()
1451 			mrioc->unrecoverable = 1;  in mpi3mr_bring_ioc_ready()
1457 	} while (--timeout);  in mpi3mr_bring_ioc_ready()
1475 					readl(&mrioc->sysif_regs->host_diagnostic);  in mpi3mr_bring_ioc_ready()
1479 				if (!pci_device_is_present(mrioc->pdev)) {  in mpi3mr_bring_ioc_ready()
1480 					mrioc->unrecoverable = 1;  in mpi3mr_bring_ioc_ready()
1485 			} while (--timeout);  in mpi3mr_bring_ioc_ready()
1514 	ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);  in mpi3mr_bring_ioc_ready()
1516 	writel(ioc_config, &mrioc->sysif_regs->ioc_configuration);  in mpi3mr_bring_ioc_ready()
1521 	timeout = mrioc->ready_timeout * 10;  in mpi3mr_bring_ioc_ready()
1530 		ioc_status = readl(&mrioc->sysif_regs->ioc_status);  in mpi3mr_bring_ioc_ready()
1536 		if (!pci_device_is_present(mrioc->pdev)) {  in mpi3mr_bring_ioc_ready()
1537 			mrioc->unrecoverable = 1;  in mpi3mr_bring_ioc_ready()
1540 			retval = -1;  in mpi3mr_bring_ioc_ready()
1544 		elapsed_time_sec = jiffies_to_msecs(jiffies - start_time)/1000;  in mpi3mr_bring_ioc_ready()
1545 	} while (elapsed_time_sec < mrioc->ready_timeout);  in mpi3mr_bring_ioc_ready()
1548 	elapsed_time_sec = jiffies_to_msecs(jiffies - start_time)/1000;  in mpi3mr_bring_ioc_ready()
1549 	if ((retry < 2) && (elapsed_time_sec < (mrioc->ready_timeout - 60))) {  in mpi3mr_bring_ioc_ready()
1566  * mpi3mr_soft_reset_success - Check softreset is success or not
1585  * mpi3mr_diagfault_success - Check diag fault is success or not
1600 	fault = readl(&mrioc->sysif_regs->fault) & MPI3_SYSIF_FAULT_CODE_MASK;  in mpi3mr_diagfault_success()
1609  * mpi3mr_set_diagsave - Set diag save bit for snapdump
1621 	ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);  in mpi3mr_set_diagsave()
1623 	writel(ioc_config, &mrioc->sysif_regs->ioc_configuration);  in mpi3mr_set_diagsave()
1627  * mpi3mr_issue_reset - Issue reset to the controller
1637  * Return: 0 on success, non-zero on failure.
1642 	int retval = -1;  in mpi3mr_issue_reset()
1650 	if (mrioc->unrecoverable)  in mpi3mr_issue_reset()
1671 			mrioc->unrecoverable = 1;  in mpi3mr_issue_reset()
1676 		    &mrioc->sysif_regs->write_sequence);  in mpi3mr_issue_reset()
1678 		    &mrioc->sysif_regs->write_sequence);  in mpi3mr_issue_reset()
1680 		    &mrioc->sysif_regs->write_sequence);  in mpi3mr_issue_reset()
1682 		    &mrioc->sysif_regs->write_sequence);  in mpi3mr_issue_reset()
1684 		    &mrioc->sysif_regs->write_sequence);  in mpi3mr_issue_reset()
1686 		    &mrioc->sysif_regs->write_sequence);  in mpi3mr_issue_reset()
1688 		    &mrioc->sysif_regs->write_sequence);  in mpi3mr_issue_reset()
1690 		host_diagnostic = readl(&mrioc->sysif_regs->host_diagnostic);  in mpi3mr_issue_reset()
1697 	    MPI3MR_RESET_REASON_OSTYPE_SHIFT) | (mrioc->facts.ioc_num <<  in mpi3mr_issue_reset()
1699 	writel(reset_reason, &mrioc->sysif_regs->scratchpad[0]);  in mpi3mr_issue_reset()
1701 	    &mrioc->sysif_regs->host_diagnostic);  in mpi3mr_issue_reset()
1705 			ioc_status = readl(&mrioc->sysif_regs->ioc_status);  in mpi3mr_issue_reset()
1707 			    readl(&mrioc->sysif_regs->ioc_configuration);  in mpi3mr_issue_reset()
1716 		} while (--timeout);  in mpi3mr_issue_reset()
1721 			ioc_status = readl(&mrioc->sysif_regs->ioc_status);  in mpi3mr_issue_reset()
1727 		} while (--timeout);  in mpi3mr_issue_reset()
1734 	    &mrioc->sysif_regs->write_sequence);  in mpi3mr_issue_reset()
1736 	ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);  in mpi3mr_issue_reset()
1737 	ioc_status = readl(&mrioc->sysif_regs->ioc_status);  in mpi3mr_issue_reset()
1743 		mrioc->unrecoverable = 1;  in mpi3mr_issue_reset()
1748  * mpi3mr_admin_request_post - Post request to admin queue
1751  * @admin_req_sz: Request size
1758  * Return: 0 on success, non-zero on failure.
1768 	if (mrioc->unrecoverable) {  in mpi3mr_admin_request_post()
1770 		return -EFAULT;  in mpi3mr_admin_request_post()
1773 	spin_lock_irqsave(&mrioc->admin_req_lock, flags);  in mpi3mr_admin_request_post()
1774 	areq_pi = mrioc->admin_req_pi;  in mpi3mr_admin_request_post()
1775 	areq_ci = mrioc->admin_req_ci;  in mpi3mr_admin_request_post()
1776 	max_entries = mrioc->num_admin_req;  in mpi3mr_admin_request_post()
1778 	    (areq_pi == (max_entries - 1)))) {  in mpi3mr_admin_request_post()
1780 		retval = -EAGAIN;  in mpi3mr_admin_request_post()
1783 	if (!ignore_reset && mrioc->reset_in_progress) {  in mpi3mr_admin_request_post()
1785 		retval = -EAGAIN;  in mpi3mr_admin_request_post()
1788 	if (mrioc->pci_err_recovery) {  in mpi3mr_admin_request_post()
1790 		retval = -EAGAIN;  in mpi3mr_admin_request_post()
1794 	areq_entry = (u8 *)mrioc->admin_req_base +  in mpi3mr_admin_request_post()
1801 	mrioc->admin_req_pi = areq_pi;  in mpi3mr_admin_request_post()
1803 	writel(mrioc->admin_req_pi, &mrioc->sysif_regs->admin_request_queue_pi);  in mpi3mr_admin_request_post()
1806 	spin_unlock_irqrestore(&mrioc->admin_req_lock, flags);  in mpi3mr_admin_request_post()
1812  * mpi3mr_free_op_req_q_segments - free request memory segments
1823 	int size;  in mpi3mr_free_op_req_q_segments()  local
1826 	segments = mrioc->req_qinfo[q_idx].q_segments;  in mpi3mr_free_op_req_q_segments()
1830 	if (mrioc->enable_segqueue) {  in mpi3mr_free_op_req_q_segments()
1831 		size = MPI3MR_OP_REQ_Q_SEG_SIZE;  in mpi3mr_free_op_req_q_segments()
1832 		if (mrioc->req_qinfo[q_idx].q_segment_list) {  in mpi3mr_free_op_req_q_segments()
1833 			dma_free_coherent(&mrioc->pdev->dev,  in mpi3mr_free_op_req_q_segments()
1835 			    mrioc->req_qinfo[q_idx].q_segment_list,  in mpi3mr_free_op_req_q_segments()
1836 			    mrioc->req_qinfo[q_idx].q_segment_list_dma);  in mpi3mr_free_op_req_q_segments()
1837 			mrioc->req_qinfo[q_idx].q_segment_list = NULL;  in mpi3mr_free_op_req_q_segments()
1840 		size = mrioc->req_qinfo[q_idx].segment_qd *  in mpi3mr_free_op_req_q_segments()
1841 		    mrioc->facts.op_req_sz;  in mpi3mr_free_op_req_q_segments()
1843 	for (j = 0; j < mrioc->req_qinfo[q_idx].num_segments; j++) {  in mpi3mr_free_op_req_q_segments()
1846 		dma_free_coherent(&mrioc->pdev->dev,  in mpi3mr_free_op_req_q_segments()
1847 		    size, segments[j].segment, segments[j].segment_dma);  in mpi3mr_free_op_req_q_segments()
1850 	kfree(mrioc->req_qinfo[q_idx].q_segments);  in mpi3mr_free_op_req_q_segments()
1851 	mrioc->req_qinfo[q_idx].q_segments = NULL;  in mpi3mr_free_op_req_q_segments()
1852 	mrioc->req_qinfo[q_idx].qid = 0;  in mpi3mr_free_op_req_q_segments()
1856  * mpi3mr_free_op_reply_q_segments - free reply memory segments
1867 	int size;  in mpi3mr_free_op_reply_q_segments()  local
1870 	segments = mrioc->op_reply_qinfo[q_idx].q_segments;  in mpi3mr_free_op_reply_q_segments()
1874 	if (mrioc->enable_segqueue) {  in mpi3mr_free_op_reply_q_segments()
1875 		size = MPI3MR_OP_REP_Q_SEG_SIZE;  in mpi3mr_free_op_reply_q_segments()
1876 		if (mrioc->op_reply_qinfo[q_idx].q_segment_list) {  in mpi3mr_free_op_reply_q_segments()
1877 			dma_free_coherent(&mrioc->pdev->dev,  in mpi3mr_free_op_reply_q_segments()
1879 			    mrioc->op_reply_qinfo[q_idx].q_segment_list,  in mpi3mr_free_op_reply_q_segments()
1880 			    mrioc->op_reply_qinfo[q_idx].q_segment_list_dma);  in mpi3mr_free_op_reply_q_segments()
1881 			mrioc->op_reply_qinfo[q_idx].q_segment_list = NULL;  in mpi3mr_free_op_reply_q_segments()
1884 		size = mrioc->op_reply_qinfo[q_idx].segment_qd *  in mpi3mr_free_op_reply_q_segments()
1885 		    mrioc->op_reply_desc_sz;  in mpi3mr_free_op_reply_q_segments()
1887 	for (j = 0; j < mrioc->op_reply_qinfo[q_idx].num_segments; j++) {  in mpi3mr_free_op_reply_q_segments()
1890 		dma_free_coherent(&mrioc->pdev->dev,  in mpi3mr_free_op_reply_q_segments()
1891 		    size, segments[j].segment, segments[j].segment_dma);  in mpi3mr_free_op_reply_q_segments()
1895 	kfree(mrioc->op_reply_qinfo[q_idx].q_segments);  in mpi3mr_free_op_reply_q_segments()
1896 	mrioc->op_reply_qinfo[q_idx].q_segments = NULL;  in mpi3mr_free_op_reply_q_segments()
1897 	mrioc->op_reply_qinfo[q_idx].qid = 0;  in mpi3mr_free_op_reply_q_segments()
1901  * mpi3mr_delete_op_reply_q - delete operational reply queue
1908  * Return:  0 on success, non-zero on failure.
1913 	struct op_reply_qinfo *op_reply_q = mrioc->op_reply_qinfo + qidx;  in mpi3mr_delete_op_reply_q()
1917 	reply_qid = op_reply_q->qid;  in mpi3mr_delete_op_reply_q()
1919 	midx = REPLY_QUEUE_IDX_TO_MSIX_IDX(qidx, mrioc->op_reply_q_offset);  in mpi3mr_delete_op_reply_q()
1922 		retval = -1;  in mpi3mr_delete_op_reply_q()
1927 	(op_reply_q->qtype == MPI3MR_DEFAULT_QUEUE) ? mrioc->default_qcount-- :  in mpi3mr_delete_op_reply_q()
1928 	    mrioc->active_poll_qcount--;  in mpi3mr_delete_op_reply_q()
1931 	mutex_lock(&mrioc->init_cmds.mutex);  in mpi3mr_delete_op_reply_q()
1932 	if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {  in mpi3mr_delete_op_reply_q()
1933 		retval = -1;  in mpi3mr_delete_op_reply_q()
1935 		mutex_unlock(&mrioc->init_cmds.mutex);  in mpi3mr_delete_op_reply_q()
1938 	mrioc->init_cmds.state = MPI3MR_CMD_PENDING;  in mpi3mr_delete_op_reply_q()
1939 	mrioc->init_cmds.is_waiting = 1;  in mpi3mr_delete_op_reply_q()
1940 	mrioc->init_cmds.callback = NULL;  in mpi3mr_delete_op_reply_q()
1945 	init_completion(&mrioc->init_cmds.done);  in mpi3mr_delete_op_reply_q()
1952 	wait_for_completion_timeout(&mrioc->init_cmds.done,  in mpi3mr_delete_op_reply_q()
1954 	if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {  in mpi3mr_delete_op_reply_q()
1958 		retval = -1;  in mpi3mr_delete_op_reply_q()
1961 	if ((mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)  in mpi3mr_delete_op_reply_q()
1965 		    (mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),  in mpi3mr_delete_op_reply_q()
1966 		    mrioc->init_cmds.ioc_loginfo);  in mpi3mr_delete_op_reply_q()
1967 		retval = -1;  in mpi3mr_delete_op_reply_q()
1970 	mrioc->intr_info[midx].op_reply_q = NULL;  in mpi3mr_delete_op_reply_q()
1974 	mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;  in mpi3mr_delete_op_reply_q()
1975 	mutex_unlock(&mrioc->init_cmds.mutex);  in mpi3mr_delete_op_reply_q()
1982  * mpi3mr_alloc_op_reply_q_segments -Alloc segmented reply pool
1989  * Return: 0 on success, non-zero on failure.
1993 	struct op_reply_qinfo *op_reply_q = mrioc->op_reply_qinfo + qidx;  in mpi3mr_alloc_op_reply_q_segments()
1994 	int i, size;  in mpi3mr_alloc_op_reply_q_segments()  local
1998 	if (mrioc->enable_segqueue) {  in mpi3mr_alloc_op_reply_q_segments()
1999 		op_reply_q->segment_qd =  in mpi3mr_alloc_op_reply_q_segments()
2000 		    MPI3MR_OP_REP_Q_SEG_SIZE / mrioc->op_reply_desc_sz;  in mpi3mr_alloc_op_reply_q_segments()
2002 		size = MPI3MR_OP_REP_Q_SEG_SIZE;  in mpi3mr_alloc_op_reply_q_segments()
2004 		op_reply_q->q_segment_list = dma_alloc_coherent(&mrioc->pdev->dev,  in mpi3mr_alloc_op_reply_q_segments()
2005 		    MPI3MR_MAX_SEG_LIST_SIZE, &op_reply_q->q_segment_list_dma,  in mpi3mr_alloc_op_reply_q_segments()
2007 		if (!op_reply_q->q_segment_list)  in mpi3mr_alloc_op_reply_q_segments()
2008 			return -ENOMEM;  in mpi3mr_alloc_op_reply_q_segments()
2009 		q_segment_list_entry = (u64 *)op_reply_q->q_segment_list;  in mpi3mr_alloc_op_reply_q_segments()
2011 		op_reply_q->segment_qd = op_reply_q->num_replies;  in mpi3mr_alloc_op_reply_q_segments()
2012 		size = op_reply_q->num_replies * mrioc->op_reply_desc_sz;  in mpi3mr_alloc_op_reply_q_segments()
2015 	op_reply_q->num_segments = DIV_ROUND_UP(op_reply_q->num_replies,  in mpi3mr_alloc_op_reply_q_segments()
2016 	    op_reply_q->segment_qd);  in mpi3mr_alloc_op_reply_q_segments()
2018 	op_reply_q->q_segments = kcalloc(op_reply_q->num_segments,  in mpi3mr_alloc_op_reply_q_segments()
2020 	if (!op_reply_q->q_segments)  in mpi3mr_alloc_op_reply_q_segments()
2021 		return -ENOMEM;  in mpi3mr_alloc_op_reply_q_segments()
2023 	segments = op_reply_q->q_segments;  in mpi3mr_alloc_op_reply_q_segments()
2024 	for (i = 0; i < op_reply_q->num_segments; i++) {  in mpi3mr_alloc_op_reply_q_segments()
2026 		    dma_alloc_coherent(&mrioc->pdev->dev,  in mpi3mr_alloc_op_reply_q_segments()
2027 		    size, &segments[i].segment_dma, GFP_KERNEL);  in mpi3mr_alloc_op_reply_q_segments()
2029 			return -ENOMEM;  in mpi3mr_alloc_op_reply_q_segments()
2030 		if (mrioc->enable_segqueue)  in mpi3mr_alloc_op_reply_q_segments()
2039  * mpi3mr_alloc_op_req_q_segments - Alloc segmented req pool.
2046  * Return: 0 on success, non-zero on failure.
2050 	struct op_req_qinfo *op_req_q = mrioc->req_qinfo + qidx;  in mpi3mr_alloc_op_req_q_segments()
2051 	int i, size;  in mpi3mr_alloc_op_req_q_segments()  local
2055 	if (mrioc->enable_segqueue) {  in mpi3mr_alloc_op_req_q_segments()
2056 		op_req_q->segment_qd =  in mpi3mr_alloc_op_req_q_segments()
2057 		    MPI3MR_OP_REQ_Q_SEG_SIZE / mrioc->facts.op_req_sz;  in mpi3mr_alloc_op_req_q_segments()
2059 		size = MPI3MR_OP_REQ_Q_SEG_SIZE;  in mpi3mr_alloc_op_req_q_segments()
2061 		op_req_q->q_segment_list = dma_alloc_coherent(&mrioc->pdev->dev,  in mpi3mr_alloc_op_req_q_segments()
2062 		    MPI3MR_MAX_SEG_LIST_SIZE, &op_req_q->q_segment_list_dma,  in mpi3mr_alloc_op_req_q_segments()
2064 		if (!op_req_q->q_segment_list)  in mpi3mr_alloc_op_req_q_segments()
2065 			return -ENOMEM;  in mpi3mr_alloc_op_req_q_segments()
2066 		q_segment_list_entry = (u64 *)op_req_q->q_segment_list;  in mpi3mr_alloc_op_req_q_segments()
2069 		op_req_q->segment_qd = op_req_q->num_requests;  in mpi3mr_alloc_op_req_q_segments()
2070 		size = op_req_q->num_requests * mrioc->facts.op_req_sz;  in mpi3mr_alloc_op_req_q_segments()
2073 	op_req_q->num_segments = DIV_ROUND_UP(op_req_q->num_requests,  in mpi3mr_alloc_op_req_q_segments()
2074 	    op_req_q->segment_qd);  in mpi3mr_alloc_op_req_q_segments()
2076 	op_req_q->q_segments = kcalloc(op_req_q->num_segments,  in mpi3mr_alloc_op_req_q_segments()
2078 	if (!op_req_q->q_segments)  in mpi3mr_alloc_op_req_q_segments()
2079 		return -ENOMEM;  in mpi3mr_alloc_op_req_q_segments()
2081 	segments = op_req_q->q_segments;  in mpi3mr_alloc_op_req_q_segments()
2082 	for (i = 0; i < op_req_q->num_segments; i++) {  in mpi3mr_alloc_op_req_q_segments()
2084 		    dma_alloc_coherent(&mrioc->pdev->dev,  in mpi3mr_alloc_op_req_q_segments()
2085 		    size, &segments[i].segment_dma, GFP_KERNEL);  in mpi3mr_alloc_op_req_q_segments()
2087 			return -ENOMEM;  in mpi3mr_alloc_op_req_q_segments()
2088 		if (mrioc->enable_segqueue)  in mpi3mr_alloc_op_req_q_segments()
2097  * mpi3mr_create_op_reply_q - create operational reply queue
2104  * Return:  0 on success, non-zero on failure.
2109 	struct op_reply_qinfo *op_reply_q = mrioc->op_reply_qinfo + qidx;  in mpi3mr_create_op_reply_q()
2113 	reply_qid = op_reply_q->qid;  in mpi3mr_create_op_reply_q()
2115 	midx = REPLY_QUEUE_IDX_TO_MSIX_IDX(qidx, mrioc->op_reply_q_offset);  in mpi3mr_create_op_reply_q()
2118 		retval = -1;  in mpi3mr_create_op_reply_q()
2127 	if (mrioc->pdev->device == MPI3_MFGPAGE_DEVID_SAS4116) {  in mpi3mr_create_op_reply_q()
2128 		if (mrioc->pdev->revision)  in mpi3mr_create_op_reply_q()
2129 			op_reply_q->num_replies = MPI3MR_OP_REP_Q_QD;  in mpi3mr_create_op_reply_q()
2131 			op_reply_q->num_replies = MPI3MR_OP_REP_Q_QD4K;  in mpi3mr_create_op_reply_q()
2133 		op_reply_q->num_replies = MPI3MR_OP_REP_Q_QD2K;  in mpi3mr_create_op_reply_q()
2135 	op_reply_q->ci = 0;  in mpi3mr_create_op_reply_q()
2136 	op_reply_q->ephase = 1;  in mpi3mr_create_op_reply_q()
2137 	atomic_set(&op_reply_q->pend_ios, 0);  in mpi3mr_create_op_reply_q()
2138 	atomic_set(&op_reply_q->in_use, 0);  in mpi3mr_create_op_reply_q()
2139 	op_reply_q->enable_irq_poll = false;  in mpi3mr_create_op_reply_q()
2140 	op_reply_q->qfull_watermark =  in mpi3mr_create_op_reply_q()
2141 		op_reply_q->num_replies - (MPI3MR_THRESHOLD_REPLY_COUNT * 2);  in mpi3mr_create_op_reply_q()
2143 	if (!op_reply_q->q_segments) {  in mpi3mr_create_op_reply_q()
2152 	mutex_lock(&mrioc->init_cmds.mutex);  in mpi3mr_create_op_reply_q()
2153 	if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {  in mpi3mr_create_op_reply_q()
2154 		retval = -1;  in mpi3mr_create_op_reply_q()
2158 	mrioc->init_cmds.state = MPI3MR_CMD_PENDING;  in mpi3mr_create_op_reply_q()
2159 	mrioc->init_cmds.is_waiting = 1;  in mpi3mr_create_op_reply_q()
2160 	mrioc->init_cmds.callback = NULL;  in mpi3mr_create_op_reply_q()
2165 	if (midx < (mrioc->intr_info_count - mrioc->requested_poll_qcount))  in mpi3mr_create_op_reply_q()
2166 		op_reply_q->qtype = MPI3MR_DEFAULT_QUEUE;  in mpi3mr_create_op_reply_q()
2168 		op_reply_q->qtype = MPI3MR_POLL_QUEUE;  in mpi3mr_create_op_reply_q()
2170 	if (op_reply_q->qtype == MPI3MR_DEFAULT_QUEUE) {  in mpi3mr_create_op_reply_q()
2174 			cpu_to_le16(mrioc->intr_info[midx].msix_index);  in mpi3mr_create_op_reply_q()
2176 		create_req.msix_index = cpu_to_le16(mrioc->intr_info_count - 1);  in mpi3mr_create_op_reply_q()
2179 		if (!mrioc->active_poll_qcount)  in mpi3mr_create_op_reply_q()
2180 			disable_irq_nosync(pci_irq_vector(mrioc->pdev,  in mpi3mr_create_op_reply_q()
2181 			    mrioc->intr_info_count - 1));  in mpi3mr_create_op_reply_q()
2184 	if (mrioc->enable_segqueue) {  in mpi3mr_create_op_reply_q()
2188 		    op_reply_q->q_segment_list_dma);  in mpi3mr_create_op_reply_q()
2191 		    op_reply_q->q_segments[0].segment_dma);  in mpi3mr_create_op_reply_q()
2193 	create_req.size = cpu_to_le16(op_reply_q->num_replies);  in mpi3mr_create_op_reply_q()
2195 	init_completion(&mrioc->init_cmds.done);  in mpi3mr_create_op_reply_q()
2202 	wait_for_completion_timeout(&mrioc->init_cmds.done,  in mpi3mr_create_op_reply_q()
2204 	if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {  in mpi3mr_create_op_reply_q()
2208 		retval = -1;  in mpi3mr_create_op_reply_q()
2211 	if ((mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)  in mpi3mr_create_op_reply_q()
2215 		    (mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),  in mpi3mr_create_op_reply_q()
2216 		    mrioc->init_cmds.ioc_loginfo);  in mpi3mr_create_op_reply_q()
2217 		retval = -1;  in mpi3mr_create_op_reply_q()
2220 	op_reply_q->qid = reply_qid;  in mpi3mr_create_op_reply_q()
2221 	if (midx < mrioc->intr_info_count)  in mpi3mr_create_op_reply_q()
2222 		mrioc->intr_info[midx].op_reply_q = op_reply_q;  in mpi3mr_create_op_reply_q()
2224 	(op_reply_q->qtype == MPI3MR_DEFAULT_QUEUE) ? mrioc->default_qcount++ :  in mpi3mr_create_op_reply_q()
2225 	    mrioc->active_poll_qcount++;  in mpi3mr_create_op_reply_q()
2228 	mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;  in mpi3mr_create_op_reply_q()
2229 	mutex_unlock(&mrioc->init_cmds.mutex);  in mpi3mr_create_op_reply_q()
2236  * mpi3mr_create_op_req_q - create operational request queue
2244  * Return:  0 on success, non-zero on failure.
2250 	struct op_req_qinfo *op_req_q = mrioc->req_qinfo + idx;  in mpi3mr_create_op_req_q()
2254 	req_qid = op_req_q->qid;  in mpi3mr_create_op_req_q()
2257 		retval = -1;  in mpi3mr_create_op_req_q()
2265 	op_req_q->num_requests = MPI3MR_OP_REQ_Q_QD;  in mpi3mr_create_op_req_q()
2266 	op_req_q->ci = 0;  in mpi3mr_create_op_req_q()
2267 	op_req_q->pi = 0;  in mpi3mr_create_op_req_q()
2268 	op_req_q->reply_qid = reply_qid;  in mpi3mr_create_op_req_q()
2269 	spin_lock_init(&op_req_q->q_lock);  in mpi3mr_create_op_req_q()
2271 	if (!op_req_q->q_segments) {  in mpi3mr_create_op_req_q()
2280 	mutex_lock(&mrioc->init_cmds.mutex);  in mpi3mr_create_op_req_q()
2281 	if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {  in mpi3mr_create_op_req_q()
2282 		retval = -1;  in mpi3mr_create_op_req_q()
2286 	mrioc->init_cmds.state = MPI3MR_CMD_PENDING;  in mpi3mr_create_op_req_q()
2287 	mrioc->init_cmds.is_waiting = 1;  in mpi3mr_create_op_req_q()
2288 	mrioc->init_cmds.callback = NULL;  in mpi3mr_create_op_req_q()
2292 	if (mrioc->enable_segqueue) {  in mpi3mr_create_op_req_q()
2296 		    op_req_q->q_segment_list_dma);  in mpi3mr_create_op_req_q()
2299 		    op_req_q->q_segments[0].segment_dma);  in mpi3mr_create_op_req_q()
2301 	create_req.size = cpu_to_le16(op_req_q->num_requests);  in mpi3mr_create_op_req_q()
2303 	init_completion(&mrioc->init_cmds.done);  in mpi3mr_create_op_req_q()
2310 	wait_for_completion_timeout(&mrioc->init_cmds.done,  in mpi3mr_create_op_req_q()
2312 	if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {  in mpi3mr_create_op_req_q()
2316 		retval = -1;  in mpi3mr_create_op_req_q()
2319 	if ((mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)  in mpi3mr_create_op_req_q()
2323 		    (mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),  in mpi3mr_create_op_req_q()
2324 		    mrioc->init_cmds.ioc_loginfo);  in mpi3mr_create_op_req_q()
2325 		retval = -1;  in mpi3mr_create_op_req_q()
2328 	op_req_q->qid = req_qid;  in mpi3mr_create_op_req_q()
2331 	mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;  in mpi3mr_create_op_req_q()
2332 	mutex_unlock(&mrioc->init_cmds.mutex);  in mpi3mr_create_op_req_q()
2339  * mpi3mr_create_op_queues - create operational queue pairs
2345  * Return: 0 on success, non-zero on failures.
2352 	num_queues = min_t(int, mrioc->facts.max_op_reply_q,  in mpi3mr_create_op_queues()
2353 	    mrioc->facts.max_op_req_q);  in mpi3mr_create_op_queues()
2356 	    mrioc->intr_info_count - mrioc->op_reply_q_offset;  in mpi3mr_create_op_queues()
2357 	if (!mrioc->num_queues)  in mpi3mr_create_op_queues()
2358 		mrioc->num_queues = min_t(int, num_queues, msix_count_op_q);  in mpi3mr_create_op_queues()
2363 	num_queues = mrioc->num_op_reply_q ?  in mpi3mr_create_op_queues()
2364 	    mrioc->num_op_reply_q : mrioc->num_queues;  in mpi3mr_create_op_queues()
2368 	if (!mrioc->req_qinfo) {  in mpi3mr_create_op_queues()
2369 		mrioc->req_qinfo = kcalloc(num_queues,  in mpi3mr_create_op_queues()
2371 		if (!mrioc->req_qinfo) {  in mpi3mr_create_op_queues()
2372 			retval = -1;  in mpi3mr_create_op_queues()
2376 		mrioc->op_reply_qinfo = kzalloc(sizeof(struct op_reply_qinfo) *  in mpi3mr_create_op_queues()
2378 		if (!mrioc->op_reply_qinfo) {  in mpi3mr_create_op_queues()
2379 			retval = -1;  in mpi3mr_create_op_queues()
2384 	if (mrioc->enable_segqueue)  in mpi3mr_create_op_queues()
2394 		    mrioc->op_reply_qinfo[i].qid)) {  in mpi3mr_create_op_queues()
2403 		retval = -1;  in mpi3mr_create_op_queues()
2406 	mrioc->num_op_reply_q = mrioc->num_op_req_q = i;  in mpi3mr_create_op_queues()
2409 	    mrioc->num_op_reply_q, mrioc->default_qcount,  in mpi3mr_create_op_queues()
2410 	    mrioc->active_poll_qcount);  in mpi3mr_create_op_queues()
2414 	kfree(mrioc->req_qinfo);  in mpi3mr_create_op_queues()
2415 	mrioc->req_qinfo = NULL;  in mpi3mr_create_op_queues()
2417 	kfree(mrioc->op_reply_qinfo);  in mpi3mr_create_op_queues()
2418 	mrioc->op_reply_qinfo = NULL;  in mpi3mr_create_op_queues()
2424  * mpi3mr_op_request_post - Post request to operational queue
2433  * Return: 0 on success, non-zero on failure.
2443 	u16 req_sz = mrioc->facts.op_req_sz;  in mpi3mr_op_request_post()
2444 	struct segments *segments = op_req_q->q_segments;  in mpi3mr_op_request_post()
2447 	reply_qidx = op_req_q->reply_qid - 1;  in mpi3mr_op_request_post()
2448 	op_reply_q = mrioc->op_reply_qinfo + reply_qidx;  in mpi3mr_op_request_post()
2450 	if (mrioc->unrecoverable)  in mpi3mr_op_request_post()
2451 		return -EFAULT;  in mpi3mr_op_request_post()
2453 	spin_lock_irqsave(&op_req_q->q_lock, flags);  in mpi3mr_op_request_post()
2454 	pi = op_req_q->pi;  in mpi3mr_op_request_post()
2455 	max_entries = op_req_q->num_requests;  in mpi3mr_op_request_post()
2459 		    reply_qidx, mrioc->op_reply_q_offset);  in mpi3mr_op_request_post()
2460 		mpi3mr_process_op_reply_q(mrioc, mrioc->intr_info[midx].op_reply_q);  in mpi3mr_op_request_post()
2463 			retval = -EAGAIN;  in mpi3mr_op_request_post()
2468 	if (mrioc->reset_in_progress) {  in mpi3mr_op_request_post()
2470 		retval = -EAGAIN;  in mpi3mr_op_request_post()
2473 	if (mrioc->pci_err_recovery) {  in mpi3mr_op_request_post()
2475 		retval = -EAGAIN;  in mpi3mr_op_request_post()
2479 	/* Reply queue is nearing to get full, push back IOs to SML */  in mpi3mr_op_request_post()
2480 	if ((mrioc->prevent_reply_qfull == true) &&  in mpi3mr_op_request_post()
2481 		(atomic_read(&op_reply_q->pend_ios) >  in mpi3mr_op_request_post()
2482 	     (op_reply_q->qfull_watermark))) {  in mpi3mr_op_request_post()
2483 		atomic_inc(&mrioc->reply_qfull_count);  in mpi3mr_op_request_post()
2484 		retval = -EAGAIN;  in mpi3mr_op_request_post()
2488 	segment_base_addr = segments[pi / op_req_q->segment_qd].segment;  in mpi3mr_op_request_post()
2490 	    ((pi % op_req_q->segment_qd) * req_sz);  in mpi3mr_op_request_post()
2497 	op_req_q->pi = pi;  in mpi3mr_op_request_post()
2500 	if (atomic_inc_return(&mrioc->op_reply_qinfo[reply_qidx].pend_ios)  in mpi3mr_op_request_post()
2502 		mrioc->op_reply_qinfo[reply_qidx].enable_irq_poll = true;  in mpi3mr_op_request_post()
2504 	atomic_inc_return(&mrioc->op_reply_qinfo[reply_qidx].pend_ios);  in mpi3mr_op_request_post()
2507 	writel(op_req_q->pi,  in mpi3mr_op_request_post()
2508 	    &mrioc->sysif_regs->oper_queue_indexes[reply_qidx].producer_index);  in mpi3mr_op_request_post()
2511 	spin_unlock_irqrestore(&op_req_q->q_lock, flags);  in mpi3mr_op_request_post()
2516  * mpi3mr_check_rh_fault_ioc - check reset history and fault
2534 	if (mrioc->unrecoverable) {  in mpi3mr_check_rh_fault_ioc()
2539 	if (!pci_device_is_present(mrioc->pdev)) {  in mpi3mr_check_rh_fault_ioc()
2540 		mrioc->unrecoverable = 1;  in mpi3mr_check_rh_fault_ioc()
2545 	ioc_status = readl(&mrioc->sysif_regs->ioc_status);  in mpi3mr_check_rh_fault_ioc()
2552 		trigger_data.fault = (readl(&mrioc->sysif_regs->fault) &  in mpi3mr_check_rh_fault_ioc()
2564 	trigger_data.fault = (readl(&mrioc->sysif_regs->fault) &  in mpi3mr_check_rh_fault_ioc()
2570 		host_diagnostic = readl(&mrioc->sysif_regs->host_diagnostic);  in mpi3mr_check_rh_fault_ioc()
2574 	} while (--timeout);  in mpi3mr_check_rh_fault_ioc()
2578  * mpi3mr_sync_timestamp - Issue time stamp sync request
2584  * Return: 0 on success, non-zero on failure.
2593 	mutex_lock(&mrioc->init_cmds.mutex);  in mpi3mr_sync_timestamp()
2594 	if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {  in mpi3mr_sync_timestamp()
2595 		retval = -1;  in mpi3mr_sync_timestamp()
2597 		mutex_unlock(&mrioc->init_cmds.mutex);  in mpi3mr_sync_timestamp()
2600 	mrioc->init_cmds.state = MPI3MR_CMD_PENDING;  in mpi3mr_sync_timestamp()
2601 	mrioc->init_cmds.is_waiting = 1;  in mpi3mr_sync_timestamp()
2602 	mrioc->init_cmds.callback = NULL;  in mpi3mr_sync_timestamp()
2609 	init_completion(&mrioc->init_cmds.done);  in mpi3mr_sync_timestamp()
2617 	wait_for_completion_timeout(&mrioc->init_cmds.done,  in mpi3mr_sync_timestamp()
2619 	if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {  in mpi3mr_sync_timestamp()
2621 		mrioc->init_cmds.is_waiting = 0;  in mpi3mr_sync_timestamp()
2622 		if (!(mrioc->init_cmds.state & MPI3MR_CMD_RESET))  in mpi3mr_sync_timestamp()
2625 		retval = -1;  in mpi3mr_sync_timestamp()
2628 	if ((mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)  in mpi3mr_sync_timestamp()
2632 		    (mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),  in mpi3mr_sync_timestamp()
2633 		    mrioc->init_cmds.ioc_loginfo);  in mpi3mr_sync_timestamp()
2634 		retval = -1;  in mpi3mr_sync_timestamp()
2639 	mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;  in mpi3mr_sync_timestamp()
2640 	mutex_unlock(&mrioc->init_cmds.mutex);  in mpi3mr_sync_timestamp()
2647  * mpi3mr_print_pkg_ver - display controller fw package version
2653  * Return: 0 on success and non-zero on failure.
2658 	int retval = -1;  in mpi3mr_print_pkg_ver()
2665 	data = dma_alloc_coherent(&mrioc->pdev->dev, data_len, &data_dma,  in mpi3mr_print_pkg_ver()
2668 		return -ENOMEM;  in mpi3mr_print_pkg_ver()
2671 	mutex_lock(&mrioc->init_cmds.mutex);  in mpi3mr_print_pkg_ver()
2672 	if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {  in mpi3mr_print_pkg_ver()
2674 		mutex_unlock(&mrioc->init_cmds.mutex);  in mpi3mr_print_pkg_ver()
2677 	mrioc->init_cmds.state = MPI3MR_CMD_PENDING;  in mpi3mr_print_pkg_ver()
2678 	mrioc->init_cmds.is_waiting = 1;  in mpi3mr_print_pkg_ver()
2679 	mrioc->init_cmds.callback = NULL;  in mpi3mr_print_pkg_ver()
2689 	init_completion(&mrioc->init_cmds.done);  in mpi3mr_print_pkg_ver()
2696 	wait_for_completion_timeout(&mrioc->init_cmds.done,  in mpi3mr_print_pkg_ver()
2698 	if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {  in mpi3mr_print_pkg_ver()
2702 		retval = -1;  in mpi3mr_print_pkg_ver()
2705 	if ((mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)  in mpi3mr_print_pkg_ver()
2708 		if (manifest->manifest_type == MPI3_CI_MANIFEST_TYPE_MPI) {  in mpi3mr_print_pkg_ver()
2710 			    "firmware package version(%d.%d.%d.%d.%05d-%05d)\n",  in mpi3mr_print_pkg_ver()
2711 			    manifest->package_version.gen_major,  in mpi3mr_print_pkg_ver()
2712 			    manifest->package_version.gen_minor,  in mpi3mr_print_pkg_ver()
2713 			    manifest->package_version.phase_major,  in mpi3mr_print_pkg_ver()
2714 			    manifest->package_version.phase_minor,  in mpi3mr_print_pkg_ver()
2715 			    manifest->package_version.customer_id,  in mpi3mr_print_pkg_ver()
2716 			    manifest->package_version.build_num);  in mpi3mr_print_pkg_ver()
2721 	mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;  in mpi3mr_print_pkg_ver()
2722 	mutex_unlock(&mrioc->init_cmds.mutex);  in mpi3mr_print_pkg_ver()
2726 		dma_free_coherent(&mrioc->pdev->dev, data_len, data,  in mpi3mr_print_pkg_ver()
2732  * mpi3mr_watchdog_work - watchdog thread to monitor faults
2751 	if (mrioc->reset_in_progress || mrioc->pci_err_recovery)  in mpi3mr_watchdog_work()
2754 	if (!mrioc->unrecoverable && !pci_device_is_present(mrioc->pdev)) {  in mpi3mr_watchdog_work()
2756 		mrioc->unrecoverable = 1;  in mpi3mr_watchdog_work()
2759 	if (mrioc->unrecoverable) {  in mpi3mr_watchdog_work()
2766 	if (atomic_read(&mrioc->admin_pend_isr)) {  in mpi3mr_watchdog_work()
2772 	if (!(mrioc->facts.ioc_capabilities &  in mpi3mr_watchdog_work()
2774 		(mrioc->ts_update_counter++ >= mrioc->ts_update_interval)) {  in mpi3mr_watchdog_work()
2776 		mrioc->ts_update_counter = 0;  in mpi3mr_watchdog_work()
2780 	if ((mrioc->prepare_for_reset) &&  in mpi3mr_watchdog_work()
2781 	    ((mrioc->prepare_for_reset_timeout_counter++) >=  in mpi3mr_watchdog_work()
2789 	ioc_status = readl(&mrioc->sysif_regs->ioc_status);  in mpi3mr_watchdog_work()
2802 	trigger_data.fault = readl(&mrioc->sysif_regs->fault) & MPI3_SYSIF_FAULT_CODE_MASK;  in mpi3mr_watchdog_work()
2805 	host_diagnostic = readl(&mrioc->sysif_regs->host_diagnostic);  in mpi3mr_watchdog_work()
2807 		if (!mrioc->diagsave_timeout) {  in mpi3mr_watchdog_work()
2811 		if ((mrioc->diagsave_timeout++) <= MPI3_SYSIF_DIAG_SAVE_TIMEOUT)  in mpi3mr_watchdog_work()
2816 	mrioc->diagsave_timeout = 0;  in mpi3mr_watchdog_work()
2819 		mrioc->unrecoverable = 1;  in mpi3mr_watchdog_work()
2828 		mrioc->unrecoverable = 1;  in mpi3mr_watchdog_work()
2842 	spin_lock_irqsave(&mrioc->watchdog_lock, flags);  in mpi3mr_watchdog_work()
2843 	if (mrioc->watchdog_work_q)  in mpi3mr_watchdog_work()
2844 		queue_delayed_work(mrioc->watchdog_work_q,  in mpi3mr_watchdog_work()
2845 		    &mrioc->watchdog_work,  in mpi3mr_watchdog_work()
2847 	spin_unlock_irqrestore(&mrioc->watchdog_lock, flags);  in mpi3mr_watchdog_work()
2852  * mpi3mr_start_watchdog - Start watchdog
2862 	if (mrioc->watchdog_work_q)  in mpi3mr_start_watchdog()
2865 	INIT_DELAYED_WORK(&mrioc->watchdog_work, mpi3mr_watchdog_work);  in mpi3mr_start_watchdog()
2866 	snprintf(mrioc->watchdog_work_q_name,  in mpi3mr_start_watchdog()
2867 	    sizeof(mrioc->watchdog_work_q_name), "watchdog_%s%d", mrioc->name,  in mpi3mr_start_watchdog()
2868 	    mrioc->id);  in mpi3mr_start_watchdog()
2869 	mrioc->watchdog_work_q = alloc_ordered_workqueue(  in mpi3mr_start_watchdog()
2870 		"%s", WQ_MEM_RECLAIM, mrioc->watchdog_work_q_name);  in mpi3mr_start_watchdog()
2871 	if (!mrioc->watchdog_work_q) {  in mpi3mr_start_watchdog()
2876 	if (mrioc->watchdog_work_q)  in mpi3mr_start_watchdog()
2877 		queue_delayed_work(mrioc->watchdog_work_q,  in mpi3mr_start_watchdog()
2878 		    &mrioc->watchdog_work,  in mpi3mr_start_watchdog()
2883  * mpi3mr_stop_watchdog - Stop watchdog
2896 	spin_lock_irqsave(&mrioc->watchdog_lock, flags);  in mpi3mr_stop_watchdog()
2897 	wq = mrioc->watchdog_work_q;  in mpi3mr_stop_watchdog()
2898 	mrioc->watchdog_work_q = NULL;  in mpi3mr_stop_watchdog()
2899 	spin_unlock_irqrestore(&mrioc->watchdog_lock, flags);  in mpi3mr_stop_watchdog()
2901 		if (!cancel_delayed_work_sync(&mrioc->watchdog_work))  in mpi3mr_stop_watchdog()
2908  * mpi3mr_setup_admin_qpair - Setup admin queue pair
2914  * Return: 0 on success, non-zero on failures.
2921 	mrioc->admin_req_q_sz = MPI3MR_ADMIN_REQ_Q_SIZE;  in mpi3mr_setup_admin_qpair()
2922 	mrioc->num_admin_req = mrioc->admin_req_q_sz /  in mpi3mr_setup_admin_qpair()
2924 	mrioc->admin_req_ci = mrioc->admin_req_pi = 0;  in mpi3mr_setup_admin_qpair()
2926 	mrioc->admin_reply_q_sz = MPI3MR_ADMIN_REPLY_Q_SIZE;  in mpi3mr_setup_admin_qpair()
2927 	mrioc->num_admin_replies = mrioc->admin_reply_q_sz /  in mpi3mr_setup_admin_qpair()
2929 	mrioc->admin_reply_ci = 0;  in mpi3mr_setup_admin_qpair()
2930 	mrioc->admin_reply_ephase = 1;  in mpi3mr_setup_admin_qpair()
2931 	atomic_set(&mrioc->admin_reply_q_in_use, 0);  in mpi3mr_setup_admin_qpair()
2932 	atomic_set(&mrioc->admin_pend_isr, 0);  in mpi3mr_setup_admin_qpair()
2934 	if (!mrioc->admin_req_base) {  in mpi3mr_setup_admin_qpair()
2935 		mrioc->admin_req_base = dma_alloc_coherent(&mrioc->pdev->dev,  in mpi3mr_setup_admin_qpair()
2936 		    mrioc->admin_req_q_sz, &mrioc->admin_req_dma, GFP_KERNEL);  in mpi3mr_setup_admin_qpair()
2938 		if (!mrioc->admin_req_base) {  in mpi3mr_setup_admin_qpair()
2939 			retval = -1;  in mpi3mr_setup_admin_qpair()
2943 		mrioc->admin_reply_base = dma_alloc_coherent(&mrioc->pdev->dev,  in mpi3mr_setup_admin_qpair()
2944 		    mrioc->admin_reply_q_sz, &mrioc->admin_reply_dma,  in mpi3mr_setup_admin_qpair()
2947 		if (!mrioc->admin_reply_base) {  in mpi3mr_setup_admin_qpair()
2948 			retval = -1;  in mpi3mr_setup_admin_qpair()
2953 	num_admin_entries = (mrioc->num_admin_replies << 16) |  in mpi3mr_setup_admin_qpair()
2954 	    (mrioc->num_admin_req);  in mpi3mr_setup_admin_qpair()
2955 	writel(num_admin_entries, &mrioc->sysif_regs->admin_queue_num_entries);  in mpi3mr_setup_admin_qpair()
2956 	mpi3mr_writeq(mrioc->admin_req_dma,  in mpi3mr_setup_admin_qpair()
2957 	    &mrioc->sysif_regs->admin_request_queue_address);  in mpi3mr_setup_admin_qpair()
2958 	mpi3mr_writeq(mrioc->admin_reply_dma,  in mpi3mr_setup_admin_qpair()
2959 	    &mrioc->sysif_regs->admin_reply_queue_address);  in mpi3mr_setup_admin_qpair()
2960 	writel(mrioc->admin_req_pi, &mrioc->sysif_regs->admin_request_queue_pi);  in mpi3mr_setup_admin_qpair()
2961 	writel(mrioc->admin_reply_ci, &mrioc->sysif_regs->admin_reply_queue_ci);  in mpi3mr_setup_admin_qpair()
2966 	if (mrioc->admin_reply_base) {  in mpi3mr_setup_admin_qpair()
2967 		dma_free_coherent(&mrioc->pdev->dev, mrioc->admin_reply_q_sz,  in mpi3mr_setup_admin_qpair()
2968 		    mrioc->admin_reply_base, mrioc->admin_reply_dma);  in mpi3mr_setup_admin_qpair()
2969 		mrioc->admin_reply_base = NULL;  in mpi3mr_setup_admin_qpair()
2971 	if (mrioc->admin_req_base) {  in mpi3mr_setup_admin_qpair()
2972 		dma_free_coherent(&mrioc->pdev->dev, mrioc->admin_req_q_sz,  in mpi3mr_setup_admin_qpair()
2973 		    mrioc->admin_req_base, mrioc->admin_req_dma);  in mpi3mr_setup_admin_qpair()
2974 		mrioc->admin_req_base = NULL;  in mpi3mr_setup_admin_qpair()
2980  * mpi3mr_issue_iocfacts - Send IOC Facts
2987  * Return: 0 on success, non-zero on failures.
2999 	data = dma_alloc_coherent(&mrioc->pdev->dev, data_len, &data_dma,  in mpi3mr_issue_iocfacts()
3003 		retval = -1;  in mpi3mr_issue_iocfacts()
3008 	mutex_lock(&mrioc->init_cmds.mutex);  in mpi3mr_issue_iocfacts()
3009 	if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {  in mpi3mr_issue_iocfacts()
3010 		retval = -1;  in mpi3mr_issue_iocfacts()
3012 		mutex_unlock(&mrioc->init_cmds.mutex);  in mpi3mr_issue_iocfacts()
3015 	mrioc->init_cmds.state = MPI3MR_CMD_PENDING;  in mpi3mr_issue_iocfacts()
3016 	mrioc->init_cmds.is_waiting = 1;  in mpi3mr_issue_iocfacts()
3017 	mrioc->init_cmds.callback = NULL;  in mpi3mr_issue_iocfacts()
3024 	init_completion(&mrioc->init_cmds.done);  in mpi3mr_issue_iocfacts()
3031 	wait_for_completion_timeout(&mrioc->init_cmds.done,  in mpi3mr_issue_iocfacts()
3033 	if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {  in mpi3mr_issue_iocfacts()
3037 		retval = -1;  in mpi3mr_issue_iocfacts()
3040 	if ((mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)  in mpi3mr_issue_iocfacts()
3044 		    (mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),  in mpi3mr_issue_iocfacts()
3045 		    mrioc->init_cmds.ioc_loginfo);  in mpi3mr_issue_iocfacts()
3046 		retval = -1;  in mpi3mr_issue_iocfacts()
3052 	mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;  in mpi3mr_issue_iocfacts()
3053 	mutex_unlock(&mrioc->init_cmds.mutex);  in mpi3mr_issue_iocfacts()
3057 		dma_free_coherent(&mrioc->pdev->dev, data_len, data, data_dma);  in mpi3mr_issue_iocfacts()
3063  * mpi3mr_check_reset_dma_mask - Process IOC facts data
3069  * Return: 0 on success, non-zero on failure.
3073 	struct pci_dev *pdev = mrioc->pdev;  in mpi3mr_check_reset_dma_mask()
3075 	u64 facts_dma_mask = DMA_BIT_MASK(mrioc->facts.dma_mask);  in mpi3mr_check_reset_dma_mask()
3077 	if (!mrioc->facts.dma_mask || (mrioc->dma_mask <= facts_dma_mask))  in mpi3mr_check_reset_dma_mask()
3081 	    mrioc->dma_mask, facts_dma_mask);  in mpi3mr_check_reset_dma_mask()
3083 	r = dma_set_mask_and_coherent(&pdev->dev, facts_dma_mask);  in mpi3mr_check_reset_dma_mask()
3089 	mrioc->dma_mask = facts_dma_mask;  in mpi3mr_check_reset_dma_mask()
3094  * mpi3mr_process_factsdata - Process IOC facts data
3108 	if ((le16_to_cpu(facts_data->ioc_facts_data_length)) !=  in mpi3mr_process_factsdata()
3113 		    le16_to_cpu(facts_data->ioc_facts_data_length) * 4);  in mpi3mr_process_factsdata()
3116 	ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);  in mpi3mr_process_factsdata()
3119 	if (le16_to_cpu(facts_data->ioc_request_frame_size) != (req_sz / 4)) {  in mpi3mr_process_factsdata()
3122 		    req_sz / 4, le16_to_cpu(facts_data->ioc_request_frame_size));  in mpi3mr_process_factsdata()
3125 	memset(&mrioc->facts, 0, sizeof(mrioc->facts));  in mpi3mr_process_factsdata()
3127 	facts_flags = le32_to_cpu(facts_data->flags);  in mpi3mr_process_factsdata()
3128 	mrioc->facts.op_req_sz = req_sz;  in mpi3mr_process_factsdata()
3129 	mrioc->op_reply_desc_sz = 1 << ((ioc_config &  in mpi3mr_process_factsdata()
3133 	mrioc->facts.ioc_num = facts_data->ioc_number;  in mpi3mr_process_factsdata()
3134 	mrioc->facts.who_init = facts_data->who_init;  in mpi3mr_process_factsdata()
3135 	mrioc->facts.max_msix_vectors = le16_to_cpu(facts_data->max_msix_vectors);  in mpi3mr_process_factsdata()
3136 	mrioc->facts.personality = (facts_flags &  in mpi3mr_process_factsdata()
3138 	mrioc->facts.dma_mask = (facts_flags &  in mpi3mr_process_factsdata()
3141 	mrioc->facts.dma_mask = (facts_flags &  in mpi3mr_process_factsdata()
3144 	mrioc->facts.protocol_flags = facts_data->protocol_flags;  in mpi3mr_process_factsdata()
3145 	mrioc->facts.mpi_version = le32_to_cpu(facts_data->mpi_version.word);  in mpi3mr_process_factsdata()
3146 	mrioc->facts.max_reqs = le16_to_cpu(facts_data->max_outstanding_requests);  in mpi3mr_process_factsdata()
3147 	mrioc->facts.product_id = le16_to_cpu(facts_data->product_id);  in mpi3mr_process_factsdata()
3148 	mrioc->facts.reply_sz = le16_to_cpu(facts_data->reply_frame_size) * 4;  in mpi3mr_process_factsdata()
3149 	mrioc->facts.exceptions = le16_to_cpu(facts_data->ioc_exceptions);  in mpi3mr_process_factsdata()
3150 	mrioc->facts.max_perids = le16_to_cpu(facts_data->max_persistent_id);  in mpi3mr_process_factsdata()
3151 	mrioc->facts.max_vds = le16_to_cpu(facts_data->max_vds);  in mpi3mr_process_factsdata()
3152 	mrioc->facts.max_hpds = le16_to_cpu(facts_data->max_host_pds);  in mpi3mr_process_factsdata()
3153 	mrioc->facts.max_advhpds = le16_to_cpu(facts_data->max_adv_host_pds);  in mpi3mr_process_factsdata()
3154 	mrioc->facts.max_raid_pds = le16_to_cpu(facts_data->max_raid_pds);  in mpi3mr_process_factsdata()
3155 	mrioc->facts.max_nvme = le16_to_cpu(facts_data->max_nvme);  in mpi3mr_process_factsdata()
3156 	mrioc->facts.max_pcie_switches =  in mpi3mr_process_factsdata()
3157 	    le16_to_cpu(facts_data->max_pcie_switches);  in mpi3mr_process_factsdata()
3158 	mrioc->facts.max_sasexpanders =  in mpi3mr_process_factsdata()
3159 	    le16_to_cpu(facts_data->max_sas_expanders);  in mpi3mr_process_factsdata()
3160 	mrioc->facts.max_data_length = le16_to_cpu(facts_data->max_data_length);  in mpi3mr_process_factsdata()
3161 	mrioc->facts.max_sasinitiators =  in mpi3mr_process_factsdata()
3162 	    le16_to_cpu(facts_data->max_sas_initiators);  in mpi3mr_process_factsdata()
3163 	mrioc->facts.max_enclosures = le16_to_cpu(facts_data->max_enclosures);  in mpi3mr_process_factsdata()
3164 	mrioc->facts.min_devhandle = le16_to_cpu(facts_data->min_dev_handle);  in mpi3mr_process_factsdata()
3165 	mrioc->facts.max_devhandle = le16_to_cpu(facts_data->max_dev_handle);  in mpi3mr_process_factsdata()
3166 	mrioc->facts.max_op_req_q =  in mpi3mr_process_factsdata()
3167 	    le16_to_cpu(facts_data->max_operational_request_queues);  in mpi3mr_process_factsdata()
3168 	mrioc->facts.max_op_reply_q =  in mpi3mr_process_factsdata()
3169 	    le16_to_cpu(facts_data->max_operational_reply_queues);  in mpi3mr_process_factsdata()
3170 	mrioc->facts.ioc_capabilities =  in mpi3mr_process_factsdata()
3171 	    le32_to_cpu(facts_data->ioc_capabilities);  in mpi3mr_process_factsdata()
3172 	mrioc->facts.fw_ver.build_num =  in mpi3mr_process_factsdata()
3173 	    le16_to_cpu(facts_data->fw_version.build_num);  in mpi3mr_process_factsdata()
3174 	mrioc->facts.fw_ver.cust_id =  in mpi3mr_process_factsdata()
3175 	    le16_to_cpu(facts_data->fw_version.customer_id);  in mpi3mr_process_factsdata()
3176 	mrioc->facts.fw_ver.ph_minor = facts_data->fw_version.phase_minor;  in mpi3mr_process_factsdata()
3177 	mrioc->facts.fw_ver.ph_major = facts_data->fw_version.phase_major;  in mpi3mr_process_factsdata()
3178 	mrioc->facts.fw_ver.gen_minor = facts_data->fw_version.gen_minor;  in mpi3mr_process_factsdata()
3179 	mrioc->facts.fw_ver.gen_major = facts_data->fw_version.gen_major;  in mpi3mr_process_factsdata()
3180 	mrioc->msix_count = min_t(int, mrioc->msix_count,  in mpi3mr_process_factsdata()
3181 	    mrioc->facts.max_msix_vectors);  in mpi3mr_process_factsdata()
3182 	mrioc->facts.sge_mod_mask = facts_data->sge_modifier_mask;  in mpi3mr_process_factsdata()
3183 	mrioc->facts.sge_mod_value = facts_data->sge_modifier_value;  in mpi3mr_process_factsdata()
3184 	mrioc->facts.sge_mod_shift = facts_data->sge_modifier_shift;  in mpi3mr_process_factsdata()
3185 	mrioc->facts.shutdown_timeout =  in mpi3mr_process_factsdata()
3186 	    le16_to_cpu(facts_data->shutdown_timeout);  in mpi3mr_process_factsdata()
3187 	mrioc->facts.diag_trace_sz =  in mpi3mr_process_factsdata()
3188 	    le32_to_cpu(facts_data->diag_trace_size);  in mpi3mr_process_factsdata()
3189 	mrioc->facts.diag_fw_sz =  in mpi3mr_process_factsdata()
3190 	    le32_to_cpu(facts_data->diag_fw_size);  in mpi3mr_process_factsdata()
3191 	mrioc->facts.diag_drvr_sz = le32_to_cpu(facts_data->diag_driver_size);  in mpi3mr_process_factsdata()
3192 	mrioc->facts.max_dev_per_tg =  in mpi3mr_process_factsdata()
3193 	    facts_data->max_devices_per_throttle_group;  in mpi3mr_process_factsdata()
3194 	mrioc->facts.io_throttle_data_length =  in mpi3mr_process_factsdata()
3195 	    le16_to_cpu(facts_data->io_throttle_data_length);  in mpi3mr_process_factsdata()
3196 	mrioc->facts.max_io_throttle_group =  in mpi3mr_process_factsdata()
3197 	    le16_to_cpu(facts_data->max_io_throttle_group);  in mpi3mr_process_factsdata()
3198 	mrioc->facts.io_throttle_low = le16_to_cpu(facts_data->io_throttle_low);  in mpi3mr_process_factsdata()
3199 	mrioc->facts.io_throttle_high =  in mpi3mr_process_factsdata()
3200 	    le16_to_cpu(facts_data->io_throttle_high);  in mpi3mr_process_factsdata()
3202 	if (mrioc->facts.max_data_length ==  in mpi3mr_process_factsdata()
3204 		mrioc->facts.max_data_length = MPI3MR_DEFAULT_MAX_IO_SIZE;  in mpi3mr_process_factsdata()
3206 		mrioc->facts.max_data_length *= MPI3MR_PAGE_SIZE_4K;  in mpi3mr_process_factsdata()
3208 	if (mrioc->facts.io_throttle_data_length)  in mpi3mr_process_factsdata()
3209 		mrioc->io_throttle_data_length =  in mpi3mr_process_factsdata()
3210 		    (mrioc->facts.io_throttle_data_length * 2 * 4);  in mpi3mr_process_factsdata()
3213 		mrioc->io_throttle_data_length = (mrioc->facts.max_data_length / 512) + 2;  in mpi3mr_process_factsdata()
3215 	mrioc->io_throttle_high = (mrioc->facts.io_throttle_high * 2 * 1024);  in mpi3mr_process_factsdata()
3216 	mrioc->io_throttle_low = (mrioc->facts.io_throttle_low * 2 * 1024);  in mpi3mr_process_factsdata()
3219 	    mrioc->facts.ioc_num, mrioc->facts.max_op_req_q,  in mpi3mr_process_factsdata()
3220 	    mrioc->facts.max_op_reply_q, mrioc->facts.max_devhandle);  in mpi3mr_process_factsdata()
3223 	    mrioc->facts.max_reqs, mrioc->facts.min_devhandle,  in mpi3mr_process_factsdata()
3224 	    mrioc->facts.max_msix_vectors, mrioc->facts.max_perids);  in mpi3mr_process_factsdata()
3226 	    mrioc->facts.sge_mod_mask, mrioc->facts.sge_mod_value,  in mpi3mr_process_factsdata()
3227 	    mrioc->facts.sge_mod_shift);  in mpi3mr_process_factsdata()
3229 	    mrioc->facts.dma_mask, (facts_flags &  in mpi3mr_process_factsdata()
3230 	    MPI3_IOCFACTS_FLAGS_INITIAL_PORT_ENABLE_MASK), mrioc->facts.max_data_length);  in mpi3mr_process_factsdata()
3233 	    mrioc->facts.max_dev_per_tg, mrioc->facts.max_io_throttle_group);  in mpi3mr_process_factsdata()
3236 	   mrioc->facts.io_throttle_data_length * 4,  in mpi3mr_process_factsdata()
3237 	   mrioc->facts.io_throttle_high, mrioc->facts.io_throttle_low);  in mpi3mr_process_factsdata()
3241  * mpi3mr_alloc_reply_sense_bufs - Send IOC Init
3247  * Return: 0 on success, non-zero on failures.
3254 	if (mrioc->init_cmds.reply)  in mpi3mr_alloc_reply_sense_bufs()
3257 	mrioc->init_cmds.reply = kzalloc(mrioc->reply_sz, GFP_KERNEL);  in mpi3mr_alloc_reply_sense_bufs()
3258 	if (!mrioc->init_cmds.reply)  in mpi3mr_alloc_reply_sense_bufs()
3261 	mrioc->bsg_cmds.reply = kzalloc(mrioc->reply_sz, GFP_KERNEL);  in mpi3mr_alloc_reply_sense_bufs()
3262 	if (!mrioc->bsg_cmds.reply)  in mpi3mr_alloc_reply_sense_bufs()
3265 	mrioc->transport_cmds.reply = kzalloc(mrioc->reply_sz, GFP_KERNEL);  in mpi3mr_alloc_reply_sense_bufs()
3266 	if (!mrioc->transport_cmds.reply)  in mpi3mr_alloc_reply_sense_bufs()
3270 		mrioc->dev_rmhs_cmds[i].reply = kzalloc(mrioc->reply_sz,  in mpi3mr_alloc_reply_sense_bufs()
3272 		if (!mrioc->dev_rmhs_cmds[i].reply)  in mpi3mr_alloc_reply_sense_bufs()
3277 		mrioc->evtack_cmds[i].reply = kzalloc(mrioc->reply_sz,  in mpi3mr_alloc_reply_sense_bufs()
3279 		if (!mrioc->evtack_cmds[i].reply)  in mpi3mr_alloc_reply_sense_bufs()
3283 	mrioc->host_tm_cmds.reply = kzalloc(mrioc->reply_sz, GFP_KERNEL);  in mpi3mr_alloc_reply_sense_bufs()
3284 	if (!mrioc->host_tm_cmds.reply)  in mpi3mr_alloc_reply_sense_bufs()
3287 	mrioc->pel_cmds.reply = kzalloc(mrioc->reply_sz, GFP_KERNEL);  in mpi3mr_alloc_reply_sense_bufs()
3288 	if (!mrioc->pel_cmds.reply)  in mpi3mr_alloc_reply_sense_bufs()
3291 	mrioc->pel_abort_cmd.reply = kzalloc(mrioc->reply_sz, GFP_KERNEL);  in mpi3mr_alloc_reply_sense_bufs()
3292 	if (!mrioc->pel_abort_cmd.reply)  in mpi3mr_alloc_reply_sense_bufs()
3295 	mrioc->dev_handle_bitmap_bits = mrioc->facts.max_devhandle;  in mpi3mr_alloc_reply_sense_bufs()
3296 	mrioc->removepend_bitmap = bitmap_zalloc(mrioc->dev_handle_bitmap_bits,  in mpi3mr_alloc_reply_sense_bufs()
3298 	if (!mrioc->removepend_bitmap)  in mpi3mr_alloc_reply_sense_bufs()
3301 	mrioc->devrem_bitmap = bitmap_zalloc(MPI3MR_NUM_DEVRMCMD, GFP_KERNEL);  in mpi3mr_alloc_reply_sense_bufs()
3302 	if (!mrioc->devrem_bitmap)  in mpi3mr_alloc_reply_sense_bufs()
3305 	mrioc->evtack_cmds_bitmap = bitmap_zalloc(MPI3MR_NUM_EVTACKCMD,  in mpi3mr_alloc_reply_sense_bufs()
3307 	if (!mrioc->evtack_cmds_bitmap)  in mpi3mr_alloc_reply_sense_bufs()
3310 	mrioc->num_reply_bufs = mrioc->facts.max_reqs + MPI3MR_NUM_EVT_REPLIES;  in mpi3mr_alloc_reply_sense_bufs()
3311 	mrioc->reply_free_qsz = mrioc->num_reply_bufs + 1;  in mpi3mr_alloc_reply_sense_bufs()
3312 	mrioc->num_sense_bufs = mrioc->facts.max_reqs / MPI3MR_SENSEBUF_FACTOR;  in mpi3mr_alloc_reply_sense_bufs()
3313 	mrioc->sense_buf_q_sz = mrioc->num_sense_bufs + 1;  in mpi3mr_alloc_reply_sense_bufs()
3316 	sz = mrioc->num_reply_bufs * mrioc->reply_sz;  in mpi3mr_alloc_reply_sense_bufs()
3317 	mrioc->reply_buf_pool = dma_pool_create("reply_buf pool",  in mpi3mr_alloc_reply_sense_bufs()
3318 	    &mrioc->pdev->dev, sz, 16, 0);  in mpi3mr_alloc_reply_sense_bufs()
3319 	if (!mrioc->reply_buf_pool) {  in mpi3mr_alloc_reply_sense_bufs()
3324 	mrioc->reply_buf = dma_pool_zalloc(mrioc->reply_buf_pool, GFP_KERNEL,  in mpi3mr_alloc_reply_sense_bufs()
3325 	    &mrioc->reply_buf_dma);  in mpi3mr_alloc_reply_sense_bufs()
3326 	if (!mrioc->reply_buf)  in mpi3mr_alloc_reply_sense_bufs()
3329 	mrioc->reply_buf_dma_max_address = mrioc->reply_buf_dma + sz;  in mpi3mr_alloc_reply_sense_bufs()
3332 	sz = mrioc->reply_free_qsz * 8;  in mpi3mr_alloc_reply_sense_bufs()
3333 	mrioc->reply_free_q_pool = dma_pool_create("reply_free_q pool",  in mpi3mr_alloc_reply_sense_bufs()
3334 	    &mrioc->pdev->dev, sz, 8, 0);  in mpi3mr_alloc_reply_sense_bufs()
3335 	if (!mrioc->reply_free_q_pool) {  in mpi3mr_alloc_reply_sense_bufs()
3339 	mrioc->reply_free_q = dma_pool_zalloc(mrioc->reply_free_q_pool,  in mpi3mr_alloc_reply_sense_bufs()
3340 	    GFP_KERNEL, &mrioc->reply_free_q_dma);  in mpi3mr_alloc_reply_sense_bufs()
3341 	if (!mrioc->reply_free_q)  in mpi3mr_alloc_reply_sense_bufs()
3345 	sz = mrioc->num_sense_bufs * MPI3MR_SENSE_BUF_SZ;  in mpi3mr_alloc_reply_sense_bufs()
3346 	mrioc->sense_buf_pool = dma_pool_create("sense_buf pool",  in mpi3mr_alloc_reply_sense_bufs()
3347 	    &mrioc->pdev->dev, sz, 4, 0);  in mpi3mr_alloc_reply_sense_bufs()
3348 	if (!mrioc->sense_buf_pool) {  in mpi3mr_alloc_reply_sense_bufs()
3352 	mrioc->sense_buf = dma_pool_zalloc(mrioc->sense_buf_pool, GFP_KERNEL,  in mpi3mr_alloc_reply_sense_bufs()
3353 	    &mrioc->sense_buf_dma);  in mpi3mr_alloc_reply_sense_bufs()
3354 	if (!mrioc->sense_buf)  in mpi3mr_alloc_reply_sense_bufs()
3358 	sz = mrioc->sense_buf_q_sz * 8;  in mpi3mr_alloc_reply_sense_bufs()
3359 	mrioc->sense_buf_q_pool = dma_pool_create("sense_buf_q pool",  in mpi3mr_alloc_reply_sense_bufs()
3360 	    &mrioc->pdev->dev, sz, 8, 0);  in mpi3mr_alloc_reply_sense_bufs()
3361 	if (!mrioc->sense_buf_q_pool) {  in mpi3mr_alloc_reply_sense_bufs()
3365 	mrioc->sense_buf_q = dma_pool_zalloc(mrioc->sense_buf_q_pool,  in mpi3mr_alloc_reply_sense_bufs()
3366 	    GFP_KERNEL, &mrioc->sense_buf_q_dma);  in mpi3mr_alloc_reply_sense_bufs()
3367 	if (!mrioc->sense_buf_q)  in mpi3mr_alloc_reply_sense_bufs()
3373 	retval = -1;  in mpi3mr_alloc_reply_sense_bufs()
3378  * mpimr_initialize_reply_sbuf_queues - initialize reply sense
3392 	sz = mrioc->num_reply_bufs * mrioc->reply_sz;  in mpimr_initialize_reply_sbuf_queues()
3395 	    mrioc->reply_buf, mrioc->num_reply_bufs, mrioc->reply_sz,  in mpimr_initialize_reply_sbuf_queues()
3396 	    (sz / 1024), (unsigned long long)mrioc->reply_buf_dma);  in mpimr_initialize_reply_sbuf_queues()
3397 	sz = mrioc->reply_free_qsz * 8;  in mpimr_initialize_reply_sbuf_queues()
3400 	    mrioc->reply_free_q, mrioc->reply_free_qsz, 8, (sz / 1024),  in mpimr_initialize_reply_sbuf_queues()
3401 	    (unsigned long long)mrioc->reply_free_q_dma);  in mpimr_initialize_reply_sbuf_queues()
3402 	sz = mrioc->num_sense_bufs * MPI3MR_SENSE_BUF_SZ;  in mpimr_initialize_reply_sbuf_queues()
3405 	    mrioc->sense_buf, mrioc->num_sense_bufs, MPI3MR_SENSE_BUF_SZ,  in mpimr_initialize_reply_sbuf_queues()
3406 	    (sz / 1024), (unsigned long long)mrioc->sense_buf_dma);  in mpimr_initialize_reply_sbuf_queues()
3407 	sz = mrioc->sense_buf_q_sz * 8;  in mpimr_initialize_reply_sbuf_queues()
3410 	    mrioc->sense_buf_q, mrioc->sense_buf_q_sz, 8, (sz / 1024),  in mpimr_initialize_reply_sbuf_queues()
3411 	    (unsigned long long)mrioc->sense_buf_q_dma);  in mpimr_initialize_reply_sbuf_queues()
3414 	for (i = 0, phy_addr = mrioc->reply_buf_dma;  in mpimr_initialize_reply_sbuf_queues()
3415 	    i < mrioc->num_reply_bufs; i++, phy_addr += mrioc->reply_sz)  in mpimr_initialize_reply_sbuf_queues()
3416 		mrioc->reply_free_q[i] = cpu_to_le64(phy_addr);  in mpimr_initialize_reply_sbuf_queues()
3417 	mrioc->reply_free_q[i] = cpu_to_le64(0);  in mpimr_initialize_reply_sbuf_queues()
3420 	for (i = 0, phy_addr = mrioc->sense_buf_dma;  in mpimr_initialize_reply_sbuf_queues()
3421 	    i < mrioc->num_sense_bufs; i++, phy_addr += MPI3MR_SENSE_BUF_SZ)  in mpimr_initialize_reply_sbuf_queues()
3422 		mrioc->sense_buf_q[i] = cpu_to_le64(phy_addr);  in mpimr_initialize_reply_sbuf_queues()
3423 	mrioc->sense_buf_q[i] = cpu_to_le64(0);  in mpimr_initialize_reply_sbuf_queues()
3427  * mpi3mr_issue_iocinit - Send IOC Init
3433  * Return: 0 on success, non-zero on failures.
3444 	drv_info = dma_alloc_coherent(&mrioc->pdev->dev, data_len, &data_dma,  in mpi3mr_issue_iocinit()
3447 		retval = -1;  in mpi3mr_issue_iocinit()
3452 	drv_info->information_length = cpu_to_le32(data_len);  in mpi3mr_issue_iocinit()
3453 	strscpy(drv_info->driver_signature, "Broadcom", sizeof(drv_info->driver_signature));  in mpi3mr_issue_iocinit()
3454 	strscpy(drv_info->os_name, utsname()->sysname, sizeof(drv_info->os_name));  in mpi3mr_issue_iocinit()
3455 	strscpy(drv_info->os_version, utsname()->release, sizeof(drv_info->os_version));  in mpi3mr_issue_iocinit()
3456 	strscpy(drv_info->driver_name, MPI3MR_DRIVER_NAME, sizeof(drv_info->driver_name));  in mpi3mr_issue_iocinit()
3457 	strscpy(drv_info->driver_version, MPI3MR_DRIVER_VERSION, sizeof(drv_info->driver_version));  in mpi3mr_issue_iocinit()
3458 	strscpy(drv_info->driver_release_date, MPI3MR_DRIVER_RELDATE,  in mpi3mr_issue_iocinit()
3459 	    sizeof(drv_info->driver_release_date));  in mpi3mr_issue_iocinit()
3460 	drv_info->driver_capabilities = 0;  in mpi3mr_issue_iocinit()
3461 	memcpy((u8 *)&mrioc->driver_info, (u8 *)drv_info,  in mpi3mr_issue_iocinit()
3462 	    sizeof(mrioc->driver_info));  in mpi3mr_issue_iocinit()
3465 	mutex_lock(&mrioc->init_cmds.mutex);  in mpi3mr_issue_iocinit()
3466 	if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {  in mpi3mr_issue_iocinit()
3467 		retval = -1;  in mpi3mr_issue_iocinit()
3469 		mutex_unlock(&mrioc->init_cmds.mutex);  in mpi3mr_issue_iocinit()
3472 	mrioc->init_cmds.state = MPI3MR_CMD_PENDING;  in mpi3mr_issue_iocinit()
3473 	mrioc->init_cmds.is_waiting = 1;  in mpi3mr_issue_iocinit()
3474 	mrioc->init_cmds.callback = NULL;  in mpi3mr_issue_iocinit()
3482 	iocinit_req.reply_free_queue_depth = cpu_to_le16(mrioc->reply_free_qsz);  in mpi3mr_issue_iocinit()
3484 	    cpu_to_le64(mrioc->reply_free_q_dma);  in mpi3mr_issue_iocinit()
3487 	    cpu_to_le16(mrioc->sense_buf_q_sz);  in mpi3mr_issue_iocinit()
3489 	    cpu_to_le64(mrioc->sense_buf_q_dma);  in mpi3mr_issue_iocinit()
3500 	init_completion(&mrioc->init_cmds.done);  in mpi3mr_issue_iocinit()
3507 	wait_for_completion_timeout(&mrioc->init_cmds.done,  in mpi3mr_issue_iocinit()
3509 	if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {  in mpi3mr_issue_iocinit()
3513 		retval = -1;  in mpi3mr_issue_iocinit()
3516 	if ((mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)  in mpi3mr_issue_iocinit()
3520 		    (mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),  in mpi3mr_issue_iocinit()
3521 		    mrioc->init_cmds.ioc_loginfo);  in mpi3mr_issue_iocinit()
3522 		retval = -1;  in mpi3mr_issue_iocinit()
3526 	mrioc->reply_free_queue_host_index = mrioc->num_reply_bufs;  in mpi3mr_issue_iocinit()
3527 	writel(mrioc->reply_free_queue_host_index,  in mpi3mr_issue_iocinit()
3528 	    &mrioc->sysif_regs->reply_free_host_index);  in mpi3mr_issue_iocinit()
3530 	mrioc->sbq_host_index = mrioc->num_sense_bufs;  in mpi3mr_issue_iocinit()
3531 	writel(mrioc->sbq_host_index,  in mpi3mr_issue_iocinit()
3532 	    &mrioc->sysif_regs->sense_buffer_free_host_index);  in mpi3mr_issue_iocinit()
3534 	mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;  in mpi3mr_issue_iocinit()
3535 	mutex_unlock(&mrioc->init_cmds.mutex);  in mpi3mr_issue_iocinit()
3539 		dma_free_coherent(&mrioc->pdev->dev, data_len, drv_info,  in mpi3mr_issue_iocinit()
3546  * mpi3mr_unmask_events - Unmask events in event mask bitmap
3553  * Return: 0 on success, non-zero on failures.
3566 	mrioc->event_masks[word] &= ~desired_event;  in mpi3mr_unmask_events()
3570  * mpi3mr_issue_event_notification - Send event notification
3576  * Return: 0 on success, non-zero on failures.
3585 	mutex_lock(&mrioc->init_cmds.mutex);  in mpi3mr_issue_event_notification()
3586 	if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {  in mpi3mr_issue_event_notification()
3587 		retval = -1;  in mpi3mr_issue_event_notification()
3589 		mutex_unlock(&mrioc->init_cmds.mutex);  in mpi3mr_issue_event_notification()
3592 	mrioc->init_cmds.state = MPI3MR_CMD_PENDING;  in mpi3mr_issue_event_notification()
3593 	mrioc->init_cmds.is_waiting = 1;  in mpi3mr_issue_event_notification()
3594 	mrioc->init_cmds.callback = NULL;  in mpi3mr_issue_event_notification()
3599 		    cpu_to_le32(mrioc->event_masks[i]);  in mpi3mr_issue_event_notification()
3600 	init_completion(&mrioc->init_cmds.done);  in mpi3mr_issue_event_notification()
3607 	wait_for_completion_timeout(&mrioc->init_cmds.done,  in mpi3mr_issue_event_notification()
3609 	if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {  in mpi3mr_issue_event_notification()
3613 		retval = -1;  in mpi3mr_issue_event_notification()
3616 	if ((mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)  in mpi3mr_issue_event_notification()
3620 		    (mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),  in mpi3mr_issue_event_notification()
3621 		    mrioc->init_cmds.ioc_loginfo);  in mpi3mr_issue_event_notification()
3622 		retval = -1;  in mpi3mr_issue_event_notification()
3627 	mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;  in mpi3mr_issue_event_notification()
3628 	mutex_unlock(&mrioc->init_cmds.mutex);  in mpi3mr_issue_event_notification()
3634  * mpi3mr_process_event_ack - Process event acknowledgment
3642  * Return: 0 on success, non-zero on failures.
3651 	mutex_lock(&mrioc->init_cmds.mutex);  in mpi3mr_process_event_ack()
3652 	if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {  in mpi3mr_process_event_ack()
3653 		retval = -1;  in mpi3mr_process_event_ack()
3655 		mutex_unlock(&mrioc->init_cmds.mutex);  in mpi3mr_process_event_ack()
3658 	mrioc->init_cmds.state = MPI3MR_CMD_PENDING;  in mpi3mr_process_event_ack()
3659 	mrioc->init_cmds.is_waiting = 1;  in mpi3mr_process_event_ack()
3660 	mrioc->init_cmds.callback = NULL;  in mpi3mr_process_event_ack()
3666 	init_completion(&mrioc->init_cmds.done);  in mpi3mr_process_event_ack()
3673 	wait_for_completion_timeout(&mrioc->init_cmds.done,  in mpi3mr_process_event_ack()
3675 	if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {  in mpi3mr_process_event_ack()
3677 		if (!(mrioc->init_cmds.state & MPI3MR_CMD_RESET))  in mpi3mr_process_event_ack()
3680 		retval = -1;  in mpi3mr_process_event_ack()
3683 	if ((mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)  in mpi3mr_process_event_ack()
3687 		    (mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),  in mpi3mr_process_event_ack()
3688 		    mrioc->init_cmds.ioc_loginfo);  in mpi3mr_process_event_ack()
3689 		retval = -1;  in mpi3mr_process_event_ack()
3694 	mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;  in mpi3mr_process_event_ack()
3695 	mutex_unlock(&mrioc->init_cmds.mutex);  in mpi3mr_process_event_ack()
3701  * mpi3mr_alloc_chain_bufs - Allocate chain buffers
3708  * Return: 0 on success, non-zero on failure
3716 	if (mrioc->chain_sgl_list)  in mpi3mr_alloc_chain_bufs()
3719 	num_chains = mrioc->max_host_ios / MPI3MR_CHAINBUF_FACTOR;  in mpi3mr_alloc_chain_bufs()
3727 	mrioc->chain_buf_count = num_chains;  in mpi3mr_alloc_chain_bufs()
3729 	mrioc->chain_sgl_list = kzalloc(sz, GFP_KERNEL);  in mpi3mr_alloc_chain_bufs()
3730 	if (!mrioc->chain_sgl_list)  in mpi3mr_alloc_chain_bufs()
3733 	if (mrioc->max_sgl_entries > (mrioc->facts.max_data_length /  in mpi3mr_alloc_chain_bufs()
3735 		mrioc->max_sgl_entries = mrioc->facts.max_data_length /  in mpi3mr_alloc_chain_bufs()
3737 	sz = mrioc->max_sgl_entries * sizeof(struct mpi3_sge_common);  in mpi3mr_alloc_chain_bufs()
3738 	ioc_info(mrioc, "number of sgl entries=%d chain buffer size=%dKB\n",  in mpi3mr_alloc_chain_bufs()
3739 			mrioc->max_sgl_entries, sz/1024);  in mpi3mr_alloc_chain_bufs()
3741 	mrioc->chain_buf_pool = dma_pool_create("chain_buf pool",  in mpi3mr_alloc_chain_bufs()
3742 	    &mrioc->pdev->dev, sz, 16, 0);  in mpi3mr_alloc_chain_bufs()
3743 	if (!mrioc->chain_buf_pool) {  in mpi3mr_alloc_chain_bufs()
3749 		mrioc->chain_sgl_list[i].addr =  in mpi3mr_alloc_chain_bufs()
3750 		    dma_pool_zalloc(mrioc->chain_buf_pool, GFP_KERNEL,  in mpi3mr_alloc_chain_bufs()
3751 		    &mrioc->chain_sgl_list[i].dma_addr);  in mpi3mr_alloc_chain_bufs()
3753 		if (!mrioc->chain_sgl_list[i].addr)  in mpi3mr_alloc_chain_bufs()
3756 	mrioc->chain_bitmap = bitmap_zalloc(num_chains, GFP_KERNEL);  in mpi3mr_alloc_chain_bufs()
3757 	if (!mrioc->chain_bitmap)  in mpi3mr_alloc_chain_bufs()
3761 	retval = -1;  in mpi3mr_alloc_chain_bufs()
3766  * mpi3mr_port_enable_complete - Mark port enable complete
3778 	drv_cmd->callback = NULL;  in mpi3mr_port_enable_complete()
3779 	mrioc->scan_started = 0;  in mpi3mr_port_enable_complete()
3780 	if (drv_cmd->state & MPI3MR_CMD_RESET)  in mpi3mr_port_enable_complete()
3781 		mrioc->scan_failed = MPI3_IOCSTATUS_INTERNAL_ERROR;  in mpi3mr_port_enable_complete()
3783 		mrioc->scan_failed = drv_cmd->ioc_status;  in mpi3mr_port_enable_complete()
3784 	drv_cmd->state = MPI3MR_CMD_NOTUSED;  in mpi3mr_port_enable_complete()
3788  * mpi3mr_issue_port_enable - Issue Port Enable
3796  * Return: 0 on success, non-zero on failures.
3805 	mutex_lock(&mrioc->init_cmds.mutex);  in mpi3mr_issue_port_enable()
3806 	if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {  in mpi3mr_issue_port_enable()
3807 		retval = -1;  in mpi3mr_issue_port_enable()
3809 		mutex_unlock(&mrioc->init_cmds.mutex);  in mpi3mr_issue_port_enable()
3812 	mrioc->init_cmds.state = MPI3MR_CMD_PENDING;  in mpi3mr_issue_port_enable()
3814 		mrioc->init_cmds.is_waiting = 0;  in mpi3mr_issue_port_enable()
3815 		mrioc->init_cmds.callback = mpi3mr_port_enable_complete;  in mpi3mr_issue_port_enable()
3817 		mrioc->init_cmds.is_waiting = 1;  in mpi3mr_issue_port_enable()
3818 		mrioc->init_cmds.callback = NULL;  in mpi3mr_issue_port_enable()
3819 		init_completion(&mrioc->init_cmds.done);  in mpi3mr_issue_port_enable()
3830 		mutex_unlock(&mrioc->init_cmds.mutex);  in mpi3mr_issue_port_enable()
3834 	wait_for_completion_timeout(&mrioc->init_cmds.done, (pe_timeout * HZ));  in mpi3mr_issue_port_enable()
3835 	if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {  in mpi3mr_issue_port_enable()
3837 		retval = -1;  in mpi3mr_issue_port_enable()
3841 	mpi3mr_port_enable_complete(mrioc, &mrioc->init_cmds);  in mpi3mr_issue_port_enable()
3844 	mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;  in mpi3mr_issue_port_enable()
3845 	mutex_unlock(&mrioc->init_cmds.mutex);  in mpi3mr_issue_port_enable()
3870  * mpi3mr_repost_diag_bufs - repost host diag buffers
3876  * Return: 0 on success, non-zero on failures.
3891 		return -1;  in mpi3mr_repost_diag_bufs()
3897 	    trace_hdb->status != MPI3MR_HDB_BUFSTATUS_NOT_ALLOCATED &&  in mpi3mr_repost_diag_bufs()
3898 	    trace_hdb->trigger_type != MPI3MR_HDB_TRIGGER_TYPE_GLOBAL &&  in mpi3mr_repost_diag_bufs()
3899 	    trace_hdb->trigger_type != MPI3MR_HDB_TRIGGER_TYPE_ELEMENT)  in mpi3mr_repost_diag_bufs()
3904 	if (fw_hdb && fw_hdb->status != MPI3MR_HDB_BUFSTATUS_NOT_ALLOCATED &&  in mpi3mr_repost_diag_bufs()
3905 	    fw_hdb->trigger_type != MPI3MR_HDB_TRIGGER_TYPE_GLOBAL &&  in mpi3mr_repost_diag_bufs()
3906 	    fw_hdb->trigger_type != MPI3MR_HDB_TRIGGER_TYPE_ELEMENT)  in mpi3mr_repost_diag_bufs()
3910 		global_trigger = le64_to_cpu(mrioc->driver_pg2->global_trigger);  in mpi3mr_repost_diag_bufs()
3920 		prev_trigger_type = trace_hdb->trigger_type;  in mpi3mr_repost_diag_bufs()
3921 		memcpy(&prev_trigger_data, &trace_hdb->trigger_data,  in mpi3mr_repost_diag_bufs()
3922 		    sizeof(trace_hdb->trigger_data));  in mpi3mr_repost_diag_bufs()
3929 			trace_hdb->trigger_type = prev_trigger_type;  in mpi3mr_repost_diag_bufs()
3930 			memcpy(&trace_hdb->trigger_data, &prev_trigger_data,  in mpi3mr_repost_diag_bufs()
3933 			return -1;  in mpi3mr_repost_diag_bufs()
3938 		prev_trigger_type = fw_hdb->trigger_type;  in mpi3mr_repost_diag_bufs()
3939 		memcpy(&prev_trigger_data, &fw_hdb->trigger_data,  in mpi3mr_repost_diag_bufs()
3940 		    sizeof(fw_hdb->trigger_data));  in mpi3mr_repost_diag_bufs()
3947 			fw_hdb->trigger_type = prev_trigger_type;  in mpi3mr_repost_diag_bufs()
3948 			memcpy(&fw_hdb->trigger_data, &prev_trigger_data,  in mpi3mr_repost_diag_bufs()
3951 			return -1;  in mpi3mr_repost_diag_bufs()
3958  * mpi3mr_read_tsu_interval - Update time stamp interval
3973 	mrioc->ts_update_interval = MPI3MR_TSUPDATE_INTERVAL;  in mpi3mr_read_tsu_interval()
3977 		mrioc->ts_update_interval = (driver_pg1.time_stamp_update * 60);  in mpi3mr_read_tsu_interval()
3981  * mpi3mr_print_ioc_info - Display controller information
3996 	struct mpi3mr_compimg_ver *fwver = &mrioc->facts.fw_ver;  in mpi3mr_print_ioc_info()
3998 	switch (mrioc->facts.personality) {  in mpi3mr_print_ioc_info()
4013 	    fwver->gen_major, fwver->gen_minor, fwver->ph_major,  in mpi3mr_print_ioc_info()
4014 	    fwver->ph_minor, fwver->cust_id, fwver->build_num);  in mpi3mr_print_ioc_info()
4017 		if (mrioc->facts.protocol_flags &  in mpi3mr_print_ioc_info()
4020 				    sizeof(protocol) - bytes_written, "%s%s",  in mpi3mr_print_ioc_info()
4028 		if (mrioc->facts.protocol_flags &  in mpi3mr_print_ioc_info()
4031 				    sizeof(capabilities) - bytes_written, "%s%s",  in mpi3mr_print_ioc_info()
4042  * mpi3mr_cleanup_resources - Free PCI resources
4047  * Return: 0 on success and non-zero on failure.
4051 	struct pci_dev *pdev = mrioc->pdev;  in mpi3mr_cleanup_resources()
4055 	if (mrioc->sysif_regs) {  in mpi3mr_cleanup_resources()
4056 		iounmap((void __iomem *)mrioc->sysif_regs);  in mpi3mr_cleanup_resources()
4057 		mrioc->sysif_regs = NULL;  in mpi3mr_cleanup_resources()
4061 		if (mrioc->bars)  in mpi3mr_cleanup_resources()
4062 			pci_release_selected_regions(pdev, mrioc->bars);  in mpi3mr_cleanup_resources()
4068  * mpi3mr_setup_resources - Enable PCI resources
4071  * Enable PCI device memory, MSI-x registers and set DMA mask.
4073  * Return: 0 on success and non-zero on failure.
4077 	struct pci_dev *pdev = mrioc->pdev;  in mpi3mr_setup_resources()
4081 	u64 dma_mask = mrioc->dma_mask ? mrioc->dma_mask :  in mpi3mr_setup_resources()
4086 		retval = -ENODEV;  in mpi3mr_setup_resources()
4092 		ioc_err(mrioc, "Unable to find MSI-X Capabilities\n");  in mpi3mr_setup_resources()
4093 		retval = -ENODEV;  in mpi3mr_setup_resources()
4096 	mrioc->bars = pci_select_bars(pdev, IORESOURCE_MEM);  in mpi3mr_setup_resources()
4098 	if (pci_request_selected_regions(pdev, mrioc->bars,  in mpi3mr_setup_resources()
4099 	    mrioc->driver_name)) {  in mpi3mr_setup_resources()
4101 		retval = -ENODEV;  in mpi3mr_setup_resources()
4107 			mrioc->sysif_regs_phys = pci_resource_start(pdev, i);  in mpi3mr_setup_resources()
4109 			mrioc->sysif_regs =  in mpi3mr_setup_resources()
4110 			    ioremap(mrioc->sysif_regs_phys, memap_sz);  in mpi3mr_setup_resources()
4117 	retval = dma_set_mask_and_coherent(&pdev->dev, dma_mask);  in mpi3mr_setup_resources()
4122 			retval = dma_set_mask_and_coherent(&pdev->dev,  in mpi3mr_setup_resources()
4126 			mrioc->dma_mask = 0;  in mpi3mr_setup_resources()
4131 	mrioc->dma_mask = dma_mask;  in mpi3mr_setup_resources()
4133 	if (!mrioc->sysif_regs) {  in mpi3mr_setup_resources()
4136 		retval = -EINVAL;  in mpi3mr_setup_resources()
4141 	mrioc->msix_count = (message_control & 0x3FF) + 1;  in mpi3mr_setup_resources()
4145 	pci_set_drvdata(pdev, mrioc->shost);  in mpi3mr_setup_resources()
4149 	ioc_info(mrioc, "iomem(0x%016llx), mapped(0x%p), size(%d)\n",  in mpi3mr_setup_resources()
4150 	    (unsigned long long)mrioc->sysif_regs_phys,  in mpi3mr_setup_resources()
4151 	    mrioc->sysif_regs, memap_sz);  in mpi3mr_setup_resources()
4152 	ioc_info(mrioc, "Number of MSI-X vectors found in capabilities: (%d)\n",  in mpi3mr_setup_resources()
4153 	    mrioc->msix_count);  in mpi3mr_setup_resources()
4156 		mrioc->requested_poll_qcount = min_t(int, poll_queues,  in mpi3mr_setup_resources()
4157 				mrioc->msix_count - 2);  in mpi3mr_setup_resources()
4166  * mpi3mr_enable_events - Enable required events
4173  * Return: 0 on success and non-zero on failure.
4181 		mrioc->event_masks[i] = -1;  in mpi3mr_enable_events()
4207  * mpi3mr_init_ioc - Initialize the controller
4219  * Return: 0 on success and non-zero on failure.
4250 	mrioc->max_host_ios = mrioc->facts.max_reqs - MPI3MR_INTERNAL_CMDS_RESVD;  in mpi3mr_init_ioc()
4251 	mrioc->shost->max_sectors = mrioc->facts.max_data_length / 512;  in mpi3mr_init_ioc()
4252 	mrioc->num_io_throttle_group = mrioc->facts.max_io_throttle_group;  in mpi3mr_init_ioc()
4253 	atomic_set(&mrioc->pend_large_data_sz, 0);  in mpi3mr_init_ioc()
4256 		mrioc->max_host_ios = min_t(int, mrioc->max_host_ios,  in mpi3mr_init_ioc()
4259 	if (!(mrioc->facts.ioc_capabilities &  in mpi3mr_init_ioc()
4261 		mrioc->sas_transport_enabled = 1;  in mpi3mr_init_ioc()
4262 		mrioc->scsi_device_channel = 1;  in mpi3mr_init_ioc()
4263 		mrioc->shost->max_channel = 1;  in mpi3mr_init_ioc()
4264 		mrioc->shost->transportt = mpi3mr_transport_template;  in mpi3mr_init_ioc()
4267 	if (mrioc->facts.max_req_limit)  in mpi3mr_init_ioc()
4268 		mrioc->prevent_reply_qfull = true;  in mpi3mr_init_ioc()
4270 	if (mrioc->facts.ioc_capabilities &  in mpi3mr_init_ioc()
4272 		mrioc->seg_tb_support = true;  in mpi3mr_init_ioc()
4274 	mrioc->reply_sz = mrioc->facts.reply_sz;  in mpi3mr_init_ioc()
4298 	if (!mrioc->init_cmds.reply) {  in mpi3mr_init_ioc()
4308 	if (!mrioc->chain_sgl_list) {  in mpi3mr_init_ioc()
4332 		ioc_err(mrioc, "Failed to re-setup ISR, error %d\n",  in mpi3mr_init_ioc()
4344 	if (!mrioc->pel_seqnum_virt) {  in mpi3mr_init_ioc()
4346 		mrioc->pel_seqnum_sz = sizeof(struct mpi3_pel_seq);  in mpi3mr_init_ioc()
4347 		mrioc->pel_seqnum_virt = dma_alloc_coherent(&mrioc->pdev->dev,  in mpi3mr_init_ioc()
4348 		    mrioc->pel_seqnum_sz, &mrioc->pel_seqnum_dma,  in mpi3mr_init_ioc()
4350 		if (!mrioc->pel_seqnum_virt) {  in mpi3mr_init_ioc()
4351 			retval = -ENOMEM;  in mpi3mr_init_ioc()
4356 	if (!mrioc->throttle_groups && mrioc->num_io_throttle_group) {  in mpi3mr_init_ioc()
4359 		mrioc->throttle_groups = kcalloc(mrioc->num_io_throttle_group, sz, GFP_KERNEL);  in mpi3mr_init_ioc()
4360 		if (!mrioc->throttle_groups) {  in mpi3mr_init_ioc()
4361 			retval = -1;  in mpi3mr_init_ioc()
4389 	retval = -1;  in mpi3mr_init_ioc()
4394 	mrioc->unrecoverable = 1;  in mpi3mr_init_ioc()
4399  * mpi3mr_reinit_ioc - Re-Initialize the controller
4403  * This the controller re-initialization routine, executed from
4410  * Return: 0 on success and non-zero on failure.
4430 	mrioc->io_admin_reset_sync = 0;  in mpi3mr_reinit_ioc()
4431 	if (is_resume || mrioc->block_on_pci_err) {  in mpi3mr_reinit_ioc()
4483 	if (is_resume || mrioc->block_on_pci_err) {  in mpi3mr_reinit_ioc()
4487 			ioc_err(mrioc, "failed to re-setup ISR\n");  in mpi3mr_reinit_ioc()
4499 	if (!mrioc->pel_seqnum_virt) {  in mpi3mr_reinit_ioc()
4501 		mrioc->pel_seqnum_sz = sizeof(struct mpi3_pel_seq);  in mpi3mr_reinit_ioc()
4502 		mrioc->pel_seqnum_virt = dma_alloc_coherent(&mrioc->pdev->dev,  in mpi3mr_reinit_ioc()
4503 		    mrioc->pel_seqnum_sz, &mrioc->pel_seqnum_dma,  in mpi3mr_reinit_ioc()
4505 		if (!mrioc->pel_seqnum_virt) {  in mpi3mr_reinit_ioc()
4506 			retval = -ENOMEM;  in mpi3mr_reinit_ioc()
4511 	if (mrioc->shost->nr_hw_queues > mrioc->num_op_reply_q) {  in mpi3mr_reinit_ioc()
4514 		    mrioc->shost->nr_hw_queues, mrioc->num_op_reply_q);  in mpi3mr_reinit_ioc()
4515 		retval = -1;  in mpi3mr_reinit_ioc()
4526 	mrioc->device_refresh_on = 1;  in mpi3mr_reinit_ioc()
4537 		if (mrioc->init_cmds.state == MPI3MR_CMD_NOTUSED)  in mpi3mr_reinit_ioc()
4539 		if (!pci_device_is_present(mrioc->pdev))  in mpi3mr_reinit_ioc()
4540 			mrioc->unrecoverable = 1;  in mpi3mr_reinit_ioc()
4541 		if (mrioc->unrecoverable) {  in mpi3mr_reinit_ioc()
4542 			retval = -1;  in mpi3mr_reinit_ioc()
4545 		ioc_status = readl(&mrioc->sysif_regs->ioc_status);  in mpi3mr_reinit_ioc()
4549 			mrioc->init_cmds.is_waiting = 0;  in mpi3mr_reinit_ioc()
4550 			mrioc->init_cmds.callback = NULL;  in mpi3mr_reinit_ioc()
4551 			mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;  in mpi3mr_reinit_ioc()
4554 	} while (--pe_timeout);  in mpi3mr_reinit_ioc()
4560 		mrioc->init_cmds.is_waiting = 0;  in mpi3mr_reinit_ioc()
4561 		mrioc->init_cmds.callback = NULL;  in mpi3mr_reinit_ioc()
4562 		mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;  in mpi3mr_reinit_ioc()
4564 	} else if (mrioc->scan_failed) {  in mpi3mr_reinit_ioc()
4567 		    mrioc->scan_failed);  in mpi3mr_reinit_ioc()
4572 	    (is_resume)?"resume":"re-initialization");  in mpi3mr_reinit_ioc()
4578 		    (is_resume)?"resume":"re-initialization", retry);  in mpi3mr_reinit_ioc()
4582 	retval = -1;  in mpi3mr_reinit_ioc()
4585 	    (is_resume)?"resume":"re-initialization");  in mpi3mr_reinit_ioc()
4588 	mrioc->unrecoverable = 1;  in mpi3mr_reinit_ioc()
4593  * mpi3mr_memset_op_reply_q_buffers - memset the operational reply queue's
4602 	struct op_reply_qinfo *op_reply_q = mrioc->op_reply_qinfo + qidx;  in mpi3mr_memset_op_reply_q_buffers()
4604 	int i, size;  in mpi3mr_memset_op_reply_q_buffers()  local
4606 	if (!op_reply_q->q_segments)  in mpi3mr_memset_op_reply_q_buffers()
4609 	size = op_reply_q->segment_qd * mrioc->op_reply_desc_sz;  in mpi3mr_memset_op_reply_q_buffers()
4610 	segments = op_reply_q->q_segments;  in mpi3mr_memset_op_reply_q_buffers()
4611 	for (i = 0; i < op_reply_q->num_segments; i++)  in mpi3mr_memset_op_reply_q_buffers()
4612 		memset(segments[i].segment, 0, size);  in mpi3mr_memset_op_reply_q_buffers()
4616  * mpi3mr_memset_op_req_q_buffers - memset the operational request queue's
4625 	struct op_req_qinfo *op_req_q = mrioc->req_qinfo + qidx;  in mpi3mr_memset_op_req_q_buffers()
4627 	int i, size;  in mpi3mr_memset_op_req_q_buffers()  local
4629 	if (!op_req_q->q_segments)  in mpi3mr_memset_op_req_q_buffers()
4632 	size = op_req_q->segment_qd * mrioc->facts.op_req_sz;  in mpi3mr_memset_op_req_q_buffers()
4633 	segments = op_req_q->q_segments;  in mpi3mr_memset_op_req_q_buffers()
4634 	for (i = 0; i < op_req_q->num_segments; i++)  in mpi3mr_memset_op_req_q_buffers()
4635 		memset(segments[i].segment, 0, size);  in mpi3mr_memset_op_req_q_buffers()
4639  * mpi3mr_memset_buffers - memset memory for a controller
4653 	mrioc->change_count = 0;  in mpi3mr_memset_buffers()
4654 	mrioc->active_poll_qcount = 0;  in mpi3mr_memset_buffers()
4655 	mrioc->default_qcount = 0;  in mpi3mr_memset_buffers()
4656 	if (mrioc->admin_req_base)  in mpi3mr_memset_buffers()
4657 		memset(mrioc->admin_req_base, 0, mrioc->admin_req_q_sz);  in mpi3mr_memset_buffers()
4658 	if (mrioc->admin_reply_base)  in mpi3mr_memset_buffers()
4659 		memset(mrioc->admin_reply_base, 0, mrioc->admin_reply_q_sz);  in mpi3mr_memset_buffers()
4660 	atomic_set(&mrioc->admin_reply_q_in_use, 0);  in mpi3mr_memset_buffers()
4661 	atomic_set(&mrioc->admin_pend_isr, 0);  in mpi3mr_memset_buffers()
4663 	if (mrioc->init_cmds.reply) {  in mpi3mr_memset_buffers()
4664 		memset(mrioc->init_cmds.reply, 0, sizeof(*mrioc->init_cmds.reply));  in mpi3mr_memset_buffers()
4665 		memset(mrioc->bsg_cmds.reply, 0,  in mpi3mr_memset_buffers()
4666 		    sizeof(*mrioc->bsg_cmds.reply));  in mpi3mr_memset_buffers()
4667 		memset(mrioc->host_tm_cmds.reply, 0,  in mpi3mr_memset_buffers()
4668 		    sizeof(*mrioc->host_tm_cmds.reply));  in mpi3mr_memset_buffers()
4669 		memset(mrioc->pel_cmds.reply, 0,  in mpi3mr_memset_buffers()
4670 		    sizeof(*mrioc->pel_cmds.reply));  in mpi3mr_memset_buffers()
4671 		memset(mrioc->pel_abort_cmd.reply, 0,  in mpi3mr_memset_buffers()
4672 		    sizeof(*mrioc->pel_abort_cmd.reply));  in mpi3mr_memset_buffers()
4673 		memset(mrioc->transport_cmds.reply, 0,  in mpi3mr_memset_buffers()
4674 		    sizeof(*mrioc->transport_cmds.reply));  in mpi3mr_memset_buffers()
4676 			memset(mrioc->dev_rmhs_cmds[i].reply, 0,  in mpi3mr_memset_buffers()
4677 			    sizeof(*mrioc->dev_rmhs_cmds[i].reply));  in mpi3mr_memset_buffers()
4679 			memset(mrioc->evtack_cmds[i].reply, 0,  in mpi3mr_memset_buffers()
4680 			    sizeof(*mrioc->evtack_cmds[i].reply));  in mpi3mr_memset_buffers()
4681 		bitmap_clear(mrioc->removepend_bitmap, 0,  in mpi3mr_memset_buffers()
4682 			     mrioc->dev_handle_bitmap_bits);  in mpi3mr_memset_buffers()
4683 		bitmap_clear(mrioc->devrem_bitmap, 0, MPI3MR_NUM_DEVRMCMD);  in mpi3mr_memset_buffers()
4684 		bitmap_clear(mrioc->evtack_cmds_bitmap, 0,  in mpi3mr_memset_buffers()
4688 	for (i = 0; i < mrioc->num_queues; i++) {  in mpi3mr_memset_buffers()
4689 		mrioc->op_reply_qinfo[i].qid = 0;  in mpi3mr_memset_buffers()
4690 		mrioc->op_reply_qinfo[i].ci = 0;  in mpi3mr_memset_buffers()
4691 		mrioc->op_reply_qinfo[i].num_replies = 0;  in mpi3mr_memset_buffers()
4692 		mrioc->op_reply_qinfo[i].ephase = 0;  in mpi3mr_memset_buffers()
4693 		atomic_set(&mrioc->op_reply_qinfo[i].pend_ios, 0);  in mpi3mr_memset_buffers()
4694 		atomic_set(&mrioc->op_reply_qinfo[i].in_use, 0);  in mpi3mr_memset_buffers()
4697 		mrioc->req_qinfo[i].ci = 0;  in mpi3mr_memset_buffers()
4698 		mrioc->req_qinfo[i].pi = 0;  in mpi3mr_memset_buffers()
4699 		mrioc->req_qinfo[i].num_requests = 0;  in mpi3mr_memset_buffers()
4700 		mrioc->req_qinfo[i].qid = 0;  in mpi3mr_memset_buffers()
4701 		mrioc->req_qinfo[i].reply_qid = 0;  in mpi3mr_memset_buffers()
4702 		spin_lock_init(&mrioc->req_qinfo[i].q_lock);  in mpi3mr_memset_buffers()
4706 	atomic_set(&mrioc->pend_large_data_sz, 0);  in mpi3mr_memset_buffers()
4707 	if (mrioc->throttle_groups) {  in mpi3mr_memset_buffers()
4708 		tg = mrioc->throttle_groups;  in mpi3mr_memset_buffers()
4709 		for (i = 0; i < mrioc->num_io_throttle_group; i++, tg++) {  in mpi3mr_memset_buffers()
4710 			tg->id = 0;  in mpi3mr_memset_buffers()
4711 			tg->fw_qd = 0;  in mpi3mr_memset_buffers()
4712 			tg->modified_qd = 0;  in mpi3mr_memset_buffers()
4713 			tg->io_divert = 0;  in mpi3mr_memset_buffers()
4714 			tg->need_qd_reduction = 0;  in mpi3mr_memset_buffers()
4715 			tg->high = 0;  in mpi3mr_memset_buffers()
4716 			tg->low = 0;  in mpi3mr_memset_buffers()
4717 			tg->qd_reduction = 0;  in mpi3mr_memset_buffers()
4718 			atomic_set(&tg->pend_large_data_sz, 0);  in mpi3mr_memset_buffers()
4724  * mpi3mr_free_mem - Free memory allocated for a controller
4740 	if (mrioc->sense_buf_pool) {  in mpi3mr_free_mem()
4741 		if (mrioc->sense_buf)  in mpi3mr_free_mem()
4742 			dma_pool_free(mrioc->sense_buf_pool, mrioc->sense_buf,  in mpi3mr_free_mem()
4743 			    mrioc->sense_buf_dma);  in mpi3mr_free_mem()
4744 		dma_pool_destroy(mrioc->sense_buf_pool);  in mpi3mr_free_mem()
4745 		mrioc->sense_buf = NULL;  in mpi3mr_free_mem()
4746 		mrioc->sense_buf_pool = NULL;  in mpi3mr_free_mem()
4748 	if (mrioc->sense_buf_q_pool) {  in mpi3mr_free_mem()
4749 		if (mrioc->sense_buf_q)  in mpi3mr_free_mem()
4750 			dma_pool_free(mrioc->sense_buf_q_pool,  in mpi3mr_free_mem()
4751 			    mrioc->sense_buf_q, mrioc->sense_buf_q_dma);  in mpi3mr_free_mem()
4752 		dma_pool_destroy(mrioc->sense_buf_q_pool);  in mpi3mr_free_mem()
4753 		mrioc->sense_buf_q = NULL;  in mpi3mr_free_mem()
4754 		mrioc->sense_buf_q_pool = NULL;  in mpi3mr_free_mem()
4757 	if (mrioc->reply_buf_pool) {  in mpi3mr_free_mem()
4758 		if (mrioc->reply_buf)  in mpi3mr_free_mem()
4759 			dma_pool_free(mrioc->reply_buf_pool, mrioc->reply_buf,  in mpi3mr_free_mem()
4760 			    mrioc->reply_buf_dma);  in mpi3mr_free_mem()
4761 		dma_pool_destroy(mrioc->reply_buf_pool);  in mpi3mr_free_mem()
4762 		mrioc->reply_buf = NULL;  in mpi3mr_free_mem()
4763 		mrioc->reply_buf_pool = NULL;  in mpi3mr_free_mem()
4765 	if (mrioc->reply_free_q_pool) {  in mpi3mr_free_mem()
4766 		if (mrioc->reply_free_q)  in mpi3mr_free_mem()
4767 			dma_pool_free(mrioc->reply_free_q_pool,  in mpi3mr_free_mem()
4768 			    mrioc->reply_free_q, mrioc->reply_free_q_dma);  in mpi3mr_free_mem()
4769 		dma_pool_destroy(mrioc->reply_free_q_pool);  in mpi3mr_free_mem()
4770 		mrioc->reply_free_q = NULL;  in mpi3mr_free_mem()
4771 		mrioc->reply_free_q_pool = NULL;  in mpi3mr_free_mem()
4774 	for (i = 0; i < mrioc->num_op_req_q; i++)  in mpi3mr_free_mem()
4777 	for (i = 0; i < mrioc->num_op_reply_q; i++)  in mpi3mr_free_mem()
4780 	for (i = 0; i < mrioc->intr_info_count; i++) {  in mpi3mr_free_mem()
4781 		intr_info = mrioc->intr_info + i;  in mpi3mr_free_mem()
4782 		intr_info->op_reply_q = NULL;  in mpi3mr_free_mem()
4785 	kfree(mrioc->req_qinfo);  in mpi3mr_free_mem()
4786 	mrioc->req_qinfo = NULL;  in mpi3mr_free_mem()
4787 	mrioc->num_op_req_q = 0;  in mpi3mr_free_mem()
4789 	kfree(mrioc->op_reply_qinfo);  in mpi3mr_free_mem()
4790 	mrioc->op_reply_qinfo = NULL;  in mpi3mr_free_mem()
4791 	mrioc->num_op_reply_q = 0;  in mpi3mr_free_mem()
4793 	kfree(mrioc->init_cmds.reply);  in mpi3mr_free_mem()
4794 	mrioc->init_cmds.reply = NULL;  in mpi3mr_free_mem()
4796 	kfree(mrioc->bsg_cmds.reply);  in mpi3mr_free_mem()
4797 	mrioc->bsg_cmds.reply = NULL;  in mpi3mr_free_mem()
4799 	kfree(mrioc->host_tm_cmds.reply);  in mpi3mr_free_mem()
4800 	mrioc->host_tm_cmds.reply = NULL;  in mpi3mr_free_mem()
4802 	kfree(mrioc->pel_cmds.reply);  in mpi3mr_free_mem()
4803 	mrioc->pel_cmds.reply = NULL;  in mpi3mr_free_mem()
4805 	kfree(mrioc->pel_abort_cmd.reply);  in mpi3mr_free_mem()
4806 	mrioc->pel_abort_cmd.reply = NULL;  in mpi3mr_free_mem()
4809 		kfree(mrioc->evtack_cmds[i].reply);  in mpi3mr_free_mem()
4810 		mrioc->evtack_cmds[i].reply = NULL;  in mpi3mr_free_mem()
4813 	bitmap_free(mrioc->removepend_bitmap);  in mpi3mr_free_mem()
4814 	mrioc->removepend_bitmap = NULL;  in mpi3mr_free_mem()
4816 	bitmap_free(mrioc->devrem_bitmap);  in mpi3mr_free_mem()
4817 	mrioc->devrem_bitmap = NULL;  in mpi3mr_free_mem()
4819 	bitmap_free(mrioc->evtack_cmds_bitmap);  in mpi3mr_free_mem()
4820 	mrioc->evtack_cmds_bitmap = NULL;  in mpi3mr_free_mem()
4822 	bitmap_free(mrioc->chain_bitmap);  in mpi3mr_free_mem()
4823 	mrioc->chain_bitmap = NULL;  in mpi3mr_free_mem()
4825 	kfree(mrioc->transport_cmds.reply);  in mpi3mr_free_mem()
4826 	mrioc->transport_cmds.reply = NULL;  in mpi3mr_free_mem()
4829 		kfree(mrioc->dev_rmhs_cmds[i].reply);  in mpi3mr_free_mem()
4830 		mrioc->dev_rmhs_cmds[i].reply = NULL;  in mpi3mr_free_mem()
4833 	if (mrioc->chain_buf_pool) {  in mpi3mr_free_mem()
4834 		for (i = 0; i < mrioc->chain_buf_count; i++) {  in mpi3mr_free_mem()
4835 			if (mrioc->chain_sgl_list[i].addr) {  in mpi3mr_free_mem()
4836 				dma_pool_free(mrioc->chain_buf_pool,  in mpi3mr_free_mem()
4837 				    mrioc->chain_sgl_list[i].addr,  in mpi3mr_free_mem()
4838 				    mrioc->chain_sgl_list[i].dma_addr);  in mpi3mr_free_mem()
4839 				mrioc->chain_sgl_list[i].addr = NULL;  in mpi3mr_free_mem()
4842 		dma_pool_destroy(mrioc->chain_buf_pool);  in mpi3mr_free_mem()
4843 		mrioc->chain_buf_pool = NULL;  in mpi3mr_free_mem()
4846 	kfree(mrioc->chain_sgl_list);  in mpi3mr_free_mem()
4847 	mrioc->chain_sgl_list = NULL;  in mpi3mr_free_mem()
4849 	if (mrioc->admin_reply_base) {  in mpi3mr_free_mem()
4850 		dma_free_coherent(&mrioc->pdev->dev, mrioc->admin_reply_q_sz,  in mpi3mr_free_mem()
4851 		    mrioc->admin_reply_base, mrioc->admin_reply_dma);  in mpi3mr_free_mem()
4852 		mrioc->admin_reply_base = NULL;  in mpi3mr_free_mem()
4854 	if (mrioc->admin_req_base) {  in mpi3mr_free_mem()
4855 		dma_free_coherent(&mrioc->pdev->dev, mrioc->admin_req_q_sz,  in mpi3mr_free_mem()
4856 		    mrioc->admin_req_base, mrioc->admin_req_dma);  in mpi3mr_free_mem()
4857 		mrioc->admin_req_base = NULL;  in mpi3mr_free_mem()
4860 	if (mrioc->pel_seqnum_virt) {  in mpi3mr_free_mem()
4861 		dma_free_coherent(&mrioc->pdev->dev, mrioc->pel_seqnum_sz,  in mpi3mr_free_mem()
4862 		    mrioc->pel_seqnum_virt, mrioc->pel_seqnum_dma);  in mpi3mr_free_mem()
4863 		mrioc->pel_seqnum_virt = NULL;  in mpi3mr_free_mem()
4867 		diag_buffer = &mrioc->diag_buffers[i];  in mpi3mr_free_mem()
4868 		if ((i == 0) && mrioc->seg_tb_support) {  in mpi3mr_free_mem()
4869 			if (mrioc->trace_buf_pool) {  in mpi3mr_free_mem()
4870 				for (j = 0; j < mrioc->num_tb_segs; j++) {  in mpi3mr_free_mem()
4871 					if (mrioc->trace_buf[j].segment) {  in mpi3mr_free_mem()
4872 						dma_pool_free(mrioc->trace_buf_pool,  in mpi3mr_free_mem()
4873 						    mrioc->trace_buf[j].segment,  in mpi3mr_free_mem()
4874 						    mrioc->trace_buf[j].segment_dma);  in mpi3mr_free_mem()
4875 						mrioc->trace_buf[j].segment = NULL;  in mpi3mr_free_mem()
4878 					mrioc->trace_buf[j].segment = NULL;  in mpi3mr_free_mem()
4880 				dma_pool_destroy(mrioc->trace_buf_pool);  in mpi3mr_free_mem()
4881 				mrioc->trace_buf_pool = NULL;  in mpi3mr_free_mem()
4884 			kfree(mrioc->trace_buf);  in mpi3mr_free_mem()
4885 			mrioc->trace_buf = NULL;  in mpi3mr_free_mem()
4886 			diag_buffer->size = sizeof(u64) * mrioc->num_tb_segs;  in mpi3mr_free_mem()
4888 		if (diag_buffer->addr) {  in mpi3mr_free_mem()
4889 			dma_free_coherent(&mrioc->pdev->dev,  in mpi3mr_free_mem()
4890 			    diag_buffer->size, diag_buffer->addr,  in mpi3mr_free_mem()
4891 			    diag_buffer->dma_addr);  in mpi3mr_free_mem()
4892 			diag_buffer->addr = NULL;  in mpi3mr_free_mem()
4893 			diag_buffer->size = 0;  in mpi3mr_free_mem()
4894 			diag_buffer->type = 0;  in mpi3mr_free_mem()
4895 			diag_buffer->status = 0;  in mpi3mr_free_mem()
4899 	kfree(mrioc->throttle_groups);  in mpi3mr_free_mem()
4900 	mrioc->throttle_groups = NULL;  in mpi3mr_free_mem()
4902 	kfree(mrioc->logdata_buf);  in mpi3mr_free_mem()
4903 	mrioc->logdata_buf = NULL;  in mpi3mr_free_mem()
4908  * mpi3mr_issue_ioc_shutdown - shutdown controller
4923 	if (mrioc->unrecoverable) {  in mpi3mr_issue_ioc_shutdown()
4928 	ioc_status = readl(&mrioc->sysif_regs->ioc_status);  in mpi3mr_issue_ioc_shutdown()
4935 	ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);  in mpi3mr_issue_ioc_shutdown()
4939 	writel(ioc_config, &mrioc->sysif_regs->ioc_configuration);  in mpi3mr_issue_ioc_shutdown()
4941 	if (mrioc->facts.shutdown_timeout)  in mpi3mr_issue_ioc_shutdown()
4942 		timeout = mrioc->facts.shutdown_timeout * 10;  in mpi3mr_issue_ioc_shutdown()
4945 		ioc_status = readl(&mrioc->sysif_regs->ioc_status);  in mpi3mr_issue_ioc_shutdown()
4952 	} while (--timeout);  in mpi3mr_issue_ioc_shutdown()
4954 	ioc_status = readl(&mrioc->sysif_regs->ioc_status);  in mpi3mr_issue_ioc_shutdown()
4955 	ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);  in mpi3mr_issue_ioc_shutdown()
4971  * mpi3mr_cleanup_ioc - Cleanup controller
4988 	if (!mrioc->unrecoverable && !mrioc->reset_in_progress &&  in mpi3mr_cleanup_ioc()
4989 	    !mrioc->pci_err_recovery &&  in mpi3mr_cleanup_ioc()
5002  * mpi3mr_drv_cmd_comp_reset - Flush a internal driver command
5014 	if (cmdptr->state & MPI3MR_CMD_PENDING) {  in mpi3mr_drv_cmd_comp_reset()
5015 		cmdptr->state |= MPI3MR_CMD_RESET;  in mpi3mr_drv_cmd_comp_reset()
5016 		cmdptr->state &= ~MPI3MR_CMD_PENDING;  in mpi3mr_drv_cmd_comp_reset()
5017 		if (cmdptr->is_waiting) {  in mpi3mr_drv_cmd_comp_reset()
5018 			complete(&cmdptr->done);  in mpi3mr_drv_cmd_comp_reset()
5019 			cmdptr->is_waiting = 0;  in mpi3mr_drv_cmd_comp_reset()
5020 		} else if (cmdptr->callback)  in mpi3mr_drv_cmd_comp_reset()
5021 			cmdptr->callback(mrioc, cmdptr);  in mpi3mr_drv_cmd_comp_reset()
5026  * mpi3mr_flush_drv_cmds - Flush internaldriver commands
5038 	cmdptr = &mrioc->init_cmds;  in mpi3mr_flush_drv_cmds()
5041 	cmdptr = &mrioc->cfg_cmds;  in mpi3mr_flush_drv_cmds()
5044 	cmdptr = &mrioc->bsg_cmds;  in mpi3mr_flush_drv_cmds()
5046 	cmdptr = &mrioc->host_tm_cmds;  in mpi3mr_flush_drv_cmds()
5050 		cmdptr = &mrioc->dev_rmhs_cmds[i];  in mpi3mr_flush_drv_cmds()
5055 		cmdptr = &mrioc->evtack_cmds[i];  in mpi3mr_flush_drv_cmds()
5059 	cmdptr = &mrioc->pel_cmds;  in mpi3mr_flush_drv_cmds()
5062 	cmdptr = &mrioc->pel_abort_cmd;  in mpi3mr_flush_drv_cmds()
5065 	cmdptr = &mrioc->transport_cmds;  in mpi3mr_flush_drv_cmds()
5070  * mpi3mr_pel_wait_post - Issue PEL Wait
5083 	mrioc->pel_abort_requested = false;  in mpi3mr_pel_wait_post()
5086 	drv_cmd->state = MPI3MR_CMD_PENDING;  in mpi3mr_pel_wait_post()
5087 	drv_cmd->is_waiting = 0;  in mpi3mr_pel_wait_post()
5088 	drv_cmd->callback = mpi3mr_pel_wait_complete;  in mpi3mr_pel_wait_post()
5089 	drv_cmd->ioc_status = 0;  in mpi3mr_pel_wait_post()
5090 	drv_cmd->ioc_loginfo = 0;  in mpi3mr_pel_wait_post()
5094 	pel_wait.starting_sequence_number = cpu_to_le32(mrioc->pel_newest_seqnum);  in mpi3mr_pel_wait_post()
5095 	pel_wait.locale = cpu_to_le16(mrioc->pel_locale);  in mpi3mr_pel_wait_post()
5096 	pel_wait.class = cpu_to_le16(mrioc->pel_class);  in mpi3mr_pel_wait_post()
5099 	    mrioc->pel_newest_seqnum, mrioc->pel_class, mrioc->pel_locale);  in mpi3mr_pel_wait_post()
5104 		drv_cmd->state = MPI3MR_CMD_NOTUSED;  in mpi3mr_pel_wait_post()
5105 		drv_cmd->callback = NULL;  in mpi3mr_pel_wait_post()
5106 		drv_cmd->retry_count = 0;  in mpi3mr_pel_wait_post()
5107 		mrioc->pel_enabled = false;  in mpi3mr_pel_wait_post()
5112  * mpi3mr_pel_get_seqnum_post - Issue PEL Get Sequence number
5119  * Return: 0 on success, non-zero on failure.
5129 	mrioc->pel_cmds.state = MPI3MR_CMD_PENDING;  in mpi3mr_pel_get_seqnum_post()
5130 	mrioc->pel_cmds.is_waiting = 0;  in mpi3mr_pel_get_seqnum_post()
5131 	mrioc->pel_cmds.ioc_status = 0;  in mpi3mr_pel_get_seqnum_post()
5132 	mrioc->pel_cmds.ioc_loginfo = 0;  in mpi3mr_pel_get_seqnum_post()
5133 	mrioc->pel_cmds.callback = mpi3mr_pel_get_seqnum_complete;  in mpi3mr_pel_get_seqnum_post()
5138 	    mrioc->pel_seqnum_sz, mrioc->pel_seqnum_dma);  in mpi3mr_pel_get_seqnum_post()
5144 			drv_cmd->state = MPI3MR_CMD_NOTUSED;  in mpi3mr_pel_get_seqnum_post()
5145 			drv_cmd->callback = NULL;  in mpi3mr_pel_get_seqnum_post()
5146 			drv_cmd->retry_count = 0;  in mpi3mr_pel_get_seqnum_post()
5148 		mrioc->pel_enabled = false;  in mpi3mr_pel_get_seqnum_post()
5155  * mpi3mr_pel_wait_complete - PELWait Completion callback
5174 	if (drv_cmd->state & MPI3MR_CMD_RESET)  in mpi3mr_pel_wait_complete()
5177 	ioc_status = drv_cmd->ioc_status & MPI3_IOCSTATUS_STATUS_MASK;  in mpi3mr_pel_wait_complete()
5180 			__func__, ioc_status, drv_cmd->ioc_loginfo);  in mpi3mr_pel_wait_complete()
5183 		    ioc_status, drv_cmd->ioc_loginfo);  in mpi3mr_pel_wait_complete()
5187 	if (drv_cmd->state & MPI3MR_CMD_REPLY_VALID)  in mpi3mr_pel_wait_complete()
5188 		pel_reply = (struct mpi3_pel_reply *)drv_cmd->reply;  in mpi3mr_pel_wait_complete()
5196 	pe_log_status = le16_to_cpu(pel_reply->pe_log_status);  in mpi3mr_pel_wait_complete()
5208 		if (drv_cmd->retry_count < MPI3MR_PEL_RETRY_COUNT) {  in mpi3mr_pel_wait_complete()
5209 			drv_cmd->retry_count++;  in mpi3mr_pel_wait_complete()
5211 			    drv_cmd->retry_count);  in mpi3mr_pel_wait_complete()
5217 		    drv_cmd->retry_count);  in mpi3mr_pel_wait_complete()
5221 	if (!mrioc->pel_abort_requested) {  in mpi3mr_pel_wait_complete()
5222 		mrioc->pel_cmds.retry_count = 0;  in mpi3mr_pel_wait_complete()
5223 		mpi3mr_pel_get_seqnum_post(mrioc, &mrioc->pel_cmds);  in mpi3mr_pel_wait_complete()
5228 	mrioc->pel_enabled = false;  in mpi3mr_pel_wait_complete()
5230 	drv_cmd->state = MPI3MR_CMD_NOTUSED;  in mpi3mr_pel_wait_complete()
5231 	drv_cmd->callback = NULL;  in mpi3mr_pel_wait_complete()
5232 	drv_cmd->retry_count = 0;  in mpi3mr_pel_wait_complete()
5236  * mpi3mr_pel_get_seqnum_complete - PELGetSeqNum Completion callback
5254 	pel_seqnum_virt = (struct mpi3_pel_seq *)mrioc->pel_seqnum_virt;  in mpi3mr_pel_get_seqnum_complete()
5256 	if (drv_cmd->state & MPI3MR_CMD_RESET)  in mpi3mr_pel_get_seqnum_complete()
5259 	ioc_status = drv_cmd->ioc_status & MPI3_IOCSTATUS_STATUS_MASK;  in mpi3mr_pel_get_seqnum_complete()
5263 		    ioc_status, drv_cmd->ioc_loginfo);  in mpi3mr_pel_get_seqnum_complete()
5267 	if (drv_cmd->state & MPI3MR_CMD_REPLY_VALID)  in mpi3mr_pel_get_seqnum_complete()
5268 		pel_reply = (struct mpi3_pel_reply *)drv_cmd->reply;  in mpi3mr_pel_get_seqnum_complete()
5275 	if (le16_to_cpu(pel_reply->pe_log_status) != MPI3_PEL_STATUS_SUCCESS) {  in mpi3mr_pel_get_seqnum_complete()
5278 		    le16_to_cpu(pel_reply->pe_log_status));  in mpi3mr_pel_get_seqnum_complete()
5283 		if (drv_cmd->retry_count < MPI3MR_PEL_RETRY_COUNT) {  in mpi3mr_pel_get_seqnum_complete()
5284 			drv_cmd->retry_count++;  in mpi3mr_pel_get_seqnum_complete()
5287 			    drv_cmd->retry_count);  in mpi3mr_pel_get_seqnum_complete()
5294 		    drv_cmd->retry_count);  in mpi3mr_pel_get_seqnum_complete()
5297 	mrioc->pel_newest_seqnum = le32_to_cpu(pel_seqnum_virt->newest) + 1;  in mpi3mr_pel_get_seqnum_complete()
5298 	drv_cmd->retry_count = 0;  in mpi3mr_pel_get_seqnum_complete()
5303 	mrioc->pel_enabled = false;  in mpi3mr_pel_get_seqnum_complete()
5305 	drv_cmd->state = MPI3MR_CMD_NOTUSED;  in mpi3mr_pel_get_seqnum_complete()
5306 	drv_cmd->callback = NULL;  in mpi3mr_pel_get_seqnum_complete()
5307 	drv_cmd->retry_count = 0;  in mpi3mr_pel_get_seqnum_complete()
5311  * mpi3mr_check_op_admin_proc -
5319  * Return: 0 on success, non-zero on failure.
5332 		if (atomic_read(&mrioc->admin_reply_q_in_use) == 1)  in mpi3mr_check_op_admin_proc()
5338 			for (i = 0; i < mrioc->num_queues; i++) {  in mpi3mr_check_op_admin_proc()
5339 				if (atomic_read(&mrioc->op_reply_qinfo[i].in_use) == 1) {  in mpi3mr_check_op_admin_proc()
5360  * mpi3mr_soft_reset_handler - Reset the controller
5378  * Return: 0 on success, non-zero on failure.
5391 	    mrioc->diagsave_timeout);  in mpi3mr_soft_reset_handler()
5392 	while (mrioc->diagsave_timeout)  in mpi3mr_soft_reset_handler()
5399 	if (!mutex_trylock(&mrioc->reset_mutex)) {  in mpi3mr_soft_reset_handler()
5405 		} while (mrioc->reset_in_progress == 1);  in mpi3mr_soft_reset_handler()
5408 		    mrioc->prev_reset_result,  in mpi3mr_soft_reset_handler()
5410 		return mrioc->prev_reset_result;  in mpi3mr_soft_reset_handler()
5415 	mrioc->device_refresh_on = 0;  in mpi3mr_soft_reset_handler()
5416 	mrioc->reset_in_progress = 1;  in mpi3mr_soft_reset_handler()
5417 	mrioc->stop_bsgs = 1;  in mpi3mr_soft_reset_handler()
5418 	mrioc->prev_reset_result = -1;  in mpi3mr_soft_reset_handler()
5430 			mrioc->event_masks[i] = -1;  in mpi3mr_soft_reset_handler()
5439 	mrioc->io_admin_reset_sync = 1;  in mpi3mr_soft_reset_handler()
5446 			trigger_data.fault = (readl(&mrioc->sysif_regs->fault) &  in mpi3mr_soft_reset_handler()
5450 				    readl(&mrioc->sysif_regs->host_diagnostic);  in mpi3mr_soft_reset_handler()
5455 			} while (--timeout);  in mpi3mr_soft_reset_handler()
5477 	if (mrioc->num_io_throttle_group !=  in mpi3mr_soft_reset_handler()
5478 	    mrioc->facts.max_io_throttle_group) {  in mpi3mr_soft_reset_handler()
5481 		    mrioc->num_io_throttle_group,  in mpi3mr_soft_reset_handler()
5482 		    mrioc->facts.max_io_throttle_group);  in mpi3mr_soft_reset_handler()
5483 		retval = -EPERM;  in mpi3mr_soft_reset_handler()
5489 	bitmap_clear(mrioc->devrem_bitmap, 0, MPI3MR_NUM_DEVRMCMD);  in mpi3mr_soft_reset_handler()
5490 	bitmap_clear(mrioc->removepend_bitmap, 0,  in mpi3mr_soft_reset_handler()
5491 		     mrioc->dev_handle_bitmap_bits);  in mpi3mr_soft_reset_handler()
5492 	bitmap_clear(mrioc->evtack_cmds_bitmap, 0, MPI3MR_NUM_EVTACKCMD);  in mpi3mr_soft_reset_handler()
5498 	if (mrioc->prepare_for_reset) {  in mpi3mr_soft_reset_handler()
5499 		mrioc->prepare_for_reset = 0;  in mpi3mr_soft_reset_handler()
5500 		mrioc->prepare_for_reset_timeout_counter = 0;  in mpi3mr_soft_reset_handler()
5504 	mrioc->fw_release_trigger_active = false;  in mpi3mr_soft_reset_handler()
5505 	mrioc->trace_release_trigger_active = false;  in mpi3mr_soft_reset_handler()
5506 	mrioc->snapdump_trigger_active = false;  in mpi3mr_soft_reset_handler()
5515 		    mrioc->name, reset_reason);  in mpi3mr_soft_reset_handler()
5522 		mrioc->diagsave_timeout = 0;  in mpi3mr_soft_reset_handler()
5523 		mrioc->reset_in_progress = 0;  in mpi3mr_soft_reset_handler()
5524 		mrioc->pel_abort_requested = 0;  in mpi3mr_soft_reset_handler()
5525 		if (mrioc->pel_enabled) {  in mpi3mr_soft_reset_handler()
5526 			mrioc->pel_cmds.retry_count = 0;  in mpi3mr_soft_reset_handler()
5527 			mpi3mr_pel_wait_post(mrioc, &mrioc->pel_cmds);  in mpi3mr_soft_reset_handler()
5530 		mrioc->device_refresh_on = 0;  in mpi3mr_soft_reset_handler()
5532 		mrioc->ts_update_counter = 0;  in mpi3mr_soft_reset_handler()
5533 		spin_lock_irqsave(&mrioc->watchdog_lock, flags);  in mpi3mr_soft_reset_handler()
5534 		if (mrioc->watchdog_work_q)  in mpi3mr_soft_reset_handler()
5535 			queue_delayed_work(mrioc->watchdog_work_q,  in mpi3mr_soft_reset_handler()
5536 			    &mrioc->watchdog_work,  in mpi3mr_soft_reset_handler()
5538 		spin_unlock_irqrestore(&mrioc->watchdog_lock, flags);  in mpi3mr_soft_reset_handler()
5539 		mrioc->stop_bsgs = 0;  in mpi3mr_soft_reset_handler()
5540 		if (mrioc->pel_enabled)  in mpi3mr_soft_reset_handler()
5545 		mrioc->device_refresh_on = 0;  in mpi3mr_soft_reset_handler()
5546 		mrioc->unrecoverable = 1;  in mpi3mr_soft_reset_handler()
5547 		mrioc->reset_in_progress = 0;  in mpi3mr_soft_reset_handler()
5548 		mrioc->stop_bsgs = 0;  in mpi3mr_soft_reset_handler()
5549 		retval = -1;  in mpi3mr_soft_reset_handler()
5552 	mrioc->prev_reset_result = retval;  in mpi3mr_soft_reset_handler()
5553 	mutex_unlock(&mrioc->reset_mutex);  in mpi3mr_soft_reset_handler()
5560  * mpi3mr_post_cfg_req - Issue config requests and wait
5574  * Return: 0 on success, non-zero on failure.
5581 	mutex_lock(&mrioc->cfg_cmds.mutex);  in mpi3mr_post_cfg_req()
5582 	if (mrioc->cfg_cmds.state & MPI3MR_CMD_PENDING) {  in mpi3mr_post_cfg_req()
5583 		retval = -1;  in mpi3mr_post_cfg_req()
5585 		mutex_unlock(&mrioc->cfg_cmds.mutex);  in mpi3mr_post_cfg_req()
5588 	mrioc->cfg_cmds.state = MPI3MR_CMD_PENDING;  in mpi3mr_post_cfg_req()
5589 	mrioc->cfg_cmds.is_waiting = 1;  in mpi3mr_post_cfg_req()
5590 	mrioc->cfg_cmds.callback = NULL;  in mpi3mr_post_cfg_req()
5591 	mrioc->cfg_cmds.ioc_status = 0;  in mpi3mr_post_cfg_req()
5592 	mrioc->cfg_cmds.ioc_loginfo = 0;  in mpi3mr_post_cfg_req()
5594 	cfg_req->host_tag = cpu_to_le16(MPI3MR_HOSTTAG_CFG_CMDS);  in mpi3mr_post_cfg_req()
5595 	cfg_req->function = MPI3_FUNCTION_CONFIG;  in mpi3mr_post_cfg_req()
5597 	init_completion(&mrioc->cfg_cmds.done);  in mpi3mr_post_cfg_req()
5599 	if (mrioc->logging_level & MPI3_DEBUG_CFG_INFO)  in mpi3mr_post_cfg_req()
5607 	wait_for_completion_timeout(&mrioc->cfg_cmds.done, (timeout * HZ));  in mpi3mr_post_cfg_req()
5608 	if (!(mrioc->cfg_cmds.state & MPI3MR_CMD_COMPLETE)) {  in mpi3mr_post_cfg_req()
5612 		retval = -1;  in mpi3mr_post_cfg_req()
5615 	*ioc_status = mrioc->cfg_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK;  in mpi3mr_post_cfg_req()
5619 		    *ioc_status, mrioc->cfg_cmds.ioc_loginfo);  in mpi3mr_post_cfg_req()
5622 	mrioc->cfg_cmds.state = MPI3MR_CMD_NOTUSED;  in mpi3mr_post_cfg_req()
5623 	mutex_unlock(&mrioc->cfg_cmds.mutex);  in mpi3mr_post_cfg_req()
5630  * mpi3mr_process_cfg_req - config page request processor
5637  * @cfg_buf_sz: Size of the memory to get config page or header
5650  * This allocates dmaable memory based on the size of the config
5654  * the cfg_buf and size of the data has to be mentioned in the
5659  * into the cfg_buf limited to a minimum of actual page size and
5663  * Return: 0 on success, non-zero on failure.
5671 	int retval = -1;  in mpi3mr_process_cfg_req()
5677 	if (cfg_req->action == MPI3_CONFIG_ACTION_PAGE_HEADER)  in mpi3mr_process_cfg_req()
5678 		mem_desc.size = sizeof(struct mpi3_config_page_header);  in mpi3mr_process_cfg_req()
5682 			    cfg_req->action, cfg_req->page_type,  in mpi3mr_process_cfg_req()
5683 			    cfg_req->page_number);  in mpi3mr_process_cfg_req()
5686 		switch (cfg_hdr->page_attribute & MPI3_CONFIG_PAGEATTR_MASK) {  in mpi3mr_process_cfg_req()
5688 			if (cfg_req->action  in mpi3mr_process_cfg_req()
5693 			if ((cfg_req->action ==  in mpi3mr_process_cfg_req()
5695 			    (cfg_req->action ==  in mpi3mr_process_cfg_req()
5706 			    cfg_req->action, cfg_req->page_type,  in mpi3mr_process_cfg_req()
5707 			    cfg_req->page_number, cfg_hdr->page_attribute);  in mpi3mr_process_cfg_req()
5710 		mem_desc.size = le16_to_cpu(cfg_hdr->page_length) * 4;  in mpi3mr_process_cfg_req()
5711 		cfg_req->page_length = cfg_hdr->page_length;  in mpi3mr_process_cfg_req()
5712 		cfg_req->page_version = cfg_hdr->page_version;  in mpi3mr_process_cfg_req()
5715 	mem_desc.addr = dma_alloc_coherent(&mrioc->pdev->dev,  in mpi3mr_process_cfg_req()
5716 		mem_desc.size, &mem_desc.dma_addr, GFP_KERNEL);  in mpi3mr_process_cfg_req()
5721 	mpi3mr_add_sg_single(&cfg_req->sgl, sgl_flags, mem_desc.size,  in mpi3mr_process_cfg_req()
5724 	if ((cfg_req->action == MPI3_CONFIG_ACTION_WRITE_PERSISTENT) ||  in mpi3mr_process_cfg_req()
5725 	    (cfg_req->action == MPI3_CONFIG_ACTION_WRITE_CURRENT)) {  in mpi3mr_process_cfg_req()
5726 		memcpy(mem_desc.addr, cfg_buf, min_t(u16, mem_desc.size,  in mpi3mr_process_cfg_req()
5729 		if (mrioc->logging_level & MPI3_DEBUG_CFG_INFO)  in mpi3mr_process_cfg_req()
5730 			dprint_dump(mem_desc.addr, mem_desc.size, "cfg_buf");  in mpi3mr_process_cfg_req()
5738 	    (cfg_req->action != MPI3_CONFIG_ACTION_WRITE_PERSISTENT) &&  in mpi3mr_process_cfg_req()
5739 	    (cfg_req->action != MPI3_CONFIG_ACTION_WRITE_CURRENT)) {  in mpi3mr_process_cfg_req()
5740 		memcpy(cfg_buf, mem_desc.addr, min_t(u16, mem_desc.size,  in mpi3mr_process_cfg_req()
5743 		if (mrioc->logging_level & MPI3_DEBUG_CFG_INFO)  in mpi3mr_process_cfg_req()
5744 			dprint_dump(mem_desc.addr, mem_desc.size, "cfg_buf");  in mpi3mr_process_cfg_req()
5749 		dma_free_coherent(&mrioc->pdev->dev, mem_desc.size,  in mpi3mr_process_cfg_req()
5758  * mpi3mr_cfg_get_dev_pg0 - Read current device page0
5762  * @pg_sz: Size of the memory allocated to the page pointer
5772  * Return: 0 on success, non-zero on failure.
5812 	return -1;  in mpi3mr_cfg_get_dev_pg0()
5817  * mpi3mr_cfg_get_sas_phy_pg0 - Read current SAS Phy page0
5821  * @pg_sz: Size of the memory allocated to the page pointer
5831  * Return: 0 on success, non-zero on failure.
5872 	return -1;  in mpi3mr_cfg_get_sas_phy_pg0()
5876  * mpi3mr_cfg_get_sas_phy_pg1 - Read current SAS Phy page1
5880  * @pg_sz: Size of the memory allocated to the page pointer
5890  * Return: 0 on success, non-zero on failure.
5931 	return -1;  in mpi3mr_cfg_get_sas_phy_pg1()
5936  * mpi3mr_cfg_get_sas_exp_pg0 - Read current SAS Expander page0
5940  * @pg_sz: Size of the memory allocated to the page pointer
5950  * Return: 0 on success, non-zero on failure.
5992 	return -1;  in mpi3mr_cfg_get_sas_exp_pg0()
5996  * mpi3mr_cfg_get_sas_exp_pg1 - Read current SAS Expander page1
6000  * @pg_sz: Size of the memory allocated to the page pointer
6010  * Return: 0 on success, non-zero on failure.
6052 	return -1;  in mpi3mr_cfg_get_sas_exp_pg1()
6056  * mpi3mr_cfg_get_enclosure_pg0 - Read current Enclosure page0
6060  * @pg_sz: Size of the memory allocated to the page pointer
6070  * Return: 0 on success, non-zero on failure.
6111 	return -1;  in mpi3mr_cfg_get_enclosure_pg0()
6116  * mpi3mr_cfg_get_sas_io_unit_pg0 - Read current SASIOUnit page0
6119  * @pg_sz: Size of the memory allocated to the page pointer
6125  * Return: 0 on success, non-zero on failure.
6168 	return -1;  in mpi3mr_cfg_get_sas_io_unit_pg0()
6172  * mpi3mr_cfg_get_sas_io_unit_pg1 - Read current SASIOUnit page1
6175  * @pg_sz: Size of the memory allocated to the page pointer
6181  * Return: 0 on success, non-zero on failure.
6224 	return -1;  in mpi3mr_cfg_get_sas_io_unit_pg1()
6228  * mpi3mr_cfg_set_sas_io_unit_pg1 - Write SASIOUnit page1
6231  * @pg_sz: Size of the memory allocated to the page pointer
6238  * Return: 0 on success, non-zero on failure.
6293 	return -1;  in mpi3mr_cfg_set_sas_io_unit_pg1()
6297  * mpi3mr_cfg_get_driver_pg1 - Read current Driver page1
6300  * @pg_sz: Size of the memory allocated to the page pointer
6306  * Return: 0 on success, non-zero on failure.
6349 	return -1;  in mpi3mr_cfg_get_driver_pg1()
6353  * mpi3mr_cfg_get_driver_pg2 - Read current driver page2
6356  * @pg_sz: Size of the memory allocated to the page pointer
6363  * Return: 0 on success, non-zero on failure.
6409 	return -1;  in mpi3mr_cfg_get_driver_pg2()