1 // SPDX-License-Identifier: GPL-2.0-or-later
5  * Copyright (C) 2017-2023 Broadcom Inc.
6  *  (mailto: mpi3mr-linuxdrv.pdl@broadcom.com)
11 #include <linux/io-64-nonatomic-lo-hi.h>
23 MODULE_PARM_DESC(poll_queues, "Number of queues for io_uring poll mode. (Range 1 - 126)");
46 	pi = op_req_q->pi;  in mpi3mr_check_req_qfull()
47 	ci = READ_ONCE(op_req_q->ci);  in mpi3mr_check_req_qfull()
48 	max_entries = op_req_q->num_requests;  in mpi3mr_check_req_qfull()
50 	if ((ci == (pi + 1)) || ((!ci) && (pi == (max_entries - 1))))  in mpi3mr_check_req_qfull()
60 	max_vectors = mrioc->intr_info_count;  in mpi3mr_sync_irqs()
63 		synchronize_irq(pci_irq_vector(mrioc->pdev, i));  in mpi3mr_sync_irqs()
68 	mrioc->intr_enabled = 0;  in mpi3mr_ioc_disable_intr()
74 	mrioc->intr_enabled = 1;  in mpi3mr_ioc_enable_intr()
83 	if (!mrioc->intr_info)  in mpi3mr_cleanup_isr()
86 	for (i = 0; i < mrioc->intr_info_count; i++)  in mpi3mr_cleanup_isr()
87 		free_irq(pci_irq_vector(mrioc->pdev, i),  in mpi3mr_cleanup_isr()
88 		    (mrioc->intr_info + i));  in mpi3mr_cleanup_isr()
90 	kfree(mrioc->intr_info);  in mpi3mr_cleanup_isr()
91 	mrioc->intr_info = NULL;  in mpi3mr_cleanup_isr()
92 	mrioc->intr_info_count = 0;  in mpi3mr_cleanup_isr()
93 	mrioc->is_intr_info_set = false;  in mpi3mr_cleanup_isr()
94 	pci_free_irq_vectors(mrioc->pdev);  in mpi3mr_cleanup_isr()
102 	sgel->flags = flags;  in mpi3mr_add_sg_single()
103 	sgel->length = cpu_to_le32(length);  in mpi3mr_add_sg_single()
104 	sgel->address = cpu_to_le64(dma_addr);  in mpi3mr_add_sg_single()
111 	mpi3mr_add_sg_single(paddr, sgl_flags, 0, -1);  in mpi3mr_build_zero_len_sge()
120 	if ((phys_addr < mrioc->reply_buf_dma) ||  in mpi3mr_get_reply_virt_addr()
121 	    (phys_addr > mrioc->reply_buf_dma_max_address))  in mpi3mr_get_reply_virt_addr()
124 	return mrioc->reply_buf + (phys_addr - mrioc->reply_buf_dma);  in mpi3mr_get_reply_virt_addr()
133 	return mrioc->sense_buf + (phys_addr - mrioc->sense_buf_dma);  in mpi3mr_get_sensebuf_virt_addr()
142 	spin_lock_irqsave(&mrioc->reply_free_queue_lock, flags);  in mpi3mr_repost_reply_buf()
143 	old_idx  =  mrioc->reply_free_queue_host_index;  in mpi3mr_repost_reply_buf()
144 	mrioc->reply_free_queue_host_index = (  in mpi3mr_repost_reply_buf()
145 	    (mrioc->reply_free_queue_host_index ==  in mpi3mr_repost_reply_buf()
146 	    (mrioc->reply_free_qsz - 1)) ? 0 :  in mpi3mr_repost_reply_buf()
147 	    (mrioc->reply_free_queue_host_index + 1));  in mpi3mr_repost_reply_buf()
148 	mrioc->reply_free_q[old_idx] = cpu_to_le64(reply_dma);  in mpi3mr_repost_reply_buf()
149 	writel(mrioc->reply_free_queue_host_index,  in mpi3mr_repost_reply_buf()
150 	    &mrioc->sysif_regs->reply_free_host_index);  in mpi3mr_repost_reply_buf()
151 	spin_unlock_irqrestore(&mrioc->reply_free_queue_lock, flags);  in mpi3mr_repost_reply_buf()
160 	spin_lock_irqsave(&mrioc->sbq_lock, flags);  in mpi3mr_repost_sense_buf()
161 	old_idx  =  mrioc->sbq_host_index;  in mpi3mr_repost_sense_buf()
162 	mrioc->sbq_host_index = ((mrioc->sbq_host_index ==  in mpi3mr_repost_sense_buf()
163 	    (mrioc->sense_buf_q_sz - 1)) ? 0 :  in mpi3mr_repost_sense_buf()
164 	    (mrioc->sbq_host_index + 1));  in mpi3mr_repost_sense_buf()
165 	mrioc->sense_buf_q[old_idx] = cpu_to_le64(sense_buf_dma);  in mpi3mr_repost_sense_buf()
166 	writel(mrioc->sbq_host_index,  in mpi3mr_repost_sense_buf()
167 	    &mrioc->sysif_regs->sense_buffer_free_host_index);  in mpi3mr_repost_sense_buf()
168 	spin_unlock_irqrestore(&mrioc->sbq_lock, flags);  in mpi3mr_repost_sense_buf()
177 	event = event_reply->event;  in mpi3mr_print_event_data()
198 		    (struct mpi3_device_page0 *)event_reply->event_data;  in mpi3mr_print_event_data()
200 		    event_data->dev_handle, event_data->device_form);  in mpi3mr_print_event_data()
206 		    (struct mpi3_device_page0 *)event_reply->event_data;  in mpi3mr_print_event_data()
208 		    event_data->dev_handle, event_data->device_form);  in mpi3mr_print_event_data()
214 		    (struct mpi3_event_data_device_status_change *)event_reply->event_data;  in mpi3mr_print_event_data()
216 		    event_data->dev_handle, event_data->reason_code);  in mpi3mr_print_event_data()
222 		    (struct mpi3_event_data_sas_discovery *)event_reply->event_data;  in mpi3mr_print_event_data()
224 		    (event_data->reason_code == MPI3_EVENT_SAS_DISC_RC_STARTED) ?  in mpi3mr_print_event_data()
226 		    le32_to_cpu(event_data->discovery_status));  in mpi3mr_print_event_data()
265 		    (struct mpi3_event_data_pcie_enumeration *)event_reply->event_data;  in mpi3mr_print_event_data()
267 		    (event_data->reason_code ==  in mpi3mr_print_event_data()
269 		if (event_data->enumeration_status)  in mpi3mr_print_event_data()
271 			    le32_to_cpu(event_data->enumeration_status));  in mpi3mr_print_event_data()
294 	mrioc->change_count = le16_to_cpu(event_reply->ioc_change_count);  in mpi3mr_handle_events()
307 		return &mrioc->init_cmds;  in mpi3mr_get_drv_cmd()
309 		return &mrioc->cfg_cmds;  in mpi3mr_get_drv_cmd()
311 		return &mrioc->bsg_cmds;  in mpi3mr_get_drv_cmd()
313 		return &mrioc->host_tm_cmds;  in mpi3mr_get_drv_cmd()
315 		return &mrioc->pel_abort_cmd;  in mpi3mr_get_drv_cmd()
317 		return &mrioc->pel_cmds;  in mpi3mr_get_drv_cmd()
319 		return &mrioc->transport_cmds;  in mpi3mr_get_drv_cmd()
321 		if (def_reply && def_reply->function ==  in mpi3mr_get_drv_cmd()
330 		idx = host_tag - MPI3MR_HOSTTAG_DEVRMCMD_MIN;  in mpi3mr_get_drv_cmd()
331 		return &mrioc->dev_rmhs_cmds[idx];  in mpi3mr_get_drv_cmd()
336 		idx = host_tag - MPI3MR_HOSTTAG_EVTACKCMD_MIN;  in mpi3mr_get_drv_cmd()
337 		return &mrioc->evtack_cmds[idx];  in mpi3mr_get_drv_cmd()
360 	reply_desc_type = le16_to_cpu(reply_desc->reply_flags) &  in mpi3mr_process_admin_reply_desc()
365 		host_tag = le16_to_cpu(status_desc->host_tag);  in mpi3mr_process_admin_reply_desc()
366 		ioc_status = le16_to_cpu(status_desc->ioc_status);  in mpi3mr_process_admin_reply_desc()
369 			ioc_loginfo = le32_to_cpu(status_desc->ioc_log_info);  in mpi3mr_process_admin_reply_desc()
375 		*reply_dma = le64_to_cpu(addr_desc->reply_frame_address);  in mpi3mr_process_admin_reply_desc()
379 		host_tag = le16_to_cpu(def_reply->host_tag);  in mpi3mr_process_admin_reply_desc()
380 		ioc_status = le16_to_cpu(def_reply->ioc_status);  in mpi3mr_process_admin_reply_desc()
383 			ioc_loginfo = le32_to_cpu(def_reply->ioc_log_info);  in mpi3mr_process_admin_reply_desc()
385 		if (def_reply->function == MPI3_FUNCTION_SCSI_IO) {  in mpi3mr_process_admin_reply_desc()
388 			    le64_to_cpu(scsi_reply->sense_data_buffer_address));  in mpi3mr_process_admin_reply_desc()
389 			sense_count = le32_to_cpu(scsi_reply->sense_count);  in mpi3mr_process_admin_reply_desc()
401 		host_tag = le16_to_cpu(success_desc->host_tag);  in mpi3mr_process_admin_reply_desc()
409 		if (cmdptr->state & MPI3MR_CMD_PENDING) {  in mpi3mr_process_admin_reply_desc()
410 			cmdptr->state |= MPI3MR_CMD_COMPLETE;  in mpi3mr_process_admin_reply_desc()
411 			cmdptr->ioc_loginfo = ioc_loginfo;  in mpi3mr_process_admin_reply_desc()
413 				cmdptr->ioc_status = ioc_status;  in mpi3mr_process_admin_reply_desc()
415 				cmdptr->ioc_status = masked_ioc_status;  in mpi3mr_process_admin_reply_desc()
416 			cmdptr->state &= ~MPI3MR_CMD_PENDING;  in mpi3mr_process_admin_reply_desc()
418 				cmdptr->state |= MPI3MR_CMD_REPLY_VALID;  in mpi3mr_process_admin_reply_desc()
419 				memcpy((u8 *)cmdptr->reply, (u8 *)def_reply,  in mpi3mr_process_admin_reply_desc()
420 				    mrioc->reply_sz);  in mpi3mr_process_admin_reply_desc()
422 			if (sense_buf && cmdptr->sensebuf) {  in mpi3mr_process_admin_reply_desc()
423 				cmdptr->is_sense = 1;  in mpi3mr_process_admin_reply_desc()
424 				memcpy(cmdptr->sensebuf, sense_buf,  in mpi3mr_process_admin_reply_desc()
427 			if (cmdptr->is_waiting) {  in mpi3mr_process_admin_reply_desc()
428 				complete(&cmdptr->done);  in mpi3mr_process_admin_reply_desc()
429 				cmdptr->is_waiting = 0;  in mpi3mr_process_admin_reply_desc()
430 			} else if (cmdptr->callback)  in mpi3mr_process_admin_reply_desc()
431 				cmdptr->callback(mrioc, cmdptr);  in mpi3mr_process_admin_reply_desc()
437 		    le64_to_cpu(scsi_reply->sense_data_buffer_address));  in mpi3mr_process_admin_reply_desc()
442 	u32 exp_phase = mrioc->admin_reply_ephase;  in mpi3mr_process_admin_reply_q()
443 	u32 admin_reply_ci = mrioc->admin_reply_ci;  in mpi3mr_process_admin_reply_q()
449 	if (!atomic_add_unless(&mrioc->admin_reply_q_in_use, 1, 1))  in mpi3mr_process_admin_reply_q()
452 	reply_desc = (struct mpi3_default_reply_descriptor *)mrioc->admin_reply_base +  in mpi3mr_process_admin_reply_q()
455 	if ((le16_to_cpu(reply_desc->reply_flags) &  in mpi3mr_process_admin_reply_q()
457 		atomic_dec(&mrioc->admin_reply_q_in_use);  in mpi3mr_process_admin_reply_q()
462 		if (mrioc->unrecoverable)  in mpi3mr_process_admin_reply_q()
465 		mrioc->admin_req_ci = le16_to_cpu(reply_desc->request_queue_ci);  in mpi3mr_process_admin_reply_q()
471 		if (++admin_reply_ci == mrioc->num_admin_replies) {  in mpi3mr_process_admin_reply_q()
476 		    (struct mpi3_default_reply_descriptor *)mrioc->admin_reply_base +  in mpi3mr_process_admin_reply_q()
478 		if ((le16_to_cpu(reply_desc->reply_flags) &  in mpi3mr_process_admin_reply_q()
483 			    &mrioc->sysif_regs->admin_reply_queue_ci);  in mpi3mr_process_admin_reply_q()
488 	writel(admin_reply_ci, &mrioc->sysif_regs->admin_reply_queue_ci);  in mpi3mr_process_admin_reply_q()
489 	mrioc->admin_reply_ci = admin_reply_ci;  in mpi3mr_process_admin_reply_q()
490 	mrioc->admin_reply_ephase = exp_phase;  in mpi3mr_process_admin_reply_q()
491 	atomic_dec(&mrioc->admin_reply_q_in_use);  in mpi3mr_process_admin_reply_q()
497  * mpi3mr_get_reply_desc - get reply descriptor frame corresponding to
508 	struct segments *segments = op_reply_q->q_segments;  in mpi3mr_get_reply_desc()
512 	    segments[reply_ci / op_reply_q->segment_qd].segment;  in mpi3mr_get_reply_desc()
514 	    (reply_ci % op_reply_q->segment_qd);  in mpi3mr_get_reply_desc()
519  * mpi3mr_process_op_reply_q - Operational reply queue handler
541 	reply_qidx = op_reply_q->qid - 1;  in mpi3mr_process_op_reply_q()
543 	if (!atomic_add_unless(&op_reply_q->in_use, 1, 1))  in mpi3mr_process_op_reply_q()
546 	exp_phase = op_reply_q->ephase;  in mpi3mr_process_op_reply_q()
547 	reply_ci = op_reply_q->ci;  in mpi3mr_process_op_reply_q()
550 	if ((le16_to_cpu(reply_desc->reply_flags) &  in mpi3mr_process_op_reply_q()
552 		atomic_dec(&op_reply_q->in_use);  in mpi3mr_process_op_reply_q()
557 		if (mrioc->unrecoverable)  in mpi3mr_process_op_reply_q()
560 		req_q_idx = le16_to_cpu(reply_desc->request_queue_id) - 1;  in mpi3mr_process_op_reply_q()
561 		op_req_q = &mrioc->req_qinfo[req_q_idx];  in mpi3mr_process_op_reply_q()
563 		WRITE_ONCE(op_req_q->ci, le16_to_cpu(reply_desc->request_queue_ci));  in mpi3mr_process_op_reply_q()
566 		atomic_dec(&op_reply_q->pend_ios);  in mpi3mr_process_op_reply_q()
572 		if (++reply_ci == op_reply_q->num_replies) {  in mpi3mr_process_op_reply_q()
579 		if ((le16_to_cpu(reply_desc->reply_flags) &  in mpi3mr_process_op_reply_q()
587 		if (num_op_reply > mrioc->max_host_ios) {  in mpi3mr_process_op_reply_q()
588 			op_reply_q->enable_irq_poll = true;  in mpi3mr_process_op_reply_q()
594 			    &mrioc->sysif_regs->oper_queue_indexes[reply_qidx].consumer_index);  in mpi3mr_process_op_reply_q()
595 			atomic_sub(threshold_comps, &op_reply_q->pend_ios);  in mpi3mr_process_op_reply_q()
601 	    &mrioc->sysif_regs->oper_queue_indexes[reply_qidx].consumer_index);  in mpi3mr_process_op_reply_q()
602 	op_reply_q->ci = reply_ci;  in mpi3mr_process_op_reply_q()
603 	op_reply_q->ephase = exp_phase;  in mpi3mr_process_op_reply_q()
604 	atomic_sub(threshold_comps, &op_reply_q->pend_ios);  in mpi3mr_process_op_reply_q()
605 	atomic_dec(&op_reply_q->in_use);  in mpi3mr_process_op_reply_q()
610  * mpi3mr_blk_mq_poll - Operational reply queue handler
626 	mrioc = (struct mpi3mr_ioc *)shost->hostdata;  in mpi3mr_blk_mq_poll()
628 	if ((mrioc->reset_in_progress || mrioc->prepare_for_reset ||  in mpi3mr_blk_mq_poll()
629 	    mrioc->unrecoverable || mrioc->pci_err_recovery))  in mpi3mr_blk_mq_poll()
633 			&mrioc->op_reply_qinfo[queue_num]);  in mpi3mr_blk_mq_poll()
648 	mrioc = intr_info->mrioc;  in mpi3mr_isr_primary()
650 	if (!mrioc->intr_enabled)  in mpi3mr_isr_primary()
653 	midx = intr_info->msix_index;  in mpi3mr_isr_primary()
657 	if (intr_info->op_reply_q)  in mpi3mr_isr_primary()
659 		    intr_info->op_reply_q);  in mpi3mr_isr_primary()
684 	if (!intr_info->op_reply_q)  in mpi3mr_isr()
687 	if (!intr_info->op_reply_q->enable_irq_poll ||  in mpi3mr_isr()
688 	    !atomic_read(&intr_info->op_reply_q->pend_ios))  in mpi3mr_isr()
691 	disable_irq_nosync(intr_info->os_irq);  in mpi3mr_isr()
697  * mpi3mr_isr_poll - Reply queue polling routine
713 	if (!intr_info || !intr_info->op_reply_q)  in mpi3mr_isr_poll()
716 	mrioc = intr_info->mrioc;  in mpi3mr_isr_poll()
717 	midx = intr_info->msix_index;  in mpi3mr_isr_poll()
721 		if (!mrioc->intr_enabled || mrioc->unrecoverable)  in mpi3mr_isr_poll()
726 		if (intr_info->op_reply_q)  in mpi3mr_isr_poll()
729 				intr_info->op_reply_q);  in mpi3mr_isr_poll()
733 	} while (atomic_read(&intr_info->op_reply_q->pend_ios) &&  in mpi3mr_isr_poll()
734 	    (num_op_reply < mrioc->max_host_ios));  in mpi3mr_isr_poll()
736 	intr_info->op_reply_q->enable_irq_poll = false;  in mpi3mr_isr_poll()
737 	enable_irq(intr_info->os_irq);  in mpi3mr_isr_poll()
745  * mpi3mr_request_irq - Request IRQ and register ISR
755 	struct pci_dev *pdev = mrioc->pdev;  in mpi3mr_request_irq()
756 	struct mpi3mr_intr_info *intr_info = mrioc->intr_info + index;  in mpi3mr_request_irq()
759 	intr_info->mrioc = mrioc;  in mpi3mr_request_irq()
760 	intr_info->msix_index = index;  in mpi3mr_request_irq()
761 	intr_info->op_reply_q = NULL;  in mpi3mr_request_irq()
763 	snprintf(intr_info->name, MPI3MR_NAME_LENGTH, "%s%d-msix%d",  in mpi3mr_request_irq()
764 	    mrioc->driver_name, mrioc->id, index);  in mpi3mr_request_irq()
768 	    mpi3mr_isr_poll, IRQF_SHARED, intr_info->name, intr_info);  in mpi3mr_request_irq()
771 	    NULL, IRQF_SHARED, intr_info->name, intr_info);  in mpi3mr_request_irq()
775 		    intr_info->name, pci_irq_vector(pdev, index));  in mpi3mr_request_irq()
779 	intr_info->os_irq = pci_irq_vector(pdev, index);  in mpi3mr_request_irq()
785 	if (!mrioc->requested_poll_qcount)  in mpi3mr_calc_poll_queues()
790 		(mrioc->requested_poll_qcount < max_vectors - 2)) {  in mpi3mr_calc_poll_queues()
793 		    mrioc->requested_poll_qcount, max_vectors);  in mpi3mr_calc_poll_queues()
797 		    mrioc->requested_poll_qcount, max_vectors);  in mpi3mr_calc_poll_queues()
798 		mrioc->requested_poll_qcount = 0;  in mpi3mr_calc_poll_queues()
803  * mpi3mr_setup_isr - Setup ISR for the controller
819 	if (mrioc->is_intr_info_set)  in mpi3mr_setup_isr()
826 		retval = pci_alloc_irq_vectors(mrioc->pdev,  in mpi3mr_setup_isr()
835 		    min_t(int, mrioc->cpu_count + 1 +  in mpi3mr_setup_isr()
836 			mrioc->requested_poll_qcount, mrioc->msix_count);  in mpi3mr_setup_isr()
841 		    "MSI-X vectors supported: %d, no of cores: %d,",  in mpi3mr_setup_isr()
842 		    mrioc->msix_count, mrioc->cpu_count);  in mpi3mr_setup_isr()
844 		    "MSI-x vectors requested: %d poll_queues %d\n",  in mpi3mr_setup_isr()
845 		    max_vectors, mrioc->requested_poll_qcount);  in mpi3mr_setup_isr()
847 		desc.post_vectors = mrioc->requested_poll_qcount;  in mpi3mr_setup_isr()
851 		retval = pci_alloc_irq_vectors_affinity(mrioc->pdev,  in mpi3mr_setup_isr()
862 		 * If only one MSI-x is allocated, then MSI-x 0 will be shared  in mpi3mr_setup_isr()
866 			mrioc->op_reply_q_offset = 0;  in mpi3mr_setup_isr()
874 		mrioc->op_reply_q_offset = (max_vectors > 1) ? 1 : 0;  in mpi3mr_setup_isr()
880 	mrioc->intr_info = kzalloc(sizeof(struct mpi3mr_intr_info) * max_vectors,  in mpi3mr_setup_isr()
882 	if (!mrioc->intr_info) {  in mpi3mr_setup_isr()
883 		retval = -ENOMEM;  in mpi3mr_setup_isr()
884 		pci_free_irq_vectors(mrioc->pdev);  in mpi3mr_setup_isr()
890 			mrioc->intr_info_count = i;  in mpi3mr_setup_isr()
895 		mrioc->is_intr_info_set = true;  in mpi3mr_setup_isr()
896 	mrioc->intr_info_count = max_vectors;  in mpi3mr_setup_isr()
968 		"get package version timeout"
986  * mpi3mr_reset_rc_name - get reset reason code name
1013 	{ MPI3_SYSIF_HOST_DIAG_RESET_ACTION_DIAG_FAULT, "diag fault" },
1017  * mpi3mr_reset_type_name - get reset type name
1039  * mpi3mr_is_fault_recoverable - Read fault code and decide
1048 	fault = (readl(&mrioc->sysif_regs->fault) &  in mpi3mr_is_fault_recoverable()
1069  * mpi3mr_print_fault_info - Display fault information
1081 	ioc_status = readl(&mrioc->sysif_regs->ioc_status);  in mpi3mr_print_fault_info()
1084 		code = readl(&mrioc->sysif_regs->fault);  in mpi3mr_print_fault_info()
1085 		code1 = readl(&mrioc->sysif_regs->fault_info[0]);  in mpi3mr_print_fault_info()
1086 		code2 = readl(&mrioc->sysif_regs->fault_info[1]);  in mpi3mr_print_fault_info()
1087 		code3 = readl(&mrioc->sysif_regs->fault_info[2]);  in mpi3mr_print_fault_info()
1096  * mpi3mr_get_iocstate - Get IOC State
1109 	ioc_status = readl(&mrioc->sysif_regs->ioc_status);  in mpi3mr_get_iocstate()
1110 	ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);  in mpi3mr_get_iocstate()
1112 	if (mrioc->unrecoverable)  in mpi3mr_get_iocstate()
1131  * mpi3mr_free_ioctl_dma_memory - free memory for ioctl dma
1143 	if (!mrioc->ioctl_dma_pool)  in mpi3mr_free_ioctl_dma_memory()
1147 		mem_desc = &mrioc->ioctl_sge[i];  in mpi3mr_free_ioctl_dma_memory()
1148 		if (mem_desc->addr) {  in mpi3mr_free_ioctl_dma_memory()
1149 			dma_pool_free(mrioc->ioctl_dma_pool,  in mpi3mr_free_ioctl_dma_memory()
1150 				      mem_desc->addr,  in mpi3mr_free_ioctl_dma_memory()
1151 				      mem_desc->dma_addr);  in mpi3mr_free_ioctl_dma_memory()
1152 			mem_desc->addr = NULL;  in mpi3mr_free_ioctl_dma_memory()
1155 	dma_pool_destroy(mrioc->ioctl_dma_pool);  in mpi3mr_free_ioctl_dma_memory()
1156 	mrioc->ioctl_dma_pool = NULL;  in mpi3mr_free_ioctl_dma_memory()
1157 	mem_desc = &mrioc->ioctl_chain_sge;  in mpi3mr_free_ioctl_dma_memory()
1159 	if (mem_desc->addr) {  in mpi3mr_free_ioctl_dma_memory()
1160 		dma_free_coherent(&mrioc->pdev->dev, mem_desc->size,  in mpi3mr_free_ioctl_dma_memory()
1161 				  mem_desc->addr, mem_desc->dma_addr);  in mpi3mr_free_ioctl_dma_memory()
1162 		mem_desc->addr = NULL;  in mpi3mr_free_ioctl_dma_memory()
1164 	mem_desc = &mrioc->ioctl_resp_sge;  in mpi3mr_free_ioctl_dma_memory()
1165 	if (mem_desc->addr) {  in mpi3mr_free_ioctl_dma_memory()
1166 		dma_free_coherent(&mrioc->pdev->dev, mem_desc->size,  in mpi3mr_free_ioctl_dma_memory()
1167 				  mem_desc->addr, mem_desc->dma_addr);  in mpi3mr_free_ioctl_dma_memory()
1168 		mem_desc->addr = NULL;  in mpi3mr_free_ioctl_dma_memory()
1171 	mrioc->ioctl_sges_allocated = false;  in mpi3mr_free_ioctl_dma_memory()
1175  * mpi3mr_alloc_ioctl_dma_memory - Alloc memory for ioctl dma
1189 	mrioc->ioctl_dma_pool = dma_pool_create("ioctl dma pool",  in mpi3mr_alloc_ioctl_dma_memory()
1190 						&mrioc->pdev->dev,  in mpi3mr_alloc_ioctl_dma_memory()
1194 	if (!mrioc->ioctl_dma_pool) {  in mpi3mr_alloc_ioctl_dma_memory()
1200 		mem_desc = &mrioc->ioctl_sge[i];  in mpi3mr_alloc_ioctl_dma_memory()
1201 		mem_desc->size = MPI3MR_IOCTL_SGE_SIZE;  in mpi3mr_alloc_ioctl_dma_memory()
1202 		mem_desc->addr = dma_pool_zalloc(mrioc->ioctl_dma_pool,  in mpi3mr_alloc_ioctl_dma_memory()
1204 						 &mem_desc->dma_addr);  in mpi3mr_alloc_ioctl_dma_memory()
1205 		if (!mem_desc->addr)  in mpi3mr_alloc_ioctl_dma_memory()
1209 	mem_desc = &mrioc->ioctl_chain_sge;  in mpi3mr_alloc_ioctl_dma_memory()
1210 	mem_desc->size = MPI3MR_PAGE_SIZE_4K;  in mpi3mr_alloc_ioctl_dma_memory()
1211 	mem_desc->addr = dma_alloc_coherent(&mrioc->pdev->dev,  in mpi3mr_alloc_ioctl_dma_memory()
1212 					    mem_desc->size,  in mpi3mr_alloc_ioctl_dma_memory()
1213 					    &mem_desc->dma_addr,  in mpi3mr_alloc_ioctl_dma_memory()
1215 	if (!mem_desc->addr)  in mpi3mr_alloc_ioctl_dma_memory()
1218 	mem_desc = &mrioc->ioctl_resp_sge;  in mpi3mr_alloc_ioctl_dma_memory()
1219 	mem_desc->size = MPI3MR_PAGE_SIZE_4K;  in mpi3mr_alloc_ioctl_dma_memory()
1220 	mem_desc->addr = dma_alloc_coherent(&mrioc->pdev->dev,  in mpi3mr_alloc_ioctl_dma_memory()
1221 					    mem_desc->size,  in mpi3mr_alloc_ioctl_dma_memory()
1222 					    &mem_desc->dma_addr,  in mpi3mr_alloc_ioctl_dma_memory()
1224 	if (!mem_desc->addr)  in mpi3mr_alloc_ioctl_dma_memory()
1227 	mrioc->ioctl_sges_allocated = true;  in mpi3mr_alloc_ioctl_dma_memory()
1237  * mpi3mr_clear_reset_history - clear reset history
1249 	ioc_status = readl(&mrioc->sysif_regs->ioc_status);  in mpi3mr_clear_reset_history()
1251 		writel(ioc_status, &mrioc->sysif_regs->ioc_status);  in mpi3mr_clear_reset_history()
1255  * mpi3mr_issue_and_process_mur - Message unit Reset handler
1262  * Return: 0 on success, -1 on failure.
1268 	int retval = -1;  in mpi3mr_issue_and_process_mur()
1271 	if (mrioc->unrecoverable) {  in mpi3mr_issue_and_process_mur()
1278 			(mrioc->facts.ioc_num <<  in mpi3mr_issue_and_process_mur()
1280 	writel(scratch_pad0, &mrioc->sysif_regs->scratchpad[0]);  in mpi3mr_issue_and_process_mur()
1281 	ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);  in mpi3mr_issue_and_process_mur()
1283 	writel(ioc_config, &mrioc->sysif_regs->ioc_configuration);  in mpi3mr_issue_and_process_mur()
1287 		ioc_status = readl(&mrioc->sysif_regs->ioc_status);  in mpi3mr_issue_and_process_mur()
1297 	} while (--timeout);  in mpi3mr_issue_and_process_mur()
1299 	ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);  in mpi3mr_issue_and_process_mur()
1311  * mpi3mr_revalidate_factsdata - validate IOCFacts parameters
1316  * older values else return -EPERM
1323 	if (mrioc->facts.reply_sz > mrioc->reply_sz) {  in mpi3mr_revalidate_factsdata()
1326 		    mrioc->reply_sz, mrioc->facts.reply_sz);  in mpi3mr_revalidate_factsdata()
1327 		return -EPERM;  in mpi3mr_revalidate_factsdata()
1330 	if (mrioc->facts.max_op_reply_q < mrioc->num_op_reply_q) {  in mpi3mr_revalidate_factsdata()
1333 		    mrioc->num_op_reply_q,  in mpi3mr_revalidate_factsdata()
1334 		    mrioc->facts.max_op_reply_q);  in mpi3mr_revalidate_factsdata()
1335 		return -EPERM;  in mpi3mr_revalidate_factsdata()
1338 	if (mrioc->facts.max_op_req_q < mrioc->num_op_req_q) {  in mpi3mr_revalidate_factsdata()
1341 		    mrioc->num_op_req_q, mrioc->facts.max_op_req_q);  in mpi3mr_revalidate_factsdata()
1342 		return -EPERM;  in mpi3mr_revalidate_factsdata()
1345 	if (mrioc->shost->max_sectors != (mrioc->facts.max_data_length / 512))  in mpi3mr_revalidate_factsdata()
1349 			    mrioc->shost->max_sectors * 512, mrioc->facts.max_data_length);  in mpi3mr_revalidate_factsdata()
1351 	if ((mrioc->sas_transport_enabled) && (mrioc->facts.ioc_capabilities &  in mpi3mr_revalidate_factsdata()
1358 	if (mrioc->facts.max_devhandle > mrioc->dev_handle_bitmap_bits) {  in mpi3mr_revalidate_factsdata()
1359 		removepend_bitmap = bitmap_zalloc(mrioc->facts.max_devhandle,  in mpi3mr_revalidate_factsdata()
1364 				mrioc->dev_handle_bitmap_bits,  in mpi3mr_revalidate_factsdata()
1365 				mrioc->facts.max_devhandle);  in mpi3mr_revalidate_factsdata()
1366 			return -EPERM;  in mpi3mr_revalidate_factsdata()
1368 		bitmap_free(mrioc->removepend_bitmap);  in mpi3mr_revalidate_factsdata()
1369 		mrioc->removepend_bitmap = removepend_bitmap;  in mpi3mr_revalidate_factsdata()
1372 			 mrioc->dev_handle_bitmap_bits,  in mpi3mr_revalidate_factsdata()
1373 			 mrioc->facts.max_devhandle);  in mpi3mr_revalidate_factsdata()
1374 		mrioc->dev_handle_bitmap_bits = mrioc->facts.max_devhandle;  in mpi3mr_revalidate_factsdata()
1381  * mpi3mr_bring_ioc_ready - Bring controller to ready state
1400 	ioc_status = readl(&mrioc->sysif_regs->ioc_status);  in mpi3mr_bring_ioc_ready()
1401 	ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);  in mpi3mr_bring_ioc_ready()
1402 	base_info = lo_hi_readq(&mrioc->sysif_regs->ioc_information);  in mpi3mr_bring_ioc_ready()
1407 		mrioc->unrecoverable = 1;  in mpi3mr_bring_ioc_ready()
1412 	mrioc->ready_timeout =  in mpi3mr_bring_ioc_ready()
1416 	ioc_info(mrioc, "ready timeout: %d seconds\n", mrioc->ready_timeout);  in mpi3mr_bring_ioc_ready()
1422 	timeout = mrioc->ready_timeout * 10;  in mpi3mr_bring_ioc_ready()
1431 		if (!pci_device_is_present(mrioc->pdev)) {  in mpi3mr_bring_ioc_ready()
1432 			mrioc->unrecoverable = 1;  in mpi3mr_bring_ioc_ready()
1438 	} while (--timeout);  in mpi3mr_bring_ioc_ready()
1456 					readl(&mrioc->sysif_regs->host_diagnostic);  in mpi3mr_bring_ioc_ready()
1460 				if (!pci_device_is_present(mrioc->pdev)) {  in mpi3mr_bring_ioc_ready()
1461 					mrioc->unrecoverable = 1;  in mpi3mr_bring_ioc_ready()
1466 			} while (--timeout);  in mpi3mr_bring_ioc_ready()
1495 	ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);  in mpi3mr_bring_ioc_ready()
1497 	writel(ioc_config, &mrioc->sysif_regs->ioc_configuration);  in mpi3mr_bring_ioc_ready()
1502 	timeout = mrioc->ready_timeout * 10;  in mpi3mr_bring_ioc_ready()
1511 		ioc_status = readl(&mrioc->sysif_regs->ioc_status);  in mpi3mr_bring_ioc_ready()
1517 		if (!pci_device_is_present(mrioc->pdev)) {  in mpi3mr_bring_ioc_ready()
1518 			mrioc->unrecoverable = 1;  in mpi3mr_bring_ioc_ready()
1521 			retval = -1;  in mpi3mr_bring_ioc_ready()
1525 		elapsed_time_sec = jiffies_to_msecs(jiffies - start_time)/1000;  in mpi3mr_bring_ioc_ready()
1526 	} while (elapsed_time_sec < mrioc->ready_timeout);  in mpi3mr_bring_ioc_ready()
1529 	elapsed_time_sec = jiffies_to_msecs(jiffies - start_time)/1000;  in mpi3mr_bring_ioc_ready()
1530 	if ((retry < 2) && (elapsed_time_sec < (mrioc->ready_timeout - 60))) {  in mpi3mr_bring_ioc_ready()
1547  * mpi3mr_soft_reset_success - Check softreset is success or not
1566  * mpi3mr_diagfault_success - Check diag fault is success or not
1570  * Check whether the controller hit diag reset fault code.
1572  * Return: True when there is diag fault, false otherwise.
1581 	fault = readl(&mrioc->sysif_regs->fault) & MPI3_SYSIF_FAULT_CODE_MASK;  in mpi3mr_diagfault_success()
1590  * mpi3mr_set_diagsave - Set diag save bit for snapdump
1593  * Set diag save bit in IOC configuration register to enable
1602 	ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);  in mpi3mr_set_diagsave()
1604 	writel(ioc_config, &mrioc->sysif_regs->ioc_configuration);  in mpi3mr_set_diagsave()
1608  * mpi3mr_issue_reset - Issue reset to the controller
1618  * Return: 0 on success, non-zero on failure.
1623 	int retval = -1;  in mpi3mr_issue_reset()
1631 	if (mrioc->unrecoverable)  in mpi3mr_issue_reset()
1645 		    "Write magic sequence to unlock host diag register (retry=%d)\n",  in mpi3mr_issue_reset()
1652 			mrioc->unrecoverable = 1;  in mpi3mr_issue_reset()
1657 		    &mrioc->sysif_regs->write_sequence);  in mpi3mr_issue_reset()
1659 		    &mrioc->sysif_regs->write_sequence);  in mpi3mr_issue_reset()
1661 		    &mrioc->sysif_regs->write_sequence);  in mpi3mr_issue_reset()
1663 		    &mrioc->sysif_regs->write_sequence);  in mpi3mr_issue_reset()
1665 		    &mrioc->sysif_regs->write_sequence);  in mpi3mr_issue_reset()
1667 		    &mrioc->sysif_regs->write_sequence);  in mpi3mr_issue_reset()
1669 		    &mrioc->sysif_regs->write_sequence);  in mpi3mr_issue_reset()
1671 		host_diagnostic = readl(&mrioc->sysif_regs->host_diagnostic);  in mpi3mr_issue_reset()
1678 	    MPI3MR_RESET_REASON_OSTYPE_SHIFT) | (mrioc->facts.ioc_num <<  in mpi3mr_issue_reset()
1680 	writel(reset_reason, &mrioc->sysif_regs->scratchpad[0]);  in mpi3mr_issue_reset()
1682 	    &mrioc->sysif_regs->host_diagnostic);  in mpi3mr_issue_reset()
1686 			ioc_status = readl(&mrioc->sysif_regs->ioc_status);  in mpi3mr_issue_reset()
1688 			    readl(&mrioc->sysif_regs->ioc_configuration);  in mpi3mr_issue_reset()
1697 		} while (--timeout);  in mpi3mr_issue_reset()
1702 			ioc_status = readl(&mrioc->sysif_regs->ioc_status);  in mpi3mr_issue_reset()
1708 		} while (--timeout);  in mpi3mr_issue_reset()
1715 	    &mrioc->sysif_regs->write_sequence);  in mpi3mr_issue_reset()
1717 	ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);  in mpi3mr_issue_reset()
1718 	ioc_status = readl(&mrioc->sysif_regs->ioc_status);  in mpi3mr_issue_reset()
1724 		mrioc->unrecoverable = 1;  in mpi3mr_issue_reset()
1729  * mpi3mr_admin_request_post - Post request to admin queue
1739  * Return: 0 on success, non-zero on failure.
1749 	if (mrioc->unrecoverable) {  in mpi3mr_admin_request_post()
1751 		return -EFAULT;  in mpi3mr_admin_request_post()
1754 	spin_lock_irqsave(&mrioc->admin_req_lock, flags);  in mpi3mr_admin_request_post()
1755 	areq_pi = mrioc->admin_req_pi;  in mpi3mr_admin_request_post()
1756 	areq_ci = mrioc->admin_req_ci;  in mpi3mr_admin_request_post()
1757 	max_entries = mrioc->num_admin_req;  in mpi3mr_admin_request_post()
1759 	    (areq_pi == (max_entries - 1)))) {  in mpi3mr_admin_request_post()
1761 		retval = -EAGAIN;  in mpi3mr_admin_request_post()
1764 	if (!ignore_reset && mrioc->reset_in_progress) {  in mpi3mr_admin_request_post()
1766 		retval = -EAGAIN;  in mpi3mr_admin_request_post()
1769 	if (mrioc->pci_err_recovery) {  in mpi3mr_admin_request_post()
1771 		retval = -EAGAIN;  in mpi3mr_admin_request_post()
1775 	areq_entry = (u8 *)mrioc->admin_req_base +  in mpi3mr_admin_request_post()
1782 	mrioc->admin_req_pi = areq_pi;  in mpi3mr_admin_request_post()
1784 	writel(mrioc->admin_req_pi, &mrioc->sysif_regs->admin_request_queue_pi);  in mpi3mr_admin_request_post()
1787 	spin_unlock_irqrestore(&mrioc->admin_req_lock, flags);  in mpi3mr_admin_request_post()
1793  * mpi3mr_free_op_req_q_segments - free request memory segments
1807 	segments = mrioc->req_qinfo[q_idx].q_segments;  in mpi3mr_free_op_req_q_segments()
1811 	if (mrioc->enable_segqueue) {  in mpi3mr_free_op_req_q_segments()
1813 		if (mrioc->req_qinfo[q_idx].q_segment_list) {  in mpi3mr_free_op_req_q_segments()
1814 			dma_free_coherent(&mrioc->pdev->dev,  in mpi3mr_free_op_req_q_segments()
1816 			    mrioc->req_qinfo[q_idx].q_segment_list,  in mpi3mr_free_op_req_q_segments()
1817 			    mrioc->req_qinfo[q_idx].q_segment_list_dma);  in mpi3mr_free_op_req_q_segments()
1818 			mrioc->req_qinfo[q_idx].q_segment_list = NULL;  in mpi3mr_free_op_req_q_segments()
1821 		size = mrioc->req_qinfo[q_idx].segment_qd *  in mpi3mr_free_op_req_q_segments()
1822 		    mrioc->facts.op_req_sz;  in mpi3mr_free_op_req_q_segments()
1824 	for (j = 0; j < mrioc->req_qinfo[q_idx].num_segments; j++) {  in mpi3mr_free_op_req_q_segments()
1827 		dma_free_coherent(&mrioc->pdev->dev,  in mpi3mr_free_op_req_q_segments()
1831 	kfree(mrioc->req_qinfo[q_idx].q_segments);  in mpi3mr_free_op_req_q_segments()
1832 	mrioc->req_qinfo[q_idx].q_segments = NULL;  in mpi3mr_free_op_req_q_segments()
1833 	mrioc->req_qinfo[q_idx].qid = 0;  in mpi3mr_free_op_req_q_segments()
1837  * mpi3mr_free_op_reply_q_segments - free reply memory segments
1851 	segments = mrioc->op_reply_qinfo[q_idx].q_segments;  in mpi3mr_free_op_reply_q_segments()
1855 	if (mrioc->enable_segqueue) {  in mpi3mr_free_op_reply_q_segments()
1857 		if (mrioc->op_reply_qinfo[q_idx].q_segment_list) {  in mpi3mr_free_op_reply_q_segments()
1858 			dma_free_coherent(&mrioc->pdev->dev,  in mpi3mr_free_op_reply_q_segments()
1860 			    mrioc->op_reply_qinfo[q_idx].q_segment_list,  in mpi3mr_free_op_reply_q_segments()
1861 			    mrioc->op_reply_qinfo[q_idx].q_segment_list_dma);  in mpi3mr_free_op_reply_q_segments()
1862 			mrioc->op_reply_qinfo[q_idx].q_segment_list = NULL;  in mpi3mr_free_op_reply_q_segments()
1865 		size = mrioc->op_reply_qinfo[q_idx].segment_qd *  in mpi3mr_free_op_reply_q_segments()
1866 		    mrioc->op_reply_desc_sz;  in mpi3mr_free_op_reply_q_segments()
1868 	for (j = 0; j < mrioc->op_reply_qinfo[q_idx].num_segments; j++) {  in mpi3mr_free_op_reply_q_segments()
1871 		dma_free_coherent(&mrioc->pdev->dev,  in mpi3mr_free_op_reply_q_segments()
1876 	kfree(mrioc->op_reply_qinfo[q_idx].q_segments);  in mpi3mr_free_op_reply_q_segments()
1877 	mrioc->op_reply_qinfo[q_idx].q_segments = NULL;  in mpi3mr_free_op_reply_q_segments()
1878 	mrioc->op_reply_qinfo[q_idx].qid = 0;  in mpi3mr_free_op_reply_q_segments()
1882  * mpi3mr_delete_op_reply_q - delete operational reply queue
1889  * Return:  0 on success, non-zero on failure.
1894 	struct op_reply_qinfo *op_reply_q = mrioc->op_reply_qinfo + qidx;  in mpi3mr_delete_op_reply_q()
1898 	reply_qid = op_reply_q->qid;  in mpi3mr_delete_op_reply_q()
1900 	midx = REPLY_QUEUE_IDX_TO_MSIX_IDX(qidx, mrioc->op_reply_q_offset);  in mpi3mr_delete_op_reply_q()
1903 		retval = -1;  in mpi3mr_delete_op_reply_q()
1908 	(op_reply_q->qtype == MPI3MR_DEFAULT_QUEUE) ? mrioc->default_qcount-- :  in mpi3mr_delete_op_reply_q()
1909 	    mrioc->active_poll_qcount--;  in mpi3mr_delete_op_reply_q()
1912 	mutex_lock(&mrioc->init_cmds.mutex);  in mpi3mr_delete_op_reply_q()
1913 	if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {  in mpi3mr_delete_op_reply_q()
1914 		retval = -1;  in mpi3mr_delete_op_reply_q()
1916 		mutex_unlock(&mrioc->init_cmds.mutex);  in mpi3mr_delete_op_reply_q()
1919 	mrioc->init_cmds.state = MPI3MR_CMD_PENDING;  in mpi3mr_delete_op_reply_q()
1920 	mrioc->init_cmds.is_waiting = 1;  in mpi3mr_delete_op_reply_q()
1921 	mrioc->init_cmds.callback = NULL;  in mpi3mr_delete_op_reply_q()
1926 	init_completion(&mrioc->init_cmds.done);  in mpi3mr_delete_op_reply_q()
1933 	wait_for_completion_timeout(&mrioc->init_cmds.done,  in mpi3mr_delete_op_reply_q()
1935 	if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {  in mpi3mr_delete_op_reply_q()
1939 		retval = -1;  in mpi3mr_delete_op_reply_q()
1942 	if ((mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)  in mpi3mr_delete_op_reply_q()
1946 		    (mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),  in mpi3mr_delete_op_reply_q()
1947 		    mrioc->init_cmds.ioc_loginfo);  in mpi3mr_delete_op_reply_q()
1948 		retval = -1;  in mpi3mr_delete_op_reply_q()
1951 	mrioc->intr_info[midx].op_reply_q = NULL;  in mpi3mr_delete_op_reply_q()
1955 	mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;  in mpi3mr_delete_op_reply_q()
1956 	mutex_unlock(&mrioc->init_cmds.mutex);  in mpi3mr_delete_op_reply_q()
1963  * mpi3mr_alloc_op_reply_q_segments -Alloc segmented reply pool
1970  * Return: 0 on success, non-zero on failure.
1974 	struct op_reply_qinfo *op_reply_q = mrioc->op_reply_qinfo + qidx;  in mpi3mr_alloc_op_reply_q_segments()
1979 	if (mrioc->enable_segqueue) {  in mpi3mr_alloc_op_reply_q_segments()
1980 		op_reply_q->segment_qd =  in mpi3mr_alloc_op_reply_q_segments()
1981 		    MPI3MR_OP_REP_Q_SEG_SIZE / mrioc->op_reply_desc_sz;  in mpi3mr_alloc_op_reply_q_segments()
1985 		op_reply_q->q_segment_list = dma_alloc_coherent(&mrioc->pdev->dev,  in mpi3mr_alloc_op_reply_q_segments()
1986 		    MPI3MR_MAX_SEG_LIST_SIZE, &op_reply_q->q_segment_list_dma,  in mpi3mr_alloc_op_reply_q_segments()
1988 		if (!op_reply_q->q_segment_list)  in mpi3mr_alloc_op_reply_q_segments()
1989 			return -ENOMEM;  in mpi3mr_alloc_op_reply_q_segments()
1990 		q_segment_list_entry = (u64 *)op_reply_q->q_segment_list;  in mpi3mr_alloc_op_reply_q_segments()
1992 		op_reply_q->segment_qd = op_reply_q->num_replies;  in mpi3mr_alloc_op_reply_q_segments()
1993 		size = op_reply_q->num_replies * mrioc->op_reply_desc_sz;  in mpi3mr_alloc_op_reply_q_segments()
1996 	op_reply_q->num_segments = DIV_ROUND_UP(op_reply_q->num_replies,  in mpi3mr_alloc_op_reply_q_segments()
1997 	    op_reply_q->segment_qd);  in mpi3mr_alloc_op_reply_q_segments()
1999 	op_reply_q->q_segments = kcalloc(op_reply_q->num_segments,  in mpi3mr_alloc_op_reply_q_segments()
2001 	if (!op_reply_q->q_segments)  in mpi3mr_alloc_op_reply_q_segments()
2002 		return -ENOMEM;  in mpi3mr_alloc_op_reply_q_segments()
2004 	segments = op_reply_q->q_segments;  in mpi3mr_alloc_op_reply_q_segments()
2005 	for (i = 0; i < op_reply_q->num_segments; i++) {  in mpi3mr_alloc_op_reply_q_segments()
2007 		    dma_alloc_coherent(&mrioc->pdev->dev,  in mpi3mr_alloc_op_reply_q_segments()
2010 			return -ENOMEM;  in mpi3mr_alloc_op_reply_q_segments()
2011 		if (mrioc->enable_segqueue)  in mpi3mr_alloc_op_reply_q_segments()
2020  * mpi3mr_alloc_op_req_q_segments - Alloc segmented req pool.
2027  * Return: 0 on success, non-zero on failure.
2031 	struct op_req_qinfo *op_req_q = mrioc->req_qinfo + qidx;  in mpi3mr_alloc_op_req_q_segments()
2036 	if (mrioc->enable_segqueue) {  in mpi3mr_alloc_op_req_q_segments()
2037 		op_req_q->segment_qd =  in mpi3mr_alloc_op_req_q_segments()
2038 		    MPI3MR_OP_REQ_Q_SEG_SIZE / mrioc->facts.op_req_sz;  in mpi3mr_alloc_op_req_q_segments()
2042 		op_req_q->q_segment_list = dma_alloc_coherent(&mrioc->pdev->dev,  in mpi3mr_alloc_op_req_q_segments()
2043 		    MPI3MR_MAX_SEG_LIST_SIZE, &op_req_q->q_segment_list_dma,  in mpi3mr_alloc_op_req_q_segments()
2045 		if (!op_req_q->q_segment_list)  in mpi3mr_alloc_op_req_q_segments()
2046 			return -ENOMEM;  in mpi3mr_alloc_op_req_q_segments()
2047 		q_segment_list_entry = (u64 *)op_req_q->q_segment_list;  in mpi3mr_alloc_op_req_q_segments()
2050 		op_req_q->segment_qd = op_req_q->num_requests;  in mpi3mr_alloc_op_req_q_segments()
2051 		size = op_req_q->num_requests * mrioc->facts.op_req_sz;  in mpi3mr_alloc_op_req_q_segments()
2054 	op_req_q->num_segments = DIV_ROUND_UP(op_req_q->num_requests,  in mpi3mr_alloc_op_req_q_segments()
2055 	    op_req_q->segment_qd);  in mpi3mr_alloc_op_req_q_segments()
2057 	op_req_q->q_segments = kcalloc(op_req_q->num_segments,  in mpi3mr_alloc_op_req_q_segments()
2059 	if (!op_req_q->q_segments)  in mpi3mr_alloc_op_req_q_segments()
2060 		return -ENOMEM;  in mpi3mr_alloc_op_req_q_segments()
2062 	segments = op_req_q->q_segments;  in mpi3mr_alloc_op_req_q_segments()
2063 	for (i = 0; i < op_req_q->num_segments; i++) {  in mpi3mr_alloc_op_req_q_segments()
2065 		    dma_alloc_coherent(&mrioc->pdev->dev,  in mpi3mr_alloc_op_req_q_segments()
2068 			return -ENOMEM;  in mpi3mr_alloc_op_req_q_segments()
2069 		if (mrioc->enable_segqueue)  in mpi3mr_alloc_op_req_q_segments()
2078  * mpi3mr_create_op_reply_q - create operational reply queue
2085  * Return:  0 on success, non-zero on failure.
2090 	struct op_reply_qinfo *op_reply_q = mrioc->op_reply_qinfo + qidx;  in mpi3mr_create_op_reply_q()
2094 	reply_qid = op_reply_q->qid;  in mpi3mr_create_op_reply_q()
2096 	midx = REPLY_QUEUE_IDX_TO_MSIX_IDX(qidx, mrioc->op_reply_q_offset);  in mpi3mr_create_op_reply_q()
2099 		retval = -1;  in mpi3mr_create_op_reply_q()
2107 	op_reply_q->num_replies = MPI3MR_OP_REP_Q_QD;  in mpi3mr_create_op_reply_q()
2108 	if ((mrioc->pdev->device == MPI3_MFGPAGE_DEVID_SAS4116) &&  in mpi3mr_create_op_reply_q()
2109 		!mrioc->pdev->revision)  in mpi3mr_create_op_reply_q()
2110 		op_reply_q->num_replies = MPI3MR_OP_REP_Q_QD4K;  in mpi3mr_create_op_reply_q()
2111 	op_reply_q->ci = 0;  in mpi3mr_create_op_reply_q()
2112 	op_reply_q->ephase = 1;  in mpi3mr_create_op_reply_q()
2113 	atomic_set(&op_reply_q->pend_ios, 0);  in mpi3mr_create_op_reply_q()
2114 	atomic_set(&op_reply_q->in_use, 0);  in mpi3mr_create_op_reply_q()
2115 	op_reply_q->enable_irq_poll = false;  in mpi3mr_create_op_reply_q()
2117 	if (!op_reply_q->q_segments) {  in mpi3mr_create_op_reply_q()
2126 	mutex_lock(&mrioc->init_cmds.mutex);  in mpi3mr_create_op_reply_q()
2127 	if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {  in mpi3mr_create_op_reply_q()
2128 		retval = -1;  in mpi3mr_create_op_reply_q()
2132 	mrioc->init_cmds.state = MPI3MR_CMD_PENDING;  in mpi3mr_create_op_reply_q()
2133 	mrioc->init_cmds.is_waiting = 1;  in mpi3mr_create_op_reply_q()
2134 	mrioc->init_cmds.callback = NULL;  in mpi3mr_create_op_reply_q()
2139 	if (midx < (mrioc->intr_info_count - mrioc->requested_poll_qcount))  in mpi3mr_create_op_reply_q()
2140 		op_reply_q->qtype = MPI3MR_DEFAULT_QUEUE;  in mpi3mr_create_op_reply_q()
2142 		op_reply_q->qtype = MPI3MR_POLL_QUEUE;  in mpi3mr_create_op_reply_q()
2144 	if (op_reply_q->qtype == MPI3MR_DEFAULT_QUEUE) {  in mpi3mr_create_op_reply_q()
2148 			cpu_to_le16(mrioc->intr_info[midx].msix_index);  in mpi3mr_create_op_reply_q()
2150 		create_req.msix_index = cpu_to_le16(mrioc->intr_info_count - 1);  in mpi3mr_create_op_reply_q()
2153 		if (!mrioc->active_poll_qcount)  in mpi3mr_create_op_reply_q()
2154 			disable_irq_nosync(pci_irq_vector(mrioc->pdev,  in mpi3mr_create_op_reply_q()
2155 			    mrioc->intr_info_count - 1));  in mpi3mr_create_op_reply_q()
2158 	if (mrioc->enable_segqueue) {  in mpi3mr_create_op_reply_q()
2162 		    op_reply_q->q_segment_list_dma);  in mpi3mr_create_op_reply_q()
2165 		    op_reply_q->q_segments[0].segment_dma);  in mpi3mr_create_op_reply_q()
2167 	create_req.size = cpu_to_le16(op_reply_q->num_replies);  in mpi3mr_create_op_reply_q()
2169 	init_completion(&mrioc->init_cmds.done);  in mpi3mr_create_op_reply_q()
2176 	wait_for_completion_timeout(&mrioc->init_cmds.done,  in mpi3mr_create_op_reply_q()
2178 	if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {  in mpi3mr_create_op_reply_q()
2182 		retval = -1;  in mpi3mr_create_op_reply_q()
2185 	if ((mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)  in mpi3mr_create_op_reply_q()
2189 		    (mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),  in mpi3mr_create_op_reply_q()
2190 		    mrioc->init_cmds.ioc_loginfo);  in mpi3mr_create_op_reply_q()
2191 		retval = -1;  in mpi3mr_create_op_reply_q()
2194 	op_reply_q->qid = reply_qid;  in mpi3mr_create_op_reply_q()
2195 	if (midx < mrioc->intr_info_count)  in mpi3mr_create_op_reply_q()
2196 		mrioc->intr_info[midx].op_reply_q = op_reply_q;  in mpi3mr_create_op_reply_q()
2198 	(op_reply_q->qtype == MPI3MR_DEFAULT_QUEUE) ? mrioc->default_qcount++ :  in mpi3mr_create_op_reply_q()
2199 	    mrioc->active_poll_qcount++;  in mpi3mr_create_op_reply_q()
2202 	mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;  in mpi3mr_create_op_reply_q()
2203 	mutex_unlock(&mrioc->init_cmds.mutex);  in mpi3mr_create_op_reply_q()
2210  * mpi3mr_create_op_req_q - create operational request queue
2218  * Return:  0 on success, non-zero on failure.
2224 	struct op_req_qinfo *op_req_q = mrioc->req_qinfo + idx;  in mpi3mr_create_op_req_q()
2228 	req_qid = op_req_q->qid;  in mpi3mr_create_op_req_q()
2231 		retval = -1;  in mpi3mr_create_op_req_q()
2239 	op_req_q->num_requests = MPI3MR_OP_REQ_Q_QD;  in mpi3mr_create_op_req_q()
2240 	op_req_q->ci = 0;  in mpi3mr_create_op_req_q()
2241 	op_req_q->pi = 0;  in mpi3mr_create_op_req_q()
2242 	op_req_q->reply_qid = reply_qid;  in mpi3mr_create_op_req_q()
2243 	spin_lock_init(&op_req_q->q_lock);  in mpi3mr_create_op_req_q()
2245 	if (!op_req_q->q_segments) {  in mpi3mr_create_op_req_q()
2254 	mutex_lock(&mrioc->init_cmds.mutex);  in mpi3mr_create_op_req_q()
2255 	if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {  in mpi3mr_create_op_req_q()
2256 		retval = -1;  in mpi3mr_create_op_req_q()
2260 	mrioc->init_cmds.state = MPI3MR_CMD_PENDING;  in mpi3mr_create_op_req_q()
2261 	mrioc->init_cmds.is_waiting = 1;  in mpi3mr_create_op_req_q()
2262 	mrioc->init_cmds.callback = NULL;  in mpi3mr_create_op_req_q()
2266 	if (mrioc->enable_segqueue) {  in mpi3mr_create_op_req_q()
2270 		    op_req_q->q_segment_list_dma);  in mpi3mr_create_op_req_q()
2273 		    op_req_q->q_segments[0].segment_dma);  in mpi3mr_create_op_req_q()
2275 	create_req.size = cpu_to_le16(op_req_q->num_requests);  in mpi3mr_create_op_req_q()
2277 	init_completion(&mrioc->init_cmds.done);  in mpi3mr_create_op_req_q()
2284 	wait_for_completion_timeout(&mrioc->init_cmds.done,  in mpi3mr_create_op_req_q()
2286 	if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {  in mpi3mr_create_op_req_q()
2290 		retval = -1;  in mpi3mr_create_op_req_q()
2293 	if ((mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)  in mpi3mr_create_op_req_q()
2297 		    (mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),  in mpi3mr_create_op_req_q()
2298 		    mrioc->init_cmds.ioc_loginfo);  in mpi3mr_create_op_req_q()
2299 		retval = -1;  in mpi3mr_create_op_req_q()
2302 	op_req_q->qid = req_qid;  in mpi3mr_create_op_req_q()
2305 	mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;  in mpi3mr_create_op_req_q()
2306 	mutex_unlock(&mrioc->init_cmds.mutex);  in mpi3mr_create_op_req_q()
2313  * mpi3mr_create_op_queues - create operational queue pairs
2319  * Return: 0 on success, non-zero on failures.
2326 	num_queues = min_t(int, mrioc->facts.max_op_reply_q,  in mpi3mr_create_op_queues()
2327 	    mrioc->facts.max_op_req_q);  in mpi3mr_create_op_queues()
2330 	    mrioc->intr_info_count - mrioc->op_reply_q_offset;  in mpi3mr_create_op_queues()
2331 	if (!mrioc->num_queues)  in mpi3mr_create_op_queues()
2332 		mrioc->num_queues = min_t(int, num_queues, msix_count_op_q);  in mpi3mr_create_op_queues()
2337 	num_queues = mrioc->num_op_reply_q ?  in mpi3mr_create_op_queues()
2338 	    mrioc->num_op_reply_q : mrioc->num_queues;  in mpi3mr_create_op_queues()
2342 	if (!mrioc->req_qinfo) {  in mpi3mr_create_op_queues()
2343 		mrioc->req_qinfo = kcalloc(num_queues,  in mpi3mr_create_op_queues()
2345 		if (!mrioc->req_qinfo) {  in mpi3mr_create_op_queues()
2346 			retval = -1;  in mpi3mr_create_op_queues()
2350 		mrioc->op_reply_qinfo = kzalloc(sizeof(struct op_reply_qinfo) *  in mpi3mr_create_op_queues()
2352 		if (!mrioc->op_reply_qinfo) {  in mpi3mr_create_op_queues()
2353 			retval = -1;  in mpi3mr_create_op_queues()
2358 	if (mrioc->enable_segqueue)  in mpi3mr_create_op_queues()
2368 		    mrioc->op_reply_qinfo[i].qid)) {  in mpi3mr_create_op_queues()
2377 		retval = -1;  in mpi3mr_create_op_queues()
2380 	mrioc->num_op_reply_q = mrioc->num_op_req_q = i;  in mpi3mr_create_op_queues()
2383 	    mrioc->num_op_reply_q, mrioc->default_qcount,  in mpi3mr_create_op_queues()
2384 	    mrioc->active_poll_qcount);  in mpi3mr_create_op_queues()
2388 	kfree(mrioc->req_qinfo);  in mpi3mr_create_op_queues()
2389 	mrioc->req_qinfo = NULL;  in mpi3mr_create_op_queues()
2391 	kfree(mrioc->op_reply_qinfo);  in mpi3mr_create_op_queues()
2392 	mrioc->op_reply_qinfo = NULL;  in mpi3mr_create_op_queues()
2398  * mpi3mr_op_request_post - Post request to operational queue
2407  * Return: 0 on success, non-zero on failure.
2417 	u16 req_sz = mrioc->facts.op_req_sz;  in mpi3mr_op_request_post()
2418 	struct segments *segments = op_req_q->q_segments;  in mpi3mr_op_request_post()
2420 	reply_qidx = op_req_q->reply_qid - 1;  in mpi3mr_op_request_post()
2422 	if (mrioc->unrecoverable)  in mpi3mr_op_request_post()
2423 		return -EFAULT;  in mpi3mr_op_request_post()
2425 	spin_lock_irqsave(&op_req_q->q_lock, flags);  in mpi3mr_op_request_post()
2426 	pi = op_req_q->pi;  in mpi3mr_op_request_post()
2427 	max_entries = op_req_q->num_requests;  in mpi3mr_op_request_post()
2431 		    reply_qidx, mrioc->op_reply_q_offset);  in mpi3mr_op_request_post()
2432 		mpi3mr_process_op_reply_q(mrioc, mrioc->intr_info[midx].op_reply_q);  in mpi3mr_op_request_post()
2435 			retval = -EAGAIN;  in mpi3mr_op_request_post()
2440 	if (mrioc->reset_in_progress) {  in mpi3mr_op_request_post()
2442 		retval = -EAGAIN;  in mpi3mr_op_request_post()
2445 	if (mrioc->pci_err_recovery) {  in mpi3mr_op_request_post()
2447 		retval = -EAGAIN;  in mpi3mr_op_request_post()
2451 	segment_base_addr = segments[pi / op_req_q->segment_qd].segment;  in mpi3mr_op_request_post()
2453 	    ((pi % op_req_q->segment_qd) * req_sz);  in mpi3mr_op_request_post()
2460 	op_req_q->pi = pi;  in mpi3mr_op_request_post()
2463 	if (atomic_inc_return(&mrioc->op_reply_qinfo[reply_qidx].pend_ios)  in mpi3mr_op_request_post()
2465 		mrioc->op_reply_qinfo[reply_qidx].enable_irq_poll = true;  in mpi3mr_op_request_post()
2467 	atomic_inc_return(&mrioc->op_reply_qinfo[reply_qidx].pend_ios);  in mpi3mr_op_request_post()
2470 	writel(op_req_q->pi,  in mpi3mr_op_request_post()
2471 	    &mrioc->sysif_regs->oper_queue_indexes[reply_qidx].producer_index);  in mpi3mr_op_request_post()
2474 	spin_unlock_irqrestore(&op_req_q->q_lock, flags);  in mpi3mr_op_request_post()
2479  * mpi3mr_check_rh_fault_ioc - check reset history and fault
2497 	if (mrioc->unrecoverable) {  in mpi3mr_check_rh_fault_ioc()
2502 	if (!pci_device_is_present(mrioc->pdev)) {  in mpi3mr_check_rh_fault_ioc()
2503 		mrioc->unrecoverable = 1;  in mpi3mr_check_rh_fault_ioc()
2508 	ioc_status = readl(&mrioc->sysif_regs->ioc_status);  in mpi3mr_check_rh_fault_ioc()
2515 		trigger_data.fault = (readl(&mrioc->sysif_regs->fault) &  in mpi3mr_check_rh_fault_ioc()
2527 	trigger_data.fault = (readl(&mrioc->sysif_regs->fault) &  in mpi3mr_check_rh_fault_ioc()
2533 		host_diagnostic = readl(&mrioc->sysif_regs->host_diagnostic);  in mpi3mr_check_rh_fault_ioc()
2537 	} while (--timeout);  in mpi3mr_check_rh_fault_ioc()
2541  * mpi3mr_sync_timestamp - Issue time stamp sync request
2547  * Return: 0 on success, non-zero on failure.
2556 	mutex_lock(&mrioc->init_cmds.mutex);  in mpi3mr_sync_timestamp()
2557 	if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {  in mpi3mr_sync_timestamp()
2558 		retval = -1;  in mpi3mr_sync_timestamp()
2560 		mutex_unlock(&mrioc->init_cmds.mutex);  in mpi3mr_sync_timestamp()
2563 	mrioc->init_cmds.state = MPI3MR_CMD_PENDING;  in mpi3mr_sync_timestamp()
2564 	mrioc->init_cmds.is_waiting = 1;  in mpi3mr_sync_timestamp()
2565 	mrioc->init_cmds.callback = NULL;  in mpi3mr_sync_timestamp()
2572 	init_completion(&mrioc->init_cmds.done);  in mpi3mr_sync_timestamp()
2580 	wait_for_completion_timeout(&mrioc->init_cmds.done,  in mpi3mr_sync_timestamp()
2582 	if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {  in mpi3mr_sync_timestamp()
2584 		mrioc->init_cmds.is_waiting = 0;  in mpi3mr_sync_timestamp()
2585 		if (!(mrioc->init_cmds.state & MPI3MR_CMD_RESET))  in mpi3mr_sync_timestamp()
2588 		retval = -1;  in mpi3mr_sync_timestamp()
2591 	if ((mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)  in mpi3mr_sync_timestamp()
2595 		    (mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),  in mpi3mr_sync_timestamp()
2596 		    mrioc->init_cmds.ioc_loginfo);  in mpi3mr_sync_timestamp()
2597 		retval = -1;  in mpi3mr_sync_timestamp()
2602 	mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;  in mpi3mr_sync_timestamp()
2603 	mutex_unlock(&mrioc->init_cmds.mutex);  in mpi3mr_sync_timestamp()
2610  * mpi3mr_print_pkg_ver - display controller fw package version
2613  * Retrieve firmware package version from the component image
2616  * Return: 0 on success and non-zero on failure.
2621 	int retval = -1;  in mpi3mr_print_pkg_ver()
2628 	data = dma_alloc_coherent(&mrioc->pdev->dev, data_len, &data_dma,  in mpi3mr_print_pkg_ver()
2631 		return -ENOMEM;  in mpi3mr_print_pkg_ver()
2634 	mutex_lock(&mrioc->init_cmds.mutex);  in mpi3mr_print_pkg_ver()
2635 	if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {  in mpi3mr_print_pkg_ver()
2636 		ioc_err(mrioc, "sending get package version failed due to command in use\n");  in mpi3mr_print_pkg_ver()
2637 		mutex_unlock(&mrioc->init_cmds.mutex);  in mpi3mr_print_pkg_ver()
2640 	mrioc->init_cmds.state = MPI3MR_CMD_PENDING;  in mpi3mr_print_pkg_ver()
2641 	mrioc->init_cmds.is_waiting = 1;  in mpi3mr_print_pkg_ver()
2642 	mrioc->init_cmds.callback = NULL;  in mpi3mr_print_pkg_ver()
2652 	init_completion(&mrioc->init_cmds.done);  in mpi3mr_print_pkg_ver()
2656 		ioc_err(mrioc, "posting get package version failed\n");  in mpi3mr_print_pkg_ver()
2659 	wait_for_completion_timeout(&mrioc->init_cmds.done,  in mpi3mr_print_pkg_ver()
2661 	if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {  in mpi3mr_print_pkg_ver()
2662 		ioc_err(mrioc, "get package version timed out\n");  in mpi3mr_print_pkg_ver()
2665 		retval = -1;  in mpi3mr_print_pkg_ver()
2668 	if ((mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)  in mpi3mr_print_pkg_ver()
2671 		if (manifest->manifest_type == MPI3_CI_MANIFEST_TYPE_MPI) {  in mpi3mr_print_pkg_ver()
2673 			    "firmware package version(%d.%d.%d.%d.%05d-%05d)\n",  in mpi3mr_print_pkg_ver()
2674 			    manifest->package_version.gen_major,  in mpi3mr_print_pkg_ver()
2675 			    manifest->package_version.gen_minor,  in mpi3mr_print_pkg_ver()
2676 			    manifest->package_version.phase_major,  in mpi3mr_print_pkg_ver()
2677 			    manifest->package_version.phase_minor,  in mpi3mr_print_pkg_ver()
2678 			    manifest->package_version.customer_id,  in mpi3mr_print_pkg_ver()
2679 			    manifest->package_version.build_num);  in mpi3mr_print_pkg_ver()
2684 	mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;  in mpi3mr_print_pkg_ver()
2685 	mutex_unlock(&mrioc->init_cmds.mutex);  in mpi3mr_print_pkg_ver()
2689 		dma_free_coherent(&mrioc->pdev->dev, data_len, data,  in mpi3mr_print_pkg_ver()
2695  * mpi3mr_watchdog_work - watchdog thread to monitor faults
2714 	if (mrioc->reset_in_progress || mrioc->pci_err_recovery)  in mpi3mr_watchdog_work()
2717 	if (!mrioc->unrecoverable && !pci_device_is_present(mrioc->pdev)) {  in mpi3mr_watchdog_work()
2719 		mrioc->unrecoverable = 1;  in mpi3mr_watchdog_work()
2722 	if (mrioc->unrecoverable) {  in mpi3mr_watchdog_work()
2729 	if (mrioc->ts_update_counter++ >= mrioc->ts_update_interval) {  in mpi3mr_watchdog_work()
2730 		mrioc->ts_update_counter = 0;  in mpi3mr_watchdog_work()
2734 	if ((mrioc->prepare_for_reset) &&  in mpi3mr_watchdog_work()
2735 	    ((mrioc->prepare_for_reset_timeout_counter++) >=  in mpi3mr_watchdog_work()
2743 	ioc_status = readl(&mrioc->sysif_regs->ioc_status);  in mpi3mr_watchdog_work()
2756 	trigger_data.fault = readl(&mrioc->sysif_regs->fault) & MPI3_SYSIF_FAULT_CODE_MASK;  in mpi3mr_watchdog_work()
2759 	host_diagnostic = readl(&mrioc->sysif_regs->host_diagnostic);  in mpi3mr_watchdog_work()
2761 		if (!mrioc->diagsave_timeout) {  in mpi3mr_watchdog_work()
2763 			ioc_warn(mrioc, "diag save in progress\n");  in mpi3mr_watchdog_work()
2765 		if ((mrioc->diagsave_timeout++) <= MPI3_SYSIF_DIAG_SAVE_TIMEOUT)  in mpi3mr_watchdog_work()
2770 	mrioc->diagsave_timeout = 0;  in mpi3mr_watchdog_work()
2773 		mrioc->unrecoverable = 1;  in mpi3mr_watchdog_work()
2782 		mrioc->unrecoverable = 1;  in mpi3mr_watchdog_work()
2796 	spin_lock_irqsave(&mrioc->watchdog_lock, flags);  in mpi3mr_watchdog_work()
2797 	if (mrioc->watchdog_work_q)  in mpi3mr_watchdog_work()
2798 		queue_delayed_work(mrioc->watchdog_work_q,  in mpi3mr_watchdog_work()
2799 		    &mrioc->watchdog_work,  in mpi3mr_watchdog_work()
2801 	spin_unlock_irqrestore(&mrioc->watchdog_lock, flags);  in mpi3mr_watchdog_work()
2806  * mpi3mr_start_watchdog - Start watchdog
2816 	if (mrioc->watchdog_work_q)  in mpi3mr_start_watchdog()
2819 	INIT_DELAYED_WORK(&mrioc->watchdog_work, mpi3mr_watchdog_work);  in mpi3mr_start_watchdog()
2820 	snprintf(mrioc->watchdog_work_q_name,  in mpi3mr_start_watchdog()
2821 	    sizeof(mrioc->watchdog_work_q_name), "watchdog_%s%d", mrioc->name,  in mpi3mr_start_watchdog()
2822 	    mrioc->id);  in mpi3mr_start_watchdog()
2823 	mrioc->watchdog_work_q = alloc_ordered_workqueue(  in mpi3mr_start_watchdog()
2824 		"%s", WQ_MEM_RECLAIM, mrioc->watchdog_work_q_name);  in mpi3mr_start_watchdog()
2825 	if (!mrioc->watchdog_work_q) {  in mpi3mr_start_watchdog()
2830 	if (mrioc->watchdog_work_q)  in mpi3mr_start_watchdog()
2831 		queue_delayed_work(mrioc->watchdog_work_q,  in mpi3mr_start_watchdog()
2832 		    &mrioc->watchdog_work,  in mpi3mr_start_watchdog()
2837  * mpi3mr_stop_watchdog - Stop watchdog
2850 	spin_lock_irqsave(&mrioc->watchdog_lock, flags);  in mpi3mr_stop_watchdog()
2851 	wq = mrioc->watchdog_work_q;  in mpi3mr_stop_watchdog()
2852 	mrioc->watchdog_work_q = NULL;  in mpi3mr_stop_watchdog()
2853 	spin_unlock_irqrestore(&mrioc->watchdog_lock, flags);  in mpi3mr_stop_watchdog()
2855 		if (!cancel_delayed_work_sync(&mrioc->watchdog_work))  in mpi3mr_stop_watchdog()
2862  * mpi3mr_setup_admin_qpair - Setup admin queue pair
2868  * Return: 0 on success, non-zero on failures.
2875 	mrioc->admin_req_q_sz = MPI3MR_ADMIN_REQ_Q_SIZE;  in mpi3mr_setup_admin_qpair()
2876 	mrioc->num_admin_req = mrioc->admin_req_q_sz /  in mpi3mr_setup_admin_qpair()
2878 	mrioc->admin_req_ci = mrioc->admin_req_pi = 0;  in mpi3mr_setup_admin_qpair()
2880 	mrioc->admin_reply_q_sz = MPI3MR_ADMIN_REPLY_Q_SIZE;  in mpi3mr_setup_admin_qpair()
2881 	mrioc->num_admin_replies = mrioc->admin_reply_q_sz /  in mpi3mr_setup_admin_qpair()
2883 	mrioc->admin_reply_ci = 0;  in mpi3mr_setup_admin_qpair()
2884 	mrioc->admin_reply_ephase = 1;  in mpi3mr_setup_admin_qpair()
2885 	atomic_set(&mrioc->admin_reply_q_in_use, 0);  in mpi3mr_setup_admin_qpair()
2887 	if (!mrioc->admin_req_base) {  in mpi3mr_setup_admin_qpair()
2888 		mrioc->admin_req_base = dma_alloc_coherent(&mrioc->pdev->dev,  in mpi3mr_setup_admin_qpair()
2889 		    mrioc->admin_req_q_sz, &mrioc->admin_req_dma, GFP_KERNEL);  in mpi3mr_setup_admin_qpair()
2891 		if (!mrioc->admin_req_base) {  in mpi3mr_setup_admin_qpair()
2892 			retval = -1;  in mpi3mr_setup_admin_qpair()
2896 		mrioc->admin_reply_base = dma_alloc_coherent(&mrioc->pdev->dev,  in mpi3mr_setup_admin_qpair()
2897 		    mrioc->admin_reply_q_sz, &mrioc->admin_reply_dma,  in mpi3mr_setup_admin_qpair()
2900 		if (!mrioc->admin_reply_base) {  in mpi3mr_setup_admin_qpair()
2901 			retval = -1;  in mpi3mr_setup_admin_qpair()
2906 	num_admin_entries = (mrioc->num_admin_replies << 16) |  in mpi3mr_setup_admin_qpair()
2907 	    (mrioc->num_admin_req);  in mpi3mr_setup_admin_qpair()
2908 	writel(num_admin_entries, &mrioc->sysif_regs->admin_queue_num_entries);  in mpi3mr_setup_admin_qpair()
2909 	mpi3mr_writeq(mrioc->admin_req_dma,  in mpi3mr_setup_admin_qpair()
2910 	    &mrioc->sysif_regs->admin_request_queue_address);  in mpi3mr_setup_admin_qpair()
2911 	mpi3mr_writeq(mrioc->admin_reply_dma,  in mpi3mr_setup_admin_qpair()
2912 	    &mrioc->sysif_regs->admin_reply_queue_address);  in mpi3mr_setup_admin_qpair()
2913 	writel(mrioc->admin_req_pi, &mrioc->sysif_regs->admin_request_queue_pi);  in mpi3mr_setup_admin_qpair()
2914 	writel(mrioc->admin_reply_ci, &mrioc->sysif_regs->admin_reply_queue_ci);  in mpi3mr_setup_admin_qpair()
2919 	if (mrioc->admin_reply_base) {  in mpi3mr_setup_admin_qpair()
2920 		dma_free_coherent(&mrioc->pdev->dev, mrioc->admin_reply_q_sz,  in mpi3mr_setup_admin_qpair()
2921 		    mrioc->admin_reply_base, mrioc->admin_reply_dma);  in mpi3mr_setup_admin_qpair()
2922 		mrioc->admin_reply_base = NULL;  in mpi3mr_setup_admin_qpair()
2924 	if (mrioc->admin_req_base) {  in mpi3mr_setup_admin_qpair()
2925 		dma_free_coherent(&mrioc->pdev->dev, mrioc->admin_req_q_sz,  in mpi3mr_setup_admin_qpair()
2926 		    mrioc->admin_req_base, mrioc->admin_req_dma);  in mpi3mr_setup_admin_qpair()
2927 		mrioc->admin_req_base = NULL;  in mpi3mr_setup_admin_qpair()
2933  * mpi3mr_issue_iocfacts - Send IOC Facts
2940  * Return: 0 on success, non-zero on failures.
2952 	data = dma_alloc_coherent(&mrioc->pdev->dev, data_len, &data_dma,  in mpi3mr_issue_iocfacts()
2956 		retval = -1;  in mpi3mr_issue_iocfacts()
2961 	mutex_lock(&mrioc->init_cmds.mutex);  in mpi3mr_issue_iocfacts()
2962 	if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {  in mpi3mr_issue_iocfacts()
2963 		retval = -1;  in mpi3mr_issue_iocfacts()
2965 		mutex_unlock(&mrioc->init_cmds.mutex);  in mpi3mr_issue_iocfacts()
2968 	mrioc->init_cmds.state = MPI3MR_CMD_PENDING;  in mpi3mr_issue_iocfacts()
2969 	mrioc->init_cmds.is_waiting = 1;  in mpi3mr_issue_iocfacts()
2970 	mrioc->init_cmds.callback = NULL;  in mpi3mr_issue_iocfacts()
2977 	init_completion(&mrioc->init_cmds.done);  in mpi3mr_issue_iocfacts()
2984 	wait_for_completion_timeout(&mrioc->init_cmds.done,  in mpi3mr_issue_iocfacts()
2986 	if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {  in mpi3mr_issue_iocfacts()
2990 		retval = -1;  in mpi3mr_issue_iocfacts()
2993 	if ((mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)  in mpi3mr_issue_iocfacts()
2997 		    (mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),  in mpi3mr_issue_iocfacts()
2998 		    mrioc->init_cmds.ioc_loginfo);  in mpi3mr_issue_iocfacts()
2999 		retval = -1;  in mpi3mr_issue_iocfacts()
3005 	mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;  in mpi3mr_issue_iocfacts()
3006 	mutex_unlock(&mrioc->init_cmds.mutex);  in mpi3mr_issue_iocfacts()
3010 		dma_free_coherent(&mrioc->pdev->dev, data_len, data, data_dma);  in mpi3mr_issue_iocfacts()
3016  * mpi3mr_check_reset_dma_mask - Process IOC facts data
3022  * Return: 0 on success, non-zero on failure.
3026 	struct pci_dev *pdev = mrioc->pdev;  in mpi3mr_check_reset_dma_mask()
3028 	u64 facts_dma_mask = DMA_BIT_MASK(mrioc->facts.dma_mask);  in mpi3mr_check_reset_dma_mask()
3030 	if (!mrioc->facts.dma_mask || (mrioc->dma_mask <= facts_dma_mask))  in mpi3mr_check_reset_dma_mask()
3034 	    mrioc->dma_mask, facts_dma_mask);  in mpi3mr_check_reset_dma_mask()
3036 	r = dma_set_mask_and_coherent(&pdev->dev, facts_dma_mask);  in mpi3mr_check_reset_dma_mask()
3042 	mrioc->dma_mask = facts_dma_mask;  in mpi3mr_check_reset_dma_mask()
3047  * mpi3mr_process_factsdata - Process IOC facts data
3061 	if ((le16_to_cpu(facts_data->ioc_facts_data_length)) !=  in mpi3mr_process_factsdata()
3066 		    le16_to_cpu(facts_data->ioc_facts_data_length) * 4);  in mpi3mr_process_factsdata()
3069 	ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);  in mpi3mr_process_factsdata()
3072 	if (le16_to_cpu(facts_data->ioc_request_frame_size) != (req_sz / 4)) {  in mpi3mr_process_factsdata()
3075 		    req_sz / 4, le16_to_cpu(facts_data->ioc_request_frame_size));  in mpi3mr_process_factsdata()
3078 	memset(&mrioc->facts, 0, sizeof(mrioc->facts));  in mpi3mr_process_factsdata()
3080 	facts_flags = le32_to_cpu(facts_data->flags);  in mpi3mr_process_factsdata()
3081 	mrioc->facts.op_req_sz = req_sz;  in mpi3mr_process_factsdata()
3082 	mrioc->op_reply_desc_sz = 1 << ((ioc_config &  in mpi3mr_process_factsdata()
3086 	mrioc->facts.ioc_num = facts_data->ioc_number;  in mpi3mr_process_factsdata()
3087 	mrioc->facts.who_init = facts_data->who_init;  in mpi3mr_process_factsdata()
3088 	mrioc->facts.max_msix_vectors = le16_to_cpu(facts_data->max_msix_vectors);  in mpi3mr_process_factsdata()
3089 	mrioc->facts.personality = (facts_flags &  in mpi3mr_process_factsdata()
3091 	mrioc->facts.dma_mask = (facts_flags &  in mpi3mr_process_factsdata()
3094 	mrioc->facts.protocol_flags = facts_data->protocol_flags;  in mpi3mr_process_factsdata()
3095 	mrioc->facts.mpi_version = le32_to_cpu(facts_data->mpi_version.word);  in mpi3mr_process_factsdata()
3096 	mrioc->facts.max_reqs = le16_to_cpu(facts_data->max_outstanding_requests);  in mpi3mr_process_factsdata()
3097 	mrioc->facts.product_id = le16_to_cpu(facts_data->product_id);  in mpi3mr_process_factsdata()
3098 	mrioc->facts.reply_sz = le16_to_cpu(facts_data->reply_frame_size) * 4;  in mpi3mr_process_factsdata()
3099 	mrioc->facts.exceptions = le16_to_cpu(facts_data->ioc_exceptions);  in mpi3mr_process_factsdata()
3100 	mrioc->facts.max_perids = le16_to_cpu(facts_data->max_persistent_id);  in mpi3mr_process_factsdata()
3101 	mrioc->facts.max_vds = le16_to_cpu(facts_data->max_vds);  in mpi3mr_process_factsdata()
3102 	mrioc->facts.max_hpds = le16_to_cpu(facts_data->max_host_pds);  in mpi3mr_process_factsdata()
3103 	mrioc->facts.max_advhpds = le16_to_cpu(facts_data->max_adv_host_pds);  in mpi3mr_process_factsdata()
3104 	mrioc->facts.max_raid_pds = le16_to_cpu(facts_data->max_raid_pds);  in mpi3mr_process_factsdata()
3105 	mrioc->facts.max_nvme = le16_to_cpu(facts_data->max_nvme);  in mpi3mr_process_factsdata()
3106 	mrioc->facts.max_pcie_switches =  in mpi3mr_process_factsdata()
3107 	    le16_to_cpu(facts_data->max_pcie_switches);  in mpi3mr_process_factsdata()
3108 	mrioc->facts.max_sasexpanders =  in mpi3mr_process_factsdata()
3109 	    le16_to_cpu(facts_data->max_sas_expanders);  in mpi3mr_process_factsdata()
3110 	mrioc->facts.max_data_length = le16_to_cpu(facts_data->max_data_length);  in mpi3mr_process_factsdata()
3111 	mrioc->facts.max_sasinitiators =  in mpi3mr_process_factsdata()
3112 	    le16_to_cpu(facts_data->max_sas_initiators);  in mpi3mr_process_factsdata()
3113 	mrioc->facts.max_enclosures = le16_to_cpu(facts_data->max_enclosures);  in mpi3mr_process_factsdata()
3114 	mrioc->facts.min_devhandle = le16_to_cpu(facts_data->min_dev_handle);  in mpi3mr_process_factsdata()
3115 	mrioc->facts.max_devhandle = le16_to_cpu(facts_data->max_dev_handle);  in mpi3mr_process_factsdata()
3116 	mrioc->facts.max_op_req_q =  in mpi3mr_process_factsdata()
3117 	    le16_to_cpu(facts_data->max_operational_request_queues);  in mpi3mr_process_factsdata()
3118 	mrioc->facts.max_op_reply_q =  in mpi3mr_process_factsdata()
3119 	    le16_to_cpu(facts_data->max_operational_reply_queues);  in mpi3mr_process_factsdata()
3120 	mrioc->facts.ioc_capabilities =  in mpi3mr_process_factsdata()
3121 	    le32_to_cpu(facts_data->ioc_capabilities);  in mpi3mr_process_factsdata()
3122 	mrioc->facts.fw_ver.build_num =  in mpi3mr_process_factsdata()
3123 	    le16_to_cpu(facts_data->fw_version.build_num);  in mpi3mr_process_factsdata()
3124 	mrioc->facts.fw_ver.cust_id =  in mpi3mr_process_factsdata()
3125 	    le16_to_cpu(facts_data->fw_version.customer_id);  in mpi3mr_process_factsdata()
3126 	mrioc->facts.fw_ver.ph_minor = facts_data->fw_version.phase_minor;  in mpi3mr_process_factsdata()
3127 	mrioc->facts.fw_ver.ph_major = facts_data->fw_version.phase_major;  in mpi3mr_process_factsdata()
3128 	mrioc->facts.fw_ver.gen_minor = facts_data->fw_version.gen_minor;  in mpi3mr_process_factsdata()
3129 	mrioc->facts.fw_ver.gen_major = facts_data->fw_version.gen_major;  in mpi3mr_process_factsdata()
3130 	mrioc->msix_count = min_t(int, mrioc->msix_count,  in mpi3mr_process_factsdata()
3131 	    mrioc->facts.max_msix_vectors);  in mpi3mr_process_factsdata()
3132 	mrioc->facts.sge_mod_mask = facts_data->sge_modifier_mask;  in mpi3mr_process_factsdata()
3133 	mrioc->facts.sge_mod_value = facts_data->sge_modifier_value;  in mpi3mr_process_factsdata()
3134 	mrioc->facts.sge_mod_shift = facts_data->sge_modifier_shift;  in mpi3mr_process_factsdata()
3135 	mrioc->facts.shutdown_timeout =  in mpi3mr_process_factsdata()
3136 	    le16_to_cpu(facts_data->shutdown_timeout);  in mpi3mr_process_factsdata()
3137 	mrioc->facts.diag_trace_sz =  in mpi3mr_process_factsdata()
3138 	    le32_to_cpu(facts_data->diag_trace_size);  in mpi3mr_process_factsdata()
3139 	mrioc->facts.diag_fw_sz =  in mpi3mr_process_factsdata()
3140 	    le32_to_cpu(facts_data->diag_fw_size);  in mpi3mr_process_factsdata()
3141 	mrioc->facts.diag_drvr_sz = le32_to_cpu(facts_data->diag_driver_size);  in mpi3mr_process_factsdata()
3142 	mrioc->facts.max_dev_per_tg =  in mpi3mr_process_factsdata()
3143 	    facts_data->max_devices_per_throttle_group;  in mpi3mr_process_factsdata()
3144 	mrioc->facts.io_throttle_data_length =  in mpi3mr_process_factsdata()
3145 	    le16_to_cpu(facts_data->io_throttle_data_length);  in mpi3mr_process_factsdata()
3146 	mrioc->facts.max_io_throttle_group =  in mpi3mr_process_factsdata()
3147 	    le16_to_cpu(facts_data->max_io_throttle_group);  in mpi3mr_process_factsdata()
3148 	mrioc->facts.io_throttle_low = le16_to_cpu(facts_data->io_throttle_low);  in mpi3mr_process_factsdata()
3149 	mrioc->facts.io_throttle_high =  in mpi3mr_process_factsdata()
3150 	    le16_to_cpu(facts_data->io_throttle_high);  in mpi3mr_process_factsdata()
3152 	if (mrioc->facts.max_data_length ==  in mpi3mr_process_factsdata()
3154 		mrioc->facts.max_data_length = MPI3MR_DEFAULT_MAX_IO_SIZE;  in mpi3mr_process_factsdata()
3156 		mrioc->facts.max_data_length *= MPI3MR_PAGE_SIZE_4K;  in mpi3mr_process_factsdata()
3158 	if (mrioc->facts.io_throttle_data_length)  in mpi3mr_process_factsdata()
3159 		mrioc->io_throttle_data_length =  in mpi3mr_process_factsdata()
3160 		    (mrioc->facts.io_throttle_data_length * 2 * 4);  in mpi3mr_process_factsdata()
3163 		mrioc->io_throttle_data_length = (mrioc->facts.max_data_length / 512) + 2;  in mpi3mr_process_factsdata()
3165 	mrioc->io_throttle_high = (mrioc->facts.io_throttle_high * 2 * 1024);  in mpi3mr_process_factsdata()
3166 	mrioc->io_throttle_low = (mrioc->facts.io_throttle_low * 2 * 1024);  in mpi3mr_process_factsdata()
3169 	    mrioc->facts.ioc_num, mrioc->facts.max_op_req_q,  in mpi3mr_process_factsdata()
3170 	    mrioc->facts.max_op_reply_q, mrioc->facts.max_devhandle);  in mpi3mr_process_factsdata()
3173 	    mrioc->facts.max_reqs, mrioc->facts.min_devhandle,  in mpi3mr_process_factsdata()
3174 	    mrioc->facts.max_msix_vectors, mrioc->facts.max_perids);  in mpi3mr_process_factsdata()
3176 	    mrioc->facts.sge_mod_mask, mrioc->facts.sge_mod_value,  in mpi3mr_process_factsdata()
3177 	    mrioc->facts.sge_mod_shift);  in mpi3mr_process_factsdata()
3179 	    mrioc->facts.dma_mask, (facts_flags &  in mpi3mr_process_factsdata()
3180 	    MPI3_IOCFACTS_FLAGS_INITIAL_PORT_ENABLE_MASK), mrioc->facts.max_data_length);  in mpi3mr_process_factsdata()
3183 	    mrioc->facts.max_dev_per_tg, mrioc->facts.max_io_throttle_group);  in mpi3mr_process_factsdata()
3186 	   mrioc->facts.io_throttle_data_length * 4,  in mpi3mr_process_factsdata()
3187 	   mrioc->facts.io_throttle_high, mrioc->facts.io_throttle_low);  in mpi3mr_process_factsdata()
3191  * mpi3mr_alloc_reply_sense_bufs - Send IOC Init
3197  * Return: 0 on success, non-zero on failures.
3204 	if (mrioc->init_cmds.reply)  in mpi3mr_alloc_reply_sense_bufs()
3207 	mrioc->init_cmds.reply = kzalloc(mrioc->reply_sz, GFP_KERNEL);  in mpi3mr_alloc_reply_sense_bufs()
3208 	if (!mrioc->init_cmds.reply)  in mpi3mr_alloc_reply_sense_bufs()
3211 	mrioc->bsg_cmds.reply = kzalloc(mrioc->reply_sz, GFP_KERNEL);  in mpi3mr_alloc_reply_sense_bufs()
3212 	if (!mrioc->bsg_cmds.reply)  in mpi3mr_alloc_reply_sense_bufs()
3215 	mrioc->transport_cmds.reply = kzalloc(mrioc->reply_sz, GFP_KERNEL);  in mpi3mr_alloc_reply_sense_bufs()
3216 	if (!mrioc->transport_cmds.reply)  in mpi3mr_alloc_reply_sense_bufs()
3220 		mrioc->dev_rmhs_cmds[i].reply = kzalloc(mrioc->reply_sz,  in mpi3mr_alloc_reply_sense_bufs()
3222 		if (!mrioc->dev_rmhs_cmds[i].reply)  in mpi3mr_alloc_reply_sense_bufs()
3227 		mrioc->evtack_cmds[i].reply = kzalloc(mrioc->reply_sz,  in mpi3mr_alloc_reply_sense_bufs()
3229 		if (!mrioc->evtack_cmds[i].reply)  in mpi3mr_alloc_reply_sense_bufs()
3233 	mrioc->host_tm_cmds.reply = kzalloc(mrioc->reply_sz, GFP_KERNEL);  in mpi3mr_alloc_reply_sense_bufs()
3234 	if (!mrioc->host_tm_cmds.reply)  in mpi3mr_alloc_reply_sense_bufs()
3237 	mrioc->pel_cmds.reply = kzalloc(mrioc->reply_sz, GFP_KERNEL);  in mpi3mr_alloc_reply_sense_bufs()
3238 	if (!mrioc->pel_cmds.reply)  in mpi3mr_alloc_reply_sense_bufs()
3241 	mrioc->pel_abort_cmd.reply = kzalloc(mrioc->reply_sz, GFP_KERNEL);  in mpi3mr_alloc_reply_sense_bufs()
3242 	if (!mrioc->pel_abort_cmd.reply)  in mpi3mr_alloc_reply_sense_bufs()
3245 	mrioc->dev_handle_bitmap_bits = mrioc->facts.max_devhandle;  in mpi3mr_alloc_reply_sense_bufs()
3246 	mrioc->removepend_bitmap = bitmap_zalloc(mrioc->dev_handle_bitmap_bits,  in mpi3mr_alloc_reply_sense_bufs()
3248 	if (!mrioc->removepend_bitmap)  in mpi3mr_alloc_reply_sense_bufs()
3251 	mrioc->devrem_bitmap = bitmap_zalloc(MPI3MR_NUM_DEVRMCMD, GFP_KERNEL);  in mpi3mr_alloc_reply_sense_bufs()
3252 	if (!mrioc->devrem_bitmap)  in mpi3mr_alloc_reply_sense_bufs()
3255 	mrioc->evtack_cmds_bitmap = bitmap_zalloc(MPI3MR_NUM_EVTACKCMD,  in mpi3mr_alloc_reply_sense_bufs()
3257 	if (!mrioc->evtack_cmds_bitmap)  in mpi3mr_alloc_reply_sense_bufs()
3260 	mrioc->num_reply_bufs = mrioc->facts.max_reqs + MPI3MR_NUM_EVT_REPLIES;  in mpi3mr_alloc_reply_sense_bufs()
3261 	mrioc->reply_free_qsz = mrioc->num_reply_bufs + 1;  in mpi3mr_alloc_reply_sense_bufs()
3262 	mrioc->num_sense_bufs = mrioc->facts.max_reqs / MPI3MR_SENSEBUF_FACTOR;  in mpi3mr_alloc_reply_sense_bufs()
3263 	mrioc->sense_buf_q_sz = mrioc->num_sense_bufs + 1;  in mpi3mr_alloc_reply_sense_bufs()
3266 	sz = mrioc->num_reply_bufs * mrioc->reply_sz;  in mpi3mr_alloc_reply_sense_bufs()
3267 	mrioc->reply_buf_pool = dma_pool_create("reply_buf pool",  in mpi3mr_alloc_reply_sense_bufs()
3268 	    &mrioc->pdev->dev, sz, 16, 0);  in mpi3mr_alloc_reply_sense_bufs()
3269 	if (!mrioc->reply_buf_pool) {  in mpi3mr_alloc_reply_sense_bufs()
3274 	mrioc->reply_buf = dma_pool_zalloc(mrioc->reply_buf_pool, GFP_KERNEL,  in mpi3mr_alloc_reply_sense_bufs()
3275 	    &mrioc->reply_buf_dma);  in mpi3mr_alloc_reply_sense_bufs()
3276 	if (!mrioc->reply_buf)  in mpi3mr_alloc_reply_sense_bufs()
3279 	mrioc->reply_buf_dma_max_address = mrioc->reply_buf_dma + sz;  in mpi3mr_alloc_reply_sense_bufs()
3282 	sz = mrioc->reply_free_qsz * 8;  in mpi3mr_alloc_reply_sense_bufs()
3283 	mrioc->reply_free_q_pool = dma_pool_create("reply_free_q pool",  in mpi3mr_alloc_reply_sense_bufs()
3284 	    &mrioc->pdev->dev, sz, 8, 0);  in mpi3mr_alloc_reply_sense_bufs()
3285 	if (!mrioc->reply_free_q_pool) {  in mpi3mr_alloc_reply_sense_bufs()
3289 	mrioc->reply_free_q = dma_pool_zalloc(mrioc->reply_free_q_pool,  in mpi3mr_alloc_reply_sense_bufs()
3290 	    GFP_KERNEL, &mrioc->reply_free_q_dma);  in mpi3mr_alloc_reply_sense_bufs()
3291 	if (!mrioc->reply_free_q)  in mpi3mr_alloc_reply_sense_bufs()
3295 	sz = mrioc->num_sense_bufs * MPI3MR_SENSE_BUF_SZ;  in mpi3mr_alloc_reply_sense_bufs()
3296 	mrioc->sense_buf_pool = dma_pool_create("sense_buf pool",  in mpi3mr_alloc_reply_sense_bufs()
3297 	    &mrioc->pdev->dev, sz, 4, 0);  in mpi3mr_alloc_reply_sense_bufs()
3298 	if (!mrioc->sense_buf_pool) {  in mpi3mr_alloc_reply_sense_bufs()
3302 	mrioc->sense_buf = dma_pool_zalloc(mrioc->sense_buf_pool, GFP_KERNEL,  in mpi3mr_alloc_reply_sense_bufs()
3303 	    &mrioc->sense_buf_dma);  in mpi3mr_alloc_reply_sense_bufs()
3304 	if (!mrioc->sense_buf)  in mpi3mr_alloc_reply_sense_bufs()
3308 	sz = mrioc->sense_buf_q_sz * 8;  in mpi3mr_alloc_reply_sense_bufs()
3309 	mrioc->sense_buf_q_pool = dma_pool_create("sense_buf_q pool",  in mpi3mr_alloc_reply_sense_bufs()
3310 	    &mrioc->pdev->dev, sz, 8, 0);  in mpi3mr_alloc_reply_sense_bufs()
3311 	if (!mrioc->sense_buf_q_pool) {  in mpi3mr_alloc_reply_sense_bufs()
3315 	mrioc->sense_buf_q = dma_pool_zalloc(mrioc->sense_buf_q_pool,  in mpi3mr_alloc_reply_sense_bufs()
3316 	    GFP_KERNEL, &mrioc->sense_buf_q_dma);  in mpi3mr_alloc_reply_sense_bufs()
3317 	if (!mrioc->sense_buf_q)  in mpi3mr_alloc_reply_sense_bufs()
3323 	retval = -1;  in mpi3mr_alloc_reply_sense_bufs()
3328  * mpimr_initialize_reply_sbuf_queues - initialize reply sense
3342 	sz = mrioc->num_reply_bufs * mrioc->reply_sz;  in mpimr_initialize_reply_sbuf_queues()
3345 	    mrioc->reply_buf, mrioc->num_reply_bufs, mrioc->reply_sz,  in mpimr_initialize_reply_sbuf_queues()
3346 	    (sz / 1024), (unsigned long long)mrioc->reply_buf_dma);  in mpimr_initialize_reply_sbuf_queues()
3347 	sz = mrioc->reply_free_qsz * 8;  in mpimr_initialize_reply_sbuf_queues()
3350 	    mrioc->reply_free_q, mrioc->reply_free_qsz, 8, (sz / 1024),  in mpimr_initialize_reply_sbuf_queues()
3351 	    (unsigned long long)mrioc->reply_free_q_dma);  in mpimr_initialize_reply_sbuf_queues()
3352 	sz = mrioc->num_sense_bufs * MPI3MR_SENSE_BUF_SZ;  in mpimr_initialize_reply_sbuf_queues()
3355 	    mrioc->sense_buf, mrioc->num_sense_bufs, MPI3MR_SENSE_BUF_SZ,  in mpimr_initialize_reply_sbuf_queues()
3356 	    (sz / 1024), (unsigned long long)mrioc->sense_buf_dma);  in mpimr_initialize_reply_sbuf_queues()
3357 	sz = mrioc->sense_buf_q_sz * 8;  in mpimr_initialize_reply_sbuf_queues()
3360 	    mrioc->sense_buf_q, mrioc->sense_buf_q_sz, 8, (sz / 1024),  in mpimr_initialize_reply_sbuf_queues()
3361 	    (unsigned long long)mrioc->sense_buf_q_dma);  in mpimr_initialize_reply_sbuf_queues()
3364 	for (i = 0, phy_addr = mrioc->reply_buf_dma;  in mpimr_initialize_reply_sbuf_queues()
3365 	    i < mrioc->num_reply_bufs; i++, phy_addr += mrioc->reply_sz)  in mpimr_initialize_reply_sbuf_queues()
3366 		mrioc->reply_free_q[i] = cpu_to_le64(phy_addr);  in mpimr_initialize_reply_sbuf_queues()
3367 	mrioc->reply_free_q[i] = cpu_to_le64(0);  in mpimr_initialize_reply_sbuf_queues()
3370 	for (i = 0, phy_addr = mrioc->sense_buf_dma;  in mpimr_initialize_reply_sbuf_queues()
3371 	    i < mrioc->num_sense_bufs; i++, phy_addr += MPI3MR_SENSE_BUF_SZ)  in mpimr_initialize_reply_sbuf_queues()
3372 		mrioc->sense_buf_q[i] = cpu_to_le64(phy_addr);  in mpimr_initialize_reply_sbuf_queues()
3373 	mrioc->sense_buf_q[i] = cpu_to_le64(0);  in mpimr_initialize_reply_sbuf_queues()
3377  * mpi3mr_issue_iocinit - Send IOC Init
3383  * Return: 0 on success, non-zero on failures.
3394 	drv_info = dma_alloc_coherent(&mrioc->pdev->dev, data_len, &data_dma,  in mpi3mr_issue_iocinit()
3397 		retval = -1;  in mpi3mr_issue_iocinit()
3402 	drv_info->information_length = cpu_to_le32(data_len);  in mpi3mr_issue_iocinit()
3403 	strscpy(drv_info->driver_signature, "Broadcom", sizeof(drv_info->driver_signature));  in mpi3mr_issue_iocinit()
3404 	strscpy(drv_info->os_name, utsname()->sysname, sizeof(drv_info->os_name));  in mpi3mr_issue_iocinit()
3405 	strscpy(drv_info->os_version, utsname()->release, sizeof(drv_info->os_version));  in mpi3mr_issue_iocinit()
3406 	strscpy(drv_info->driver_name, MPI3MR_DRIVER_NAME, sizeof(drv_info->driver_name));  in mpi3mr_issue_iocinit()
3407 	strscpy(drv_info->driver_version, MPI3MR_DRIVER_VERSION, sizeof(drv_info->driver_version));  in mpi3mr_issue_iocinit()
3408 	strscpy(drv_info->driver_release_date, MPI3MR_DRIVER_RELDATE,  in mpi3mr_issue_iocinit()
3409 	    sizeof(drv_info->driver_release_date));  in mpi3mr_issue_iocinit()
3410 	drv_info->driver_capabilities = 0;  in mpi3mr_issue_iocinit()
3411 	memcpy((u8 *)&mrioc->driver_info, (u8 *)drv_info,  in mpi3mr_issue_iocinit()
3412 	    sizeof(mrioc->driver_info));  in mpi3mr_issue_iocinit()
3415 	mutex_lock(&mrioc->init_cmds.mutex);  in mpi3mr_issue_iocinit()
3416 	if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {  in mpi3mr_issue_iocinit()
3417 		retval = -1;  in mpi3mr_issue_iocinit()
3419 		mutex_unlock(&mrioc->init_cmds.mutex);  in mpi3mr_issue_iocinit()
3422 	mrioc->init_cmds.state = MPI3MR_CMD_PENDING;  in mpi3mr_issue_iocinit()
3423 	mrioc->init_cmds.is_waiting = 1;  in mpi3mr_issue_iocinit()
3424 	mrioc->init_cmds.callback = NULL;  in mpi3mr_issue_iocinit()
3432 	iocinit_req.reply_free_queue_depth = cpu_to_le16(mrioc->reply_free_qsz);  in mpi3mr_issue_iocinit()
3434 	    cpu_to_le64(mrioc->reply_free_q_dma);  in mpi3mr_issue_iocinit()
3437 	    cpu_to_le16(mrioc->sense_buf_q_sz);  in mpi3mr_issue_iocinit()
3439 	    cpu_to_le64(mrioc->sense_buf_q_dma);  in mpi3mr_issue_iocinit()
3450 	init_completion(&mrioc->init_cmds.done);  in mpi3mr_issue_iocinit()
3457 	wait_for_completion_timeout(&mrioc->init_cmds.done,  in mpi3mr_issue_iocinit()
3459 	if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {  in mpi3mr_issue_iocinit()
3463 		retval = -1;  in mpi3mr_issue_iocinit()
3466 	if ((mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)  in mpi3mr_issue_iocinit()
3470 		    (mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),  in mpi3mr_issue_iocinit()
3471 		    mrioc->init_cmds.ioc_loginfo);  in mpi3mr_issue_iocinit()
3472 		retval = -1;  in mpi3mr_issue_iocinit()
3476 	mrioc->reply_free_queue_host_index = mrioc->num_reply_bufs;  in mpi3mr_issue_iocinit()
3477 	writel(mrioc->reply_free_queue_host_index,  in mpi3mr_issue_iocinit()
3478 	    &mrioc->sysif_regs->reply_free_host_index);  in mpi3mr_issue_iocinit()
3480 	mrioc->sbq_host_index = mrioc->num_sense_bufs;  in mpi3mr_issue_iocinit()
3481 	writel(mrioc->sbq_host_index,  in mpi3mr_issue_iocinit()
3482 	    &mrioc->sysif_regs->sense_buffer_free_host_index);  in mpi3mr_issue_iocinit()
3484 	mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;  in mpi3mr_issue_iocinit()
3485 	mutex_unlock(&mrioc->init_cmds.mutex);  in mpi3mr_issue_iocinit()
3489 		dma_free_coherent(&mrioc->pdev->dev, data_len, drv_info,  in mpi3mr_issue_iocinit()
3496  * mpi3mr_unmask_events - Unmask events in event mask bitmap
3503  * Return: 0 on success, non-zero on failures.
3516 	mrioc->event_masks[word] &= ~desired_event;  in mpi3mr_unmask_events()
3520  * mpi3mr_issue_event_notification - Send event notification
3526  * Return: 0 on success, non-zero on failures.
3535 	mutex_lock(&mrioc->init_cmds.mutex);  in mpi3mr_issue_event_notification()
3536 	if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {  in mpi3mr_issue_event_notification()
3537 		retval = -1;  in mpi3mr_issue_event_notification()
3539 		mutex_unlock(&mrioc->init_cmds.mutex);  in mpi3mr_issue_event_notification()
3542 	mrioc->init_cmds.state = MPI3MR_CMD_PENDING;  in mpi3mr_issue_event_notification()
3543 	mrioc->init_cmds.is_waiting = 1;  in mpi3mr_issue_event_notification()
3544 	mrioc->init_cmds.callback = NULL;  in mpi3mr_issue_event_notification()
3549 		    cpu_to_le32(mrioc->event_masks[i]);  in mpi3mr_issue_event_notification()
3550 	init_completion(&mrioc->init_cmds.done);  in mpi3mr_issue_event_notification()
3557 	wait_for_completion_timeout(&mrioc->init_cmds.done,  in mpi3mr_issue_event_notification()
3559 	if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {  in mpi3mr_issue_event_notification()
3563 		retval = -1;  in mpi3mr_issue_event_notification()
3566 	if ((mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)  in mpi3mr_issue_event_notification()
3570 		    (mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),  in mpi3mr_issue_event_notification()
3571 		    mrioc->init_cmds.ioc_loginfo);  in mpi3mr_issue_event_notification()
3572 		retval = -1;  in mpi3mr_issue_event_notification()
3577 	mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;  in mpi3mr_issue_event_notification()
3578 	mutex_unlock(&mrioc->init_cmds.mutex);  in mpi3mr_issue_event_notification()
3584  * mpi3mr_process_event_ack - Process event acknowledgment
3592  * Return: 0 on success, non-zero on failures.
3601 	mutex_lock(&mrioc->init_cmds.mutex);  in mpi3mr_process_event_ack()
3602 	if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {  in mpi3mr_process_event_ack()
3603 		retval = -1;  in mpi3mr_process_event_ack()
3605 		mutex_unlock(&mrioc->init_cmds.mutex);  in mpi3mr_process_event_ack()
3608 	mrioc->init_cmds.state = MPI3MR_CMD_PENDING;  in mpi3mr_process_event_ack()
3609 	mrioc->init_cmds.is_waiting = 1;  in mpi3mr_process_event_ack()
3610 	mrioc->init_cmds.callback = NULL;  in mpi3mr_process_event_ack()
3616 	init_completion(&mrioc->init_cmds.done);  in mpi3mr_process_event_ack()
3623 	wait_for_completion_timeout(&mrioc->init_cmds.done,  in mpi3mr_process_event_ack()
3625 	if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {  in mpi3mr_process_event_ack()
3627 		if (!(mrioc->init_cmds.state & MPI3MR_CMD_RESET))  in mpi3mr_process_event_ack()
3630 		retval = -1;  in mpi3mr_process_event_ack()
3633 	if ((mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)  in mpi3mr_process_event_ack()
3637 		    (mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),  in mpi3mr_process_event_ack()
3638 		    mrioc->init_cmds.ioc_loginfo);  in mpi3mr_process_event_ack()
3639 		retval = -1;  in mpi3mr_process_event_ack()
3644 	mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;  in mpi3mr_process_event_ack()
3645 	mutex_unlock(&mrioc->init_cmds.mutex);  in mpi3mr_process_event_ack()
3651  * mpi3mr_alloc_chain_bufs - Allocate chain buffers
3658  * Return: 0 on success, non-zero on failure
3666 	if (mrioc->chain_sgl_list)  in mpi3mr_alloc_chain_bufs()
3669 	num_chains = mrioc->max_host_ios / MPI3MR_CHAINBUF_FACTOR;  in mpi3mr_alloc_chain_bufs()
3677 	mrioc->chain_buf_count = num_chains;  in mpi3mr_alloc_chain_bufs()
3679 	mrioc->chain_sgl_list = kzalloc(sz, GFP_KERNEL);  in mpi3mr_alloc_chain_bufs()
3680 	if (!mrioc->chain_sgl_list)  in mpi3mr_alloc_chain_bufs()
3683 	if (mrioc->max_sgl_entries > (mrioc->facts.max_data_length /  in mpi3mr_alloc_chain_bufs()
3685 		mrioc->max_sgl_entries = mrioc->facts.max_data_length /  in mpi3mr_alloc_chain_bufs()
3687 	sz = mrioc->max_sgl_entries * sizeof(struct mpi3_sge_common);  in mpi3mr_alloc_chain_bufs()
3689 			mrioc->max_sgl_entries, sz/1024);  in mpi3mr_alloc_chain_bufs()
3691 	mrioc->chain_buf_pool = dma_pool_create("chain_buf pool",  in mpi3mr_alloc_chain_bufs()
3692 	    &mrioc->pdev->dev, sz, 16, 0);  in mpi3mr_alloc_chain_bufs()
3693 	if (!mrioc->chain_buf_pool) {  in mpi3mr_alloc_chain_bufs()
3699 		mrioc->chain_sgl_list[i].addr =  in mpi3mr_alloc_chain_bufs()
3700 		    dma_pool_zalloc(mrioc->chain_buf_pool, GFP_KERNEL,  in mpi3mr_alloc_chain_bufs()
3701 		    &mrioc->chain_sgl_list[i].dma_addr);  in mpi3mr_alloc_chain_bufs()
3703 		if (!mrioc->chain_sgl_list[i].addr)  in mpi3mr_alloc_chain_bufs()
3706 	mrioc->chain_bitmap = bitmap_zalloc(num_chains, GFP_KERNEL);  in mpi3mr_alloc_chain_bufs()
3707 	if (!mrioc->chain_bitmap)  in mpi3mr_alloc_chain_bufs()
3711 	retval = -1;  in mpi3mr_alloc_chain_bufs()
3716  * mpi3mr_port_enable_complete - Mark port enable complete
3728 	drv_cmd->callback = NULL;  in mpi3mr_port_enable_complete()
3729 	mrioc->scan_started = 0;  in mpi3mr_port_enable_complete()
3730 	if (drv_cmd->state & MPI3MR_CMD_RESET)  in mpi3mr_port_enable_complete()
3731 		mrioc->scan_failed = MPI3_IOCSTATUS_INTERNAL_ERROR;  in mpi3mr_port_enable_complete()
3733 		mrioc->scan_failed = drv_cmd->ioc_status;  in mpi3mr_port_enable_complete()
3734 	drv_cmd->state = MPI3MR_CMD_NOTUSED;  in mpi3mr_port_enable_complete()
3738  * mpi3mr_issue_port_enable - Issue Port Enable
3746  * Return: 0 on success, non-zero on failures.
3755 	mutex_lock(&mrioc->init_cmds.mutex);  in mpi3mr_issue_port_enable()
3756 	if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {  in mpi3mr_issue_port_enable()
3757 		retval = -1;  in mpi3mr_issue_port_enable()
3759 		mutex_unlock(&mrioc->init_cmds.mutex);  in mpi3mr_issue_port_enable()
3762 	mrioc->init_cmds.state = MPI3MR_CMD_PENDING;  in mpi3mr_issue_port_enable()
3764 		mrioc->init_cmds.is_waiting = 0;  in mpi3mr_issue_port_enable()
3765 		mrioc->init_cmds.callback = mpi3mr_port_enable_complete;  in mpi3mr_issue_port_enable()
3767 		mrioc->init_cmds.is_waiting = 1;  in mpi3mr_issue_port_enable()
3768 		mrioc->init_cmds.callback = NULL;  in mpi3mr_issue_port_enable()
3769 		init_completion(&mrioc->init_cmds.done);  in mpi3mr_issue_port_enable()
3780 		mutex_unlock(&mrioc->init_cmds.mutex);  in mpi3mr_issue_port_enable()
3784 	wait_for_completion_timeout(&mrioc->init_cmds.done, (pe_timeout * HZ));  in mpi3mr_issue_port_enable()
3785 	if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {  in mpi3mr_issue_port_enable()
3787 		retval = -1;  in mpi3mr_issue_port_enable()
3791 	mpi3mr_port_enable_complete(mrioc, &mrioc->init_cmds);  in mpi3mr_issue_port_enable()
3794 	mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;  in mpi3mr_issue_port_enable()
3795 	mutex_unlock(&mrioc->init_cmds.mutex);  in mpi3mr_issue_port_enable()
3820  * mpi3mr_repost_diag_bufs - repost host diag buffers
3823  * repost firmware and trace diag buffers based on global
3826  * Return: 0 on success, non-zero on failures.
3841 		return -1;  in mpi3mr_repost_diag_bufs()
3847 	    trace_hdb->status != MPI3MR_HDB_BUFSTATUS_NOT_ALLOCATED &&  in mpi3mr_repost_diag_bufs()
3848 	    trace_hdb->trigger_type != MPI3MR_HDB_TRIGGER_TYPE_GLOBAL &&  in mpi3mr_repost_diag_bufs()
3849 	    trace_hdb->trigger_type != MPI3MR_HDB_TRIGGER_TYPE_ELEMENT)  in mpi3mr_repost_diag_bufs()
3854 	if (fw_hdb && fw_hdb->status != MPI3MR_HDB_BUFSTATUS_NOT_ALLOCATED &&  in mpi3mr_repost_diag_bufs()
3855 	    fw_hdb->trigger_type != MPI3MR_HDB_TRIGGER_TYPE_GLOBAL &&  in mpi3mr_repost_diag_bufs()
3856 	    fw_hdb->trigger_type != MPI3MR_HDB_TRIGGER_TYPE_ELEMENT)  in mpi3mr_repost_diag_bufs()
3860 		global_trigger = le64_to_cpu(mrioc->driver_pg2->global_trigger);  in mpi3mr_repost_diag_bufs()
3870 		prev_trigger_type = trace_hdb->trigger_type;  in mpi3mr_repost_diag_bufs()
3871 		memcpy(&prev_trigger_data, &trace_hdb->trigger_data,  in mpi3mr_repost_diag_bufs()
3872 		    sizeof(trace_hdb->trigger_data));  in mpi3mr_repost_diag_bufs()
3875 			dprint_init(mrioc, "trace diag buffer reposted");  in mpi3mr_repost_diag_bufs()
3879 			trace_hdb->trigger_type = prev_trigger_type;  in mpi3mr_repost_diag_bufs()
3880 			memcpy(&trace_hdb->trigger_data, &prev_trigger_data,  in mpi3mr_repost_diag_bufs()
3882 			ioc_err(mrioc, "trace diag buffer repost failed");  in mpi3mr_repost_diag_bufs()
3883 			return -1;  in mpi3mr_repost_diag_bufs()
3888 		prev_trigger_type = fw_hdb->trigger_type;  in mpi3mr_repost_diag_bufs()
3889 		memcpy(&prev_trigger_data, &fw_hdb->trigger_data,  in mpi3mr_repost_diag_bufs()
3890 		    sizeof(fw_hdb->trigger_data));  in mpi3mr_repost_diag_bufs()
3893 			dprint_init(mrioc, "firmware diag buffer reposted");  in mpi3mr_repost_diag_bufs()
3897 			fw_hdb->trigger_type = prev_trigger_type;  in mpi3mr_repost_diag_bufs()
3898 			memcpy(&fw_hdb->trigger_data, &prev_trigger_data,  in mpi3mr_repost_diag_bufs()
3900 			ioc_err(mrioc, "firmware diag buffer repost failed");  in mpi3mr_repost_diag_bufs()
3901 			return -1;  in mpi3mr_repost_diag_bufs()
3908  * mpi3mr_read_tsu_interval - Update time stamp interval
3923 	mrioc->ts_update_interval = MPI3MR_TSUPDATE_INTERVAL;  in mpi3mr_read_tsu_interval()
3927 		mrioc->ts_update_interval = (driver_pg1.time_stamp_update * 60);  in mpi3mr_read_tsu_interval()
3931  * mpi3mr_print_ioc_info - Display controller information
3946 	struct mpi3mr_compimg_ver *fwver = &mrioc->facts.fw_ver;  in mpi3mr_print_ioc_info()
3948 	switch (mrioc->facts.personality) {  in mpi3mr_print_ioc_info()
3962 	ioc_info(mrioc, "FW version(%d.%d.%d.%d.%d.%d)\n",  in mpi3mr_print_ioc_info()
3963 	    fwver->gen_major, fwver->gen_minor, fwver->ph_major,  in mpi3mr_print_ioc_info()
3964 	    fwver->ph_minor, fwver->cust_id, fwver->build_num);  in mpi3mr_print_ioc_info()
3967 		if (mrioc->facts.protocol_flags &  in mpi3mr_print_ioc_info()
3970 				    sizeof(protocol) - bytes_written, "%s%s",  in mpi3mr_print_ioc_info()
3978 		if (mrioc->facts.protocol_flags &  in mpi3mr_print_ioc_info()
3981 				    sizeof(capabilities) - bytes_written, "%s%s",  in mpi3mr_print_ioc_info()
3992  * mpi3mr_cleanup_resources - Free PCI resources
3997  * Return: 0 on success and non-zero on failure.
4001 	struct pci_dev *pdev = mrioc->pdev;  in mpi3mr_cleanup_resources()
4005 	if (mrioc->sysif_regs) {  in mpi3mr_cleanup_resources()
4006 		iounmap((void __iomem *)mrioc->sysif_regs);  in mpi3mr_cleanup_resources()
4007 		mrioc->sysif_regs = NULL;  in mpi3mr_cleanup_resources()
4011 		if (mrioc->bars)  in mpi3mr_cleanup_resources()
4012 			pci_release_selected_regions(pdev, mrioc->bars);  in mpi3mr_cleanup_resources()
4018  * mpi3mr_setup_resources - Enable PCI resources
4021  * Enable PCI device memory, MSI-x registers and set DMA mask.
4023  * Return: 0 on success and non-zero on failure.
4027 	struct pci_dev *pdev = mrioc->pdev;  in mpi3mr_setup_resources()
4031 	u64 dma_mask = mrioc->dma_mask ? mrioc->dma_mask :  in mpi3mr_setup_resources()
4036 		retval = -ENODEV;  in mpi3mr_setup_resources()
4042 		ioc_err(mrioc, "Unable to find MSI-X Capabilities\n");  in mpi3mr_setup_resources()
4043 		retval = -ENODEV;  in mpi3mr_setup_resources()
4046 	mrioc->bars = pci_select_bars(pdev, IORESOURCE_MEM);  in mpi3mr_setup_resources()
4048 	if (pci_request_selected_regions(pdev, mrioc->bars,  in mpi3mr_setup_resources()
4049 	    mrioc->driver_name)) {  in mpi3mr_setup_resources()
4051 		retval = -ENODEV;  in mpi3mr_setup_resources()
4057 			mrioc->sysif_regs_phys = pci_resource_start(pdev, i);  in mpi3mr_setup_resources()
4059 			mrioc->sysif_regs =  in mpi3mr_setup_resources()
4060 			    ioremap(mrioc->sysif_regs_phys, memap_sz);  in mpi3mr_setup_resources()
4067 	retval = dma_set_mask_and_coherent(&pdev->dev, dma_mask);  in mpi3mr_setup_resources()
4072 			retval = dma_set_mask_and_coherent(&pdev->dev,  in mpi3mr_setup_resources()
4076 			mrioc->dma_mask = 0;  in mpi3mr_setup_resources()
4081 	mrioc->dma_mask = dma_mask;  in mpi3mr_setup_resources()
4083 	if (!mrioc->sysif_regs) {  in mpi3mr_setup_resources()
4086 		retval = -EINVAL;  in mpi3mr_setup_resources()
4091 	mrioc->msix_count = (message_control & 0x3FF) + 1;  in mpi3mr_setup_resources()
4095 	pci_set_drvdata(pdev, mrioc->shost);  in mpi3mr_setup_resources()
4100 	    (unsigned long long)mrioc->sysif_regs_phys,  in mpi3mr_setup_resources()
4101 	    mrioc->sysif_regs, memap_sz);  in mpi3mr_setup_resources()
4102 	ioc_info(mrioc, "Number of MSI-X vectors found in capabilities: (%d)\n",  in mpi3mr_setup_resources()
4103 	    mrioc->msix_count);  in mpi3mr_setup_resources()
4106 		mrioc->requested_poll_qcount = min_t(int, poll_queues,  in mpi3mr_setup_resources()
4107 				mrioc->msix_count - 2);  in mpi3mr_setup_resources()
4116  * mpi3mr_enable_events - Enable required events
4123  * Return: 0 on success and non-zero on failure.
4131 		mrioc->event_masks[i] = -1;  in mpi3mr_enable_events()
4157  * mpi3mr_init_ioc - Initialize the controller
4169  * Return: 0 on success and non-zero on failure.
4200 	mrioc->max_host_ios = mrioc->facts.max_reqs - MPI3MR_INTERNAL_CMDS_RESVD;  in mpi3mr_init_ioc()
4201 	mrioc->shost->max_sectors = mrioc->facts.max_data_length / 512;  in mpi3mr_init_ioc()
4202 	mrioc->num_io_throttle_group = mrioc->facts.max_io_throttle_group;  in mpi3mr_init_ioc()
4203 	atomic_set(&mrioc->pend_large_data_sz, 0);  in mpi3mr_init_ioc()
4206 		mrioc->max_host_ios = min_t(int, mrioc->max_host_ios,  in mpi3mr_init_ioc()
4209 	if (!(mrioc->facts.ioc_capabilities &  in mpi3mr_init_ioc()
4211 		mrioc->sas_transport_enabled = 1;  in mpi3mr_init_ioc()
4212 		mrioc->scsi_device_channel = 1;  in mpi3mr_init_ioc()
4213 		mrioc->shost->max_channel = 1;  in mpi3mr_init_ioc()
4214 		mrioc->shost->transportt = mpi3mr_transport_template;  in mpi3mr_init_ioc()
4217 	mrioc->reply_sz = mrioc->facts.reply_sz;  in mpi3mr_init_ioc()
4229 	dprint_init(mrioc, "allocating host diag buffers\n");  in mpi3mr_init_ioc()
4235 	dprint_init(mrioc, "posting host diag buffers\n");  in mpi3mr_init_ioc()
4239 		ioc_warn(mrioc, "failed to post host diag buffers\n");  in mpi3mr_init_ioc()
4241 	if (!mrioc->init_cmds.reply) {  in mpi3mr_init_ioc()
4251 	if (!mrioc->chain_sgl_list) {  in mpi3mr_init_ioc()
4269 		ioc_err(mrioc, "failed to get package version\n");  in mpi3mr_init_ioc()
4275 		ioc_err(mrioc, "Failed to re-setup ISR, error %d\n",  in mpi3mr_init_ioc()
4287 	if (!mrioc->pel_seqnum_virt) {  in mpi3mr_init_ioc()
4289 		mrioc->pel_seqnum_sz = sizeof(struct mpi3_pel_seq);  in mpi3mr_init_ioc()
4290 		mrioc->pel_seqnum_virt = dma_alloc_coherent(&mrioc->pdev->dev,  in mpi3mr_init_ioc()
4291 		    mrioc->pel_seqnum_sz, &mrioc->pel_seqnum_dma,  in mpi3mr_init_ioc()
4293 		if (!mrioc->pel_seqnum_virt) {  in mpi3mr_init_ioc()
4294 			retval = -ENOMEM;  in mpi3mr_init_ioc()
4299 	if (!mrioc->throttle_groups && mrioc->num_io_throttle_group) {  in mpi3mr_init_ioc()
4302 		mrioc->throttle_groups = kcalloc(mrioc->num_io_throttle_group, sz, GFP_KERNEL);  in mpi3mr_init_ioc()
4303 		if (!mrioc->throttle_groups) {  in mpi3mr_init_ioc()
4304 			retval = -1;  in mpi3mr_init_ioc()
4332 	retval = -1;  in mpi3mr_init_ioc()
4337 	mrioc->unrecoverable = 1;  in mpi3mr_init_ioc()
4342  * mpi3mr_reinit_ioc - Re-Initialize the controller
4346  * This the controller re-initialization routine, executed from
4353  * Return: 0 on success and non-zero on failure.
4373 	if (is_resume || mrioc->block_on_pci_err) {  in mpi3mr_reinit_ioc()
4401 		dprint_reset(mrioc, "posting host diag buffers\n");  in mpi3mr_reinit_ioc()
4404 			ioc_warn(mrioc, "failed to post host diag buffers\n");  in mpi3mr_reinit_ioc()
4408 			ioc_warn(mrioc, "failed to re post host diag buffers\n");  in mpi3mr_reinit_ioc()
4418 	dprint_reset(mrioc, "getting package version\n");  in mpi3mr_reinit_ioc()
4421 		ioc_err(mrioc, "failed to get package version\n");  in mpi3mr_reinit_ioc()
4425 	if (is_resume || mrioc->block_on_pci_err) {  in mpi3mr_reinit_ioc()
4429 			ioc_err(mrioc, "failed to re-setup ISR\n");  in mpi3mr_reinit_ioc()
4441 	if (!mrioc->pel_seqnum_virt) {  in mpi3mr_reinit_ioc()
4443 		mrioc->pel_seqnum_sz = sizeof(struct mpi3_pel_seq);  in mpi3mr_reinit_ioc()
4444 		mrioc->pel_seqnum_virt = dma_alloc_coherent(&mrioc->pdev->dev,  in mpi3mr_reinit_ioc()
4445 		    mrioc->pel_seqnum_sz, &mrioc->pel_seqnum_dma,  in mpi3mr_reinit_ioc()
4447 		if (!mrioc->pel_seqnum_virt) {  in mpi3mr_reinit_ioc()
4448 			retval = -ENOMEM;  in mpi3mr_reinit_ioc()
4453 	if (mrioc->shost->nr_hw_queues > mrioc->num_op_reply_q) {  in mpi3mr_reinit_ioc()
4456 		    mrioc->shost->nr_hw_queues, mrioc->num_op_reply_q);  in mpi3mr_reinit_ioc()
4457 		retval = -1;  in mpi3mr_reinit_ioc()
4468 	mrioc->device_refresh_on = 1;  in mpi3mr_reinit_ioc()
4479 		if (mrioc->init_cmds.state == MPI3MR_CMD_NOTUSED)  in mpi3mr_reinit_ioc()
4481 		if (!pci_device_is_present(mrioc->pdev))  in mpi3mr_reinit_ioc()
4482 			mrioc->unrecoverable = 1;  in mpi3mr_reinit_ioc()
4483 		if (mrioc->unrecoverable) {  in mpi3mr_reinit_ioc()
4484 			retval = -1;  in mpi3mr_reinit_ioc()
4487 		ioc_status = readl(&mrioc->sysif_regs->ioc_status);  in mpi3mr_reinit_ioc()
4491 			mrioc->init_cmds.is_waiting = 0;  in mpi3mr_reinit_ioc()
4492 			mrioc->init_cmds.callback = NULL;  in mpi3mr_reinit_ioc()
4493 			mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;  in mpi3mr_reinit_ioc()
4496 	} while (--pe_timeout);  in mpi3mr_reinit_ioc()
4502 		mrioc->init_cmds.is_waiting = 0;  in mpi3mr_reinit_ioc()
4503 		mrioc->init_cmds.callback = NULL;  in mpi3mr_reinit_ioc()
4504 		mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;  in mpi3mr_reinit_ioc()
4506 	} else if (mrioc->scan_failed) {  in mpi3mr_reinit_ioc()
4509 		    mrioc->scan_failed);  in mpi3mr_reinit_ioc()
4514 	    (is_resume)?"resume":"re-initialization");  in mpi3mr_reinit_ioc()
4520 		    (is_resume)?"resume":"re-initialization", retry);  in mpi3mr_reinit_ioc()
4524 	retval = -1;  in mpi3mr_reinit_ioc()
4527 	    (is_resume)?"resume":"re-initialization");  in mpi3mr_reinit_ioc()
4530 	mrioc->unrecoverable = 1;  in mpi3mr_reinit_ioc()
4535  * mpi3mr_memset_op_reply_q_buffers - memset the operational reply queue's
4544 	struct op_reply_qinfo *op_reply_q = mrioc->op_reply_qinfo + qidx;  in mpi3mr_memset_op_reply_q_buffers()
4548 	if (!op_reply_q->q_segments)  in mpi3mr_memset_op_reply_q_buffers()
4551 	size = op_reply_q->segment_qd * mrioc->op_reply_desc_sz;  in mpi3mr_memset_op_reply_q_buffers()
4552 	segments = op_reply_q->q_segments;  in mpi3mr_memset_op_reply_q_buffers()
4553 	for (i = 0; i < op_reply_q->num_segments; i++)  in mpi3mr_memset_op_reply_q_buffers()
4558  * mpi3mr_memset_op_req_q_buffers - memset the operational request queue's
4567 	struct op_req_qinfo *op_req_q = mrioc->req_qinfo + qidx;  in mpi3mr_memset_op_req_q_buffers()
4571 	if (!op_req_q->q_segments)  in mpi3mr_memset_op_req_q_buffers()
4574 	size = op_req_q->segment_qd * mrioc->facts.op_req_sz;  in mpi3mr_memset_op_req_q_buffers()
4575 	segments = op_req_q->q_segments;  in mpi3mr_memset_op_req_q_buffers()
4576 	for (i = 0; i < op_req_q->num_segments; i++)  in mpi3mr_memset_op_req_q_buffers()
4581  * mpi3mr_memset_buffers - memset memory for a controller
4595 	mrioc->change_count = 0;  in mpi3mr_memset_buffers()
4596 	mrioc->active_poll_qcount = 0;  in mpi3mr_memset_buffers()
4597 	mrioc->default_qcount = 0;  in mpi3mr_memset_buffers()
4598 	if (mrioc->admin_req_base)  in mpi3mr_memset_buffers()
4599 		memset(mrioc->admin_req_base, 0, mrioc->admin_req_q_sz);  in mpi3mr_memset_buffers()
4600 	if (mrioc->admin_reply_base)  in mpi3mr_memset_buffers()
4601 		memset(mrioc->admin_reply_base, 0, mrioc->admin_reply_q_sz);  in mpi3mr_memset_buffers()
4602 	atomic_set(&mrioc->admin_reply_q_in_use, 0);  in mpi3mr_memset_buffers()
4604 	if (mrioc->init_cmds.reply) {  in mpi3mr_memset_buffers()
4605 		memset(mrioc->init_cmds.reply, 0, sizeof(*mrioc->init_cmds.reply));  in mpi3mr_memset_buffers()
4606 		memset(mrioc->bsg_cmds.reply, 0,  in mpi3mr_memset_buffers()
4607 		    sizeof(*mrioc->bsg_cmds.reply));  in mpi3mr_memset_buffers()
4608 		memset(mrioc->host_tm_cmds.reply, 0,  in mpi3mr_memset_buffers()
4609 		    sizeof(*mrioc->host_tm_cmds.reply));  in mpi3mr_memset_buffers()
4610 		memset(mrioc->pel_cmds.reply, 0,  in mpi3mr_memset_buffers()
4611 		    sizeof(*mrioc->pel_cmds.reply));  in mpi3mr_memset_buffers()
4612 		memset(mrioc->pel_abort_cmd.reply, 0,  in mpi3mr_memset_buffers()
4613 		    sizeof(*mrioc->pel_abort_cmd.reply));  in mpi3mr_memset_buffers()
4614 		memset(mrioc->transport_cmds.reply, 0,  in mpi3mr_memset_buffers()
4615 		    sizeof(*mrioc->transport_cmds.reply));  in mpi3mr_memset_buffers()
4617 			memset(mrioc->dev_rmhs_cmds[i].reply, 0,  in mpi3mr_memset_buffers()
4618 			    sizeof(*mrioc->dev_rmhs_cmds[i].reply));  in mpi3mr_memset_buffers()
4620 			memset(mrioc->evtack_cmds[i].reply, 0,  in mpi3mr_memset_buffers()
4621 			    sizeof(*mrioc->evtack_cmds[i].reply));  in mpi3mr_memset_buffers()
4622 		bitmap_clear(mrioc->removepend_bitmap, 0,  in mpi3mr_memset_buffers()
4623 			     mrioc->dev_handle_bitmap_bits);  in mpi3mr_memset_buffers()
4624 		bitmap_clear(mrioc->devrem_bitmap, 0, MPI3MR_NUM_DEVRMCMD);  in mpi3mr_memset_buffers()
4625 		bitmap_clear(mrioc->evtack_cmds_bitmap, 0,  in mpi3mr_memset_buffers()
4629 	for (i = 0; i < mrioc->num_queues; i++) {  in mpi3mr_memset_buffers()
4630 		mrioc->op_reply_qinfo[i].qid = 0;  in mpi3mr_memset_buffers()
4631 		mrioc->op_reply_qinfo[i].ci = 0;  in mpi3mr_memset_buffers()
4632 		mrioc->op_reply_qinfo[i].num_replies = 0;  in mpi3mr_memset_buffers()
4633 		mrioc->op_reply_qinfo[i].ephase = 0;  in mpi3mr_memset_buffers()
4634 		atomic_set(&mrioc->op_reply_qinfo[i].pend_ios, 0);  in mpi3mr_memset_buffers()
4635 		atomic_set(&mrioc->op_reply_qinfo[i].in_use, 0);  in mpi3mr_memset_buffers()
4638 		mrioc->req_qinfo[i].ci = 0;  in mpi3mr_memset_buffers()
4639 		mrioc->req_qinfo[i].pi = 0;  in mpi3mr_memset_buffers()
4640 		mrioc->req_qinfo[i].num_requests = 0;  in mpi3mr_memset_buffers()
4641 		mrioc->req_qinfo[i].qid = 0;  in mpi3mr_memset_buffers()
4642 		mrioc->req_qinfo[i].reply_qid = 0;  in mpi3mr_memset_buffers()
4643 		spin_lock_init(&mrioc->req_qinfo[i].q_lock);  in mpi3mr_memset_buffers()
4647 	atomic_set(&mrioc->pend_large_data_sz, 0);  in mpi3mr_memset_buffers()
4648 	if (mrioc->throttle_groups) {  in mpi3mr_memset_buffers()
4649 		tg = mrioc->throttle_groups;  in mpi3mr_memset_buffers()
4650 		for (i = 0; i < mrioc->num_io_throttle_group; i++, tg++) {  in mpi3mr_memset_buffers()
4651 			tg->id = 0;  in mpi3mr_memset_buffers()
4652 			tg->fw_qd = 0;  in mpi3mr_memset_buffers()
4653 			tg->modified_qd = 0;  in mpi3mr_memset_buffers()
4654 			tg->io_divert = 0;  in mpi3mr_memset_buffers()
4655 			tg->need_qd_reduction = 0;  in mpi3mr_memset_buffers()
4656 			tg->high = 0;  in mpi3mr_memset_buffers()
4657 			tg->low = 0;  in mpi3mr_memset_buffers()
4658 			tg->qd_reduction = 0;  in mpi3mr_memset_buffers()
4659 			atomic_set(&tg->pend_large_data_sz, 0);  in mpi3mr_memset_buffers()
4665  * mpi3mr_free_mem - Free memory allocated for a controller
4681 	if (mrioc->sense_buf_pool) {  in mpi3mr_free_mem()
4682 		if (mrioc->sense_buf)  in mpi3mr_free_mem()
4683 			dma_pool_free(mrioc->sense_buf_pool, mrioc->sense_buf,  in mpi3mr_free_mem()
4684 			    mrioc->sense_buf_dma);  in mpi3mr_free_mem()
4685 		dma_pool_destroy(mrioc->sense_buf_pool);  in mpi3mr_free_mem()
4686 		mrioc->sense_buf = NULL;  in mpi3mr_free_mem()
4687 		mrioc->sense_buf_pool = NULL;  in mpi3mr_free_mem()
4689 	if (mrioc->sense_buf_q_pool) {  in mpi3mr_free_mem()
4690 		if (mrioc->sense_buf_q)  in mpi3mr_free_mem()
4691 			dma_pool_free(mrioc->sense_buf_q_pool,  in mpi3mr_free_mem()
4692 			    mrioc->sense_buf_q, mrioc->sense_buf_q_dma);  in mpi3mr_free_mem()
4693 		dma_pool_destroy(mrioc->sense_buf_q_pool);  in mpi3mr_free_mem()
4694 		mrioc->sense_buf_q = NULL;  in mpi3mr_free_mem()
4695 		mrioc->sense_buf_q_pool = NULL;  in mpi3mr_free_mem()
4698 	if (mrioc->reply_buf_pool) {  in mpi3mr_free_mem()
4699 		if (mrioc->reply_buf)  in mpi3mr_free_mem()
4700 			dma_pool_free(mrioc->reply_buf_pool, mrioc->reply_buf,  in mpi3mr_free_mem()
4701 			    mrioc->reply_buf_dma);  in mpi3mr_free_mem()
4702 		dma_pool_destroy(mrioc->reply_buf_pool);  in mpi3mr_free_mem()
4703 		mrioc->reply_buf = NULL;  in mpi3mr_free_mem()
4704 		mrioc->reply_buf_pool = NULL;  in mpi3mr_free_mem()
4706 	if (mrioc->reply_free_q_pool) {  in mpi3mr_free_mem()
4707 		if (mrioc->reply_free_q)  in mpi3mr_free_mem()
4708 			dma_pool_free(mrioc->reply_free_q_pool,  in mpi3mr_free_mem()
4709 			    mrioc->reply_free_q, mrioc->reply_free_q_dma);  in mpi3mr_free_mem()
4710 		dma_pool_destroy(mrioc->reply_free_q_pool);  in mpi3mr_free_mem()
4711 		mrioc->reply_free_q = NULL;  in mpi3mr_free_mem()
4712 		mrioc->reply_free_q_pool = NULL;  in mpi3mr_free_mem()
4715 	for (i = 0; i < mrioc->num_op_req_q; i++)  in mpi3mr_free_mem()
4718 	for (i = 0; i < mrioc->num_op_reply_q; i++)  in mpi3mr_free_mem()
4721 	for (i = 0; i < mrioc->intr_info_count; i++) {  in mpi3mr_free_mem()
4722 		intr_info = mrioc->intr_info + i;  in mpi3mr_free_mem()
4723 		intr_info->op_reply_q = NULL;  in mpi3mr_free_mem()
4726 	kfree(mrioc->req_qinfo);  in mpi3mr_free_mem()
4727 	mrioc->req_qinfo = NULL;  in mpi3mr_free_mem()
4728 	mrioc->num_op_req_q = 0;  in mpi3mr_free_mem()
4730 	kfree(mrioc->op_reply_qinfo);  in mpi3mr_free_mem()
4731 	mrioc->op_reply_qinfo = NULL;  in mpi3mr_free_mem()
4732 	mrioc->num_op_reply_q = 0;  in mpi3mr_free_mem()
4734 	kfree(mrioc->init_cmds.reply);  in mpi3mr_free_mem()
4735 	mrioc->init_cmds.reply = NULL;  in mpi3mr_free_mem()
4737 	kfree(mrioc->bsg_cmds.reply);  in mpi3mr_free_mem()
4738 	mrioc->bsg_cmds.reply = NULL;  in mpi3mr_free_mem()
4740 	kfree(mrioc->host_tm_cmds.reply);  in mpi3mr_free_mem()
4741 	mrioc->host_tm_cmds.reply = NULL;  in mpi3mr_free_mem()
4743 	kfree(mrioc->pel_cmds.reply);  in mpi3mr_free_mem()
4744 	mrioc->pel_cmds.reply = NULL;  in mpi3mr_free_mem()
4746 	kfree(mrioc->pel_abort_cmd.reply);  in mpi3mr_free_mem()
4747 	mrioc->pel_abort_cmd.reply = NULL;  in mpi3mr_free_mem()
4750 		kfree(mrioc->evtack_cmds[i].reply);  in mpi3mr_free_mem()
4751 		mrioc->evtack_cmds[i].reply = NULL;  in mpi3mr_free_mem()
4754 	bitmap_free(mrioc->removepend_bitmap);  in mpi3mr_free_mem()
4755 	mrioc->removepend_bitmap = NULL;  in mpi3mr_free_mem()
4757 	bitmap_free(mrioc->devrem_bitmap);  in mpi3mr_free_mem()
4758 	mrioc->devrem_bitmap = NULL;  in mpi3mr_free_mem()
4760 	bitmap_free(mrioc->evtack_cmds_bitmap);  in mpi3mr_free_mem()
4761 	mrioc->evtack_cmds_bitmap = NULL;  in mpi3mr_free_mem()
4763 	bitmap_free(mrioc->chain_bitmap);  in mpi3mr_free_mem()
4764 	mrioc->chain_bitmap = NULL;  in mpi3mr_free_mem()
4766 	kfree(mrioc->transport_cmds.reply);  in mpi3mr_free_mem()
4767 	mrioc->transport_cmds.reply = NULL;  in mpi3mr_free_mem()
4770 		kfree(mrioc->dev_rmhs_cmds[i].reply);  in mpi3mr_free_mem()
4771 		mrioc->dev_rmhs_cmds[i].reply = NULL;  in mpi3mr_free_mem()
4774 	if (mrioc->chain_buf_pool) {  in mpi3mr_free_mem()
4775 		for (i = 0; i < mrioc->chain_buf_count; i++) {  in mpi3mr_free_mem()
4776 			if (mrioc->chain_sgl_list[i].addr) {  in mpi3mr_free_mem()
4777 				dma_pool_free(mrioc->chain_buf_pool,  in mpi3mr_free_mem()
4778 				    mrioc->chain_sgl_list[i].addr,  in mpi3mr_free_mem()
4779 				    mrioc->chain_sgl_list[i].dma_addr);  in mpi3mr_free_mem()
4780 				mrioc->chain_sgl_list[i].addr = NULL;  in mpi3mr_free_mem()
4783 		dma_pool_destroy(mrioc->chain_buf_pool);  in mpi3mr_free_mem()
4784 		mrioc->chain_buf_pool = NULL;  in mpi3mr_free_mem()
4787 	kfree(mrioc->chain_sgl_list);  in mpi3mr_free_mem()
4788 	mrioc->chain_sgl_list = NULL;  in mpi3mr_free_mem()
4790 	if (mrioc->admin_reply_base) {  in mpi3mr_free_mem()
4791 		dma_free_coherent(&mrioc->pdev->dev, mrioc->admin_reply_q_sz,  in mpi3mr_free_mem()
4792 		    mrioc->admin_reply_base, mrioc->admin_reply_dma);  in mpi3mr_free_mem()
4793 		mrioc->admin_reply_base = NULL;  in mpi3mr_free_mem()
4795 	if (mrioc->admin_req_base) {  in mpi3mr_free_mem()
4796 		dma_free_coherent(&mrioc->pdev->dev, mrioc->admin_req_q_sz,  in mpi3mr_free_mem()
4797 		    mrioc->admin_req_base, mrioc->admin_req_dma);  in mpi3mr_free_mem()
4798 		mrioc->admin_req_base = NULL;  in mpi3mr_free_mem()
4801 	if (mrioc->pel_seqnum_virt) {  in mpi3mr_free_mem()
4802 		dma_free_coherent(&mrioc->pdev->dev, mrioc->pel_seqnum_sz,  in mpi3mr_free_mem()
4803 		    mrioc->pel_seqnum_virt, mrioc->pel_seqnum_dma);  in mpi3mr_free_mem()
4804 		mrioc->pel_seqnum_virt = NULL;  in mpi3mr_free_mem()
4808 		diag_buffer = &mrioc->diag_buffers[i];  in mpi3mr_free_mem()
4809 		if (diag_buffer->addr) {  in mpi3mr_free_mem()
4810 			dma_free_coherent(&mrioc->pdev->dev,  in mpi3mr_free_mem()
4811 			    diag_buffer->size, diag_buffer->addr,  in mpi3mr_free_mem()
4812 			    diag_buffer->dma_addr);  in mpi3mr_free_mem()
4813 			diag_buffer->addr = NULL;  in mpi3mr_free_mem()
4814 			diag_buffer->size = 0;  in mpi3mr_free_mem()
4815 			diag_buffer->type = 0;  in mpi3mr_free_mem()
4816 			diag_buffer->status = 0;  in mpi3mr_free_mem()
4820 	kfree(mrioc->throttle_groups);  in mpi3mr_free_mem()
4821 	mrioc->throttle_groups = NULL;  in mpi3mr_free_mem()
4823 	kfree(mrioc->logdata_buf);  in mpi3mr_free_mem()
4824 	mrioc->logdata_buf = NULL;  in mpi3mr_free_mem()
4829  * mpi3mr_issue_ioc_shutdown - shutdown controller
4844 	if (mrioc->unrecoverable) {  in mpi3mr_issue_ioc_shutdown()
4849 	ioc_status = readl(&mrioc->sysif_regs->ioc_status);  in mpi3mr_issue_ioc_shutdown()
4856 	ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);  in mpi3mr_issue_ioc_shutdown()
4860 	writel(ioc_config, &mrioc->sysif_regs->ioc_configuration);  in mpi3mr_issue_ioc_shutdown()
4862 	if (mrioc->facts.shutdown_timeout)  in mpi3mr_issue_ioc_shutdown()
4863 		timeout = mrioc->facts.shutdown_timeout * 10;  in mpi3mr_issue_ioc_shutdown()
4866 		ioc_status = readl(&mrioc->sysif_regs->ioc_status);  in mpi3mr_issue_ioc_shutdown()
4873 	} while (--timeout);  in mpi3mr_issue_ioc_shutdown()
4875 	ioc_status = readl(&mrioc->sysif_regs->ioc_status);  in mpi3mr_issue_ioc_shutdown()
4876 	ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);  in mpi3mr_issue_ioc_shutdown()
4892  * mpi3mr_cleanup_ioc - Cleanup controller
4909 	if (!mrioc->unrecoverable && !mrioc->reset_in_progress &&  in mpi3mr_cleanup_ioc()
4910 	    !mrioc->pci_err_recovery &&  in mpi3mr_cleanup_ioc()
4923  * mpi3mr_drv_cmd_comp_reset - Flush a internal driver command
4935 	if (cmdptr->state & MPI3MR_CMD_PENDING) {  in mpi3mr_drv_cmd_comp_reset()
4936 		cmdptr->state |= MPI3MR_CMD_RESET;  in mpi3mr_drv_cmd_comp_reset()
4937 		cmdptr->state &= ~MPI3MR_CMD_PENDING;  in mpi3mr_drv_cmd_comp_reset()
4938 		if (cmdptr->is_waiting) {  in mpi3mr_drv_cmd_comp_reset()
4939 			complete(&cmdptr->done);  in mpi3mr_drv_cmd_comp_reset()
4940 			cmdptr->is_waiting = 0;  in mpi3mr_drv_cmd_comp_reset()
4941 		} else if (cmdptr->callback)  in mpi3mr_drv_cmd_comp_reset()
4942 			cmdptr->callback(mrioc, cmdptr);  in mpi3mr_drv_cmd_comp_reset()
4947  * mpi3mr_flush_drv_cmds - Flush internaldriver commands
4959 	cmdptr = &mrioc->init_cmds;  in mpi3mr_flush_drv_cmds()
4962 	cmdptr = &mrioc->cfg_cmds;  in mpi3mr_flush_drv_cmds()
4965 	cmdptr = &mrioc->bsg_cmds;  in mpi3mr_flush_drv_cmds()
4967 	cmdptr = &mrioc->host_tm_cmds;  in mpi3mr_flush_drv_cmds()
4971 		cmdptr = &mrioc->dev_rmhs_cmds[i];  in mpi3mr_flush_drv_cmds()
4976 		cmdptr = &mrioc->evtack_cmds[i];  in mpi3mr_flush_drv_cmds()
4980 	cmdptr = &mrioc->pel_cmds;  in mpi3mr_flush_drv_cmds()
4983 	cmdptr = &mrioc->pel_abort_cmd;  in mpi3mr_flush_drv_cmds()
4986 	cmdptr = &mrioc->transport_cmds;  in mpi3mr_flush_drv_cmds()
4991  * mpi3mr_pel_wait_post - Issue PEL Wait
5004 	mrioc->pel_abort_requested = false;  in mpi3mr_pel_wait_post()
5007 	drv_cmd->state = MPI3MR_CMD_PENDING;  in mpi3mr_pel_wait_post()
5008 	drv_cmd->is_waiting = 0;  in mpi3mr_pel_wait_post()
5009 	drv_cmd->callback = mpi3mr_pel_wait_complete;  in mpi3mr_pel_wait_post()
5010 	drv_cmd->ioc_status = 0;  in mpi3mr_pel_wait_post()
5011 	drv_cmd->ioc_loginfo = 0;  in mpi3mr_pel_wait_post()
5015 	pel_wait.starting_sequence_number = cpu_to_le32(mrioc->pel_newest_seqnum);  in mpi3mr_pel_wait_post()
5016 	pel_wait.locale = cpu_to_le16(mrioc->pel_locale);  in mpi3mr_pel_wait_post()
5017 	pel_wait.class = cpu_to_le16(mrioc->pel_class);  in mpi3mr_pel_wait_post()
5020 	    mrioc->pel_newest_seqnum, mrioc->pel_class, mrioc->pel_locale);  in mpi3mr_pel_wait_post()
5025 		drv_cmd->state = MPI3MR_CMD_NOTUSED;  in mpi3mr_pel_wait_post()
5026 		drv_cmd->callback = NULL;  in mpi3mr_pel_wait_post()
5027 		drv_cmd->retry_count = 0;  in mpi3mr_pel_wait_post()
5028 		mrioc->pel_enabled = false;  in mpi3mr_pel_wait_post()
5033  * mpi3mr_pel_get_seqnum_post - Issue PEL Get Sequence number
5040  * Return: 0 on success, non-zero on failure.
5050 	mrioc->pel_cmds.state = MPI3MR_CMD_PENDING;  in mpi3mr_pel_get_seqnum_post()
5051 	mrioc->pel_cmds.is_waiting = 0;  in mpi3mr_pel_get_seqnum_post()
5052 	mrioc->pel_cmds.ioc_status = 0;  in mpi3mr_pel_get_seqnum_post()
5053 	mrioc->pel_cmds.ioc_loginfo = 0;  in mpi3mr_pel_get_seqnum_post()
5054 	mrioc->pel_cmds.callback = mpi3mr_pel_get_seqnum_complete;  in mpi3mr_pel_get_seqnum_post()
5059 	    mrioc->pel_seqnum_sz, mrioc->pel_seqnum_dma);  in mpi3mr_pel_get_seqnum_post()
5065 			drv_cmd->state = MPI3MR_CMD_NOTUSED;  in mpi3mr_pel_get_seqnum_post()
5066 			drv_cmd->callback = NULL;  in mpi3mr_pel_get_seqnum_post()
5067 			drv_cmd->retry_count = 0;  in mpi3mr_pel_get_seqnum_post()
5069 		mrioc->pel_enabled = false;  in mpi3mr_pel_get_seqnum_post()
5076  * mpi3mr_pel_wait_complete - PELWait Completion callback
5095 	if (drv_cmd->state & MPI3MR_CMD_RESET)  in mpi3mr_pel_wait_complete()
5098 	ioc_status = drv_cmd->ioc_status & MPI3_IOCSTATUS_STATUS_MASK;  in mpi3mr_pel_wait_complete()
5101 			__func__, ioc_status, drv_cmd->ioc_loginfo);  in mpi3mr_pel_wait_complete()
5104 		    ioc_status, drv_cmd->ioc_loginfo);  in mpi3mr_pel_wait_complete()
5108 	if (drv_cmd->state & MPI3MR_CMD_REPLY_VALID)  in mpi3mr_pel_wait_complete()
5109 		pel_reply = (struct mpi3_pel_reply *)drv_cmd->reply;  in mpi3mr_pel_wait_complete()
5117 	pe_log_status = le16_to_cpu(pel_reply->pe_log_status);  in mpi3mr_pel_wait_complete()
5129 		if (drv_cmd->retry_count < MPI3MR_PEL_RETRY_COUNT) {  in mpi3mr_pel_wait_complete()
5130 			drv_cmd->retry_count++;  in mpi3mr_pel_wait_complete()
5132 			    drv_cmd->retry_count);  in mpi3mr_pel_wait_complete()
5138 		    drv_cmd->retry_count);  in mpi3mr_pel_wait_complete()
5142 	if (!mrioc->pel_abort_requested) {  in mpi3mr_pel_wait_complete()
5143 		mrioc->pel_cmds.retry_count = 0;  in mpi3mr_pel_wait_complete()
5144 		mpi3mr_pel_get_seqnum_post(mrioc, &mrioc->pel_cmds);  in mpi3mr_pel_wait_complete()
5149 	mrioc->pel_enabled = false;  in mpi3mr_pel_wait_complete()
5151 	drv_cmd->state = MPI3MR_CMD_NOTUSED;  in mpi3mr_pel_wait_complete()
5152 	drv_cmd->callback = NULL;  in mpi3mr_pel_wait_complete()
5153 	drv_cmd->retry_count = 0;  in mpi3mr_pel_wait_complete()
5157  * mpi3mr_pel_get_seqnum_complete - PELGetSeqNum Completion callback
5175 	pel_seqnum_virt = (struct mpi3_pel_seq *)mrioc->pel_seqnum_virt;  in mpi3mr_pel_get_seqnum_complete()
5177 	if (drv_cmd->state & MPI3MR_CMD_RESET)  in mpi3mr_pel_get_seqnum_complete()
5180 	ioc_status = drv_cmd->ioc_status & MPI3_IOCSTATUS_STATUS_MASK;  in mpi3mr_pel_get_seqnum_complete()
5184 		    ioc_status, drv_cmd->ioc_loginfo);  in mpi3mr_pel_get_seqnum_complete()
5188 	if (drv_cmd->state & MPI3MR_CMD_REPLY_VALID)  in mpi3mr_pel_get_seqnum_complete()
5189 		pel_reply = (struct mpi3_pel_reply *)drv_cmd->reply;  in mpi3mr_pel_get_seqnum_complete()
5196 	if (le16_to_cpu(pel_reply->pe_log_status) != MPI3_PEL_STATUS_SUCCESS) {  in mpi3mr_pel_get_seqnum_complete()
5199 		    le16_to_cpu(pel_reply->pe_log_status));  in mpi3mr_pel_get_seqnum_complete()
5204 		if (drv_cmd->retry_count < MPI3MR_PEL_RETRY_COUNT) {  in mpi3mr_pel_get_seqnum_complete()
5205 			drv_cmd->retry_count++;  in mpi3mr_pel_get_seqnum_complete()
5208 			    drv_cmd->retry_count);  in mpi3mr_pel_get_seqnum_complete()
5215 		    drv_cmd->retry_count);  in mpi3mr_pel_get_seqnum_complete()
5218 	mrioc->pel_newest_seqnum = le32_to_cpu(pel_seqnum_virt->newest) + 1;  in mpi3mr_pel_get_seqnum_complete()
5219 	drv_cmd->retry_count = 0;  in mpi3mr_pel_get_seqnum_complete()
5224 	mrioc->pel_enabled = false;  in mpi3mr_pel_get_seqnum_complete()
5226 	drv_cmd->state = MPI3MR_CMD_NOTUSED;  in mpi3mr_pel_get_seqnum_complete()
5227 	drv_cmd->callback = NULL;  in mpi3mr_pel_get_seqnum_complete()
5228 	drv_cmd->retry_count = 0;  in mpi3mr_pel_get_seqnum_complete()
5232  * mpi3mr_soft_reset_handler - Reset the controller
5238  * reset are diag fault reset.  This is a blocking function and
5245  * In snapdump bit is set, the controller is issued with diag
5250  * Return: 0 on success, non-zero on failure.
5260 	/* Block the reset handler until diag save in progress*/  in mpi3mr_soft_reset_handler()
5263 	    mrioc->diagsave_timeout);  in mpi3mr_soft_reset_handler()
5264 	while (mrioc->diagsave_timeout)  in mpi3mr_soft_reset_handler()
5271 	if (!mutex_trylock(&mrioc->reset_mutex)) {  in mpi3mr_soft_reset_handler()
5277 		} while (mrioc->reset_in_progress == 1);  in mpi3mr_soft_reset_handler()
5280 		    mrioc->prev_reset_result,  in mpi3mr_soft_reset_handler()
5282 		return mrioc->prev_reset_result;  in mpi3mr_soft_reset_handler()
5287 	mrioc->device_refresh_on = 0;  in mpi3mr_soft_reset_handler()
5288 	mrioc->reset_in_progress = 1;  in mpi3mr_soft_reset_handler()
5289 	mrioc->stop_bsgs = 1;  in mpi3mr_soft_reset_handler()
5290 	mrioc->prev_reset_result = -1;  in mpi3mr_soft_reset_handler()
5302 			mrioc->event_masks[i] = -1;  in mpi3mr_soft_reset_handler()
5317 			trigger_data.fault = (readl(&mrioc->sysif_regs->fault) &  in mpi3mr_soft_reset_handler()
5321 				    readl(&mrioc->sysif_regs->host_diagnostic);  in mpi3mr_soft_reset_handler()
5326 			} while (--timeout);  in mpi3mr_soft_reset_handler()
5338 	if (mrioc->num_io_throttle_group !=  in mpi3mr_soft_reset_handler()
5339 	    mrioc->facts.max_io_throttle_group) {  in mpi3mr_soft_reset_handler()
5342 		    mrioc->num_io_throttle_group,  in mpi3mr_soft_reset_handler()
5343 		    mrioc->facts.max_io_throttle_group);  in mpi3mr_soft_reset_handler()
5344 		retval = -EPERM;  in mpi3mr_soft_reset_handler()
5350 	bitmap_clear(mrioc->devrem_bitmap, 0, MPI3MR_NUM_DEVRMCMD);  in mpi3mr_soft_reset_handler()
5351 	bitmap_clear(mrioc->removepend_bitmap, 0,  in mpi3mr_soft_reset_handler()
5352 		     mrioc->dev_handle_bitmap_bits);  in mpi3mr_soft_reset_handler()
5353 	bitmap_clear(mrioc->evtack_cmds_bitmap, 0, MPI3MR_NUM_EVTACKCMD);  in mpi3mr_soft_reset_handler()
5359 	if (mrioc->prepare_for_reset) {  in mpi3mr_soft_reset_handler()
5360 		mrioc->prepare_for_reset = 0;  in mpi3mr_soft_reset_handler()
5361 		mrioc->prepare_for_reset_timeout_counter = 0;  in mpi3mr_soft_reset_handler()
5365 	mrioc->fw_release_trigger_active = false;  in mpi3mr_soft_reset_handler()
5366 	mrioc->trace_release_trigger_active = false;  in mpi3mr_soft_reset_handler()
5367 	mrioc->snapdump_trigger_active = false;  in mpi3mr_soft_reset_handler()
5376 		    mrioc->name, reset_reason);  in mpi3mr_soft_reset_handler()
5383 		mrioc->diagsave_timeout = 0;  in mpi3mr_soft_reset_handler()
5384 		mrioc->reset_in_progress = 0;  in mpi3mr_soft_reset_handler()
5385 		mrioc->pel_abort_requested = 0;  in mpi3mr_soft_reset_handler()
5386 		if (mrioc->pel_enabled) {  in mpi3mr_soft_reset_handler()
5387 			mrioc->pel_cmds.retry_count = 0;  in mpi3mr_soft_reset_handler()
5388 			mpi3mr_pel_wait_post(mrioc, &mrioc->pel_cmds);  in mpi3mr_soft_reset_handler()
5391 		mrioc->device_refresh_on = 0;  in mpi3mr_soft_reset_handler()
5393 		mrioc->ts_update_counter = 0;  in mpi3mr_soft_reset_handler()
5394 		spin_lock_irqsave(&mrioc->watchdog_lock, flags);  in mpi3mr_soft_reset_handler()
5395 		if (mrioc->watchdog_work_q)  in mpi3mr_soft_reset_handler()
5396 			queue_delayed_work(mrioc->watchdog_work_q,  in mpi3mr_soft_reset_handler()
5397 			    &mrioc->watchdog_work,  in mpi3mr_soft_reset_handler()
5399 		spin_unlock_irqrestore(&mrioc->watchdog_lock, flags);  in mpi3mr_soft_reset_handler()
5400 		mrioc->stop_bsgs = 0;  in mpi3mr_soft_reset_handler()
5401 		if (mrioc->pel_enabled)  in mpi3mr_soft_reset_handler()
5406 		mrioc->device_refresh_on = 0;  in mpi3mr_soft_reset_handler()
5407 		mrioc->unrecoverable = 1;  in mpi3mr_soft_reset_handler()
5408 		mrioc->reset_in_progress = 0;  in mpi3mr_soft_reset_handler()
5409 		mrioc->stop_bsgs = 0;  in mpi3mr_soft_reset_handler()
5410 		retval = -1;  in mpi3mr_soft_reset_handler()
5413 	mrioc->prev_reset_result = retval;  in mpi3mr_soft_reset_handler()
5414 	mutex_unlock(&mrioc->reset_mutex);  in mpi3mr_soft_reset_handler()
5421  * mpi3mr_post_cfg_req - Issue config requests and wait
5435  * Return: 0 on success, non-zero on failure.
5442 	mutex_lock(&mrioc->cfg_cmds.mutex);  in mpi3mr_post_cfg_req()
5443 	if (mrioc->cfg_cmds.state & MPI3MR_CMD_PENDING) {  in mpi3mr_post_cfg_req()
5444 		retval = -1;  in mpi3mr_post_cfg_req()
5446 		mutex_unlock(&mrioc->cfg_cmds.mutex);  in mpi3mr_post_cfg_req()
5449 	mrioc->cfg_cmds.state = MPI3MR_CMD_PENDING;  in mpi3mr_post_cfg_req()
5450 	mrioc->cfg_cmds.is_waiting = 1;  in mpi3mr_post_cfg_req()
5451 	mrioc->cfg_cmds.callback = NULL;  in mpi3mr_post_cfg_req()
5452 	mrioc->cfg_cmds.ioc_status = 0;  in mpi3mr_post_cfg_req()
5453 	mrioc->cfg_cmds.ioc_loginfo = 0;  in mpi3mr_post_cfg_req()
5455 	cfg_req->host_tag = cpu_to_le16(MPI3MR_HOSTTAG_CFG_CMDS);  in mpi3mr_post_cfg_req()
5456 	cfg_req->function = MPI3_FUNCTION_CONFIG;  in mpi3mr_post_cfg_req()
5458 	init_completion(&mrioc->cfg_cmds.done);  in mpi3mr_post_cfg_req()
5460 	if (mrioc->logging_level & MPI3_DEBUG_CFG_INFO)  in mpi3mr_post_cfg_req()
5468 	wait_for_completion_timeout(&mrioc->cfg_cmds.done, (timeout * HZ));  in mpi3mr_post_cfg_req()
5469 	if (!(mrioc->cfg_cmds.state & MPI3MR_CMD_COMPLETE)) {  in mpi3mr_post_cfg_req()
5473 		retval = -1;  in mpi3mr_post_cfg_req()
5476 	*ioc_status = mrioc->cfg_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK;  in mpi3mr_post_cfg_req()
5480 		    *ioc_status, mrioc->cfg_cmds.ioc_loginfo);  in mpi3mr_post_cfg_req()
5483 	mrioc->cfg_cmds.state = MPI3MR_CMD_NOTUSED;  in mpi3mr_post_cfg_req()
5484 	mutex_unlock(&mrioc->cfg_cmds.mutex);  in mpi3mr_post_cfg_req()
5491  * mpi3mr_process_cfg_req - config page request processor
5524  * Return: 0 on success, non-zero on failure.
5532 	int retval = -1;  in mpi3mr_process_cfg_req()
5538 	if (cfg_req->action == MPI3_CONFIG_ACTION_PAGE_HEADER)  in mpi3mr_process_cfg_req()
5543 			    cfg_req->action, cfg_req->page_type,  in mpi3mr_process_cfg_req()
5544 			    cfg_req->page_number);  in mpi3mr_process_cfg_req()
5547 		switch (cfg_hdr->page_attribute & MPI3_CONFIG_PAGEATTR_MASK) {  in mpi3mr_process_cfg_req()
5549 			if (cfg_req->action  in mpi3mr_process_cfg_req()
5554 			if ((cfg_req->action ==  in mpi3mr_process_cfg_req()
5556 			    (cfg_req->action ==  in mpi3mr_process_cfg_req()
5567 			    cfg_req->action, cfg_req->page_type,  in mpi3mr_process_cfg_req()
5568 			    cfg_req->page_number, cfg_hdr->page_attribute);  in mpi3mr_process_cfg_req()
5571 		mem_desc.size = le16_to_cpu(cfg_hdr->page_length) * 4;  in mpi3mr_process_cfg_req()
5572 		cfg_req->page_length = cfg_hdr->page_length;  in mpi3mr_process_cfg_req()
5573 		cfg_req->page_version = cfg_hdr->page_version;  in mpi3mr_process_cfg_req()
5576 	mem_desc.addr = dma_alloc_coherent(&mrioc->pdev->dev,  in mpi3mr_process_cfg_req()
5582 	mpi3mr_add_sg_single(&cfg_req->sgl, sgl_flags, mem_desc.size,  in mpi3mr_process_cfg_req()
5585 	if ((cfg_req->action == MPI3_CONFIG_ACTION_WRITE_PERSISTENT) ||  in mpi3mr_process_cfg_req()
5586 	    (cfg_req->action == MPI3_CONFIG_ACTION_WRITE_CURRENT)) {  in mpi3mr_process_cfg_req()
5590 		if (mrioc->logging_level & MPI3_DEBUG_CFG_INFO)  in mpi3mr_process_cfg_req()
5599 	    (cfg_req->action != MPI3_CONFIG_ACTION_WRITE_PERSISTENT) &&  in mpi3mr_process_cfg_req()
5600 	    (cfg_req->action != MPI3_CONFIG_ACTION_WRITE_CURRENT)) {  in mpi3mr_process_cfg_req()
5604 		if (mrioc->logging_level & MPI3_DEBUG_CFG_INFO)  in mpi3mr_process_cfg_req()
5610 		dma_free_coherent(&mrioc->pdev->dev, mem_desc.size,  in mpi3mr_process_cfg_req()
5619  * mpi3mr_cfg_get_dev_pg0 - Read current device page0
5633  * Return: 0 on success, non-zero on failure.
5673 	return -1;  in mpi3mr_cfg_get_dev_pg0()
5678  * mpi3mr_cfg_get_sas_phy_pg0 - Read current SAS Phy page0
5692  * Return: 0 on success, non-zero on failure.
5733 	return -1;  in mpi3mr_cfg_get_sas_phy_pg0()
5737  * mpi3mr_cfg_get_sas_phy_pg1 - Read current SAS Phy page1
5751  * Return: 0 on success, non-zero on failure.
5792 	return -1;  in mpi3mr_cfg_get_sas_phy_pg1()
5797  * mpi3mr_cfg_get_sas_exp_pg0 - Read current SAS Expander page0
5811  * Return: 0 on success, non-zero on failure.
5853 	return -1;  in mpi3mr_cfg_get_sas_exp_pg0()
5857  * mpi3mr_cfg_get_sas_exp_pg1 - Read current SAS Expander page1
5871  * Return: 0 on success, non-zero on failure.
5913 	return -1;  in mpi3mr_cfg_get_sas_exp_pg1()
5917  * mpi3mr_cfg_get_enclosure_pg0 - Read current Enclosure page0
5931  * Return: 0 on success, non-zero on failure.
5972 	return -1;  in mpi3mr_cfg_get_enclosure_pg0()
5977  * mpi3mr_cfg_get_sas_io_unit_pg0 - Read current SASIOUnit page0
5986  * Return: 0 on success, non-zero on failure.
6029 	return -1;  in mpi3mr_cfg_get_sas_io_unit_pg0()
6033  * mpi3mr_cfg_get_sas_io_unit_pg1 - Read current SASIOUnit page1
6042  * Return: 0 on success, non-zero on failure.
6085 	return -1;  in mpi3mr_cfg_get_sas_io_unit_pg1()
6089  * mpi3mr_cfg_set_sas_io_unit_pg1 - Write SASIOUnit page1
6099  * Return: 0 on success, non-zero on failure.
6154 	return -1;  in mpi3mr_cfg_set_sas_io_unit_pg1()
6158  * mpi3mr_cfg_get_driver_pg1 - Read current Driver page1
6167  * Return: 0 on success, non-zero on failure.
6210 	return -1;  in mpi3mr_cfg_get_driver_pg1()
6214  * mpi3mr_cfg_get_driver_pg2 - Read current driver page2
6224  * Return: 0 on success, non-zero on failure.
6270 	return -1;  in mpi3mr_cfg_get_driver_pg2()