1 // SPDX-License-Identifier: GPL-2.0-only
10 #include <linux/dma-mapping.h>
16 #include <linux/mmc/mmc.h>
17 #include <linux/mmc/host.h>
18 #include <linux/mmc/card.h>
21 #include "cqhci-crypto.h"
43 	return cq_host->desc_base + (tag * cq_host->slot_sz);  in get_desc()
50 	return desc + cq_host->task_desc_len;  in get_link_desc()
55 	return cq_host->trans_desc_len * cq_host->mmc->max_segs * tag;  in get_trans_desc_offset()
62 	return cq_host->trans_desc_dma_base + offset;  in get_trans_desc_dma()
69 	return cq_host->trans_desc_base + offset;  in get_trans_desc()
80 	memset(link_temp, 0, cq_host->link_desc_len);  in setup_trans_desc()
81 	if (cq_host->link_desc_len > 8)  in setup_trans_desc()
84 	if (tag == DCMD_SLOT && (cq_host->mmc->caps2 & MMC_CAP2_CQE_DCMD)) {  in setup_trans_desc()
91 	if (cq_host->dma64) {  in setup_trans_desc()
111 	pr_err("%s: " DRV_NAME ": " f, mmc_hostname(mmc), ## x)
115 	struct mmc_host *mmc = cq_host->mmc;  in cqhci_dumpregs()  local
153 	if (cq_host->ops->dumpregs)  in cqhci_dumpregs()
154 		cq_host->ops->dumpregs(mmc);  in cqhci_dumpregs()
162  * |----------|
163  * |task desc |  |->|----------|
164  * |----------|  |  |trans desc|
165  * |link desc-|->|  |----------|
166  * |----------|          .
168  *  no. of slots      max-segs
169  *      .           |----------|
170  * |----------|
179 	if (cq_host->caps & CQHCI_TASK_DESC_SZ_128) {  in cqhci_host_alloc_tdl()
182 		cq_host->task_desc_len = 16;  in cqhci_host_alloc_tdl()
184 		cq_host->task_desc_len = 8;  in cqhci_host_alloc_tdl()
192 	if (cq_host->dma64) {  in cqhci_host_alloc_tdl()
193 		if (cq_host->quirks & CQHCI_QUIRK_SHORT_TXFR_DESC_SZ)  in cqhci_host_alloc_tdl()
194 			cq_host->trans_desc_len = 12;  in cqhci_host_alloc_tdl()
196 			cq_host->trans_desc_len = 16;  in cqhci_host_alloc_tdl()
197 		cq_host->link_desc_len = 16;  in cqhci_host_alloc_tdl()
199 		cq_host->trans_desc_len = 8;  in cqhci_host_alloc_tdl()
200 		cq_host->link_desc_len = 8;  in cqhci_host_alloc_tdl()
204 	cq_host->slot_sz = cq_host->task_desc_len + cq_host->link_desc_len;  in cqhci_host_alloc_tdl()
206 	cq_host->desc_size = cq_host->slot_sz * cq_host->num_slots;  in cqhci_host_alloc_tdl()
208 	cq_host->data_size = get_trans_desc_offset(cq_host, cq_host->mmc->cqe_qdepth);  in cqhci_host_alloc_tdl()
210 	pr_debug("%s: cqhci: desc_size: %zu data_sz: %zu slot-sz: %d\n",  in cqhci_host_alloc_tdl()
211 		 mmc_hostname(cq_host->mmc), cq_host->desc_size, cq_host->data_size,  in cqhci_host_alloc_tdl()
212 		 cq_host->slot_sz);  in cqhci_host_alloc_tdl()
215 	 * allocate a dma-mapped chunk of memory for the descriptors  in cqhci_host_alloc_tdl()
216 	 * allocate a dma-mapped chunk of memory for link descriptors  in cqhci_host_alloc_tdl()
217 	 * setup each link-desc memory offset per slot-number to  in cqhci_host_alloc_tdl()
220 	cq_host->desc_base = dmam_alloc_coherent(mmc_dev(cq_host->mmc),  in cqhci_host_alloc_tdl()
221 						 cq_host->desc_size,  in cqhci_host_alloc_tdl()
222 						 &cq_host->desc_dma_base,  in cqhci_host_alloc_tdl()
224 	if (!cq_host->desc_base)  in cqhci_host_alloc_tdl()
225 		return -ENOMEM;  in cqhci_host_alloc_tdl()
227 	cq_host->trans_desc_base = dmam_alloc_coherent(mmc_dev(cq_host->mmc),  in cqhci_host_alloc_tdl()
228 					      cq_host->data_size,  in cqhci_host_alloc_tdl()
229 					      &cq_host->trans_desc_dma_base,  in cqhci_host_alloc_tdl()
231 	if (!cq_host->trans_desc_base) {  in cqhci_host_alloc_tdl()
232 		dmam_free_coherent(mmc_dev(cq_host->mmc), cq_host->desc_size,  in cqhci_host_alloc_tdl()
233 				   cq_host->desc_base,  in cqhci_host_alloc_tdl()
234 				   cq_host->desc_dma_base);  in cqhci_host_alloc_tdl()
235 		cq_host->desc_base = NULL;  in cqhci_host_alloc_tdl()
236 		cq_host->desc_dma_base = 0;  in cqhci_host_alloc_tdl()
237 		return -ENOMEM;  in cqhci_host_alloc_tdl()
240 	pr_debug("%s: cqhci: desc-base: 0x%p trans-base: 0x%p\n desc_dma 0x%llx trans_dma: 0x%llx\n",  in cqhci_host_alloc_tdl()
241 		 mmc_hostname(cq_host->mmc), cq_host->desc_base, cq_host->trans_desc_base,  in cqhci_host_alloc_tdl()
242 		(unsigned long long)cq_host->desc_dma_base,  in cqhci_host_alloc_tdl()
243 		(unsigned long long)cq_host->trans_desc_dma_base);  in cqhci_host_alloc_tdl()
245 	for (; i < (cq_host->num_slots); i++)  in cqhci_host_alloc_tdl()
253 	struct mmc_host *mmc = cq_host->mmc;  in __cqhci_enable()  local
266 	if (mmc->caps2 & MMC_CAP2_CQE_DCMD)  in __cqhci_enable()
269 	if (cq_host->caps & CQHCI_TASK_DESC_SZ_128)  in __cqhci_enable()
272 	if (mmc->caps2 & MMC_CAP2_CRYPTO)  in __cqhci_enable()
277 	cqhci_writel(cq_host, lower_32_bits(cq_host->desc_dma_base),  in __cqhci_enable()
279 	cqhci_writel(cq_host, upper_32_bits(cq_host->desc_dma_base),  in __cqhci_enable()
282 	cqhci_writel(cq_host, cq_host->rca, CQHCI_SSC2);  in __cqhci_enable()
293 	mmc->cqe_on = true;  in __cqhci_enable()
295 	if (cq_host->ops->enable)  in __cqhci_enable()
296 		cq_host->ops->enable(mmc);  in __cqhci_enable()
303 	cq_host->activated = true;  in __cqhci_enable()
314 	cq_host->mmc->cqe_on = false;  in __cqhci_disable()
316 	cq_host->activated = false;  in __cqhci_disable()
319 int cqhci_deactivate(struct mmc_host *mmc)  in cqhci_deactivate()  argument
321 	struct cqhci_host *cq_host = mmc->cqe_private;  in cqhci_deactivate()
323 	if (cq_host->enabled && cq_host->activated)  in cqhci_deactivate()
330 int cqhci_resume(struct mmc_host *mmc)  in cqhci_resume()  argument
332 	/* Re-enable is done upon first request */  in cqhci_resume()
337 static int cqhci_enable(struct mmc_host *mmc, struct mmc_card *card)  in cqhci_enable()  argument
339 	struct cqhci_host *cq_host = mmc->cqe_private;  in cqhci_enable()
342 	if (!card->ext_csd.cmdq_en)  in cqhci_enable()
343 		return -EINVAL;  in cqhci_enable()
345 	if (cq_host->enabled)  in cqhci_enable()
348 	cq_host->rca = card->rca;  in cqhci_enable()
353 		       mmc_hostname(mmc), err);  in cqhci_enable()
359 	cq_host->enabled = true;  in cqhci_enable()
375 static void cqhci_off(struct mmc_host *mmc)  in cqhci_off()  argument
377 	struct cqhci_host *cq_host = mmc->cqe_private;  in cqhci_off()
381 	if (!cq_host->enabled || !mmc->cqe_on || cq_host->recovery_halt)  in cqhci_off()
384 	if (cq_host->ops->disable)  in cqhci_off()
385 		cq_host->ops->disable(mmc, false);  in cqhci_off()
392 		pr_err("%s: cqhci: CQE stuck on\n", mmc_hostname(mmc));  in cqhci_off()
394 		pr_debug("%s: cqhci: CQE off\n", mmc_hostname(mmc));  in cqhci_off()
396 	if (cq_host->ops->post_disable)  in cqhci_off()
397 		cq_host->ops->post_disable(mmc);  in cqhci_off()
399 	mmc->cqe_on = false;  in cqhci_off()
402 static void cqhci_disable(struct mmc_host *mmc)  in cqhci_disable()  argument
404 	struct cqhci_host *cq_host = mmc->cqe_private;  in cqhci_disable()
406 	if (!cq_host->enabled)  in cqhci_disable()
409 	cqhci_off(mmc);  in cqhci_disable()
413 	dmam_free_coherent(mmc_dev(mmc), cq_host->data_size,  in cqhci_disable()
414 			   cq_host->trans_desc_base,  in cqhci_disable()
415 			   cq_host->trans_desc_dma_base);  in cqhci_disable()
417 	dmam_free_coherent(mmc_dev(mmc), cq_host->desc_size,  in cqhci_disable()
418 			   cq_host->desc_base,  in cqhci_disable()
419 			   cq_host->desc_dma_base);  in cqhci_disable()
421 	cq_host->trans_desc_base = NULL;  in cqhci_disable()
422 	cq_host->desc_base = NULL;  in cqhci_disable()
424 	cq_host->enabled = false;  in cqhci_disable()
431 	u32 req_flags = mrq->data->flags;  in cqhci_prep_task_desc()
444 		CQHCI_BLK_COUNT(mrq->data->blocks) |  in cqhci_prep_task_desc()
445 		CQHCI_BLK_ADDR((u64)mrq->data->blk_addr);  in cqhci_prep_task_desc()
449 	if (cq_host->caps & CQHCI_TASK_DESC_SZ_128) {  in cqhci_prep_task_desc()
455 			 mmc_hostname(mrq->host), mrq->tag, desc1, desc0);  in cqhci_prep_task_desc()
458 			 mmc_hostname(mrq->host), mrq->tag, desc0);  in cqhci_prep_task_desc()
465 	struct mmc_data *data = mrq->data;  in cqhci_dma_map()
468 		return -EINVAL;  in cqhci_dma_map()
470 	sg_count = dma_map_sg(mmc_dev(host), data->sg,  in cqhci_dma_map()
471 			      data->sg_len,  in cqhci_dma_map()
472 			      (data->flags & MMC_DATA_WRITE) ?  in cqhci_dma_map()
475 		pr_err("%s: sg-len: %d\n", __func__, data->sg_len);  in cqhci_dma_map()
476 		return -ENOMEM;  in cqhci_dma_map()
508 	struct mmc_data *data = mrq->data;  in cqhci_prep_tran_desc()
511 	bool dma64 = cq_host->dma64;  in cqhci_prep_tran_desc()
516 	sg_count = cqhci_dma_map(mrq->host, mrq);  in cqhci_prep_tran_desc()
519 				mmc_hostname(mrq->host), __func__, sg_count);  in cqhci_prep_tran_desc()
525 	for_each_sg(data->sg, sg, sg_count, i) {  in cqhci_prep_tran_desc()
531 		if (cq_host->ops->set_tran_desc)  in cqhci_prep_tran_desc()
532 			cq_host->ops->set_tran_desc(cq_host, &desc, addr, len, end, dma64);  in cqhci_prep_tran_desc()
536 		desc += cq_host->trans_desc_len;  in cqhci_prep_tran_desc()
542 static void cqhci_prep_dcmd_desc(struct mmc_host *mmc,  in cqhci_prep_dcmd_desc()  argument
550 	struct cqhci_host *cq_host = mmc->cqe_private;  in cqhci_prep_dcmd_desc()
553 	if (!(mrq->cmd->flags & MMC_RSP_PRESENT)) {  in cqhci_prep_dcmd_desc()
557 		if (mrq->cmd->flags & MMC_RSP_R1B) {  in cqhci_prep_dcmd_desc()
566 	task_desc = (__le64 __force *)get_desc(cq_host, cq_host->dcmd_slot);  in cqhci_prep_dcmd_desc()
567 	memset(task_desc, 0, cq_host->task_desc_len);  in cqhci_prep_dcmd_desc()
573 		 CQHCI_CMD_INDEX(mrq->cmd->opcode) |  in cqhci_prep_dcmd_desc()
575 	if (cq_host->ops->update_dcmd_desc)  in cqhci_prep_dcmd_desc()
576 		cq_host->ops->update_dcmd_desc(mmc, mrq, &data);  in cqhci_prep_dcmd_desc()
580 		 mmc_hostname(mmc), mrq->cmd->opcode, timing, resp_type);  in cqhci_prep_dcmd_desc()
582 	dataddr[0] = cpu_to_le64((u64)mrq->cmd->arg);  in cqhci_prep_dcmd_desc()
588 	struct mmc_data *data = mrq->data;  in cqhci_post_req()
591 		dma_unmap_sg(mmc_dev(host), data->sg, data->sg_len,  in cqhci_post_req()
592 			     (data->flags & MMC_DATA_READ) ?  in cqhci_post_req()
599 	return mrq->cmd ? DCMD_SLOT : mrq->tag;  in cqhci_tag()
602 static int cqhci_request(struct mmc_host *mmc, struct mmc_request *mrq)  in cqhci_request()  argument
606 	struct cqhci_host *cq_host = mmc->cqe_private;  in cqhci_request()
609 	if (!cq_host->enabled) {  in cqhci_request()
610 		pr_err("%s: cqhci: not enabled\n", mmc_hostname(mmc));  in cqhci_request()
611 		return -EINVAL;  in cqhci_request()
614 	/* First request after resume has to re-enable */  in cqhci_request()
615 	if (!cq_host->activated)  in cqhci_request()
618 	if (!mmc->cqe_on) {  in cqhci_request()
619 		if (cq_host->ops->pre_enable)  in cqhci_request()
620 			cq_host->ops->pre_enable(mmc);  in cqhci_request()
623 		mmc->cqe_on = true;  in cqhci_request()
624 		pr_debug("%s: cqhci: CQE on\n", mmc_hostname(mmc));  in cqhci_request()
627 			       mmc_hostname(mmc));  in cqhci_request()
629 		if (cq_host->ops->enable)  in cqhci_request()
630 			cq_host->ops->enable(mmc);  in cqhci_request()
633 	if (mrq->data) {  in cqhci_request()
639 			       mmc_hostname(mmc), err);  in cqhci_request()
643 		cqhci_prep_dcmd_desc(mmc, mrq);  in cqhci_request()
646 	spin_lock_irqsave(&cq_host->lock, flags);  in cqhci_request()
648 	if (cq_host->recovery_halt) {  in cqhci_request()
649 		err = -EBUSY;  in cqhci_request()
653 	cq_host->slot[tag].mrq = mrq;  in cqhci_request()
654 	cq_host->slot[tag].flags = 0;  in cqhci_request()
656 	cq_host->qcnt += 1;  in cqhci_request()
662 			 mmc_hostname(mmc), tag);  in cqhci_request()
664 	spin_unlock_irqrestore(&cq_host->lock, flags);  in cqhci_request()
667 		cqhci_post_req(mmc, mrq);  in cqhci_request()
672 static void cqhci_recovery_needed(struct mmc_host *mmc, struct mmc_request *mrq,  in cqhci_recovery_needed()  argument
675 	struct cqhci_host *cq_host = mmc->cqe_private;  in cqhci_recovery_needed()
677 	if (!cq_host->recovery_halt) {  in cqhci_recovery_needed()
678 		cq_host->recovery_halt = true;  in cqhci_recovery_needed()
679 		pr_debug("%s: cqhci: recovery needed\n", mmc_hostname(mmc));  in cqhci_recovery_needed()
680 		wake_up(&cq_host->wait_queue);  in cqhci_recovery_needed()
681 		if (notify && mrq->recovery_notifier)  in cqhci_recovery_needed()
682 			mrq->recovery_notifier(mrq);  in cqhci_recovery_needed()
691 	case -EILSEQ:  in cqhci_error_flags()
693 	case -ETIMEDOUT:  in cqhci_error_flags()
700 static void cqhci_error_irq(struct mmc_host *mmc, u32 status, int cmd_error,  in cqhci_error_irq()  argument
703 	struct cqhci_host *cq_host = mmc->cqe_private;  in cqhci_error_irq()
709 	spin_lock(&cq_host->lock);  in cqhci_error_irq()
714 		 mmc_hostname(mmc), status, cmd_error, data_error, terri);  in cqhci_error_irq()
717 	if (cq_host->recovery_halt)  in cqhci_error_irq()
720 	if (!cq_host->qcnt) {  in cqhci_error_irq()
722 			  mmc_hostname(mmc), status, cmd_error, data_error,  in cqhci_error_irq()
729 		slot = &cq_host->slot[tag];  in cqhci_error_irq()
730 		if (slot->mrq) {  in cqhci_error_irq()
731 			slot->flags = cqhci_error_flags(cmd_error, data_error);  in cqhci_error_irq()
732 			cqhci_recovery_needed(mmc, slot->mrq, true);  in cqhci_error_irq()
738 		slot = &cq_host->slot[tag];  in cqhci_error_irq()
739 		if (slot->mrq) {  in cqhci_error_irq()
740 			slot->flags = cqhci_error_flags(data_error, cmd_error);  in cqhci_error_irq()
741 			cqhci_recovery_needed(mmc, slot->mrq, true);  in cqhci_error_irq()
747 	 * never happen, since the block layer ensures that all crypto-enabled  in cqhci_error_irq()
757 			  mmc_hostname(mmc), status, tdpe);  in cqhci_error_irq()
761 			slot = &cq_host->slot[tag];  in cqhci_error_irq()
762 			if (!slot->mrq)  in cqhci_error_irq()
764 			slot->flags = cqhci_error_flags(data_error, cmd_error);  in cqhci_error_irq()
765 			cqhci_recovery_needed(mmc, slot->mrq, true);  in cqhci_error_irq()
769 	if (!cq_host->recovery_halt) {  in cqhci_error_irq()
775 			slot = &cq_host->slot[tag];  in cqhci_error_irq()
776 			if (!slot->mrq)  in cqhci_error_irq()
778 			slot->flags = cqhci_error_flags(data_error, cmd_error);  in cqhci_error_irq()
779 			cqhci_recovery_needed(mmc, slot->mrq, true);  in cqhci_error_irq()
785 	spin_unlock(&cq_host->lock);  in cqhci_error_irq()
788 static void cqhci_finish_mrq(struct mmc_host *mmc, unsigned int tag)  in cqhci_finish_mrq()  argument
790 	struct cqhci_host *cq_host = mmc->cqe_private;  in cqhci_finish_mrq()
791 	struct cqhci_slot *slot = &cq_host->slot[tag];  in cqhci_finish_mrq()
792 	struct mmc_request *mrq = slot->mrq;  in cqhci_finish_mrq()
797 			  mmc_hostname(mmc), tag);  in cqhci_finish_mrq()
802 	if (cq_host->recovery_halt) {  in cqhci_finish_mrq()
803 		slot->flags |= CQHCI_COMPLETED;  in cqhci_finish_mrq()
807 	slot->mrq = NULL;  in cqhci_finish_mrq()
809 	cq_host->qcnt -= 1;  in cqhci_finish_mrq()
811 	data = mrq->data;  in cqhci_finish_mrq()
813 		if (data->error)  in cqhci_finish_mrq()
814 			data->bytes_xfered = 0;  in cqhci_finish_mrq()
816 			data->bytes_xfered = data->blksz * data->blocks;  in cqhci_finish_mrq()
819 	mmc_cqe_request_done(mmc, mrq);  in cqhci_finish_mrq()
822 irqreturn_t cqhci_irq(struct mmc_host *mmc, u32 intmask, int cmd_error,  in cqhci_irq()  argument
827 	struct cqhci_host *cq_host = mmc->cqe_private;  in cqhci_irq()
832 	pr_debug("%s: cqhci: IRQ status: 0x%08x\n", mmc_hostname(mmc), status);  in cqhci_irq()
837 			mmc_debugfs_err_stats_inc(mmc, MMC_ERR_CMDQ_RED);  in cqhci_irq()
839 			mmc_debugfs_err_stats_inc(mmc, MMC_ERR_CMDQ_GCE);  in cqhci_irq()
841 			mmc_debugfs_err_stats_inc(mmc, MMC_ERR_CMDQ_ICCE);  in cqhci_irq()
842 		cqhci_error_irq(mmc, status, cmd_error, data_error);  in cqhci_irq()
850 			 mmc_hostname(mmc), comp_status);  in cqhci_irq()
852 		spin_lock(&cq_host->lock);  in cqhci_irq()
854 		for_each_set_bit(tag, &comp_status, cq_host->num_slots) {  in cqhci_irq()
857 				 mmc_hostname(mmc), tag);  in cqhci_irq()
858 			cqhci_finish_mrq(mmc, tag);  in cqhci_irq()
861 		if (cq_host->waiting_for_idle && !cq_host->qcnt) {  in cqhci_irq()
862 			cq_host->waiting_for_idle = false;  in cqhci_irq()
863 			wake_up(&cq_host->wait_queue);  in cqhci_irq()
866 		spin_unlock(&cq_host->lock);  in cqhci_irq()
870 		wake_up(&cq_host->wait_queue);  in cqhci_irq()
873 		wake_up(&cq_host->wait_queue);  in cqhci_irq()
884 	spin_lock_irqsave(&cq_host->lock, flags);  in cqhci_is_idle()
885 	is_idle = !cq_host->qcnt || cq_host->recovery_halt;  in cqhci_is_idle()
886 	*ret = cq_host->recovery_halt ? -EBUSY : 0;  in cqhci_is_idle()
887 	cq_host->waiting_for_idle = !is_idle;  in cqhci_is_idle()
888 	spin_unlock_irqrestore(&cq_host->lock, flags);  in cqhci_is_idle()
893 static int cqhci_wait_for_idle(struct mmc_host *mmc)  in cqhci_wait_for_idle()  argument
895 	struct cqhci_host *cq_host = mmc->cqe_private;  in cqhci_wait_for_idle()
898 	wait_event(cq_host->wait_queue, cqhci_is_idle(cq_host, &ret));  in cqhci_wait_for_idle()
903 static bool cqhci_timeout(struct mmc_host *mmc, struct mmc_request *mrq,  in cqhci_timeout()  argument
906 	struct cqhci_host *cq_host = mmc->cqe_private;  in cqhci_timeout()
908 	struct cqhci_slot *slot = &cq_host->slot[tag];  in cqhci_timeout()
912 	spin_lock_irqsave(&cq_host->lock, flags);  in cqhci_timeout()
913 	timed_out = slot->mrq == mrq;  in cqhci_timeout()
915 		slot->flags |= CQHCI_EXTERNAL_TIMEOUT;  in cqhci_timeout()
916 		cqhci_recovery_needed(mmc, mrq, false);  in cqhci_timeout()
917 		*recovery_needed = cq_host->recovery_halt;  in cqhci_timeout()
919 	spin_unlock_irqrestore(&cq_host->lock, flags);  in cqhci_timeout()
923 		       mmc_hostname(mmc), tag, cq_host->qcnt);  in cqhci_timeout()
935 static bool cqhci_clear_all_tasks(struct mmc_host *mmc, unsigned int timeout)  in cqhci_clear_all_tasks()  argument
937 	struct cqhci_host *cq_host = mmc->cqe_private;  in cqhci_clear_all_tasks()
947 	wait_event_timeout(cq_host->wait_queue, cqhci_tasks_cleared(cq_host),  in cqhci_clear_all_tasks()
956 			mmc_hostname(mmc));  in cqhci_clear_all_tasks()
961 static bool cqhci_halt(struct mmc_host *mmc, unsigned int timeout)  in cqhci_halt()  argument
963 	struct cqhci_host *cq_host = mmc->cqe_private;  in cqhci_halt()
976 	wait_event_timeout(cq_host->wait_queue, cqhci_halted(cq_host),  in cqhci_halt()
984 		pr_warn("%s: cqhci: Failed to halt\n", mmc_hostname(mmc));  in cqhci_halt()
997 static void cqhci_recovery_start(struct mmc_host *mmc)  in cqhci_recovery_start()  argument
999 	struct cqhci_host *cq_host = mmc->cqe_private;  in cqhci_recovery_start()
1001 	pr_debug("%s: cqhci: %s\n", mmc_hostname(mmc), __func__);  in cqhci_recovery_start()
1003 	WARN_ON(!cq_host->recovery_halt);  in cqhci_recovery_start()
1005 	cqhci_halt(mmc, CQHCI_START_HALT_TIMEOUT);  in cqhci_recovery_start()
1007 	if (cq_host->ops->disable)  in cqhci_recovery_start()
1008 		cq_host->ops->disable(mmc, true);  in cqhci_recovery_start()
1010 	mmc->cqe_on = false;  in cqhci_recovery_start()
1018 	/* CRC errors might indicate re-tuning so prefer to report that */  in cqhci_error_from_flags()
1020 		return -EILSEQ;  in cqhci_error_from_flags()
1023 		return -ETIMEDOUT;  in cqhci_error_from_flags()
1025 	return -EIO;  in cqhci_error_from_flags()
1030 	struct cqhci_slot *slot = &cq_host->slot[tag];  in cqhci_recover_mrq()
1031 	struct mmc_request *mrq = slot->mrq;  in cqhci_recover_mrq()
1037 	slot->mrq = NULL;  in cqhci_recover_mrq()
1039 	cq_host->qcnt -= 1;  in cqhci_recover_mrq()
1041 	data = mrq->data;  in cqhci_recover_mrq()
1043 		data->bytes_xfered = 0;  in cqhci_recover_mrq()
1044 		data->error = cqhci_error_from_flags(slot->flags);  in cqhci_recover_mrq()
1046 		mrq->cmd->error = cqhci_error_from_flags(slot->flags);  in cqhci_recover_mrq()
1049 	mmc_cqe_request_done(cq_host->mmc, mrq);  in cqhci_recover_mrq()
1056 	for (i = 0; i < cq_host->num_slots; i++)  in cqhci_recover_mrqs()
1070 static void cqhci_recovery_finish(struct mmc_host *mmc)  in cqhci_recovery_finish()  argument
1072 	struct cqhci_host *cq_host = mmc->cqe_private;  in cqhci_recovery_finish()
1077 	pr_debug("%s: cqhci: %s\n", mmc_hostname(mmc), __func__);  in cqhci_recovery_finish()
1079 	WARN_ON(!cq_host->recovery_halt);  in cqhci_recovery_finish()
1081 	ok = cqhci_halt(mmc, CQHCI_FINISH_HALT_TIMEOUT);  in cqhci_recovery_finish()
1086 	 * be disabled/re-enabled, but not to disable before clearing tasks.  in cqhci_recovery_finish()
1089 	if (!cqhci_clear_all_tasks(mmc, CQHCI_CLEAR_TIMEOUT))  in cqhci_recovery_finish()
1101 	cqhci_halt(mmc, CQHCI_FINISH_HALT_TIMEOUT);  in cqhci_recovery_finish()
1104 		cqhci_clear_all_tasks(mmc, CQHCI_CLEAR_TIMEOUT);  in cqhci_recovery_finish()
1108 	WARN_ON(cq_host->qcnt);  in cqhci_recovery_finish()
1110 	spin_lock_irqsave(&cq_host->lock, flags);  in cqhci_recovery_finish()
1111 	cq_host->qcnt = 0;  in cqhci_recovery_finish()
1112 	cq_host->recovery_halt = false;  in cqhci_recovery_finish()
1113 	mmc->cqe_on = false;  in cqhci_recovery_finish()
1114 	spin_unlock_irqrestore(&cq_host->lock, flags);  in cqhci_recovery_finish()
1116 	/* Ensure all writes are done before interrupts are re-enabled */  in cqhci_recovery_finish()
1123 	pr_debug("%s: cqhci: recovery done\n", mmc_hostname(mmc));  in cqhci_recovery_finish()
1147 		dev_dbg(&pdev->dev, "CMDQ not supported\n");  in cqhci_pltfm_init()
1148 		return ERR_PTR(-EINVAL);  in cqhci_pltfm_init()
1151 	cq_host = devm_kzalloc(&pdev->dev, sizeof(*cq_host), GFP_KERNEL);  in cqhci_pltfm_init()
1153 		return ERR_PTR(-ENOMEM);  in cqhci_pltfm_init()
1154 	cq_host->mmio = devm_ioremap(&pdev->dev,  in cqhci_pltfm_init()
1155 				     cqhci_memres->start,  in cqhci_pltfm_init()
1157 	if (!cq_host->mmio) {  in cqhci_pltfm_init()
1158 		dev_err(&pdev->dev, "failed to remap cqhci regs\n");  in cqhci_pltfm_init()
1159 		return ERR_PTR(-EBUSY);  in cqhci_pltfm_init()
1161 	dev_dbg(&pdev->dev, "CMDQ ioremap: done\n");  in cqhci_pltfm_init()
1179 int cqhci_init(struct cqhci_host *cq_host, struct mmc_host *mmc,  in cqhci_init()  argument
1184 	cq_host->dma64 = dma64;  in cqhci_init()
1185 	cq_host->mmc = mmc;  in cqhci_init()
1186 	cq_host->mmc->cqe_private = cq_host;  in cqhci_init()
1188 	cq_host->num_slots = NUM_SLOTS;  in cqhci_init()
1189 	cq_host->dcmd_slot = DCMD_SLOT;  in cqhci_init()
1191 	mmc->cqe_ops = &cqhci_cqe_ops;  in cqhci_init()
1193 	mmc->cqe_qdepth = NUM_SLOTS;  in cqhci_init()
1194 	if (mmc->caps2 & MMC_CAP2_CQE_DCMD)  in cqhci_init()
1195 		mmc->cqe_qdepth -= 1;  in cqhci_init()
1197 	cq_host->slot = devm_kcalloc(mmc_dev(mmc), cq_host->num_slots,  in cqhci_init()
1198 				     sizeof(*cq_host->slot), GFP_KERNEL);  in cqhci_init()
1199 	if (!cq_host->slot) {  in cqhci_init()
1200 		err = -ENOMEM;  in cqhci_init()
1207 		       mmc_hostname(mmc));  in cqhci_init()
1211 	spin_lock_init(&cq_host->lock);  in cqhci_init()
1213 	init_completion(&cq_host->halt_comp);  in cqhci_init()
1214 	init_waitqueue_head(&cq_host->wait_queue);  in cqhci_init()
1217 		mmc_hostname(mmc), cqhci_ver_major(cq_host),  in cqhci_init()
1224 	       mmc_hostname(mmc), cqhci_ver_major(cq_host),  in cqhci_init()