85  * @hba: per adapter instance
91 void ufshcd_mcq_config_mac(struct ufs_hba *hba, u32 max_active_cmds)  in ufshcd_mcq_config_mac()  argument
95 	val = ufshcd_readl(hba, REG_UFS_MCQ_CFG);  in ufshcd_mcq_config_mac()
98 	ufshcd_writel(hba, val, REG_UFS_MCQ_CFG);  in ufshcd_mcq_config_mac()
105  * @hba: per adapter instance
111 struct ufs_hw_queue *ufshcd_mcq_req_to_hwq(struct ufs_hba *hba,  in ufshcd_mcq_req_to_hwq()  argument
116 	return hctx ? &hba->uhq[hctx->queue_num] : NULL;  in ufshcd_mcq_req_to_hwq()
122  * @hba: per adapter instance
126 unsigned int ufshcd_mcq_queue_cfg_addr(struct ufs_hba *hba)  in ufshcd_mcq_queue_cfg_addr()  argument
128 	return FIELD_GET(QCFGPTR, hba->mcq_capabilities) * 0x200;  in ufshcd_mcq_queue_cfg_addr()
134  * @hba: per adapter instance
143 int ufshcd_mcq_decide_queue_depth(struct ufs_hba *hba)  in ufshcd_mcq_decide_queue_depth()  argument
147 	if (!hba->vops || !hba->vops->get_hba_mac) {  in ufshcd_mcq_decide_queue_depth()
153 		hba->capabilities =  in ufshcd_mcq_decide_queue_depth()
154 			ufshcd_readl(hba, REG_CONTROLLER_CAPABILITIES);  in ufshcd_mcq_decide_queue_depth()
155 		mac = hba->capabilities & MASK_TRANSFER_REQUESTS_SLOTS_MCQ;  in ufshcd_mcq_decide_queue_depth()
158 		mac = hba->vops->get_hba_mac(hba);  in ufshcd_mcq_decide_queue_depth()
163 	WARN_ON_ONCE(!hba->dev_info.bqueuedepth);  in ufshcd_mcq_decide_queue_depth()
169 	return min_t(int, mac, hba->dev_info.bqueuedepth);  in ufshcd_mcq_decide_queue_depth()
172 	dev_err(hba->dev, "Failed to get mac, err=%d\n", mac);  in ufshcd_mcq_decide_queue_depth()
176 static int ufshcd_mcq_config_nr_queues(struct ufs_hba *hba)  in ufshcd_mcq_config_nr_queues()  argument
180 	struct Scsi_Host *host = hba->host;  in ufshcd_mcq_config_nr_queues()
183 	hba_maxq = FIELD_GET(MAX_QUEUE_SUP, hba->mcq_capabilities) + 1;  in ufshcd_mcq_config_nr_queues()
188 		dev_err(hba->dev, "Total queues (%d) exceeds HC capacity (%d)\n",  in ufshcd_mcq_config_nr_queues()
195 	 * commands via hba->dev_cmd_queue.  in ufshcd_mcq_config_nr_queues()
198 		dev_err(hba->dev, "At least one non-poll queue required\n");  in ufshcd_mcq_config_nr_queues()
205 		hba->nr_queues[HCTX_TYPE_DEFAULT] = rw_queues;  in ufshcd_mcq_config_nr_queues()
206 		rem -= hba->nr_queues[HCTX_TYPE_DEFAULT];  in ufshcd_mcq_config_nr_queues()
212 		hba->nr_queues[HCTX_TYPE_POLL] = poll_queues;  in ufshcd_mcq_config_nr_queues()
213 		rem -= hba->nr_queues[HCTX_TYPE_POLL];  in ufshcd_mcq_config_nr_queues()
217 		hba->nr_queues[HCTX_TYPE_READ] = read_queues;  in ufshcd_mcq_config_nr_queues()
218 		rem -= hba->nr_queues[HCTX_TYPE_READ];  in ufshcd_mcq_config_nr_queues()
221 	if (!hba->nr_queues[HCTX_TYPE_DEFAULT])  in ufshcd_mcq_config_nr_queues()
222 		hba->nr_queues[HCTX_TYPE_DEFAULT] = min3(rem, rw_queues,  in ufshcd_mcq_config_nr_queues()
226 		host->nr_hw_queues += hba->nr_queues[i];  in ufshcd_mcq_config_nr_queues()
228 	hba->nr_hw_queues = host->nr_hw_queues;  in ufshcd_mcq_config_nr_queues()
232 int ufshcd_mcq_memory_alloc(struct ufs_hba *hba)  in ufshcd_mcq_memory_alloc()  argument
238 	for (i = 0; i < hba->nr_hw_queues; i++) {  in ufshcd_mcq_memory_alloc()
239 		hwq = &hba->uhq[i];  in ufshcd_mcq_memory_alloc()
243 		hwq->sqe_base_addr = dmam_alloc_coherent(hba->dev, utrdl_size,  in ufshcd_mcq_memory_alloc()
247 			dev_err(hba->dev, "SQE allocation failed\n");  in ufshcd_mcq_memory_alloc()
252 		hwq->cqe_base_addr = dmam_alloc_coherent(hba->dev, cqe_size,  in ufshcd_mcq_memory_alloc()
256 			dev_err(hba->dev, "CQE allocation failed\n");  in ufshcd_mcq_memory_alloc()
264 static void __iomem *mcq_opr_base(struct ufs_hba *hba,  in mcq_opr_base()  argument
267 	struct ufshcd_mcq_opr_info_t *opr = &hba->mcq_opr[n];  in mcq_opr_base()
272 u32 ufshcd_mcq_read_cqis(struct ufs_hba *hba, int i)  in ufshcd_mcq_read_cqis()  argument
274 	return readl(mcq_opr_base(hba, OPR_CQIS, i) + REG_CQIS);  in ufshcd_mcq_read_cqis()
278 void ufshcd_mcq_write_cqis(struct ufs_hba *hba, u32 val, int i)  in ufshcd_mcq_write_cqis()  argument
280 	writel(val, mcq_opr_base(hba, OPR_CQIS, i) + REG_CQIS);  in ufshcd_mcq_write_cqis()
288 static int ufshcd_mcq_get_tag(struct ufs_hba *hba, struct cq_entry *cqe)  in ufshcd_mcq_get_tag()  argument
297 		hba->ucdl_dma_addr;  in ufshcd_mcq_get_tag()
299 	return div_u64(addr, ufshcd_get_ucd_size(hba));  in ufshcd_mcq_get_tag()
302 static void ufshcd_mcq_process_cqe(struct ufs_hba *hba,  in ufshcd_mcq_process_cqe()  argument
306 	int tag = ufshcd_mcq_get_tag(hba, cqe);  in ufshcd_mcq_process_cqe()
309 		ufshcd_compl_one_cqe(hba, tag, cqe);  in ufshcd_mcq_process_cqe()
315 void ufshcd_mcq_compl_all_cqes_lock(struct ufs_hba *hba,  in ufshcd_mcq_compl_all_cqes_lock()  argument
323 		ufshcd_mcq_process_cqe(hba, hwq);  in ufshcd_mcq_compl_all_cqes_lock()
333 unsigned long ufshcd_mcq_poll_cqe_lock(struct ufs_hba *hba,  in ufshcd_mcq_poll_cqe_lock()  argument
342 		ufshcd_mcq_process_cqe(hba, hwq);  in ufshcd_mcq_poll_cqe_lock()
355 void ufshcd_mcq_make_queues_operational(struct ufs_hba *hba)  in ufshcd_mcq_make_queues_operational()  argument
361 	for (i = 0; i < hba->nr_hw_queues; i++) {  in ufshcd_mcq_make_queues_operational()
362 		hwq = &hba->uhq[i];  in ufshcd_mcq_make_queues_operational()
367 		ufsmcq_writelx(hba, lower_32_bits(hwq->sqe_dma_addr),  in ufshcd_mcq_make_queues_operational()
370 		ufsmcq_writelx(hba, upper_32_bits(hwq->sqe_dma_addr),  in ufshcd_mcq_make_queues_operational()
373 		ufsmcq_writelx(hba, ufshcd_mcq_opr_offset(hba, OPR_SQD, i),  in ufshcd_mcq_make_queues_operational()
376 		ufsmcq_writelx(hba, ufshcd_mcq_opr_offset(hba, OPR_SQIS, i),  in ufshcd_mcq_make_queues_operational()
380 		ufsmcq_writelx(hba, lower_32_bits(hwq->cqe_dma_addr),  in ufshcd_mcq_make_queues_operational()
383 		ufsmcq_writelx(hba, upper_32_bits(hwq->cqe_dma_addr),  in ufshcd_mcq_make_queues_operational()
386 		ufsmcq_writelx(hba, ufshcd_mcq_opr_offset(hba, OPR_CQD, i),  in ufshcd_mcq_make_queues_operational()
389 		ufsmcq_writelx(hba, ufshcd_mcq_opr_offset(hba, OPR_CQIS, i),  in ufshcd_mcq_make_queues_operational()
393 		hwq->mcq_sq_head = mcq_opr_base(hba, OPR_SQD, i) + REG_SQHP;  in ufshcd_mcq_make_queues_operational()
394 		hwq->mcq_sq_tail = mcq_opr_base(hba, OPR_SQD, i) + REG_SQTP;  in ufshcd_mcq_make_queues_operational()
395 		hwq->mcq_cq_head = mcq_opr_base(hba, OPR_CQD, i) + REG_CQHP;  in ufshcd_mcq_make_queues_operational()
396 		hwq->mcq_cq_tail = mcq_opr_base(hba, OPR_CQD, i) + REG_CQTP;  in ufshcd_mcq_make_queues_operational()
402 		if (i < hba->nr_hw_queues - hba->nr_queues[HCTX_TYPE_POLL])  in ufshcd_mcq_make_queues_operational()
403 			writel(1, mcq_opr_base(hba, OPR_CQIS, i) + REG_CQIE);  in ufshcd_mcq_make_queues_operational()
406 		ufsmcq_writel(hba, (1 << QUEUE_EN_OFFSET) | qsize,  in ufshcd_mcq_make_queues_operational()
413 		ufsmcq_writel(hba, (1 << QUEUE_EN_OFFSET) | qsize |  in ufshcd_mcq_make_queues_operational()
420 void ufshcd_mcq_enable(struct ufs_hba *hba)  in ufshcd_mcq_enable()  argument
422 	ufshcd_rmwl(hba, MCQ_MODE_SELECT, MCQ_MODE_SELECT, REG_UFS_MEM_CFG);  in ufshcd_mcq_enable()
423 	hba->mcq_enabled = true;  in ufshcd_mcq_enable()
427 void ufshcd_mcq_disable(struct ufs_hba *hba)  in ufshcd_mcq_disable()  argument
429 	ufshcd_rmwl(hba, MCQ_MODE_SELECT, 0, REG_UFS_MEM_CFG);  in ufshcd_mcq_disable()
430 	hba->mcq_enabled = false;  in ufshcd_mcq_disable()
433 void ufshcd_mcq_enable_esi(struct ufs_hba *hba)  in ufshcd_mcq_enable_esi()  argument
435 	ufshcd_writel(hba, ufshcd_readl(hba, REG_UFS_MEM_CFG) | 0x2,  in ufshcd_mcq_enable_esi()
440 void ufshcd_mcq_config_esi(struct ufs_hba *hba, struct msi_msg *msg)  in ufshcd_mcq_config_esi()  argument
442 	ufshcd_writel(hba, msg->address_lo, REG_UFS_ESILBA);  in ufshcd_mcq_config_esi()
443 	ufshcd_writel(hba, msg->address_hi, REG_UFS_ESIUBA);  in ufshcd_mcq_config_esi()
447 int ufshcd_mcq_init(struct ufs_hba *hba)  in ufshcd_mcq_init()  argument
449 	struct Scsi_Host *host = hba->host;  in ufshcd_mcq_init()
453 	ret = ufshcd_mcq_config_nr_queues(hba);  in ufshcd_mcq_init()
457 	ret = ufshcd_vops_mcq_config_resource(hba);  in ufshcd_mcq_init()
461 	ret = ufshcd_mcq_vops_op_runtime_config(hba);  in ufshcd_mcq_init()
463 		dev_err(hba->dev, "Operation runtime config failed, ret=%d\n",  in ufshcd_mcq_init()
467 	hba->uhq = devm_kzalloc(hba->dev,  in ufshcd_mcq_init()
468 				hba->nr_hw_queues * sizeof(struct ufs_hw_queue),  in ufshcd_mcq_init()
470 	if (!hba->uhq) {  in ufshcd_mcq_init()
471 		dev_err(hba->dev, "ufs hw queue memory allocation failed\n");  in ufshcd_mcq_init()
475 	for (i = 0; i < hba->nr_hw_queues; i++) {  in ufshcd_mcq_init()
476 		hwq = &hba->uhq[i];  in ufshcd_mcq_init()
477 		hwq->max_entries = hba->nutrs + 1;  in ufshcd_mcq_init()
484 	hba->dev_cmd_queue = &hba->uhq[0];  in ufshcd_mcq_init()
490 static int ufshcd_mcq_sq_stop(struct ufs_hba *hba, struct ufs_hw_queue *hwq)  in ufshcd_mcq_sq_stop()  argument
496 	if (hba->quirks & UFSHCD_QUIRK_MCQ_BROKEN_RTC)  in ufshcd_mcq_sq_stop()
499 	writel(SQ_STOP, mcq_opr_base(hba, OPR_SQD, id) + REG_SQRTC);  in ufshcd_mcq_sq_stop()
500 	reg = mcq_opr_base(hba, OPR_SQD, id) + REG_SQRTS;  in ufshcd_mcq_sq_stop()
504 		dev_err(hba->dev, "%s: failed. hwq-id=%d, err=%d\n",  in ufshcd_mcq_sq_stop()
509 static int ufshcd_mcq_sq_start(struct ufs_hba *hba, struct ufs_hw_queue *hwq)  in ufshcd_mcq_sq_start()  argument
515 	if (hba->quirks & UFSHCD_QUIRK_MCQ_BROKEN_RTC)  in ufshcd_mcq_sq_start()
518 	writel(SQ_START, mcq_opr_base(hba, OPR_SQD, id) + REG_SQRTC);  in ufshcd_mcq_sq_start()
519 	reg = mcq_opr_base(hba, OPR_SQD, id) + REG_SQRTS;  in ufshcd_mcq_sq_start()
523 		dev_err(hba->dev, "%s: failed. hwq-id=%d, err=%d\n",  in ufshcd_mcq_sq_start()
531  * @hba: per adapter instance.
536 int ufshcd_mcq_sq_cleanup(struct ufs_hba *hba, int task_tag)  in ufshcd_mcq_sq_cleanup()  argument
538 	struct ufshcd_lrb *lrbp = &hba->lrb[task_tag];  in ufshcd_mcq_sq_cleanup()
545 	if (hba->quirks & UFSHCD_QUIRK_MCQ_BROKEN_RTC)  in ufshcd_mcq_sq_cleanup()
548 	if (task_tag != hba->nutrs - UFSHCD_NUM_RESERVED) {  in ufshcd_mcq_sq_cleanup()
551 		hwq = ufshcd_mcq_req_to_hwq(hba, scsi_cmd_to_rq(cmd));  in ufshcd_mcq_sq_cleanup()
555 		hwq = hba->dev_cmd_queue;  in ufshcd_mcq_sq_cleanup()
563 	err = ufshcd_mcq_sq_stop(hba, hwq);  in ufshcd_mcq_sq_cleanup()
569 	opr_sqd_base = mcq_opr_base(hba, OPR_SQD, id);  in ufshcd_mcq_sq_cleanup()
581 		dev_err(hba->dev, "%s: failed. hwq=%d, tag=%d err=%d\n",  in ufshcd_mcq_sq_cleanup()
584 		dev_info(hba->dev,  in ufshcd_mcq_sq_cleanup()
589 	if (ufshcd_mcq_sq_start(hba, hwq))  in ufshcd_mcq_sq_cleanup()
614  * @hba: per adapter instance.
621 static bool ufshcd_mcq_sqe_search(struct ufs_hba *hba,  in ufshcd_mcq_sqe_search()  argument
624 	struct ufshcd_lrb *lrbp = &hba->lrb[task_tag];  in ufshcd_mcq_sqe_search()
631 	if (hba->quirks & UFSHCD_QUIRK_MCQ_BROKEN_RTC)  in ufshcd_mcq_sqe_search()
636 	ufshcd_mcq_sq_stop(hba, hwq);  in ufshcd_mcq_sqe_search()
659 	ufshcd_mcq_sq_start(hba, hwq);  in ufshcd_mcq_sqe_search()
673 	struct ufs_hba *hba = shost_priv(host);  in ufshcd_mcq_abort()  local
675 	struct ufshcd_lrb *lrbp = &hba->lrb[tag];  in ufshcd_mcq_abort()
681 		dev_err(hba->dev,  in ufshcd_mcq_abort()
689 		dev_err(hba->dev, "%s: skip abort. tag %d failed earlier\n",  in ufshcd_mcq_abort()
694 	hwq = ufshcd_mcq_req_to_hwq(hba, scsi_cmd_to_rq(cmd));  in ufshcd_mcq_abort()
696 	if (ufshcd_mcq_sqe_search(hba, hwq, tag)) {  in ufshcd_mcq_abort()
701 		dev_err(hba->dev, "%s: cmd found in sq. hwq=%d, tag=%d\n",  in ufshcd_mcq_abort()
711 	err = ufshcd_try_to_abort_task(hba, tag);  in ufshcd_mcq_abort()
713 		dev_err(hba->dev, "%s: device abort failed %d\n", __func__, err);  in ufshcd_mcq_abort()
720 		ufshcd_release_scsi_cmd(hba, lrbp);  in ufshcd_mcq_abort()