1 // SPDX-License-Identifier: GPL-2.0
4  * MMC software queue support based on command queue interfaces
10 #include <linux/mmc/card.h>
11 #include <linux/mmc/host.h>
19 	struct mmc_host *mmc = hsq->mmc;  in mmc_hsq_retry_handler()  local
21 	mmc->ops->request(mmc, hsq->mrq);  in mmc_hsq_retry_handler()
26 	struct mmc_host *mmc = hsq->mmc;  in mmc_hsq_modify_threshold()  local
30 	mmc->hsq_depth = HSQ_NORMAL_DEPTH;  in mmc_hsq_modify_threshold()
32 		mrq = hsq->slot[tag].mrq;  in mmc_hsq_modify_threshold()
33 		if (mrq && mrq->data &&  in mmc_hsq_modify_threshold()
34 		   (mrq->data->blksz * mrq->data->blocks == 4096) &&  in mmc_hsq_modify_threshold()
35 		   (mrq->data->flags & MMC_DATA_WRITE) &&  in mmc_hsq_modify_threshold()
37 			mmc->hsq_depth = HSQ_PERFORMANCE_DEPTH;  in mmc_hsq_modify_threshold()
45 	struct mmc_host *mmc = hsq->mmc;  in mmc_hsq_pump_requests()  local
50 	spin_lock_irqsave(&hsq->lock, flags);  in mmc_hsq_pump_requests()
53 	if (hsq->mrq || hsq->recovery_halt) {  in mmc_hsq_pump_requests()
54 		spin_unlock_irqrestore(&hsq->lock, flags);  in mmc_hsq_pump_requests()
59 	if (!hsq->qcnt || !hsq->enabled) {  in mmc_hsq_pump_requests()
60 		spin_unlock_irqrestore(&hsq->lock, flags);  in mmc_hsq_pump_requests()
66 	slot = &hsq->slot[hsq->next_tag];  in mmc_hsq_pump_requests()
67 	hsq->mrq = slot->mrq;  in mmc_hsq_pump_requests()
68 	hsq->qcnt--;  in mmc_hsq_pump_requests()
70 	spin_unlock_irqrestore(&hsq->lock, flags);  in mmc_hsq_pump_requests()
72 	if (mmc->ops->request_atomic)  in mmc_hsq_pump_requests()
73 		ret = mmc->ops->request_atomic(mmc, hsq->mrq);  in mmc_hsq_pump_requests()
75 		mmc->ops->request(mmc, hsq->mrq);  in mmc_hsq_pump_requests()
79 	 * may be busy now, and we should change to non-atomic context to  in mmc_hsq_pump_requests()
80 	 * try again for this unusual case, to avoid time-consuming operations  in mmc_hsq_pump_requests()
86 	if (ret == -EBUSY)  in mmc_hsq_pump_requests()
87 		schedule_work(&hsq->retry_work);  in mmc_hsq_pump_requests()
101 		hsq->next_tag = HSQ_INVALID_TAG;  in mmc_hsq_update_next_tag()
102 		hsq->tail_tag = HSQ_INVALID_TAG;  in mmc_hsq_update_next_tag()
106 	tag = hsq->tag_slot[hsq->next_tag];  in mmc_hsq_update_next_tag()
107 	hsq->tag_slot[hsq->next_tag] = HSQ_INVALID_TAG;  in mmc_hsq_update_next_tag()
108 	hsq->next_tag = tag;  in mmc_hsq_update_next_tag()
116 	spin_lock_irqsave(&hsq->lock, flags);  in mmc_hsq_post_request()
118 	remains = hsq->qcnt;  in mmc_hsq_post_request()
119 	hsq->mrq = NULL;  in mmc_hsq_post_request()
124 	if (hsq->waiting_for_idle && !remains) {  in mmc_hsq_post_request()
125 		hsq->waiting_for_idle = false;  in mmc_hsq_post_request()
126 		wake_up(&hsq->wait_queue);  in mmc_hsq_post_request()
130 	if (hsq->recovery_halt) {  in mmc_hsq_post_request()
131 		spin_unlock_irqrestore(&hsq->lock, flags);  in mmc_hsq_post_request()
135 	spin_unlock_irqrestore(&hsq->lock, flags);  in mmc_hsq_post_request()
146  * mmc_hsq_finalize_request - finalize one request if the request is done
147  * @mmc: the host controller
153 bool mmc_hsq_finalize_request(struct mmc_host *mmc, struct mmc_request *mrq)  in mmc_hsq_finalize_request()  argument
155 	struct mmc_hsq *hsq = mmc->cqe_private;  in mmc_hsq_finalize_request()
158 	spin_lock_irqsave(&hsq->lock, flags);  in mmc_hsq_finalize_request()
160 	if (!hsq->enabled || !hsq->mrq || hsq->mrq != mrq) {  in mmc_hsq_finalize_request()
161 		spin_unlock_irqrestore(&hsq->lock, flags);  in mmc_hsq_finalize_request()
168 	hsq->slot[hsq->next_tag].mrq = NULL;  in mmc_hsq_finalize_request()
170 	spin_unlock_irqrestore(&hsq->lock, flags);  in mmc_hsq_finalize_request()
172 	mmc_cqe_request_done(mmc, hsq->mrq);  in mmc_hsq_finalize_request()
180 static void mmc_hsq_recovery_start(struct mmc_host *mmc)  in mmc_hsq_recovery_start()  argument
182 	struct mmc_hsq *hsq = mmc->cqe_private;  in mmc_hsq_recovery_start()
185 	spin_lock_irqsave(&hsq->lock, flags);  in mmc_hsq_recovery_start()
187 	hsq->recovery_halt = true;  in mmc_hsq_recovery_start()
189 	spin_unlock_irqrestore(&hsq->lock, flags);  in mmc_hsq_recovery_start()
192 static void mmc_hsq_recovery_finish(struct mmc_host *mmc)  in mmc_hsq_recovery_finish()  argument
194 	struct mmc_hsq *hsq = mmc->cqe_private;  in mmc_hsq_recovery_finish()
197 	spin_lock_irq(&hsq->lock);  in mmc_hsq_recovery_finish()
199 	hsq->recovery_halt = false;  in mmc_hsq_recovery_finish()
200 	remains = hsq->qcnt;  in mmc_hsq_recovery_finish()
202 	spin_unlock_irq(&hsq->lock);  in mmc_hsq_recovery_finish()
212 static int mmc_hsq_request(struct mmc_host *mmc, struct mmc_request *mrq)  in mmc_hsq_request()  argument
214 	struct mmc_hsq *hsq = mmc->cqe_private;  in mmc_hsq_request()
215 	int tag = mrq->tag;  in mmc_hsq_request()
217 	spin_lock_irq(&hsq->lock);  in mmc_hsq_request()
219 	if (!hsq->enabled) {  in mmc_hsq_request()
220 		spin_unlock_irq(&hsq->lock);  in mmc_hsq_request()
221 		return -ESHUTDOWN;  in mmc_hsq_request()
225 	if (hsq->recovery_halt) {  in mmc_hsq_request()
226 		spin_unlock_irq(&hsq->lock);  in mmc_hsq_request()
227 		return -EBUSY;  in mmc_hsq_request()
230 	hsq->slot[tag].mrq = mrq;  in mmc_hsq_request()
236 	if (hsq->next_tag == HSQ_INVALID_TAG) {  in mmc_hsq_request()
237 		hsq->next_tag = tag;  in mmc_hsq_request()
238 		hsq->tail_tag = tag;  in mmc_hsq_request()
239 		hsq->tag_slot[hsq->tail_tag] = HSQ_INVALID_TAG;  in mmc_hsq_request()
241 		hsq->tag_slot[hsq->tail_tag] = tag;  in mmc_hsq_request()
242 		hsq->tail_tag = tag;  in mmc_hsq_request()
245 	hsq->qcnt++;  in mmc_hsq_request()
247 	spin_unlock_irq(&hsq->lock);  in mmc_hsq_request()
254 static void mmc_hsq_post_req(struct mmc_host *mmc, struct mmc_request *mrq)  in mmc_hsq_post_req()  argument
256 	if (mmc->ops->post_req)  in mmc_hsq_post_req()
257 		mmc->ops->post_req(mmc, mrq, 0);  in mmc_hsq_post_req()
264 	spin_lock_irq(&hsq->lock);  in mmc_hsq_queue_is_idle()
266 	is_idle = (!hsq->mrq && !hsq->qcnt) ||  in mmc_hsq_queue_is_idle()
267 		hsq->recovery_halt;  in mmc_hsq_queue_is_idle()
269 	*ret = hsq->recovery_halt ? -EBUSY : 0;  in mmc_hsq_queue_is_idle()
270 	hsq->waiting_for_idle = !is_idle;  in mmc_hsq_queue_is_idle()
272 	spin_unlock_irq(&hsq->lock);  in mmc_hsq_queue_is_idle()
277 static int mmc_hsq_wait_for_idle(struct mmc_host *mmc)  in mmc_hsq_wait_for_idle()  argument
279 	struct mmc_hsq *hsq = mmc->cqe_private;  in mmc_hsq_wait_for_idle()
282 	wait_event(hsq->wait_queue,  in mmc_hsq_wait_for_idle()
288 static void mmc_hsq_disable(struct mmc_host *mmc)  in mmc_hsq_disable()  argument
290 	struct mmc_hsq *hsq = mmc->cqe_private;  in mmc_hsq_disable()
294 	spin_lock_irq(&hsq->lock);  in mmc_hsq_disable()
296 	if (!hsq->enabled) {  in mmc_hsq_disable()
297 		spin_unlock_irq(&hsq->lock);  in mmc_hsq_disable()
301 	spin_unlock_irq(&hsq->lock);  in mmc_hsq_disable()
303 	ret = wait_event_timeout(hsq->wait_queue,  in mmc_hsq_disable()
307 		pr_warn("could not stop mmc software queue\n");  in mmc_hsq_disable()
311 	spin_lock_irq(&hsq->lock);  in mmc_hsq_disable()
313 	hsq->enabled = false;  in mmc_hsq_disable()
315 	spin_unlock_irq(&hsq->lock);  in mmc_hsq_disable()
318 static int mmc_hsq_enable(struct mmc_host *mmc, struct mmc_card *card)  in mmc_hsq_enable()  argument
320 	struct mmc_hsq *hsq = mmc->cqe_private;  in mmc_hsq_enable()
322 	spin_lock_irq(&hsq->lock);  in mmc_hsq_enable()
324 	if (hsq->enabled) {  in mmc_hsq_enable()
325 		spin_unlock_irq(&hsq->lock);  in mmc_hsq_enable()
326 		return -EBUSY;  in mmc_hsq_enable()
329 	hsq->enabled = true;  in mmc_hsq_enable()
331 	spin_unlock_irq(&hsq->lock);  in mmc_hsq_enable()
346 int mmc_hsq_init(struct mmc_hsq *hsq, struct mmc_host *mmc)  in mmc_hsq_init()  argument
349 	hsq->num_slots = HSQ_NUM_SLOTS;  in mmc_hsq_init()
350 	hsq->next_tag = HSQ_INVALID_TAG;  in mmc_hsq_init()
351 	hsq->tail_tag = HSQ_INVALID_TAG;  in mmc_hsq_init()
353 	hsq->slot = devm_kcalloc(mmc_dev(mmc), hsq->num_slots,  in mmc_hsq_init()
355 	if (!hsq->slot)  in mmc_hsq_init()
356 		return -ENOMEM;  in mmc_hsq_init()
358 	hsq->mmc = mmc;  in mmc_hsq_init()
359 	hsq->mmc->cqe_private = hsq;  in mmc_hsq_init()
360 	mmc->cqe_ops = &mmc_hsq_ops;  in mmc_hsq_init()
361 	mmc->hsq_depth = HSQ_NORMAL_DEPTH;  in mmc_hsq_init()
364 		hsq->tag_slot[i] = HSQ_INVALID_TAG;  in mmc_hsq_init()
366 	INIT_WORK(&hsq->retry_work, mmc_hsq_retry_handler);  in mmc_hsq_init()
367 	spin_lock_init(&hsq->lock);  in mmc_hsq_init()
368 	init_waitqueue_head(&hsq->wait_queue);  in mmc_hsq_init()
374 void mmc_hsq_suspend(struct mmc_host *mmc)  in mmc_hsq_suspend()  argument
376 	mmc_hsq_disable(mmc);  in mmc_hsq_suspend()
380 int mmc_hsq_resume(struct mmc_host *mmc)  in mmc_hsq_resume()  argument
382 	return mmc_hsq_enable(mmc, NULL);  in mmc_hsq_resume()
386 MODULE_DESCRIPTION("MMC Host Software Queue support");