1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * 4 * MMC software queue support based on command queue interfaces 5 * 6 * Copyright (C) 2019 Linaro, Inc. 7 * Author: Baolin Wang <baolin.wang@linaro.org> 8 */ 9 10 #include <linux/devm-helpers.h> 11 #include <linux/mmc/card.h> 12 #include <linux/mmc/host.h> 13 #include <linux/module.h> 14 15 #include "mmc_hsq.h" 16 17 static void mmc_hsq_retry_handler(struct work_struct *work) 18 { 19 struct mmc_hsq *hsq = container_of(work, struct mmc_hsq, retry_work); 20 struct mmc_host *mmc = hsq->mmc; 21 22 mmc->ops->request(mmc, hsq->mrq); 23 } 24 25 static void mmc_hsq_modify_threshold(struct mmc_hsq *hsq) 26 { 27 struct mmc_host *mmc = hsq->mmc; 28 struct mmc_request *mrq; 29 unsigned int tag, need_change = 0; 30 31 mmc->hsq_depth = HSQ_NORMAL_DEPTH; 32 for (tag = 0; tag < HSQ_NUM_SLOTS; tag++) { 33 mrq = hsq->slot[tag].mrq; 34 if (mrq && mrq->data && 35 (mrq->data->blksz * mrq->data->blocks == 4096) && 36 (mrq->data->flags & MMC_DATA_WRITE) && 37 (++need_change == 2)) { 38 mmc->hsq_depth = HSQ_PERFORMANCE_DEPTH; 39 break; 40 } 41 } 42 } 43 44 static void mmc_hsq_pump_requests(struct mmc_hsq *hsq) 45 { 46 struct mmc_host *mmc = hsq->mmc; 47 struct hsq_slot *slot; 48 unsigned long flags; 49 int ret = 0; 50 51 spin_lock_irqsave(&hsq->lock, flags); 52 53 /* Make sure we are not already running a request now */ 54 if (hsq->mrq || hsq->recovery_halt) { 55 spin_unlock_irqrestore(&hsq->lock, flags); 56 return; 57 } 58 59 /* Make sure there are remain requests need to pump */ 60 if (!hsq->qcnt || !hsq->enabled) { 61 spin_unlock_irqrestore(&hsq->lock, flags); 62 return; 63 } 64 65 mmc_hsq_modify_threshold(hsq); 66 67 slot = &hsq->slot[hsq->next_tag]; 68 hsq->mrq = slot->mrq; 69 hsq->qcnt--; 70 71 spin_unlock_irqrestore(&hsq->lock, flags); 72 73 if (mmc->ops->request_atomic) 74 ret = mmc->ops->request_atomic(mmc, hsq->mrq); 75 else 76 mmc->ops->request(mmc, hsq->mrq); 77 78 /* 79 * If returning BUSY from request_atomic(), which means the card 80 * may be busy now, and we should change to non-atomic context to 81 * try again for this unusual case, to avoid time-consuming operations 82 * in the atomic context. 83 * 84 * Note: we just give a warning for other error cases, since the host 85 * driver will handle them. 86 */ 87 if (ret == -EBUSY) 88 schedule_work(&hsq->retry_work); 89 else 90 WARN_ON_ONCE(ret); 91 } 92 93 static void mmc_hsq_update_next_tag(struct mmc_hsq *hsq, int remains) 94 { 95 int tag; 96 97 /* 98 * If there are no remain requests in software queue, then set a invalid 99 * tag. 100 */ 101 if (!remains) { 102 hsq->next_tag = HSQ_INVALID_TAG; 103 hsq->tail_tag = HSQ_INVALID_TAG; 104 return; 105 } 106 107 tag = hsq->tag_slot[hsq->next_tag]; 108 hsq->tag_slot[hsq->next_tag] = HSQ_INVALID_TAG; 109 hsq->next_tag = tag; 110 } 111 112 static void mmc_hsq_post_request(struct mmc_hsq *hsq) 113 { 114 unsigned long flags; 115 int remains; 116 117 spin_lock_irqsave(&hsq->lock, flags); 118 119 remains = hsq->qcnt; 120 hsq->mrq = NULL; 121 122 /* Update the next available tag to be queued. */ 123 mmc_hsq_update_next_tag(hsq, remains); 124 125 if (hsq->waiting_for_idle && !remains) { 126 hsq->waiting_for_idle = false; 127 wake_up(&hsq->wait_queue); 128 } 129 130 /* Do not pump new request in recovery mode. */ 131 if (hsq->recovery_halt) { 132 spin_unlock_irqrestore(&hsq->lock, flags); 133 return; 134 } 135 136 spin_unlock_irqrestore(&hsq->lock, flags); 137 138 /* 139 * Try to pump new request to host controller as fast as possible, 140 * after completing previous request. 141 */ 142 if (remains > 0) 143 mmc_hsq_pump_requests(hsq); 144 } 145 146 /** 147 * mmc_hsq_finalize_request - finalize one request if the request is done 148 * @mmc: the host controller 149 * @mrq: the request need to be finalized 150 * 151 * Return true if we finalized the corresponding request in software queue, 152 * otherwise return false. 153 */ 154 bool mmc_hsq_finalize_request(struct mmc_host *mmc, struct mmc_request *mrq) 155 { 156 struct mmc_hsq *hsq = mmc->cqe_private; 157 unsigned long flags; 158 159 spin_lock_irqsave(&hsq->lock, flags); 160 161 if (!hsq->enabled || !hsq->mrq || hsq->mrq != mrq) { 162 spin_unlock_irqrestore(&hsq->lock, flags); 163 return false; 164 } 165 166 /* 167 * Clear current completed slot request to make a room for new request. 168 */ 169 hsq->slot[hsq->next_tag].mrq = NULL; 170 171 spin_unlock_irqrestore(&hsq->lock, flags); 172 173 mmc_cqe_request_done(mmc, hsq->mrq); 174 175 mmc_hsq_post_request(hsq); 176 177 return true; 178 } 179 EXPORT_SYMBOL_GPL(mmc_hsq_finalize_request); 180 181 static void mmc_hsq_recovery_start(struct mmc_host *mmc) 182 { 183 struct mmc_hsq *hsq = mmc->cqe_private; 184 unsigned long flags; 185 186 spin_lock_irqsave(&hsq->lock, flags); 187 188 hsq->recovery_halt = true; 189 190 spin_unlock_irqrestore(&hsq->lock, flags); 191 } 192 193 static void mmc_hsq_recovery_finish(struct mmc_host *mmc) 194 { 195 struct mmc_hsq *hsq = mmc->cqe_private; 196 int remains; 197 198 spin_lock_irq(&hsq->lock); 199 200 hsq->recovery_halt = false; 201 remains = hsq->qcnt; 202 203 spin_unlock_irq(&hsq->lock); 204 205 /* 206 * Try to pump new request if there are request pending in software 207 * queue after finishing recovery. 208 */ 209 if (remains > 0) 210 mmc_hsq_pump_requests(hsq); 211 } 212 213 static int mmc_hsq_request(struct mmc_host *mmc, struct mmc_request *mrq) 214 { 215 struct mmc_hsq *hsq = mmc->cqe_private; 216 int tag = mrq->tag; 217 218 spin_lock_irq(&hsq->lock); 219 220 if (!hsq->enabled) { 221 spin_unlock_irq(&hsq->lock); 222 return -ESHUTDOWN; 223 } 224 225 /* Do not queue any new requests in recovery mode. */ 226 if (hsq->recovery_halt) { 227 spin_unlock_irq(&hsq->lock); 228 return -EBUSY; 229 } 230 231 hsq->slot[tag].mrq = mrq; 232 233 /* 234 * Set the next tag as current request tag if no available 235 * next tag. 236 */ 237 if (hsq->next_tag == HSQ_INVALID_TAG) { 238 hsq->next_tag = tag; 239 hsq->tail_tag = tag; 240 hsq->tag_slot[hsq->tail_tag] = HSQ_INVALID_TAG; 241 } else { 242 hsq->tag_slot[hsq->tail_tag] = tag; 243 hsq->tail_tag = tag; 244 } 245 246 hsq->qcnt++; 247 248 spin_unlock_irq(&hsq->lock); 249 250 mmc_hsq_pump_requests(hsq); 251 252 return 0; 253 } 254 255 static void mmc_hsq_post_req(struct mmc_host *mmc, struct mmc_request *mrq) 256 { 257 if (mmc->ops->post_req) 258 mmc->ops->post_req(mmc, mrq, 0); 259 } 260 261 static bool mmc_hsq_queue_is_idle(struct mmc_hsq *hsq, int *ret) 262 { 263 bool is_idle; 264 265 spin_lock_irq(&hsq->lock); 266 267 is_idle = (!hsq->mrq && !hsq->qcnt) || 268 hsq->recovery_halt; 269 270 *ret = hsq->recovery_halt ? -EBUSY : 0; 271 hsq->waiting_for_idle = !is_idle; 272 273 spin_unlock_irq(&hsq->lock); 274 275 return is_idle; 276 } 277 278 static int mmc_hsq_wait_for_idle(struct mmc_host *mmc) 279 { 280 struct mmc_hsq *hsq = mmc->cqe_private; 281 int ret; 282 283 wait_event(hsq->wait_queue, 284 mmc_hsq_queue_is_idle(hsq, &ret)); 285 286 return ret; 287 } 288 289 static void mmc_hsq_disable(struct mmc_host *mmc) 290 { 291 struct mmc_hsq *hsq = mmc->cqe_private; 292 u32 timeout = 500; 293 int ret; 294 295 spin_lock_irq(&hsq->lock); 296 297 if (!hsq->enabled) { 298 spin_unlock_irq(&hsq->lock); 299 return; 300 } 301 302 spin_unlock_irq(&hsq->lock); 303 304 ret = wait_event_timeout(hsq->wait_queue, 305 mmc_hsq_queue_is_idle(hsq, &ret), 306 msecs_to_jiffies(timeout)); 307 if (ret == 0) { 308 pr_warn("could not stop mmc software queue\n"); 309 return; 310 } 311 312 spin_lock_irq(&hsq->lock); 313 314 hsq->enabled = false; 315 316 spin_unlock_irq(&hsq->lock); 317 } 318 319 static int mmc_hsq_enable(struct mmc_host *mmc, struct mmc_card *card) 320 { 321 struct mmc_hsq *hsq = mmc->cqe_private; 322 323 spin_lock_irq(&hsq->lock); 324 325 if (hsq->enabled) { 326 spin_unlock_irq(&hsq->lock); 327 return -EBUSY; 328 } 329 330 hsq->enabled = true; 331 332 spin_unlock_irq(&hsq->lock); 333 334 return 0; 335 } 336 337 static const struct mmc_cqe_ops mmc_hsq_ops = { 338 .cqe_enable = mmc_hsq_enable, 339 .cqe_disable = mmc_hsq_disable, 340 .cqe_request = mmc_hsq_request, 341 .cqe_post_req = mmc_hsq_post_req, 342 .cqe_wait_for_idle = mmc_hsq_wait_for_idle, 343 .cqe_recovery_start = mmc_hsq_recovery_start, 344 .cqe_recovery_finish = mmc_hsq_recovery_finish, 345 }; 346 347 int mmc_hsq_init(struct mmc_hsq *hsq, struct mmc_host *mmc) 348 { 349 int ret; 350 int i; 351 hsq->num_slots = HSQ_NUM_SLOTS; 352 hsq->next_tag = HSQ_INVALID_TAG; 353 hsq->tail_tag = HSQ_INVALID_TAG; 354 355 hsq->slot = devm_kcalloc(mmc_dev(mmc), hsq->num_slots, 356 sizeof(struct hsq_slot), GFP_KERNEL); 357 if (!hsq->slot) 358 return -ENOMEM; 359 360 hsq->mmc = mmc; 361 hsq->mmc->cqe_private = hsq; 362 mmc->cqe_ops = &mmc_hsq_ops; 363 mmc->hsq_depth = HSQ_NORMAL_DEPTH; 364 365 for (i = 0; i < HSQ_NUM_SLOTS; i++) 366 hsq->tag_slot[i] = HSQ_INVALID_TAG; 367 368 ret = devm_work_autocancel(mmc_dev(mmc), &hsq->retry_work, 369 mmc_hsq_retry_handler); 370 if (ret) 371 return ret; 372 373 spin_lock_init(&hsq->lock); 374 init_waitqueue_head(&hsq->wait_queue); 375 376 return 0; 377 } 378 EXPORT_SYMBOL_GPL(mmc_hsq_init); 379 380 void mmc_hsq_suspend(struct mmc_host *mmc) 381 { 382 mmc_hsq_disable(mmc); 383 } 384 EXPORT_SYMBOL_GPL(mmc_hsq_suspend); 385 386 int mmc_hsq_resume(struct mmc_host *mmc) 387 { 388 return mmc_hsq_enable(mmc, NULL); 389 } 390 EXPORT_SYMBOL_GPL(mmc_hsq_resume); 391 392 MODULE_DESCRIPTION("MMC Host Software Queue support"); 393 MODULE_LICENSE("GPL v2"); 394