1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (c) 2022 Qualcomm Innovation Center. All rights reserved. 4 * 5 * Authors: 6 * Asutosh Das <quic_asutoshd@quicinc.com> 7 * Can Guo <quic_cang@quicinc.com> 8 */ 9 10 #include <asm/unaligned.h> 11 #include <linux/dma-mapping.h> 12 #include <linux/module.h> 13 #include <linux/platform_device.h> 14 #include "ufshcd-priv.h" 15 #include <linux/delay.h> 16 #include <scsi/scsi_cmnd.h> 17 #include <linux/bitfield.h> 18 #include <linux/iopoll.h> 19 20 #define MAX_QUEUE_SUP GENMASK(7, 0) 21 #define UFS_MCQ_MIN_RW_QUEUES 2 22 #define UFS_MCQ_MIN_READ_QUEUES 0 23 #define UFS_MCQ_MIN_POLL_QUEUES 0 24 #define QUEUE_EN_OFFSET 31 25 #define QUEUE_ID_OFFSET 16 26 27 #define MCQ_CFG_MAC_MASK GENMASK(16, 8) 28 #define MCQ_QCFG_SIZE 0x40 29 #define MCQ_ENTRY_SIZE_IN_DWORD 8 30 #define CQE_UCD_BA GENMASK_ULL(63, 7) 31 32 /* Max mcq register polling time in microseconds */ 33 #define MCQ_POLL_US 500000 34 35 static int rw_queue_count_set(const char *val, const struct kernel_param *kp) 36 { 37 return param_set_uint_minmax(val, kp, UFS_MCQ_MIN_RW_QUEUES, 38 num_possible_cpus()); 39 } 40 41 static const struct kernel_param_ops rw_queue_count_ops = { 42 .set = rw_queue_count_set, 43 .get = param_get_uint, 44 }; 45 46 static unsigned int rw_queues; 47 module_param_cb(rw_queues, &rw_queue_count_ops, &rw_queues, 0644); 48 MODULE_PARM_DESC(rw_queues, 49 "Number of interrupt driven I/O queues used for rw. Default value is nr_cpus"); 50 51 static int read_queue_count_set(const char *val, const struct kernel_param *kp) 52 { 53 return param_set_uint_minmax(val, kp, UFS_MCQ_MIN_READ_QUEUES, 54 num_possible_cpus()); 55 } 56 57 static const struct kernel_param_ops read_queue_count_ops = { 58 .set = read_queue_count_set, 59 .get = param_get_uint, 60 }; 61 62 static unsigned int read_queues; 63 module_param_cb(read_queues, &read_queue_count_ops, &read_queues, 0644); 64 MODULE_PARM_DESC(read_queues, 65 "Number of interrupt driven read queues used for read. Default value is 0"); 66 67 static int poll_queue_count_set(const char *val, const struct kernel_param *kp) 68 { 69 return param_set_uint_minmax(val, kp, UFS_MCQ_MIN_POLL_QUEUES, 70 num_possible_cpus()); 71 } 72 73 static const struct kernel_param_ops poll_queue_count_ops = { 74 .set = poll_queue_count_set, 75 .get = param_get_uint, 76 }; 77 78 static unsigned int poll_queues = 1; 79 module_param_cb(poll_queues, &poll_queue_count_ops, &poll_queues, 0644); 80 MODULE_PARM_DESC(poll_queues, 81 "Number of poll queues used for r/w. Default value is 1"); 82 83 /** 84 * ufshcd_mcq_config_mac - Set the #Max Activ Cmds. 85 * @hba: per adapter instance 86 * @max_active_cmds: maximum # of active commands to the device at any time. 87 * 88 * The controller won't send more than the max_active_cmds to the device at 89 * any time. 90 */ 91 void ufshcd_mcq_config_mac(struct ufs_hba *hba, u32 max_active_cmds) 92 { 93 u32 val; 94 95 val = ufshcd_readl(hba, REG_UFS_MCQ_CFG); 96 val &= ~MCQ_CFG_MAC_MASK; 97 val |= FIELD_PREP(MCQ_CFG_MAC_MASK, max_active_cmds - 1); 98 ufshcd_writel(hba, val, REG_UFS_MCQ_CFG); 99 } 100 EXPORT_SYMBOL_GPL(ufshcd_mcq_config_mac); 101 102 /** 103 * ufshcd_mcq_req_to_hwq - find the hardware queue on which the 104 * request would be issued. 105 * @hba: per adapter instance 106 * @req: pointer to the request to be issued 107 * 108 * Return: the hardware queue instance on which the request would 109 * be queued. 110 */ 111 struct ufs_hw_queue *ufshcd_mcq_req_to_hwq(struct ufs_hba *hba, 112 struct request *req) 113 { 114 u32 utag = blk_mq_unique_tag(req); 115 u32 hwq = blk_mq_unique_tag_to_hwq(utag); 116 117 return &hba->uhq[hwq]; 118 } 119 120 /** 121 * ufshcd_mcq_decide_queue_depth - decide the queue depth 122 * @hba: per adapter instance 123 * 124 * Return: queue-depth on success, non-zero on error 125 * 126 * MAC - Max. Active Command of the Host Controller (HC) 127 * HC wouldn't send more than this commands to the device. 128 * It is mandatory to implement get_hba_mac() to enable MCQ mode. 129 * Calculates and adjusts the queue depth based on the depth 130 * supported by the HC and ufs device. 131 */ 132 int ufshcd_mcq_decide_queue_depth(struct ufs_hba *hba) 133 { 134 int mac; 135 136 /* Mandatory to implement get_hba_mac() */ 137 mac = ufshcd_mcq_vops_get_hba_mac(hba); 138 if (mac < 0) { 139 dev_err(hba->dev, "Failed to get mac, err=%d\n", mac); 140 return mac; 141 } 142 143 WARN_ON_ONCE(!hba->dev_info.bqueuedepth); 144 /* 145 * max. value of bqueuedepth = 256, mac is host dependent. 146 * It is mandatory for UFS device to define bQueueDepth if 147 * shared queuing architecture is enabled. 148 */ 149 return min_t(int, mac, hba->dev_info.bqueuedepth); 150 } 151 152 static int ufshcd_mcq_config_nr_queues(struct ufs_hba *hba) 153 { 154 int i; 155 u32 hba_maxq, rem, tot_queues; 156 struct Scsi_Host *host = hba->host; 157 158 /* maxq is 0 based value */ 159 hba_maxq = FIELD_GET(MAX_QUEUE_SUP, hba->mcq_capabilities) + 1; 160 161 tot_queues = read_queues + poll_queues + rw_queues; 162 163 if (hba_maxq < tot_queues) { 164 dev_err(hba->dev, "Total queues (%d) exceeds HC capacity (%d)\n", 165 tot_queues, hba_maxq); 166 return -EOPNOTSUPP; 167 } 168 169 rem = hba_maxq; 170 171 if (rw_queues) { 172 hba->nr_queues[HCTX_TYPE_DEFAULT] = rw_queues; 173 rem -= hba->nr_queues[HCTX_TYPE_DEFAULT]; 174 } else { 175 rw_queues = num_possible_cpus(); 176 } 177 178 if (poll_queues) { 179 hba->nr_queues[HCTX_TYPE_POLL] = poll_queues; 180 rem -= hba->nr_queues[HCTX_TYPE_POLL]; 181 } 182 183 if (read_queues) { 184 hba->nr_queues[HCTX_TYPE_READ] = read_queues; 185 rem -= hba->nr_queues[HCTX_TYPE_READ]; 186 } 187 188 if (!hba->nr_queues[HCTX_TYPE_DEFAULT]) 189 hba->nr_queues[HCTX_TYPE_DEFAULT] = min3(rem, rw_queues, 190 num_possible_cpus()); 191 192 for (i = 0; i < HCTX_MAX_TYPES; i++) 193 host->nr_hw_queues += hba->nr_queues[i]; 194 195 hba->nr_hw_queues = host->nr_hw_queues; 196 return 0; 197 } 198 199 int ufshcd_mcq_memory_alloc(struct ufs_hba *hba) 200 { 201 struct ufs_hw_queue *hwq; 202 size_t utrdl_size, cqe_size; 203 int i; 204 205 for (i = 0; i < hba->nr_hw_queues; i++) { 206 hwq = &hba->uhq[i]; 207 208 utrdl_size = sizeof(struct utp_transfer_req_desc) * 209 hwq->max_entries; 210 hwq->sqe_base_addr = dmam_alloc_coherent(hba->dev, utrdl_size, 211 &hwq->sqe_dma_addr, 212 GFP_KERNEL); 213 if (!hwq->sqe_dma_addr) { 214 dev_err(hba->dev, "SQE allocation failed\n"); 215 return -ENOMEM; 216 } 217 218 cqe_size = sizeof(struct cq_entry) * hwq->max_entries; 219 hwq->cqe_base_addr = dmam_alloc_coherent(hba->dev, cqe_size, 220 &hwq->cqe_dma_addr, 221 GFP_KERNEL); 222 if (!hwq->cqe_dma_addr) { 223 dev_err(hba->dev, "CQE allocation failed\n"); 224 return -ENOMEM; 225 } 226 } 227 228 return 0; 229 } 230 231 232 /* Operation and runtime registers configuration */ 233 #define MCQ_CFG_n(r, i) ((r) + MCQ_QCFG_SIZE * (i)) 234 #define MCQ_OPR_OFFSET_n(p, i) \ 235 (hba->mcq_opr[(p)].offset + hba->mcq_opr[(p)].stride * (i)) 236 237 static void __iomem *mcq_opr_base(struct ufs_hba *hba, 238 enum ufshcd_mcq_opr n, int i) 239 { 240 struct ufshcd_mcq_opr_info_t *opr = &hba->mcq_opr[n]; 241 242 return opr->base + opr->stride * i; 243 } 244 245 u32 ufshcd_mcq_read_cqis(struct ufs_hba *hba, int i) 246 { 247 return readl(mcq_opr_base(hba, OPR_CQIS, i) + REG_CQIS); 248 } 249 EXPORT_SYMBOL_GPL(ufshcd_mcq_read_cqis); 250 251 void ufshcd_mcq_write_cqis(struct ufs_hba *hba, u32 val, int i) 252 { 253 writel(val, mcq_opr_base(hba, OPR_CQIS, i) + REG_CQIS); 254 } 255 EXPORT_SYMBOL_GPL(ufshcd_mcq_write_cqis); 256 257 /* 258 * Current MCQ specification doesn't provide a Task Tag or its equivalent in 259 * the Completion Queue Entry. Find the Task Tag using an indirect method. 260 */ 261 static int ufshcd_mcq_get_tag(struct ufs_hba *hba, struct cq_entry *cqe) 262 { 263 u64 addr; 264 265 /* sizeof(struct utp_transfer_cmd_desc) must be a multiple of 128 */ 266 BUILD_BUG_ON(sizeof(struct utp_transfer_cmd_desc) & GENMASK(6, 0)); 267 268 /* Bits 63:7 UCD base address, 6:5 are reserved, 4:0 is SQ ID */ 269 addr = (le64_to_cpu(cqe->command_desc_base_addr) & CQE_UCD_BA) - 270 hba->ucdl_dma_addr; 271 272 return div_u64(addr, ufshcd_get_ucd_size(hba)); 273 } 274 275 static void ufshcd_mcq_process_cqe(struct ufs_hba *hba, 276 struct ufs_hw_queue *hwq) 277 { 278 struct cq_entry *cqe = ufshcd_mcq_cur_cqe(hwq); 279 int tag = ufshcd_mcq_get_tag(hba, cqe); 280 281 if (cqe->command_desc_base_addr) { 282 ufshcd_compl_one_cqe(hba, tag, cqe); 283 /* After processed the cqe, mark it empty (invalid) entry */ 284 cqe->command_desc_base_addr = 0; 285 } 286 } 287 288 void ufshcd_mcq_compl_all_cqes_lock(struct ufs_hba *hba, 289 struct ufs_hw_queue *hwq) 290 { 291 unsigned long flags; 292 u32 entries = hwq->max_entries; 293 294 spin_lock_irqsave(&hwq->cq_lock, flags); 295 while (entries > 0) { 296 ufshcd_mcq_process_cqe(hba, hwq); 297 ufshcd_mcq_inc_cq_head_slot(hwq); 298 entries--; 299 } 300 301 ufshcd_mcq_update_cq_tail_slot(hwq); 302 hwq->cq_head_slot = hwq->cq_tail_slot; 303 spin_unlock_irqrestore(&hwq->cq_lock, flags); 304 } 305 306 unsigned long ufshcd_mcq_poll_cqe_lock(struct ufs_hba *hba, 307 struct ufs_hw_queue *hwq) 308 { 309 unsigned long completed_reqs = 0; 310 unsigned long flags; 311 312 spin_lock_irqsave(&hwq->cq_lock, flags); 313 ufshcd_mcq_update_cq_tail_slot(hwq); 314 while (!ufshcd_mcq_is_cq_empty(hwq)) { 315 ufshcd_mcq_process_cqe(hba, hwq); 316 ufshcd_mcq_inc_cq_head_slot(hwq); 317 completed_reqs++; 318 } 319 320 if (completed_reqs) 321 ufshcd_mcq_update_cq_head(hwq); 322 spin_unlock_irqrestore(&hwq->cq_lock, flags); 323 324 return completed_reqs; 325 } 326 EXPORT_SYMBOL_GPL(ufshcd_mcq_poll_cqe_lock); 327 328 void ufshcd_mcq_make_queues_operational(struct ufs_hba *hba) 329 { 330 struct ufs_hw_queue *hwq; 331 u16 qsize; 332 int i; 333 334 for (i = 0; i < hba->nr_hw_queues; i++) { 335 hwq = &hba->uhq[i]; 336 hwq->id = i; 337 qsize = hwq->max_entries * MCQ_ENTRY_SIZE_IN_DWORD - 1; 338 339 /* Submission Queue Lower Base Address */ 340 ufsmcq_writelx(hba, lower_32_bits(hwq->sqe_dma_addr), 341 MCQ_CFG_n(REG_SQLBA, i)); 342 /* Submission Queue Upper Base Address */ 343 ufsmcq_writelx(hba, upper_32_bits(hwq->sqe_dma_addr), 344 MCQ_CFG_n(REG_SQUBA, i)); 345 /* Submission Queue Doorbell Address Offset */ 346 ufsmcq_writelx(hba, MCQ_OPR_OFFSET_n(OPR_SQD, i), 347 MCQ_CFG_n(REG_SQDAO, i)); 348 /* Submission Queue Interrupt Status Address Offset */ 349 ufsmcq_writelx(hba, MCQ_OPR_OFFSET_n(OPR_SQIS, i), 350 MCQ_CFG_n(REG_SQISAO, i)); 351 352 /* Completion Queue Lower Base Address */ 353 ufsmcq_writelx(hba, lower_32_bits(hwq->cqe_dma_addr), 354 MCQ_CFG_n(REG_CQLBA, i)); 355 /* Completion Queue Upper Base Address */ 356 ufsmcq_writelx(hba, upper_32_bits(hwq->cqe_dma_addr), 357 MCQ_CFG_n(REG_CQUBA, i)); 358 /* Completion Queue Doorbell Address Offset */ 359 ufsmcq_writelx(hba, MCQ_OPR_OFFSET_n(OPR_CQD, i), 360 MCQ_CFG_n(REG_CQDAO, i)); 361 /* Completion Queue Interrupt Status Address Offset */ 362 ufsmcq_writelx(hba, MCQ_OPR_OFFSET_n(OPR_CQIS, i), 363 MCQ_CFG_n(REG_CQISAO, i)); 364 365 /* Save the base addresses for quicker access */ 366 hwq->mcq_sq_head = mcq_opr_base(hba, OPR_SQD, i) + REG_SQHP; 367 hwq->mcq_sq_tail = mcq_opr_base(hba, OPR_SQD, i) + REG_SQTP; 368 hwq->mcq_cq_head = mcq_opr_base(hba, OPR_CQD, i) + REG_CQHP; 369 hwq->mcq_cq_tail = mcq_opr_base(hba, OPR_CQD, i) + REG_CQTP; 370 371 /* Reinitializing is needed upon HC reset */ 372 hwq->sq_tail_slot = hwq->cq_tail_slot = hwq->cq_head_slot = 0; 373 374 /* Enable Tail Entry Push Status interrupt only for non-poll queues */ 375 if (i < hba->nr_hw_queues - hba->nr_queues[HCTX_TYPE_POLL]) 376 writel(1, mcq_opr_base(hba, OPR_CQIS, i) + REG_CQIE); 377 378 /* Completion Queue Enable|Size to Completion Queue Attribute */ 379 ufsmcq_writel(hba, (1 << QUEUE_EN_OFFSET) | qsize, 380 MCQ_CFG_n(REG_CQATTR, i)); 381 382 /* 383 * Submission Qeueue Enable|Size|Completion Queue ID to 384 * Submission Queue Attribute 385 */ 386 ufsmcq_writel(hba, (1 << QUEUE_EN_OFFSET) | qsize | 387 (i << QUEUE_ID_OFFSET), 388 MCQ_CFG_n(REG_SQATTR, i)); 389 } 390 } 391 EXPORT_SYMBOL_GPL(ufshcd_mcq_make_queues_operational); 392 393 void ufshcd_mcq_enable_esi(struct ufs_hba *hba) 394 { 395 ufshcd_writel(hba, ufshcd_readl(hba, REG_UFS_MEM_CFG) | 0x2, 396 REG_UFS_MEM_CFG); 397 } 398 EXPORT_SYMBOL_GPL(ufshcd_mcq_enable_esi); 399 400 void ufshcd_mcq_enable(struct ufs_hba *hba) 401 { 402 ufshcd_rmwl(hba, MCQ_MODE_SELECT, MCQ_MODE_SELECT, REG_UFS_MEM_CFG); 403 } 404 EXPORT_SYMBOL_GPL(ufshcd_mcq_enable); 405 406 void ufshcd_mcq_config_esi(struct ufs_hba *hba, struct msi_msg *msg) 407 { 408 ufshcd_writel(hba, msg->address_lo, REG_UFS_ESILBA); 409 ufshcd_writel(hba, msg->address_hi, REG_UFS_ESIUBA); 410 } 411 EXPORT_SYMBOL_GPL(ufshcd_mcq_config_esi); 412 413 int ufshcd_mcq_init(struct ufs_hba *hba) 414 { 415 struct Scsi_Host *host = hba->host; 416 struct ufs_hw_queue *hwq; 417 int ret, i; 418 419 ret = ufshcd_mcq_config_nr_queues(hba); 420 if (ret) 421 return ret; 422 423 ret = ufshcd_vops_mcq_config_resource(hba); 424 if (ret) 425 return ret; 426 427 ret = ufshcd_mcq_vops_op_runtime_config(hba); 428 if (ret) { 429 dev_err(hba->dev, "Operation runtime config failed, ret=%d\n", 430 ret); 431 return ret; 432 } 433 hba->uhq = devm_kzalloc(hba->dev, 434 hba->nr_hw_queues * sizeof(struct ufs_hw_queue), 435 GFP_KERNEL); 436 if (!hba->uhq) { 437 dev_err(hba->dev, "ufs hw queue memory allocation failed\n"); 438 return -ENOMEM; 439 } 440 441 for (i = 0; i < hba->nr_hw_queues; i++) { 442 hwq = &hba->uhq[i]; 443 hwq->max_entries = hba->nutrs + 1; 444 spin_lock_init(&hwq->sq_lock); 445 spin_lock_init(&hwq->cq_lock); 446 mutex_init(&hwq->sq_mutex); 447 } 448 449 /* The very first HW queue serves device commands */ 450 hba->dev_cmd_queue = &hba->uhq[0]; 451 452 host->host_tagset = 1; 453 return 0; 454 } 455 456 static int ufshcd_mcq_sq_stop(struct ufs_hba *hba, struct ufs_hw_queue *hwq) 457 { 458 void __iomem *reg; 459 u32 id = hwq->id, val; 460 int err; 461 462 if (hba->quirks & UFSHCD_QUIRK_MCQ_BROKEN_RTC) 463 return -ETIMEDOUT; 464 465 writel(SQ_STOP, mcq_opr_base(hba, OPR_SQD, id) + REG_SQRTC); 466 reg = mcq_opr_base(hba, OPR_SQD, id) + REG_SQRTS; 467 err = read_poll_timeout(readl, val, val & SQ_STS, 20, 468 MCQ_POLL_US, false, reg); 469 if (err) 470 dev_err(hba->dev, "%s: failed. hwq-id=%d, err=%d\n", 471 __func__, id, err); 472 return err; 473 } 474 475 static int ufshcd_mcq_sq_start(struct ufs_hba *hba, struct ufs_hw_queue *hwq) 476 { 477 void __iomem *reg; 478 u32 id = hwq->id, val; 479 int err; 480 481 if (hba->quirks & UFSHCD_QUIRK_MCQ_BROKEN_RTC) 482 return -ETIMEDOUT; 483 484 writel(SQ_START, mcq_opr_base(hba, OPR_SQD, id) + REG_SQRTC); 485 reg = mcq_opr_base(hba, OPR_SQD, id) + REG_SQRTS; 486 err = read_poll_timeout(readl, val, !(val & SQ_STS), 20, 487 MCQ_POLL_US, false, reg); 488 if (err) 489 dev_err(hba->dev, "%s: failed. hwq-id=%d, err=%d\n", 490 __func__, id, err); 491 return err; 492 } 493 494 /** 495 * ufshcd_mcq_sq_cleanup - Clean up submission queue resources 496 * associated with the pending command. 497 * @hba: per adapter instance. 498 * @task_tag: The command's task tag. 499 * 500 * Return: 0 for success; error code otherwise. 501 */ 502 int ufshcd_mcq_sq_cleanup(struct ufs_hba *hba, int task_tag) 503 { 504 struct ufshcd_lrb *lrbp = &hba->lrb[task_tag]; 505 struct scsi_cmnd *cmd = lrbp->cmd; 506 struct ufs_hw_queue *hwq; 507 void __iomem *reg, *opr_sqd_base; 508 u32 nexus, id, val; 509 int err; 510 511 if (hba->quirks & UFSHCD_QUIRK_MCQ_BROKEN_RTC) 512 return -ETIMEDOUT; 513 514 if (task_tag != hba->nutrs - UFSHCD_NUM_RESERVED) { 515 if (!cmd) 516 return -EINVAL; 517 hwq = ufshcd_mcq_req_to_hwq(hba, scsi_cmd_to_rq(cmd)); 518 } else { 519 hwq = hba->dev_cmd_queue; 520 } 521 522 id = hwq->id; 523 524 mutex_lock(&hwq->sq_mutex); 525 526 /* stop the SQ fetching before working on it */ 527 err = ufshcd_mcq_sq_stop(hba, hwq); 528 if (err) 529 goto unlock; 530 531 /* SQCTI = EXT_IID, IID, LUN, Task Tag */ 532 nexus = lrbp->lun << 8 | task_tag; 533 opr_sqd_base = mcq_opr_base(hba, OPR_SQD, id); 534 writel(nexus, opr_sqd_base + REG_SQCTI); 535 536 /* SQRTCy.ICU = 1 */ 537 writel(SQ_ICU, opr_sqd_base + REG_SQRTC); 538 539 /* Poll SQRTSy.CUS = 1. Return result from SQRTSy.RTC */ 540 reg = opr_sqd_base + REG_SQRTS; 541 err = read_poll_timeout(readl, val, val & SQ_CUS, 20, 542 MCQ_POLL_US, false, reg); 543 if (err) 544 dev_err(hba->dev, "%s: failed. hwq=%d, tag=%d err=%ld\n", 545 __func__, id, task_tag, 546 FIELD_GET(SQ_ICU_ERR_CODE_MASK, readl(reg))); 547 548 if (ufshcd_mcq_sq_start(hba, hwq)) 549 err = -ETIMEDOUT; 550 551 unlock: 552 mutex_unlock(&hwq->sq_mutex); 553 return err; 554 } 555 556 /** 557 * ufshcd_mcq_nullify_sqe - Nullify the submission queue entry. 558 * Write the sqe's Command Type to 0xF. The host controller will not 559 * fetch any sqe with Command Type = 0xF. 560 * 561 * @utrd: UTP Transfer Request Descriptor to be nullified. 562 */ 563 static void ufshcd_mcq_nullify_sqe(struct utp_transfer_req_desc *utrd) 564 { 565 utrd->header.command_type = 0xf; 566 } 567 568 /** 569 * ufshcd_mcq_sqe_search - Search for the command in the submission queue 570 * If the command is in the submission queue and not issued to the device yet, 571 * nullify the sqe so the host controller will skip fetching the sqe. 572 * 573 * @hba: per adapter instance. 574 * @hwq: Hardware Queue to be searched. 575 * @task_tag: The command's task tag. 576 * 577 * Return: true if the SQE containing the command is present in the SQ 578 * (not fetched by the controller); returns false if the SQE is not in the SQ. 579 */ 580 static bool ufshcd_mcq_sqe_search(struct ufs_hba *hba, 581 struct ufs_hw_queue *hwq, int task_tag) 582 { 583 struct ufshcd_lrb *lrbp = &hba->lrb[task_tag]; 584 struct utp_transfer_req_desc *utrd; 585 __le64 cmd_desc_base_addr; 586 bool ret = false; 587 u64 addr, match; 588 u32 sq_head_slot; 589 590 if (hba->quirks & UFSHCD_QUIRK_MCQ_BROKEN_RTC) 591 return true; 592 593 mutex_lock(&hwq->sq_mutex); 594 595 ufshcd_mcq_sq_stop(hba, hwq); 596 sq_head_slot = ufshcd_mcq_get_sq_head_slot(hwq); 597 if (sq_head_slot == hwq->sq_tail_slot) 598 goto out; 599 600 cmd_desc_base_addr = lrbp->utr_descriptor_ptr->command_desc_base_addr; 601 addr = le64_to_cpu(cmd_desc_base_addr) & CQE_UCD_BA; 602 603 while (sq_head_slot != hwq->sq_tail_slot) { 604 utrd = hwq->sqe_base_addr + sq_head_slot; 605 match = le64_to_cpu(utrd->command_desc_base_addr) & CQE_UCD_BA; 606 if (addr == match) { 607 ufshcd_mcq_nullify_sqe(utrd); 608 ret = true; 609 goto out; 610 } 611 612 sq_head_slot++; 613 if (sq_head_slot == hwq->max_entries) 614 sq_head_slot = 0; 615 } 616 617 out: 618 ufshcd_mcq_sq_start(hba, hwq); 619 mutex_unlock(&hwq->sq_mutex); 620 return ret; 621 } 622 623 /** 624 * ufshcd_mcq_abort - Abort the command in MCQ. 625 * @cmd: The command to be aborted. 626 * 627 * Return: SUCCESS or FAILED error codes 628 */ 629 int ufshcd_mcq_abort(struct scsi_cmnd *cmd) 630 { 631 struct Scsi_Host *host = cmd->device->host; 632 struct ufs_hba *hba = shost_priv(host); 633 int tag = scsi_cmd_to_rq(cmd)->tag; 634 struct ufshcd_lrb *lrbp = &hba->lrb[tag]; 635 struct ufs_hw_queue *hwq; 636 unsigned long flags; 637 int err; 638 639 if (!ufshcd_cmd_inflight(lrbp->cmd)) { 640 dev_err(hba->dev, 641 "%s: skip abort. cmd at tag %d already completed.\n", 642 __func__, tag); 643 return FAILED; 644 } 645 646 /* Skip task abort in case previous aborts failed and report failure */ 647 if (lrbp->req_abort_skip) { 648 dev_err(hba->dev, "%s: skip abort. tag %d failed earlier\n", 649 __func__, tag); 650 return FAILED; 651 } 652 653 hwq = ufshcd_mcq_req_to_hwq(hba, scsi_cmd_to_rq(cmd)); 654 655 if (ufshcd_mcq_sqe_search(hba, hwq, tag)) { 656 /* 657 * Failure. The command should not be "stuck" in SQ for 658 * a long time which resulted in command being aborted. 659 */ 660 dev_err(hba->dev, "%s: cmd found in sq. hwq=%d, tag=%d\n", 661 __func__, hwq->id, tag); 662 return FAILED; 663 } 664 665 /* 666 * The command is not in the submission queue, and it is not 667 * in the completion queue either. Query the device to see if 668 * the command is being processed in the device. 669 */ 670 err = ufshcd_try_to_abort_task(hba, tag); 671 if (err) { 672 dev_err(hba->dev, "%s: device abort failed %d\n", __func__, err); 673 lrbp->req_abort_skip = true; 674 return FAILED; 675 } 676 677 spin_lock_irqsave(&hwq->cq_lock, flags); 678 if (ufshcd_cmd_inflight(lrbp->cmd)) 679 ufshcd_release_scsi_cmd(hba, lrbp); 680 spin_unlock_irqrestore(&hwq->cq_lock, flags); 681 682 return SUCCESS; 683 } 684