1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (c) 2022 Qualcomm Innovation Center. All rights reserved. 4 * 5 * Authors: 6 * Asutosh Das <quic_asutoshd@quicinc.com> 7 * Can Guo <quic_cang@quicinc.com> 8 */ 9 10 #include <linux/unaligned.h> 11 #include <linux/dma-mapping.h> 12 #include <linux/module.h> 13 #include <linux/platform_device.h> 14 #include "ufshcd-priv.h" 15 #include <linux/delay.h> 16 #include <scsi/scsi_cmnd.h> 17 #include <linux/bitfield.h> 18 #include <linux/iopoll.h> 19 20 #define MAX_QUEUE_SUP GENMASK(7, 0) 21 #define QCFGPTR GENMASK(23, 16) 22 #define UFS_MCQ_MIN_RW_QUEUES 2 23 #define UFS_MCQ_MIN_READ_QUEUES 0 24 #define UFS_MCQ_MIN_POLL_QUEUES 0 25 #define QUEUE_EN_OFFSET 31 26 #define QUEUE_ID_OFFSET 16 27 28 #define MCQ_CFG_MAC_MASK GENMASK(16, 8) 29 #define MCQ_ENTRY_SIZE_IN_DWORD 8 30 #define CQE_UCD_BA GENMASK_ULL(63, 7) 31 32 #define UFSHCD_ENABLE_MCQ_INTRS (UTP_TASK_REQ_COMPL |\ 33 UFSHCD_ERROR_MASK |\ 34 MCQ_CQ_EVENT_STATUS |\ 35 MCQ_IAG_EVENT_STATUS) 36 37 /* Max mcq register polling time in microseconds */ 38 #define MCQ_POLL_US 500000 39 40 static int rw_queue_count_set(const char *val, const struct kernel_param *kp) 41 { 42 return param_set_uint_minmax(val, kp, UFS_MCQ_MIN_RW_QUEUES, 43 num_possible_cpus()); 44 } 45 46 static const struct kernel_param_ops rw_queue_count_ops = { 47 .set = rw_queue_count_set, 48 .get = param_get_uint, 49 }; 50 51 static unsigned int rw_queues; 52 module_param_cb(rw_queues, &rw_queue_count_ops, &rw_queues, 0644); 53 MODULE_PARM_DESC(rw_queues, 54 "Number of interrupt driven I/O queues used for rw. Default value is nr_cpus"); 55 56 static int read_queue_count_set(const char *val, const struct kernel_param *kp) 57 { 58 return param_set_uint_minmax(val, kp, UFS_MCQ_MIN_READ_QUEUES, 59 num_possible_cpus()); 60 } 61 62 static const struct kernel_param_ops read_queue_count_ops = { 63 .set = read_queue_count_set, 64 .get = param_get_uint, 65 }; 66 67 static unsigned int read_queues; 68 module_param_cb(read_queues, &read_queue_count_ops, &read_queues, 0644); 69 MODULE_PARM_DESC(read_queues, 70 "Number of interrupt driven read queues used for read. Default value is 0"); 71 72 static int poll_queue_count_set(const char *val, const struct kernel_param *kp) 73 { 74 return param_set_uint_minmax(val, kp, UFS_MCQ_MIN_POLL_QUEUES, 75 num_possible_cpus()); 76 } 77 78 static const struct kernel_param_ops poll_queue_count_ops = { 79 .set = poll_queue_count_set, 80 .get = param_get_uint, 81 }; 82 83 static unsigned int poll_queues = 1; 84 module_param_cb(poll_queues, &poll_queue_count_ops, &poll_queues, 0644); 85 MODULE_PARM_DESC(poll_queues, 86 "Number of poll queues used for r/w. Default value is 1"); 87 88 /** 89 * ufshcd_mcq_config_mac - Set the #Max Activ Cmds. 90 * @hba: per adapter instance 91 * @max_active_cmds: maximum # of active commands to the device at any time. 92 * 93 * The controller won't send more than the max_active_cmds to the device at 94 * any time. 95 */ 96 void ufshcd_mcq_config_mac(struct ufs_hba *hba, u32 max_active_cmds) 97 { 98 u32 val; 99 100 val = ufshcd_readl(hba, REG_UFS_MCQ_CFG); 101 val &= ~MCQ_CFG_MAC_MASK; 102 val |= FIELD_PREP(MCQ_CFG_MAC_MASK, max_active_cmds - 1); 103 ufshcd_writel(hba, val, REG_UFS_MCQ_CFG); 104 } 105 EXPORT_SYMBOL_GPL(ufshcd_mcq_config_mac); 106 107 /** 108 * ufshcd_mcq_req_to_hwq - find the hardware queue on which the 109 * request would be issued. 110 * @hba: per adapter instance 111 * @req: pointer to the request to be issued 112 * 113 * Return: the hardware queue instance on which the request will be or has 114 * been queued. %NULL if the request has already been freed. 115 */ 116 struct ufs_hw_queue *ufshcd_mcq_req_to_hwq(struct ufs_hba *hba, 117 struct request *req) 118 { 119 struct blk_mq_hw_ctx *hctx = READ_ONCE(req->mq_hctx); 120 121 return hctx ? &hba->uhq[hctx->queue_num] : NULL; 122 } 123 124 /** 125 * ufshcd_mcq_queue_cfg_addr - get an start address of the MCQ Queue Config 126 * Registers. 127 * @hba: per adapter instance 128 * 129 * Return: Start address of MCQ Queue Config Registers in HCI 130 */ 131 unsigned int ufshcd_mcq_queue_cfg_addr(struct ufs_hba *hba) 132 { 133 return FIELD_GET(QCFGPTR, hba->mcq_capabilities) * 0x200; 134 } 135 EXPORT_SYMBOL_GPL(ufshcd_mcq_queue_cfg_addr); 136 137 /** 138 * ufshcd_get_hba_mac - Maximum number of commands supported by the host 139 * controller. 140 * @hba: per adapter instance 141 * 142 * Return: queue depth on success; negative upon error. 143 * 144 * MAC = Maximum number of Active Commands supported by the Host Controller. 145 */ 146 int ufshcd_get_hba_mac(struct ufs_hba *hba) 147 { 148 int mac; 149 150 if (!hba->vops || !hba->vops->get_hba_mac) { 151 /* 152 * Extract the maximum number of active transfer tasks value 153 * from the host controller capabilities register. This value is 154 * 0-based. 155 */ 156 hba->capabilities = 157 ufshcd_readl(hba, REG_CONTROLLER_CAPABILITIES); 158 mac = hba->capabilities & MASK_TRANSFER_REQUESTS_SLOTS_MCQ; 159 mac++; 160 } else { 161 mac = hba->vops->get_hba_mac(hba); 162 } 163 if (mac < 0) 164 dev_err(hba->dev, "Failed to get mac, err=%d\n", mac); 165 return mac; 166 } 167 168 static int ufshcd_mcq_config_nr_queues(struct ufs_hba *hba) 169 { 170 int i; 171 u32 hba_maxq, rem, tot_queues; 172 struct Scsi_Host *host = hba->host; 173 174 /* maxq is 0 based value */ 175 hba_maxq = FIELD_GET(MAX_QUEUE_SUP, hba->mcq_capabilities) + 1; 176 177 tot_queues = read_queues + poll_queues + rw_queues; 178 179 if (hba_maxq < tot_queues) { 180 dev_err(hba->dev, "Total queues (%d) exceeds HC capacity (%d)\n", 181 tot_queues, hba_maxq); 182 return -EOPNOTSUPP; 183 } 184 185 /* 186 * Device should support at least one I/O queue to handle device 187 * commands via hba->dev_cmd_queue. 188 */ 189 if (hba_maxq == poll_queues) { 190 dev_err(hba->dev, "At least one non-poll queue required\n"); 191 return -EOPNOTSUPP; 192 } 193 194 rem = hba_maxq; 195 196 if (rw_queues) { 197 hba->nr_queues[HCTX_TYPE_DEFAULT] = rw_queues; 198 rem -= hba->nr_queues[HCTX_TYPE_DEFAULT]; 199 } else { 200 rw_queues = num_possible_cpus(); 201 } 202 203 if (poll_queues) { 204 hba->nr_queues[HCTX_TYPE_POLL] = poll_queues; 205 rem -= hba->nr_queues[HCTX_TYPE_POLL]; 206 } 207 208 if (read_queues) { 209 hba->nr_queues[HCTX_TYPE_READ] = read_queues; 210 rem -= hba->nr_queues[HCTX_TYPE_READ]; 211 } 212 213 if (!hba->nr_queues[HCTX_TYPE_DEFAULT]) 214 hba->nr_queues[HCTX_TYPE_DEFAULT] = min3(rem, rw_queues, 215 num_possible_cpus()); 216 217 for (i = 0; i < HCTX_MAX_TYPES; i++) 218 host->nr_hw_queues += hba->nr_queues[i]; 219 220 hba->nr_hw_queues = host->nr_hw_queues; 221 return 0; 222 } 223 224 int ufshcd_mcq_memory_alloc(struct ufs_hba *hba) 225 { 226 struct ufs_hw_queue *hwq; 227 size_t utrdl_size, cqe_size; 228 int i; 229 230 for (i = 0; i < hba->nr_hw_queues; i++) { 231 hwq = &hba->uhq[i]; 232 233 utrdl_size = sizeof(struct utp_transfer_req_desc) * 234 hwq->max_entries; 235 hwq->sqe_base_addr = dmam_alloc_coherent(hba->dev, utrdl_size, 236 &hwq->sqe_dma_addr, 237 GFP_KERNEL); 238 if (!hwq->sqe_base_addr) { 239 dev_err(hba->dev, "SQE allocation failed\n"); 240 return -ENOMEM; 241 } 242 243 cqe_size = sizeof(struct cq_entry) * hwq->max_entries; 244 hwq->cqe_base_addr = dmam_alloc_coherent(hba->dev, cqe_size, 245 &hwq->cqe_dma_addr, 246 GFP_KERNEL); 247 if (!hwq->cqe_base_addr) { 248 dev_err(hba->dev, "CQE allocation failed\n"); 249 return -ENOMEM; 250 } 251 } 252 253 return 0; 254 } 255 256 static void __iomem *mcq_opr_base(struct ufs_hba *hba, 257 enum ufshcd_mcq_opr n, int i) 258 { 259 struct ufshcd_mcq_opr_info_t *opr = &hba->mcq_opr[n]; 260 261 return opr->base + opr->stride * i; 262 } 263 264 u32 ufshcd_mcq_read_cqis(struct ufs_hba *hba, int i) 265 { 266 return readl(mcq_opr_base(hba, OPR_CQIS, i) + REG_CQIS); 267 } 268 EXPORT_SYMBOL_GPL(ufshcd_mcq_read_cqis); 269 270 void ufshcd_mcq_write_cqis(struct ufs_hba *hba, u32 val, int i) 271 { 272 writel(val, mcq_opr_base(hba, OPR_CQIS, i) + REG_CQIS); 273 } 274 EXPORT_SYMBOL_GPL(ufshcd_mcq_write_cqis); 275 276 u32 ufshcd_mcq_read_mcqiacr(struct ufs_hba *hba, int i) 277 { 278 return readl(mcq_opr_base(hba, OPR_CQIS, i) + REG_MCQIACR); 279 } 280 281 void ufshcd_mcq_write_mcqiacr(struct ufs_hba *hba, u32 val, int i) 282 { 283 writel(val, mcq_opr_base(hba, OPR_CQIS, i) + REG_MCQIACR); 284 } 285 286 /* 287 * UFSHCI 4.0 MCQ specification doesn't provide a Task Tag or its equivalent in 288 * the Completion Queue Entry. Find the Task Tag using an indirect method. 289 * UFSHCI 4.1 and above can directly return the Task Tag in the Completion Queue 290 * Entry. 291 */ 292 static int ufshcd_mcq_get_tag(struct ufs_hba *hba, struct cq_entry *cqe) 293 { 294 u64 addr; 295 296 if (hba->ufs_version >= ufshci_version(4, 1)) 297 return cqe->task_tag; 298 299 /* Both UCD types must have a size that is a multiple of 128 bytes */ 300 BUILD_BUG_ON(sizeof(struct utp_transfer_cmd_desc) & GENMASK(6, 0)); 301 BUILD_BUG_ON(sizeof(struct utp_devman_cmd_desc) & GENMASK(6, 0)); 302 303 /* Bits 63:7 UCD base address, 6:5 are reserved, 4:0 is SQ ID */ 304 addr = le64_to_cpu(cqe->command_desc_base_addr) & CQE_UCD_BA; 305 306 /* The devman UCD is outside the pool; return its reserved tag. */ 307 if (unlikely(addr == hba->devman_ucd_dma_addr)) 308 return hba->dev_cmd.tag; 309 310 /* Pool entries follow the reserved tags. */ 311 return div_u64(addr - hba->ucdl_dma_addr, ufshcd_get_ucd_size(hba)) + 312 UFSHCD_NUM_RESERVED; 313 } 314 315 static void ufshcd_mcq_process_cqe(struct ufs_hba *hba, 316 struct ufs_hw_queue *hwq) 317 { 318 struct cq_entry *cqe = ufshcd_mcq_cur_cqe(hwq); 319 320 if (cqe->command_desc_base_addr) { 321 int tag = ufshcd_mcq_get_tag(hba, cqe); 322 323 ufshcd_compl_one_cqe(hba, tag, cqe); 324 /* After processed the cqe, mark it empty (invalid) entry */ 325 cqe->command_desc_base_addr = 0; 326 } else { 327 dev_err(hba->dev, "Abnormal CQ entry!\n"); 328 } 329 } 330 331 /* 332 * This function is called from the UFS error handler with the UFS host 333 * controller disabled (HCE = 0). Reading host controller registers, e.g. the 334 * CQ tail pointer (CQTPy), may not be safe with the host controller disabled. 335 * Hence, iterate over all completion queue entries. This won't result in 336 * double completions because ufshcd_mcq_process_cqe() clears a CQE after it 337 * has been processed. 338 */ 339 void ufshcd_mcq_compl_all_cqes_lock(struct ufs_hba *hba, 340 struct ufs_hw_queue *hwq) 341 { 342 unsigned long flags; 343 u32 entries = hwq->max_entries; 344 345 spin_lock_irqsave(&hwq->cq_lock, flags); 346 while (entries > 0) { 347 ufshcd_mcq_process_cqe(hba, hwq); 348 ufshcd_mcq_inc_cq_head_slot(hwq); 349 entries--; 350 } 351 352 ufshcd_mcq_update_cq_tail_slot(hwq); 353 hwq->cq_head_slot = hwq->cq_tail_slot; 354 spin_unlock_irqrestore(&hwq->cq_lock, flags); 355 } 356 357 unsigned long ufshcd_mcq_poll_cqe_lock(struct ufs_hba *hba, 358 struct ufs_hw_queue *hwq) 359 { 360 unsigned long completed_reqs = 0; 361 unsigned long flags; 362 363 spin_lock_irqsave(&hwq->cq_lock, flags); 364 ufshcd_mcq_update_cq_tail_slot(hwq); 365 while (!ufshcd_mcq_is_cq_empty(hwq)) { 366 ufshcd_mcq_process_cqe(hba, hwq); 367 ufshcd_mcq_inc_cq_head_slot(hwq); 368 completed_reqs++; 369 } 370 371 if (completed_reqs) 372 ufshcd_mcq_update_cq_head(hwq); 373 spin_unlock_irqrestore(&hwq->cq_lock, flags); 374 375 return completed_reqs; 376 } 377 EXPORT_SYMBOL_GPL(ufshcd_mcq_poll_cqe_lock); 378 379 void ufshcd_mcq_make_queues_operational(struct ufs_hba *hba) 380 { 381 struct ufs_hw_queue *hwq; 382 u32 intrs; 383 u16 qsize; 384 int i; 385 386 /* Enable required interrupts */ 387 intrs = UFSHCD_ENABLE_MCQ_INTRS; 388 if (hba->quirks & UFSHCD_QUIRK_MCQ_BROKEN_INTR) 389 intrs &= ~MCQ_CQ_EVENT_STATUS; 390 ufshcd_enable_intr(hba, intrs); 391 392 for (i = 0; i < hba->nr_hw_queues; i++) { 393 hwq = &hba->uhq[i]; 394 hwq->id = i; 395 qsize = hwq->max_entries * MCQ_ENTRY_SIZE_IN_DWORD - 1; 396 397 /* Submission Queue Lower Base Address */ 398 ufsmcq_writelx(hba, lower_32_bits(hwq->sqe_dma_addr), 399 ufshcd_mcq_cfg_offset(REG_SQLBA, i)); 400 /* Submission Queue Upper Base Address */ 401 ufsmcq_writelx(hba, upper_32_bits(hwq->sqe_dma_addr), 402 ufshcd_mcq_cfg_offset(REG_SQUBA, i)); 403 /* Submission Queue Doorbell Address Offset */ 404 ufsmcq_writelx(hba, ufshcd_mcq_opr_offset(hba, OPR_SQD, i), 405 ufshcd_mcq_cfg_offset(REG_SQDAO, i)); 406 /* Submission Queue Interrupt Status Address Offset */ 407 ufsmcq_writelx(hba, ufshcd_mcq_opr_offset(hba, OPR_SQIS, i), 408 ufshcd_mcq_cfg_offset(REG_SQISAO, i)); 409 410 /* Completion Queue Lower Base Address */ 411 ufsmcq_writelx(hba, lower_32_bits(hwq->cqe_dma_addr), 412 ufshcd_mcq_cfg_offset(REG_CQLBA, i)); 413 /* Completion Queue Upper Base Address */ 414 ufsmcq_writelx(hba, upper_32_bits(hwq->cqe_dma_addr), 415 ufshcd_mcq_cfg_offset(REG_CQUBA, i)); 416 /* Completion Queue Doorbell Address Offset */ 417 ufsmcq_writelx(hba, ufshcd_mcq_opr_offset(hba, OPR_CQD, i), 418 ufshcd_mcq_cfg_offset(REG_CQDAO, i)); 419 /* Completion Queue Interrupt Status Address Offset */ 420 ufsmcq_writelx(hba, ufshcd_mcq_opr_offset(hba, OPR_CQIS, i), 421 ufshcd_mcq_cfg_offset(REG_CQISAO, i)); 422 423 /* Save the base addresses for quicker access */ 424 hwq->mcq_sq_head = mcq_opr_base(hba, OPR_SQD, i) + REG_SQHP; 425 hwq->mcq_sq_tail = mcq_opr_base(hba, OPR_SQD, i) + REG_SQTP; 426 hwq->mcq_cq_head = mcq_opr_base(hba, OPR_CQD, i) + REG_CQHP; 427 hwq->mcq_cq_tail = mcq_opr_base(hba, OPR_CQD, i) + REG_CQTP; 428 429 /* Reinitializing is needed upon HC reset */ 430 hwq->sq_tail_slot = hwq->cq_tail_slot = hwq->cq_head_slot = 0; 431 432 /* Enable Tail Entry Push Status interrupt only for non-poll queues */ 433 if (i < hba->nr_hw_queues - hba->nr_queues[HCTX_TYPE_POLL]) 434 writel(1, mcq_opr_base(hba, OPR_CQIS, i) + REG_CQIE); 435 436 /* Completion Queue Enable|Size to Completion Queue Attribute */ 437 ufsmcq_writel(hba, (1 << QUEUE_EN_OFFSET) | qsize, 438 ufshcd_mcq_cfg_offset(REG_CQATTR, i)); 439 440 /* 441 * Submission Qeueue Enable|Size|Completion Queue ID to 442 * Submission Queue Attribute 443 */ 444 ufsmcq_writel(hba, (1 << QUEUE_EN_OFFSET) | qsize | 445 (i << QUEUE_ID_OFFSET), 446 ufshcd_mcq_cfg_offset(REG_SQATTR, i)); 447 } 448 } 449 EXPORT_SYMBOL_GPL(ufshcd_mcq_make_queues_operational); 450 451 void ufshcd_mcq_enable(struct ufs_hba *hba) 452 { 453 ufshcd_rmwl(hba, MCQ_MODE_SELECT, MCQ_MODE_SELECT, REG_UFS_MEM_CFG); 454 hba->mcq_enabled = true; 455 } 456 EXPORT_SYMBOL_GPL(ufshcd_mcq_enable); 457 458 void ufshcd_mcq_disable(struct ufs_hba *hba) 459 { 460 ufshcd_rmwl(hba, MCQ_MODE_SELECT, 0, REG_UFS_MEM_CFG); 461 hba->mcq_enabled = false; 462 } 463 464 void ufshcd_mcq_enable_esi(struct ufs_hba *hba) 465 { 466 ufshcd_rmwl(hba, ESI_ENABLE, ESI_ENABLE, REG_UFS_MEM_CFG); 467 } 468 EXPORT_SYMBOL_GPL(ufshcd_mcq_enable_esi); 469 470 void ufshcd_mcq_config_esi(struct ufs_hba *hba, struct msi_msg *msg) 471 { 472 ufshcd_writel(hba, msg->address_lo, REG_UFS_ESILBA); 473 ufshcd_writel(hba, msg->address_hi, REG_UFS_ESIUBA); 474 } 475 EXPORT_SYMBOL_GPL(ufshcd_mcq_config_esi); 476 477 int ufshcd_mcq_init(struct ufs_hba *hba) 478 { 479 struct ufs_hw_queue *hwq; 480 int ret, i; 481 482 ret = ufshcd_mcq_config_nr_queues(hba); 483 if (ret) 484 return ret; 485 486 ret = ufshcd_vops_mcq_config_resource(hba); 487 if (ret) 488 return ret; 489 490 ret = ufshcd_mcq_vops_op_runtime_config(hba); 491 if (ret) { 492 dev_err(hba->dev, "Operation runtime config failed, ret=%d\n", 493 ret); 494 return ret; 495 } 496 hba->uhq = devm_kzalloc(hba->dev, 497 hba->nr_hw_queues * sizeof(struct ufs_hw_queue), 498 GFP_KERNEL); 499 if (!hba->uhq) { 500 dev_err(hba->dev, "ufs hw queue memory allocation failed\n"); 501 return -ENOMEM; 502 } 503 504 for (i = 0; i < hba->nr_hw_queues; i++) { 505 hwq = &hba->uhq[i]; 506 hwq->max_entries = hba->nutrs + 1; 507 spin_lock_init(&hwq->sq_lock); 508 spin_lock_init(&hwq->cq_lock); 509 mutex_init(&hwq->sq_mutex); 510 } 511 512 return 0; 513 } 514 515 static int ufshcd_mcq_sq_stop(struct ufs_hba *hba, struct ufs_hw_queue *hwq) 516 { 517 void __iomem *reg; 518 u32 id = hwq->id, val; 519 int err; 520 521 if (hba->quirks & UFSHCD_QUIRK_MCQ_BROKEN_RTC) 522 return -ETIMEDOUT; 523 524 writel(SQ_STOP, mcq_opr_base(hba, OPR_SQD, id) + REG_SQRTC); 525 reg = mcq_opr_base(hba, OPR_SQD, id) + REG_SQRTS; 526 err = read_poll_timeout(readl, val, val & SQ_STS, 20, 527 MCQ_POLL_US, false, reg); 528 if (err) 529 dev_err(hba->dev, "%s: failed. hwq-id=%d, err=%d\n", 530 __func__, id, err); 531 return err; 532 } 533 534 static int ufshcd_mcq_sq_start(struct ufs_hba *hba, struct ufs_hw_queue *hwq) 535 { 536 void __iomem *reg; 537 u32 id = hwq->id, val; 538 int err; 539 540 if (hba->quirks & UFSHCD_QUIRK_MCQ_BROKEN_RTC) 541 return -ETIMEDOUT; 542 543 writel(SQ_START, mcq_opr_base(hba, OPR_SQD, id) + REG_SQRTC); 544 reg = mcq_opr_base(hba, OPR_SQD, id) + REG_SQRTS; 545 err = read_poll_timeout(readl, val, !(val & SQ_STS), 20, 546 MCQ_POLL_US, false, reg); 547 if (err) 548 dev_err(hba->dev, "%s: failed. hwq-id=%d, err=%d\n", 549 __func__, id, err); 550 return err; 551 } 552 553 /** 554 * ufshcd_mcq_sq_cleanup - Clean up submission queue resources 555 * associated with the pending command. 556 * @hba: per adapter instance. 557 * @task_tag: The command's task tag. 558 * 559 * Return: 0 for success; error code otherwise. 560 */ 561 int ufshcd_mcq_sq_cleanup(struct ufs_hba *hba, int task_tag) 562 { 563 struct scsi_cmnd *cmd = ufshcd_tag_to_cmd(hba, task_tag); 564 struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 565 struct request *rq = scsi_cmd_to_rq(cmd); 566 struct ufs_hw_queue *hwq; 567 void __iomem *reg, *opr_sqd_base; 568 u32 nexus, id, val; 569 int err; 570 571 if (hba->quirks & UFSHCD_QUIRK_MCQ_BROKEN_RTC) 572 return -ETIMEDOUT; 573 574 if (!cmd) 575 return -EINVAL; 576 577 hwq = ufshcd_mcq_req_to_hwq(hba, rq); 578 if (!hwq) 579 return 0; 580 581 id = hwq->id; 582 583 guard(mutex)(&hwq->sq_mutex); 584 585 /* stop the SQ fetching before working on it */ 586 err = ufshcd_mcq_sq_stop(hba, hwq); 587 if (err) 588 return err; 589 590 /* SQCTI = EXT_IID, IID, LUN, Task Tag */ 591 nexus = lrbp->lun << 8 | task_tag; 592 opr_sqd_base = mcq_opr_base(hba, OPR_SQD, id); 593 writel(nexus, opr_sqd_base + REG_SQCTI); 594 595 /* Initiate Cleanup */ 596 writel(readl(opr_sqd_base + REG_SQRTC) | SQ_ICU, 597 opr_sqd_base + REG_SQRTC); 598 599 /* Wait until SQRTSy.CUS = 1. Report SQRTSy.RTC. */ 600 reg = opr_sqd_base + REG_SQRTS; 601 err = read_poll_timeout(readl, val, val & SQ_CUS, 20, 602 MCQ_POLL_US, false, reg); 603 if (err) 604 dev_err(hba->dev, "%s: failed. hwq=%d, tag=%d err=%d\n", 605 __func__, id, task_tag, err); 606 else 607 dev_info(hba->dev, 608 "%s, hwq %d: cleanup return code (RTC) %ld\n", 609 __func__, id, 610 FIELD_GET(SQ_ICU_ERR_CODE_MASK, readl(reg))); 611 612 if (ufshcd_mcq_sq_start(hba, hwq)) 613 err = -ETIMEDOUT; 614 615 return err; 616 } 617 618 /** 619 * ufshcd_mcq_nullify_sqe - Nullify the submission queue entry. 620 * Write the sqe's Command Type to 0xF. The host controller will not 621 * fetch any sqe with Command Type = 0xF. 622 * 623 * @utrd: UTP Transfer Request Descriptor to be nullified. 624 */ 625 static void ufshcd_mcq_nullify_sqe(struct utp_transfer_req_desc *utrd) 626 { 627 utrd->header.command_type = 0xf; 628 } 629 630 /** 631 * ufshcd_mcq_sqe_search - Search for the command in the submission queue 632 * If the command is in the submission queue and not issued to the device yet, 633 * nullify the sqe so the host controller will skip fetching the sqe. 634 * 635 * @hba: per adapter instance. 636 * @hwq: Hardware Queue to be searched. 637 * @task_tag: The command's task tag. 638 * 639 * Return: true if the SQE containing the command is present in the SQ 640 * (not fetched by the controller); returns false if the SQE is not in the SQ. 641 */ 642 static bool ufshcd_mcq_sqe_search(struct ufs_hba *hba, 643 struct ufs_hw_queue *hwq, int task_tag) 644 { 645 struct scsi_cmnd *cmd = ufshcd_tag_to_cmd(hba, task_tag); 646 struct ufshcd_lrb *lrbp; 647 struct utp_transfer_req_desc *utrd; 648 __le64 cmd_desc_base_addr; 649 bool ret = false; 650 u64 addr, match; 651 u32 sq_head_slot; 652 653 if (hba->quirks & UFSHCD_QUIRK_MCQ_BROKEN_RTC) 654 return true; 655 656 if (!cmd) 657 return false; 658 659 lrbp = scsi_cmd_priv(cmd); 660 661 mutex_lock(&hwq->sq_mutex); 662 663 ufshcd_mcq_sq_stop(hba, hwq); 664 sq_head_slot = ufshcd_mcq_get_sq_head_slot(hwq); 665 if (sq_head_slot == hwq->sq_tail_slot) 666 goto out; 667 668 cmd_desc_base_addr = lrbp->utr_descriptor_ptr->command_desc_base_addr; 669 addr = le64_to_cpu(cmd_desc_base_addr) & CQE_UCD_BA; 670 671 while (sq_head_slot != hwq->sq_tail_slot) { 672 utrd = hwq->sqe_base_addr + sq_head_slot; 673 match = le64_to_cpu(utrd->command_desc_base_addr) & CQE_UCD_BA; 674 if (addr == match) { 675 ufshcd_mcq_nullify_sqe(utrd); 676 ret = true; 677 goto out; 678 } 679 680 sq_head_slot++; 681 if (sq_head_slot == hwq->max_entries) 682 sq_head_slot = 0; 683 } 684 685 out: 686 ufshcd_mcq_sq_start(hba, hwq); 687 mutex_unlock(&hwq->sq_mutex); 688 return ret; 689 } 690 691 /** 692 * ufshcd_mcq_abort - Abort the command in MCQ. 693 * @cmd: The command to be aborted. 694 * 695 * Return: SUCCESS or FAILED error codes 696 */ 697 int ufshcd_mcq_abort(struct scsi_cmnd *cmd) 698 { 699 struct Scsi_Host *host = cmd->device->host; 700 struct ufs_hba *hba = shost_priv(host); 701 int tag = scsi_cmd_to_rq(cmd)->tag; 702 struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); 703 struct ufs_hw_queue *hwq; 704 int err; 705 706 /* Skip task abort in case previous aborts failed and report failure */ 707 if (lrbp->req_abort_skip) { 708 dev_err(hba->dev, "%s: skip abort. tag %d failed earlier\n", 709 __func__, tag); 710 return FAILED; 711 } 712 713 hwq = ufshcd_mcq_req_to_hwq(hba, scsi_cmd_to_rq(cmd)); 714 if (!hwq) { 715 dev_err(hba->dev, "%s: skip abort. cmd at tag %d already completed.\n", 716 __func__, tag); 717 return FAILED; 718 } 719 720 if (ufshcd_mcq_sqe_search(hba, hwq, tag)) { 721 /* 722 * Failure. The command should not be "stuck" in SQ for 723 * a long time which resulted in command being aborted. 724 */ 725 dev_err(hba->dev, "%s: cmd found in sq. hwq=%d, tag=%d\n", 726 __func__, hwq->id, tag); 727 return FAILED; 728 } 729 730 /* 731 * The command is not in the submission queue, and it is not 732 * in the completion queue either. Query the device to see if 733 * the command is being processed in the device. 734 */ 735 err = ufshcd_try_to_abort_task(hba, tag); 736 if (err) { 737 dev_err(hba->dev, "%s: device abort failed %d\n", __func__, err); 738 lrbp->req_abort_skip = true; 739 return FAILED; 740 } 741 742 return SUCCESS; 743 } 744