1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright (c) 2015, The Linux Foundation. All rights reserved. 3 */ 4 5 #include <linux/delay.h> 6 #include <linux/highmem.h> 7 #include <linux/io.h> 8 #include <linux/iopoll.h> 9 #include <linux/module.h> 10 #include <linux/dma-mapping.h> 11 #include <linux/slab.h> 12 #include <linux/scatterlist.h> 13 #include <linux/platform_device.h> 14 #include <linux/ktime.h> 15 16 #include <linux/mmc/mmc.h> 17 #include <linux/mmc/host.h> 18 #include <linux/mmc/card.h> 19 20 #include "cqhci.h" 21 #include "cqhci-crypto.h" 22 23 #define DCMD_SLOT 31 24 #define NUM_SLOTS 32 25 26 struct cqhci_slot { 27 struct mmc_request *mrq; 28 unsigned int flags; 29 #define CQHCI_EXTERNAL_TIMEOUT BIT(0) 30 #define CQHCI_COMPLETED BIT(1) 31 #define CQHCI_HOST_CRC BIT(2) 32 #define CQHCI_HOST_TIMEOUT BIT(3) 33 #define CQHCI_HOST_OTHER BIT(4) 34 }; 35 36 static inline u8 *get_desc(struct cqhci_host *cq_host, u8 tag) 37 { 38 return cq_host->desc_base + (tag * cq_host->slot_sz); 39 } 40 41 static inline u8 *get_link_desc(struct cqhci_host *cq_host, u8 tag) 42 { 43 u8 *desc = get_desc(cq_host, tag); 44 45 return desc + cq_host->task_desc_len; 46 } 47 48 static inline size_t get_trans_desc_offset(struct cqhci_host *cq_host, u8 tag) 49 { 50 return cq_host->trans_desc_len * cq_host->mmc->max_segs * tag; 51 } 52 53 static inline dma_addr_t get_trans_desc_dma(struct cqhci_host *cq_host, u8 tag) 54 { 55 size_t offset = get_trans_desc_offset(cq_host, tag); 56 57 return cq_host->trans_desc_dma_base + offset; 58 } 59 60 static inline u8 *get_trans_desc(struct cqhci_host *cq_host, u8 tag) 61 { 62 size_t offset = get_trans_desc_offset(cq_host, tag); 63 64 return cq_host->trans_desc_base + offset; 65 } 66 67 static void setup_trans_desc(struct cqhci_host *cq_host, u8 tag) 68 { 69 u8 *link_temp; 70 dma_addr_t trans_temp; 71 72 link_temp = get_link_desc(cq_host, tag); 73 trans_temp = get_trans_desc_dma(cq_host, tag); 74 75 memset(link_temp, 0, cq_host->link_desc_len); 76 if (cq_host->link_desc_len > 8) 77 *(link_temp + 8) = 0; 78 79 if (tag == DCMD_SLOT && (cq_host->mmc->caps2 & MMC_CAP2_CQE_DCMD)) { 80 *link_temp = CQHCI_VALID(0) | CQHCI_ACT(0) | CQHCI_END(1); 81 return; 82 } 83 84 *link_temp = CQHCI_VALID(1) | CQHCI_ACT(0x6) | CQHCI_END(0); 85 86 if (cq_host->dma64) { 87 __le64 *data_addr = (__le64 __force *)(link_temp + 4); 88 89 data_addr[0] = cpu_to_le64(trans_temp); 90 } else { 91 __le32 *data_addr = (__le32 __force *)(link_temp + 4); 92 93 data_addr[0] = cpu_to_le32(trans_temp); 94 } 95 } 96 97 static void cqhci_set_irqs(struct cqhci_host *cq_host, u32 set) 98 { 99 cqhci_writel(cq_host, set, CQHCI_ISTE); 100 cqhci_writel(cq_host, set, CQHCI_ISGE); 101 } 102 103 #define DRV_NAME "cqhci" 104 105 #define CQHCI_DUMP(f, x...) \ 106 pr_err("%s: " DRV_NAME ": " f, mmc_hostname(mmc), ## x) 107 108 static void cqhci_dumpregs(struct cqhci_host *cq_host) 109 { 110 struct mmc_host *mmc = cq_host->mmc; 111 112 CQHCI_DUMP("============ CQHCI REGISTER DUMP ===========\n"); 113 114 CQHCI_DUMP("Caps: 0x%08x | Version: 0x%08x\n", 115 cqhci_readl(cq_host, CQHCI_CAP), 116 cqhci_readl(cq_host, CQHCI_VER)); 117 CQHCI_DUMP("Config: 0x%08x | Control: 0x%08x\n", 118 cqhci_readl(cq_host, CQHCI_CFG), 119 cqhci_readl(cq_host, CQHCI_CTL)); 120 CQHCI_DUMP("Int stat: 0x%08x | Int enab: 0x%08x\n", 121 cqhci_readl(cq_host, CQHCI_IS), 122 cqhci_readl(cq_host, CQHCI_ISTE)); 123 CQHCI_DUMP("Int sig: 0x%08x | Int Coal: 0x%08x\n", 124 cqhci_readl(cq_host, CQHCI_ISGE), 125 cqhci_readl(cq_host, CQHCI_IC)); 126 CQHCI_DUMP("TDL base: 0x%08x | TDL up32: 0x%08x\n", 127 cqhci_readl(cq_host, CQHCI_TDLBA), 128 cqhci_readl(cq_host, CQHCI_TDLBAU)); 129 CQHCI_DUMP("Doorbell: 0x%08x | TCN: 0x%08x\n", 130 cqhci_readl(cq_host, CQHCI_TDBR), 131 cqhci_readl(cq_host, CQHCI_TCN)); 132 CQHCI_DUMP("Dev queue: 0x%08x | Dev Pend: 0x%08x\n", 133 cqhci_readl(cq_host, CQHCI_DQS), 134 cqhci_readl(cq_host, CQHCI_DPT)); 135 CQHCI_DUMP("Task clr: 0x%08x | SSC1: 0x%08x\n", 136 cqhci_readl(cq_host, CQHCI_TCLR), 137 cqhci_readl(cq_host, CQHCI_SSC1)); 138 CQHCI_DUMP("SSC2: 0x%08x | DCMD rsp: 0x%08x\n", 139 cqhci_readl(cq_host, CQHCI_SSC2), 140 cqhci_readl(cq_host, CQHCI_CRDCT)); 141 CQHCI_DUMP("RED mask: 0x%08x | TERRI: 0x%08x\n", 142 cqhci_readl(cq_host, CQHCI_RMEM), 143 cqhci_readl(cq_host, CQHCI_TERRI)); 144 CQHCI_DUMP("Resp idx: 0x%08x | Resp arg: 0x%08x\n", 145 cqhci_readl(cq_host, CQHCI_CRI), 146 cqhci_readl(cq_host, CQHCI_CRA)); 147 148 if (cq_host->ops->dumpregs) 149 cq_host->ops->dumpregs(mmc); 150 else 151 CQHCI_DUMP(": ===========================================\n"); 152 } 153 154 /* 155 * The allocated descriptor table for task, link & transfer descriptors 156 * looks like: 157 * |----------| 158 * |task desc | |->|----------| 159 * |----------| | |trans desc| 160 * |link desc-|->| |----------| 161 * |----------| . 162 * . . 163 * no. of slots max-segs 164 * . |----------| 165 * |----------| 166 * The idea here is to create the [task+trans] table and mark & point the 167 * link desc to the transfer desc table on a per slot basis. 168 */ 169 static int cqhci_host_alloc_tdl(struct cqhci_host *cq_host) 170 { 171 int i = 0; 172 173 /* task descriptor can be 64/128 bit irrespective of arch */ 174 if (cq_host->caps & CQHCI_TASK_DESC_SZ_128) { 175 cqhci_writel(cq_host, cqhci_readl(cq_host, CQHCI_CFG) | 176 CQHCI_TASK_DESC_SZ, CQHCI_CFG); 177 cq_host->task_desc_len = 16; 178 } else { 179 cq_host->task_desc_len = 8; 180 } 181 182 /* 183 * 96 bits length of transfer desc instead of 128 bits which means 184 * ADMA would expect next valid descriptor at the 96th bit 185 * or 128th bit 186 */ 187 if (cq_host->dma64) { 188 if (cq_host->quirks & CQHCI_QUIRK_SHORT_TXFR_DESC_SZ) 189 cq_host->trans_desc_len = 12; 190 else 191 cq_host->trans_desc_len = 16; 192 cq_host->link_desc_len = 16; 193 } else { 194 cq_host->trans_desc_len = 8; 195 cq_host->link_desc_len = 8; 196 } 197 198 /* total size of a slot: 1 task & 1 transfer (link) */ 199 cq_host->slot_sz = cq_host->task_desc_len + cq_host->link_desc_len; 200 201 cq_host->desc_size = cq_host->slot_sz * cq_host->num_slots; 202 203 cq_host->data_size = get_trans_desc_offset(cq_host, cq_host->mmc->cqe_qdepth); 204 205 pr_debug("%s: cqhci: desc_size: %zu data_sz: %zu slot-sz: %d\n", 206 mmc_hostname(cq_host->mmc), cq_host->desc_size, cq_host->data_size, 207 cq_host->slot_sz); 208 209 /* 210 * allocate a dma-mapped chunk of memory for the descriptors 211 * allocate a dma-mapped chunk of memory for link descriptors 212 * setup each link-desc memory offset per slot-number to 213 * the descriptor table. 214 */ 215 cq_host->desc_base = dmam_alloc_coherent(mmc_dev(cq_host->mmc), 216 cq_host->desc_size, 217 &cq_host->desc_dma_base, 218 GFP_KERNEL); 219 if (!cq_host->desc_base) 220 return -ENOMEM; 221 222 cq_host->trans_desc_base = dmam_alloc_coherent(mmc_dev(cq_host->mmc), 223 cq_host->data_size, 224 &cq_host->trans_desc_dma_base, 225 GFP_KERNEL); 226 if (!cq_host->trans_desc_base) { 227 dmam_free_coherent(mmc_dev(cq_host->mmc), cq_host->desc_size, 228 cq_host->desc_base, 229 cq_host->desc_dma_base); 230 cq_host->desc_base = NULL; 231 cq_host->desc_dma_base = 0; 232 return -ENOMEM; 233 } 234 235 pr_debug("%s: cqhci: desc-base: 0x%p trans-base: 0x%p\n desc_dma 0x%llx trans_dma: 0x%llx\n", 236 mmc_hostname(cq_host->mmc), cq_host->desc_base, cq_host->trans_desc_base, 237 (unsigned long long)cq_host->desc_dma_base, 238 (unsigned long long)cq_host->trans_desc_dma_base); 239 240 for (; i < (cq_host->num_slots); i++) 241 setup_trans_desc(cq_host, i); 242 243 return 0; 244 } 245 246 static void __cqhci_enable(struct cqhci_host *cq_host) 247 { 248 struct mmc_host *mmc = cq_host->mmc; 249 u32 cqcfg; 250 251 cqcfg = cqhci_readl(cq_host, CQHCI_CFG); 252 253 /* Configuration must not be changed while enabled */ 254 if (cqcfg & CQHCI_ENABLE) { 255 cqcfg &= ~CQHCI_ENABLE; 256 cqhci_writel(cq_host, cqcfg, CQHCI_CFG); 257 } 258 259 cqcfg &= ~(CQHCI_DCMD | CQHCI_TASK_DESC_SZ); 260 261 if (mmc->caps2 & MMC_CAP2_CQE_DCMD) 262 cqcfg |= CQHCI_DCMD; 263 264 if (cq_host->caps & CQHCI_TASK_DESC_SZ_128) 265 cqcfg |= CQHCI_TASK_DESC_SZ; 266 267 if (mmc->caps2 & MMC_CAP2_CRYPTO) 268 cqcfg |= CQHCI_CRYPTO_GENERAL_ENABLE; 269 270 cqhci_writel(cq_host, cqcfg, CQHCI_CFG); 271 272 cqhci_writel(cq_host, lower_32_bits(cq_host->desc_dma_base), 273 CQHCI_TDLBA); 274 cqhci_writel(cq_host, upper_32_bits(cq_host->desc_dma_base), 275 CQHCI_TDLBAU); 276 277 cqhci_writel(cq_host, cq_host->rca, CQHCI_SSC2); 278 279 cqhci_set_irqs(cq_host, 0); 280 281 cqcfg |= CQHCI_ENABLE; 282 283 cqhci_writel(cq_host, cqcfg, CQHCI_CFG); 284 285 if (cqhci_readl(cq_host, CQHCI_CTL) & CQHCI_HALT) 286 cqhci_writel(cq_host, 0, CQHCI_CTL); 287 288 mmc->cqe_on = true; 289 290 if (cq_host->ops->enable) 291 cq_host->ops->enable(mmc); 292 293 /* Ensure all writes are done before interrupts are enabled */ 294 wmb(); 295 296 cqhci_set_irqs(cq_host, CQHCI_IS_MASK); 297 298 cq_host->activated = true; 299 } 300 301 static void __cqhci_disable(struct cqhci_host *cq_host) 302 { 303 u32 cqcfg; 304 305 cqcfg = cqhci_readl(cq_host, CQHCI_CFG); 306 cqcfg &= ~CQHCI_ENABLE; 307 cqhci_writel(cq_host, cqcfg, CQHCI_CFG); 308 309 cq_host->mmc->cqe_on = false; 310 311 cq_host->activated = false; 312 } 313 314 int cqhci_deactivate(struct mmc_host *mmc) 315 { 316 struct cqhci_host *cq_host = mmc->cqe_private; 317 318 if (cq_host->enabled && cq_host->activated) 319 __cqhci_disable(cq_host); 320 321 return 0; 322 } 323 EXPORT_SYMBOL(cqhci_deactivate); 324 325 int cqhci_resume(struct mmc_host *mmc) 326 { 327 /* Re-enable is done upon first request */ 328 return 0; 329 } 330 EXPORT_SYMBOL(cqhci_resume); 331 332 static int cqhci_enable(struct mmc_host *mmc, struct mmc_card *card) 333 { 334 struct cqhci_host *cq_host = mmc->cqe_private; 335 int err; 336 337 if (!card->ext_csd.cmdq_en) 338 return -EINVAL; 339 340 if (cq_host->enabled) 341 return 0; 342 343 cq_host->rca = card->rca; 344 345 err = cqhci_host_alloc_tdl(cq_host); 346 if (err) { 347 pr_err("%s: Failed to enable CQE, error %d\n", 348 mmc_hostname(mmc), err); 349 return err; 350 } 351 352 __cqhci_enable(cq_host); 353 354 cq_host->enabled = true; 355 356 #ifdef DEBUG 357 cqhci_dumpregs(cq_host); 358 #endif 359 return 0; 360 } 361 362 /* CQHCI is idle and should halt immediately, so set a small timeout */ 363 #define CQHCI_OFF_TIMEOUT 100 364 365 static u32 cqhci_read_ctl(struct cqhci_host *cq_host) 366 { 367 return cqhci_readl(cq_host, CQHCI_CTL); 368 } 369 370 static void cqhci_off(struct mmc_host *mmc) 371 { 372 struct cqhci_host *cq_host = mmc->cqe_private; 373 u32 reg; 374 int err; 375 376 if (!cq_host->enabled || !mmc->cqe_on || cq_host->recovery_halt) 377 return; 378 379 if (cq_host->ops->disable) 380 cq_host->ops->disable(mmc, false); 381 382 cqhci_writel(cq_host, CQHCI_HALT, CQHCI_CTL); 383 384 err = readx_poll_timeout(cqhci_read_ctl, cq_host, reg, 385 reg & CQHCI_HALT, 0, CQHCI_OFF_TIMEOUT); 386 if (err < 0) 387 pr_err("%s: cqhci: CQE stuck on\n", mmc_hostname(mmc)); 388 else 389 pr_debug("%s: cqhci: CQE off\n", mmc_hostname(mmc)); 390 391 if (cq_host->ops->post_disable) 392 cq_host->ops->post_disable(mmc); 393 394 mmc->cqe_on = false; 395 } 396 397 static void cqhci_disable(struct mmc_host *mmc) 398 { 399 struct cqhci_host *cq_host = mmc->cqe_private; 400 401 if (!cq_host->enabled) 402 return; 403 404 cqhci_off(mmc); 405 406 __cqhci_disable(cq_host); 407 408 dmam_free_coherent(mmc_dev(mmc), cq_host->data_size, 409 cq_host->trans_desc_base, 410 cq_host->trans_desc_dma_base); 411 412 dmam_free_coherent(mmc_dev(mmc), cq_host->desc_size, 413 cq_host->desc_base, 414 cq_host->desc_dma_base); 415 416 cq_host->trans_desc_base = NULL; 417 cq_host->desc_base = NULL; 418 419 cq_host->enabled = false; 420 } 421 422 static void cqhci_prep_task_desc(struct mmc_request *mrq, 423 struct cqhci_host *cq_host, int tag) 424 { 425 __le64 *task_desc = (__le64 __force *)get_desc(cq_host, tag); 426 u32 req_flags = mrq->data->flags; 427 u64 desc0; 428 429 desc0 = CQHCI_VALID(1) | 430 CQHCI_END(1) | 431 CQHCI_INT(1) | 432 CQHCI_ACT(0x5) | 433 CQHCI_FORCED_PROG(!!(req_flags & MMC_DATA_FORCED_PRG)) | 434 CQHCI_DATA_TAG(!!(req_flags & MMC_DATA_DAT_TAG)) | 435 CQHCI_DATA_DIR(!!(req_flags & MMC_DATA_READ)) | 436 CQHCI_PRIORITY(!!(req_flags & MMC_DATA_PRIO)) | 437 CQHCI_QBAR(!!(req_flags & MMC_DATA_QBR)) | 438 CQHCI_REL_WRITE(!!(req_flags & MMC_DATA_REL_WR)) | 439 CQHCI_BLK_COUNT(mrq->data->blocks) | 440 CQHCI_BLK_ADDR((u64)mrq->data->blk_addr); 441 442 task_desc[0] = cpu_to_le64(desc0); 443 444 if (cq_host->caps & CQHCI_TASK_DESC_SZ_128) { 445 u64 desc1 = cqhci_crypto_prep_task_desc(mrq); 446 447 task_desc[1] = cpu_to_le64(desc1); 448 449 pr_debug("%s: cqhci: tag %d task descriptor 0x%016llx%016llx\n", 450 mmc_hostname(mrq->host), mrq->tag, desc1, desc0); 451 } else { 452 pr_debug("%s: cqhci: tag %d task descriptor 0x%016llx\n", 453 mmc_hostname(mrq->host), mrq->tag, desc0); 454 } 455 } 456 457 static int cqhci_dma_map(struct mmc_host *host, struct mmc_request *mrq) 458 { 459 int sg_count; 460 struct mmc_data *data = mrq->data; 461 462 if (!data) 463 return -EINVAL; 464 465 sg_count = dma_map_sg(mmc_dev(host), data->sg, 466 data->sg_len, 467 (data->flags & MMC_DATA_WRITE) ? 468 DMA_TO_DEVICE : DMA_FROM_DEVICE); 469 if (!sg_count) { 470 pr_err("%s: sg-len: %d\n", __func__, data->sg_len); 471 return -ENOMEM; 472 } 473 474 return sg_count; 475 } 476 477 void cqhci_set_tran_desc(u8 *desc, dma_addr_t addr, int len, bool end, 478 bool dma64) 479 { 480 __le32 *attr = (__le32 __force *)desc; 481 482 *attr = (CQHCI_VALID(1) | 483 CQHCI_END(end ? 1 : 0) | 484 CQHCI_INT(0) | 485 CQHCI_ACT(0x4) | 486 CQHCI_DAT_LENGTH(len)); 487 488 if (dma64) { 489 __le64 *dataddr = (__le64 __force *)(desc + 4); 490 491 dataddr[0] = cpu_to_le64(addr); 492 } else { 493 __le32 *dataddr = (__le32 __force *)(desc + 4); 494 495 dataddr[0] = cpu_to_le32(addr); 496 } 497 } 498 EXPORT_SYMBOL(cqhci_set_tran_desc); 499 500 static int cqhci_prep_tran_desc(struct mmc_request *mrq, 501 struct cqhci_host *cq_host, int tag) 502 { 503 struct mmc_data *data = mrq->data; 504 int i, sg_count, len; 505 bool end = false; 506 bool dma64 = cq_host->dma64; 507 dma_addr_t addr; 508 u8 *desc; 509 struct scatterlist *sg; 510 511 sg_count = cqhci_dma_map(mrq->host, mrq); 512 if (sg_count < 0) { 513 pr_err("%s: %s: unable to map sg lists, %d\n", 514 mmc_hostname(mrq->host), __func__, sg_count); 515 return sg_count; 516 } 517 518 desc = get_trans_desc(cq_host, tag); 519 520 for_each_sg(data->sg, sg, sg_count, i) { 521 addr = sg_dma_address(sg); 522 len = sg_dma_len(sg); 523 524 if ((i+1) == sg_count) 525 end = true; 526 if (cq_host->ops->set_tran_desc) 527 cq_host->ops->set_tran_desc(cq_host, &desc, addr, len, end, dma64); 528 else 529 cqhci_set_tran_desc(desc, addr, len, end, dma64); 530 531 desc += cq_host->trans_desc_len; 532 } 533 534 return 0; 535 } 536 537 static void cqhci_prep_dcmd_desc(struct mmc_host *mmc, 538 struct mmc_request *mrq) 539 { 540 u64 *task_desc = NULL; 541 u64 data = 0; 542 u8 resp_type; 543 u8 *desc; 544 __le64 *dataddr; 545 struct cqhci_host *cq_host = mmc->cqe_private; 546 u8 timing; 547 548 if (!(mrq->cmd->flags & MMC_RSP_PRESENT)) { 549 resp_type = 0x0; 550 timing = 0x1; 551 } else { 552 if (mrq->cmd->flags & MMC_RSP_R1B) { 553 resp_type = 0x3; 554 timing = 0x0; 555 } else { 556 resp_type = 0x2; 557 timing = 0x1; 558 } 559 } 560 561 task_desc = (__le64 __force *)get_desc(cq_host, cq_host->dcmd_slot); 562 memset(task_desc, 0, cq_host->task_desc_len); 563 data |= (CQHCI_VALID(1) | 564 CQHCI_END(1) | 565 CQHCI_INT(1) | 566 CQHCI_QBAR(1) | 567 CQHCI_ACT(0x5) | 568 CQHCI_CMD_INDEX(mrq->cmd->opcode) | 569 CQHCI_CMD_TIMING(timing) | CQHCI_RESP_TYPE(resp_type)); 570 if (cq_host->ops->update_dcmd_desc) 571 cq_host->ops->update_dcmd_desc(mmc, mrq, &data); 572 *task_desc |= data; 573 desc = (u8 *)task_desc; 574 pr_debug("%s: cqhci: dcmd: cmd: %d timing: %d resp: %d\n", 575 mmc_hostname(mmc), mrq->cmd->opcode, timing, resp_type); 576 dataddr = (__le64 __force *)(desc + 4); 577 dataddr[0] = cpu_to_le64((u64)mrq->cmd->arg); 578 579 } 580 581 static void cqhci_post_req(struct mmc_host *host, struct mmc_request *mrq) 582 { 583 struct mmc_data *data = mrq->data; 584 585 if (data) { 586 dma_unmap_sg(mmc_dev(host), data->sg, data->sg_len, 587 (data->flags & MMC_DATA_READ) ? 588 DMA_FROM_DEVICE : DMA_TO_DEVICE); 589 } 590 } 591 592 static inline int cqhci_tag(struct mmc_request *mrq) 593 { 594 return mrq->cmd ? DCMD_SLOT : mrq->tag; 595 } 596 597 static int cqhci_request(struct mmc_host *mmc, struct mmc_request *mrq) 598 { 599 int err = 0; 600 int tag = cqhci_tag(mrq); 601 struct cqhci_host *cq_host = mmc->cqe_private; 602 unsigned long flags; 603 604 if (!cq_host->enabled) { 605 pr_err("%s: cqhci: not enabled\n", mmc_hostname(mmc)); 606 return -EINVAL; 607 } 608 609 /* First request after resume has to re-enable */ 610 if (!cq_host->activated) 611 __cqhci_enable(cq_host); 612 613 if (!mmc->cqe_on) { 614 if (cq_host->ops->pre_enable) 615 cq_host->ops->pre_enable(mmc); 616 617 cqhci_writel(cq_host, 0, CQHCI_CTL); 618 mmc->cqe_on = true; 619 pr_debug("%s: cqhci: CQE on\n", mmc_hostname(mmc)); 620 if (cqhci_readl(cq_host, CQHCI_CTL) && CQHCI_HALT) { 621 pr_err("%s: cqhci: CQE failed to exit halt state\n", 622 mmc_hostname(mmc)); 623 } 624 if (cq_host->ops->enable) 625 cq_host->ops->enable(mmc); 626 } 627 628 if (mrq->data) { 629 cqhci_prep_task_desc(mrq, cq_host, tag); 630 631 err = cqhci_prep_tran_desc(mrq, cq_host, tag); 632 if (err) { 633 pr_err("%s: cqhci: failed to setup tx desc: %d\n", 634 mmc_hostname(mmc), err); 635 return err; 636 } 637 } else { 638 cqhci_prep_dcmd_desc(mmc, mrq); 639 } 640 641 spin_lock_irqsave(&cq_host->lock, flags); 642 643 if (cq_host->recovery_halt) { 644 err = -EBUSY; 645 goto out_unlock; 646 } 647 648 cq_host->slot[tag].mrq = mrq; 649 cq_host->slot[tag].flags = 0; 650 651 cq_host->qcnt += 1; 652 /* Make sure descriptors are ready before ringing the doorbell */ 653 wmb(); 654 cqhci_writel(cq_host, 1 << tag, CQHCI_TDBR); 655 if (!(cqhci_readl(cq_host, CQHCI_TDBR) & (1 << tag))) 656 pr_debug("%s: cqhci: doorbell not set for tag %d\n", 657 mmc_hostname(mmc), tag); 658 out_unlock: 659 spin_unlock_irqrestore(&cq_host->lock, flags); 660 661 if (err) 662 cqhci_post_req(mmc, mrq); 663 664 return err; 665 } 666 667 static void cqhci_recovery_needed(struct mmc_host *mmc, struct mmc_request *mrq, 668 bool notify) 669 { 670 struct cqhci_host *cq_host = mmc->cqe_private; 671 672 if (!cq_host->recovery_halt) { 673 cq_host->recovery_halt = true; 674 pr_debug("%s: cqhci: recovery needed\n", mmc_hostname(mmc)); 675 wake_up(&cq_host->wait_queue); 676 if (notify && mrq->recovery_notifier) 677 mrq->recovery_notifier(mrq); 678 } 679 } 680 681 static unsigned int cqhci_error_flags(int error1, int error2) 682 { 683 int error = error1 ? error1 : error2; 684 685 switch (error) { 686 case -EILSEQ: 687 return CQHCI_HOST_CRC; 688 case -ETIMEDOUT: 689 return CQHCI_HOST_TIMEOUT; 690 default: 691 return CQHCI_HOST_OTHER; 692 } 693 } 694 695 static void cqhci_error_irq(struct mmc_host *mmc, u32 status, int cmd_error, 696 int data_error) 697 { 698 struct cqhci_host *cq_host = mmc->cqe_private; 699 struct cqhci_slot *slot; 700 u32 terri; 701 u32 tdpe; 702 int tag; 703 704 spin_lock(&cq_host->lock); 705 706 terri = cqhci_readl(cq_host, CQHCI_TERRI); 707 708 pr_debug("%s: cqhci: error IRQ status: 0x%08x cmd error %d data error %d TERRI: 0x%08x\n", 709 mmc_hostname(mmc), status, cmd_error, data_error, terri); 710 711 /* Forget about errors when recovery has already been triggered */ 712 if (cq_host->recovery_halt) 713 goto out_unlock; 714 715 if (!cq_host->qcnt) { 716 WARN_ONCE(1, "%s: cqhci: error when idle. IRQ status: 0x%08x cmd error %d data error %d TERRI: 0x%08x\n", 717 mmc_hostname(mmc), status, cmd_error, data_error, 718 terri); 719 goto out_unlock; 720 } 721 722 if (CQHCI_TERRI_C_VALID(terri)) { 723 tag = CQHCI_TERRI_C_TASK(terri); 724 slot = &cq_host->slot[tag]; 725 if (slot->mrq) { 726 slot->flags = cqhci_error_flags(cmd_error, data_error); 727 cqhci_recovery_needed(mmc, slot->mrq, true); 728 } 729 } 730 731 if (CQHCI_TERRI_D_VALID(terri)) { 732 tag = CQHCI_TERRI_D_TASK(terri); 733 slot = &cq_host->slot[tag]; 734 if (slot->mrq) { 735 slot->flags = cqhci_error_flags(data_error, cmd_error); 736 cqhci_recovery_needed(mmc, slot->mrq, true); 737 } 738 } 739 740 /* 741 * Handle ICCE ("Invalid Crypto Configuration Error"). This should 742 * never happen, since the block layer ensures that all crypto-enabled 743 * I/O requests have a valid keyslot before they reach the driver. 744 * 745 * Note that GCE ("General Crypto Error") is different; it already got 746 * handled above by checking TERRI. 747 */ 748 if (status & CQHCI_IS_ICCE) { 749 tdpe = cqhci_readl(cq_host, CQHCI_TDPE); 750 WARN_ONCE(1, 751 "%s: cqhci: invalid crypto configuration error. IRQ status: 0x%08x TDPE: 0x%08x\n", 752 mmc_hostname(mmc), status, tdpe); 753 while (tdpe != 0) { 754 tag = __ffs(tdpe); 755 tdpe &= ~(1 << tag); 756 slot = &cq_host->slot[tag]; 757 if (!slot->mrq) 758 continue; 759 slot->flags = cqhci_error_flags(data_error, cmd_error); 760 cqhci_recovery_needed(mmc, slot->mrq, true); 761 } 762 } 763 764 if (!cq_host->recovery_halt) { 765 /* 766 * The only way to guarantee forward progress is to mark at 767 * least one task in error, so if none is indicated, pick one. 768 */ 769 for (tag = 0; tag < NUM_SLOTS; tag++) { 770 slot = &cq_host->slot[tag]; 771 if (!slot->mrq) 772 continue; 773 slot->flags = cqhci_error_flags(data_error, cmd_error); 774 cqhci_recovery_needed(mmc, slot->mrq, true); 775 break; 776 } 777 } 778 779 out_unlock: 780 spin_unlock(&cq_host->lock); 781 } 782 783 static void cqhci_finish_mrq(struct mmc_host *mmc, unsigned int tag) 784 { 785 struct cqhci_host *cq_host = mmc->cqe_private; 786 struct cqhci_slot *slot = &cq_host->slot[tag]; 787 struct mmc_request *mrq = slot->mrq; 788 struct mmc_data *data; 789 790 if (!mrq) { 791 WARN_ONCE(1, "%s: cqhci: spurious TCN for tag %d\n", 792 mmc_hostname(mmc), tag); 793 return; 794 } 795 796 /* No completions allowed during recovery */ 797 if (cq_host->recovery_halt) { 798 slot->flags |= CQHCI_COMPLETED; 799 return; 800 } 801 802 slot->mrq = NULL; 803 804 cq_host->qcnt -= 1; 805 806 data = mrq->data; 807 if (data) { 808 if (data->error) 809 data->bytes_xfered = 0; 810 else 811 data->bytes_xfered = data->blksz * data->blocks; 812 } 813 814 mmc_cqe_request_done(mmc, mrq); 815 } 816 817 irqreturn_t cqhci_irq(struct mmc_host *mmc, u32 intmask, int cmd_error, 818 int data_error) 819 { 820 u32 status; 821 unsigned long tag = 0, comp_status; 822 struct cqhci_host *cq_host = mmc->cqe_private; 823 824 status = cqhci_readl(cq_host, CQHCI_IS); 825 cqhci_writel(cq_host, status, CQHCI_IS); 826 827 pr_debug("%s: cqhci: IRQ status: 0x%08x\n", mmc_hostname(mmc), status); 828 829 if ((status & (CQHCI_IS_RED | CQHCI_IS_GCE | CQHCI_IS_ICCE)) || 830 cmd_error || data_error) { 831 if (status & CQHCI_IS_RED) 832 mmc_debugfs_err_stats_inc(mmc, MMC_ERR_CMDQ_RED); 833 if (status & CQHCI_IS_GCE) 834 mmc_debugfs_err_stats_inc(mmc, MMC_ERR_CMDQ_GCE); 835 if (status & CQHCI_IS_ICCE) 836 mmc_debugfs_err_stats_inc(mmc, MMC_ERR_CMDQ_ICCE); 837 cqhci_error_irq(mmc, status, cmd_error, data_error); 838 } 839 840 if (status & CQHCI_IS_TCC) { 841 /* read TCN and complete the request */ 842 comp_status = cqhci_readl(cq_host, CQHCI_TCN); 843 cqhci_writel(cq_host, comp_status, CQHCI_TCN); 844 pr_debug("%s: cqhci: TCN: 0x%08lx\n", 845 mmc_hostname(mmc), comp_status); 846 847 spin_lock(&cq_host->lock); 848 849 for_each_set_bit(tag, &comp_status, cq_host->num_slots) { 850 /* complete the corresponding mrq */ 851 pr_debug("%s: cqhci: completing tag %lu\n", 852 mmc_hostname(mmc), tag); 853 cqhci_finish_mrq(mmc, tag); 854 } 855 856 if (cq_host->waiting_for_idle && !cq_host->qcnt) { 857 cq_host->waiting_for_idle = false; 858 wake_up(&cq_host->wait_queue); 859 } 860 861 spin_unlock(&cq_host->lock); 862 } 863 864 if (status & CQHCI_IS_TCL) 865 wake_up(&cq_host->wait_queue); 866 867 if (status & CQHCI_IS_HAC) 868 wake_up(&cq_host->wait_queue); 869 870 return IRQ_HANDLED; 871 } 872 EXPORT_SYMBOL(cqhci_irq); 873 874 static bool cqhci_is_idle(struct cqhci_host *cq_host, int *ret) 875 { 876 unsigned long flags; 877 bool is_idle; 878 879 spin_lock_irqsave(&cq_host->lock, flags); 880 is_idle = !cq_host->qcnt || cq_host->recovery_halt; 881 *ret = cq_host->recovery_halt ? -EBUSY : 0; 882 cq_host->waiting_for_idle = !is_idle; 883 spin_unlock_irqrestore(&cq_host->lock, flags); 884 885 return is_idle; 886 } 887 888 static int cqhci_wait_for_idle(struct mmc_host *mmc) 889 { 890 struct cqhci_host *cq_host = mmc->cqe_private; 891 int ret; 892 893 wait_event(cq_host->wait_queue, cqhci_is_idle(cq_host, &ret)); 894 895 return ret; 896 } 897 898 static bool cqhci_timeout(struct mmc_host *mmc, struct mmc_request *mrq, 899 bool *recovery_needed) 900 { 901 struct cqhci_host *cq_host = mmc->cqe_private; 902 int tag = cqhci_tag(mrq); 903 struct cqhci_slot *slot = &cq_host->slot[tag]; 904 unsigned long flags; 905 bool timed_out; 906 907 spin_lock_irqsave(&cq_host->lock, flags); 908 timed_out = slot->mrq == mrq; 909 if (timed_out) { 910 slot->flags |= CQHCI_EXTERNAL_TIMEOUT; 911 cqhci_recovery_needed(mmc, mrq, false); 912 *recovery_needed = cq_host->recovery_halt; 913 } 914 spin_unlock_irqrestore(&cq_host->lock, flags); 915 916 if (timed_out) { 917 pr_err("%s: cqhci: timeout for tag %d, qcnt %d\n", 918 mmc_hostname(mmc), tag, cq_host->qcnt); 919 cqhci_dumpregs(cq_host); 920 } 921 922 return timed_out; 923 } 924 925 static bool cqhci_tasks_cleared(struct cqhci_host *cq_host) 926 { 927 return !(cqhci_readl(cq_host, CQHCI_CTL) & CQHCI_CLEAR_ALL_TASKS); 928 } 929 930 static bool cqhci_clear_all_tasks(struct mmc_host *mmc, unsigned int timeout) 931 { 932 struct cqhci_host *cq_host = mmc->cqe_private; 933 bool ret; 934 u32 ctl; 935 936 cqhci_set_irqs(cq_host, CQHCI_IS_TCL); 937 938 ctl = cqhci_readl(cq_host, CQHCI_CTL); 939 ctl |= CQHCI_CLEAR_ALL_TASKS; 940 cqhci_writel(cq_host, ctl, CQHCI_CTL); 941 942 wait_event_timeout(cq_host->wait_queue, cqhci_tasks_cleared(cq_host), 943 msecs_to_jiffies(timeout) + 1); 944 945 cqhci_set_irqs(cq_host, 0); 946 947 ret = cqhci_tasks_cleared(cq_host); 948 949 if (!ret) 950 pr_warn("%s: cqhci: Failed to clear tasks\n", 951 mmc_hostname(mmc)); 952 953 return ret; 954 } 955 956 static bool cqhci_halted(struct cqhci_host *cq_host) 957 { 958 return cqhci_readl(cq_host, CQHCI_CTL) & CQHCI_HALT; 959 } 960 961 static bool cqhci_halt(struct mmc_host *mmc, unsigned int timeout) 962 { 963 struct cqhci_host *cq_host = mmc->cqe_private; 964 bool ret; 965 u32 ctl; 966 967 if (cqhci_halted(cq_host)) 968 return true; 969 970 cqhci_set_irqs(cq_host, CQHCI_IS_HAC); 971 972 ctl = cqhci_readl(cq_host, CQHCI_CTL); 973 ctl |= CQHCI_HALT; 974 cqhci_writel(cq_host, ctl, CQHCI_CTL); 975 976 wait_event_timeout(cq_host->wait_queue, cqhci_halted(cq_host), 977 msecs_to_jiffies(timeout) + 1); 978 979 cqhci_set_irqs(cq_host, 0); 980 981 ret = cqhci_halted(cq_host); 982 983 if (!ret) 984 pr_warn("%s: cqhci: Failed to halt\n", mmc_hostname(mmc)); 985 986 return ret; 987 } 988 989 /* 990 * After halting we expect to be able to use the command line. We interpret the 991 * failure to halt to mean the data lines might still be in use (and the upper 992 * layers will need to send a STOP command), however failing to halt complicates 993 * the recovery, so set a timeout that would reasonably allow I/O to complete. 994 */ 995 #define CQHCI_START_HALT_TIMEOUT 500 996 997 static void cqhci_recovery_start(struct mmc_host *mmc) 998 { 999 struct cqhci_host *cq_host = mmc->cqe_private; 1000 1001 pr_debug("%s: cqhci: %s\n", mmc_hostname(mmc), __func__); 1002 1003 WARN_ON(!cq_host->recovery_halt); 1004 1005 cqhci_halt(mmc, CQHCI_START_HALT_TIMEOUT); 1006 1007 if (cq_host->ops->disable) 1008 cq_host->ops->disable(mmc, true); 1009 1010 mmc->cqe_on = false; 1011 } 1012 1013 static int cqhci_error_from_flags(unsigned int flags) 1014 { 1015 if (!flags) 1016 return 0; 1017 1018 /* CRC errors might indicate re-tuning so prefer to report that */ 1019 if (flags & CQHCI_HOST_CRC) 1020 return -EILSEQ; 1021 1022 if (flags & (CQHCI_EXTERNAL_TIMEOUT | CQHCI_HOST_TIMEOUT)) 1023 return -ETIMEDOUT; 1024 1025 return -EIO; 1026 } 1027 1028 static void cqhci_recover_mrq(struct cqhci_host *cq_host, unsigned int tag) 1029 { 1030 struct cqhci_slot *slot = &cq_host->slot[tag]; 1031 struct mmc_request *mrq = slot->mrq; 1032 struct mmc_data *data; 1033 1034 if (!mrq) 1035 return; 1036 1037 slot->mrq = NULL; 1038 1039 cq_host->qcnt -= 1; 1040 1041 data = mrq->data; 1042 if (data) { 1043 data->bytes_xfered = 0; 1044 data->error = cqhci_error_from_flags(slot->flags); 1045 } else { 1046 mrq->cmd->error = cqhci_error_from_flags(slot->flags); 1047 } 1048 1049 mmc_cqe_request_done(cq_host->mmc, mrq); 1050 } 1051 1052 static void cqhci_recover_mrqs(struct cqhci_host *cq_host) 1053 { 1054 int i; 1055 1056 for (i = 0; i < cq_host->num_slots; i++) 1057 cqhci_recover_mrq(cq_host, i); 1058 } 1059 1060 /* 1061 * By now the command and data lines should be unused so there is no reason for 1062 * CQHCI to take a long time to halt, but if it doesn't halt there could be 1063 * problems clearing tasks, so be generous. 1064 */ 1065 #define CQHCI_FINISH_HALT_TIMEOUT 20 1066 1067 /* CQHCI could be expected to clear it's internal state pretty quickly */ 1068 #define CQHCI_CLEAR_TIMEOUT 20 1069 1070 static void cqhci_recovery_finish(struct mmc_host *mmc) 1071 { 1072 struct cqhci_host *cq_host = mmc->cqe_private; 1073 unsigned long flags; 1074 u32 cqcfg; 1075 bool ok; 1076 1077 pr_debug("%s: cqhci: %s\n", mmc_hostname(mmc), __func__); 1078 1079 WARN_ON(!cq_host->recovery_halt); 1080 1081 ok = cqhci_halt(mmc, CQHCI_FINISH_HALT_TIMEOUT); 1082 1083 /* 1084 * The specification contradicts itself, by saying that tasks cannot be 1085 * cleared if CQHCI does not halt, but if CQHCI does not halt, it should 1086 * be disabled/re-enabled, but not to disable before clearing tasks. 1087 * Have a go anyway. 1088 */ 1089 if (!cqhci_clear_all_tasks(mmc, CQHCI_CLEAR_TIMEOUT)) 1090 ok = false; 1091 1092 /* Disable to make sure tasks really are cleared */ 1093 cqcfg = cqhci_readl(cq_host, CQHCI_CFG); 1094 cqcfg &= ~CQHCI_ENABLE; 1095 cqhci_writel(cq_host, cqcfg, CQHCI_CFG); 1096 1097 cqcfg = cqhci_readl(cq_host, CQHCI_CFG); 1098 cqcfg |= CQHCI_ENABLE; 1099 cqhci_writel(cq_host, cqcfg, CQHCI_CFG); 1100 1101 cqhci_halt(mmc, CQHCI_FINISH_HALT_TIMEOUT); 1102 1103 if (!ok) 1104 cqhci_clear_all_tasks(mmc, CQHCI_CLEAR_TIMEOUT); 1105 1106 cqhci_recover_mrqs(cq_host); 1107 1108 WARN_ON(cq_host->qcnt); 1109 1110 spin_lock_irqsave(&cq_host->lock, flags); 1111 cq_host->qcnt = 0; 1112 cq_host->recovery_halt = false; 1113 mmc->cqe_on = false; 1114 spin_unlock_irqrestore(&cq_host->lock, flags); 1115 1116 /* Ensure all writes are done before interrupts are re-enabled */ 1117 wmb(); 1118 1119 cqhci_writel(cq_host, CQHCI_IS_HAC | CQHCI_IS_TCL, CQHCI_IS); 1120 1121 cqhci_set_irqs(cq_host, CQHCI_IS_MASK); 1122 1123 pr_debug("%s: cqhci: recovery done\n", mmc_hostname(mmc)); 1124 } 1125 1126 static const struct mmc_cqe_ops cqhci_cqe_ops = { 1127 .cqe_enable = cqhci_enable, 1128 .cqe_disable = cqhci_disable, 1129 .cqe_request = cqhci_request, 1130 .cqe_post_req = cqhci_post_req, 1131 .cqe_off = cqhci_off, 1132 .cqe_wait_for_idle = cqhci_wait_for_idle, 1133 .cqe_timeout = cqhci_timeout, 1134 .cqe_recovery_start = cqhci_recovery_start, 1135 .cqe_recovery_finish = cqhci_recovery_finish, 1136 }; 1137 1138 struct cqhci_host *cqhci_pltfm_init(struct platform_device *pdev) 1139 { 1140 struct cqhci_host *cq_host; 1141 struct resource *cqhci_memres = NULL; 1142 1143 /* check and setup CMDQ interface */ 1144 cqhci_memres = platform_get_resource_byname(pdev, IORESOURCE_MEM, 1145 "cqhci"); 1146 if (!cqhci_memres) { 1147 dev_dbg(&pdev->dev, "CMDQ not supported\n"); 1148 return ERR_PTR(-EINVAL); 1149 } 1150 1151 cq_host = devm_kzalloc(&pdev->dev, sizeof(*cq_host), GFP_KERNEL); 1152 if (!cq_host) 1153 return ERR_PTR(-ENOMEM); 1154 cq_host->mmio = devm_ioremap(&pdev->dev, 1155 cqhci_memres->start, 1156 resource_size(cqhci_memres)); 1157 if (!cq_host->mmio) { 1158 dev_err(&pdev->dev, "failed to remap cqhci regs\n"); 1159 return ERR_PTR(-EBUSY); 1160 } 1161 dev_dbg(&pdev->dev, "CMDQ ioremap: done\n"); 1162 1163 return cq_host; 1164 } 1165 EXPORT_SYMBOL(cqhci_pltfm_init); 1166 1167 static unsigned int cqhci_ver_major(struct cqhci_host *cq_host) 1168 { 1169 return CQHCI_VER_MAJOR(cqhci_readl(cq_host, CQHCI_VER)); 1170 } 1171 1172 static unsigned int cqhci_ver_minor(struct cqhci_host *cq_host) 1173 { 1174 u32 ver = cqhci_readl(cq_host, CQHCI_VER); 1175 1176 return CQHCI_VER_MINOR1(ver) * 10 + CQHCI_VER_MINOR2(ver); 1177 } 1178 1179 int cqhci_init(struct cqhci_host *cq_host, struct mmc_host *mmc, 1180 bool dma64) 1181 { 1182 int err; 1183 1184 cq_host->dma64 = dma64; 1185 cq_host->mmc = mmc; 1186 cq_host->mmc->cqe_private = cq_host; 1187 1188 cq_host->num_slots = NUM_SLOTS; 1189 cq_host->dcmd_slot = DCMD_SLOT; 1190 1191 mmc->cqe_ops = &cqhci_cqe_ops; 1192 1193 mmc->cqe_qdepth = NUM_SLOTS; 1194 if (mmc->caps2 & MMC_CAP2_CQE_DCMD) 1195 mmc->cqe_qdepth -= 1; 1196 1197 cq_host->slot = devm_kcalloc(mmc_dev(mmc), cq_host->num_slots, 1198 sizeof(*cq_host->slot), GFP_KERNEL); 1199 if (!cq_host->slot) { 1200 err = -ENOMEM; 1201 goto out_err; 1202 } 1203 1204 err = cqhci_crypto_init(cq_host); 1205 if (err) { 1206 pr_err("%s: CQHCI crypto initialization failed\n", 1207 mmc_hostname(mmc)); 1208 goto out_err; 1209 } 1210 1211 spin_lock_init(&cq_host->lock); 1212 1213 init_completion(&cq_host->halt_comp); 1214 init_waitqueue_head(&cq_host->wait_queue); 1215 1216 pr_info("%s: CQHCI version %u.%02u\n", 1217 mmc_hostname(mmc), cqhci_ver_major(cq_host), 1218 cqhci_ver_minor(cq_host)); 1219 1220 return 0; 1221 1222 out_err: 1223 pr_err("%s: CQHCI version %u.%02u failed to initialize, error %d\n", 1224 mmc_hostname(mmc), cqhci_ver_major(cq_host), 1225 cqhci_ver_minor(cq_host), err); 1226 return err; 1227 } 1228 EXPORT_SYMBOL(cqhci_init); 1229 1230 MODULE_AUTHOR("Venkat Gopalakrishnan <venkatg@codeaurora.org>"); 1231 MODULE_DESCRIPTION("Command Queue Host Controller Interface driver"); 1232 MODULE_LICENSE("GPL v2"); 1233