1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Synopsys DesignWare Multimedia Card Interface driver 4 * (Based on NXP driver for lpc 31xx) 5 * 6 * Copyright (C) 2009 NXP Semiconductors 7 * Copyright (C) 2009, 2010 Imagination Technologies Ltd. 8 */ 9 10 #include <linux/bitops.h> 11 #include <linux/clk.h> 12 #include <linux/debugfs.h> 13 #include <linux/delay.h> 14 #include <linux/device.h> 15 #include <linux/dma-mapping.h> 16 #include <linux/err.h> 17 #include <linux/interrupt.h> 18 #include <linux/iopoll.h> 19 #include <linux/irq.h> 20 #include <linux/ktime.h> 21 #include <linux/mmc/card.h> 22 #include <linux/mmc/host.h> 23 #include <linux/mmc/mmc.h> 24 #include <linux/mmc/sd.h> 25 #include <linux/mmc/sdio.h> 26 #include <linux/mmc/slot-gpio.h> 27 #include <linux/module.h> 28 #include <linux/of.h> 29 #include <linux/platform_device.h> 30 #include <linux/pm_runtime.h> 31 #include <linux/regulator/consumer.h> 32 33 #include "dw_mmc.h" 34 35 /* Common flag combinations */ 36 #define DW_MCI_DATA_ERROR_FLAGS (SDMMC_INT_DRTO | SDMMC_INT_DCRC | \ 37 SDMMC_INT_HTO | SDMMC_INT_SBE | \ 38 SDMMC_INT_EBE | SDMMC_INT_HLE) 39 #define DW_MCI_CMD_ERROR_FLAGS (SDMMC_INT_RTO | SDMMC_INT_RCRC | \ 40 SDMMC_INT_RESP_ERR | SDMMC_INT_HLE) 41 #define DW_MCI_ERROR_FLAGS (DW_MCI_DATA_ERROR_FLAGS | \ 42 DW_MCI_CMD_ERROR_FLAGS) 43 44 #define DW_MCI_FREQ_MAX 200000000 /* unit: HZ */ 45 #define DW_MCI_FREQ_MIN 100000 /* unit: HZ */ 46 47 #define IDMAC_INT_CLR (SDMMC_IDMAC_INT_AI | SDMMC_IDMAC_INT_NI | \ 48 SDMMC_IDMAC_INT_CES | SDMMC_IDMAC_INT_DU | \ 49 SDMMC_IDMAC_INT_FBE | SDMMC_IDMAC_INT_RI | \ 50 SDMMC_IDMAC_INT_TI) 51 52 #define DESC_RING_BUF_SZ PAGE_SIZE 53 54 struct idmac_desc_64addr { 55 u32 des0; /* Control Descriptor */ 56 #define IDMAC_OWN_CLR64(x) \ 57 !((x) & cpu_to_le32(IDMAC_DES0_OWN)) 58 59 u32 des1; /* Reserved */ 60 61 u32 des2; /*Buffer sizes */ 62 #define IDMAC_64ADDR_SET_BUFFER1_SIZE(d, s) \ 63 ((d)->des2 = ((d)->des2 & cpu_to_le32(0x03ffe000)) | \ 64 ((cpu_to_le32(s)) & cpu_to_le32(0x1fff))) 65 66 u32 des3; /* Reserved */ 67 68 u32 des4; /* Lower 32-bits of Buffer Address Pointer 1*/ 69 u32 des5; /* Upper 32-bits of Buffer Address Pointer 1*/ 70 71 u32 des6; /* Lower 32-bits of Next Descriptor Address */ 72 u32 des7; /* Upper 32-bits of Next Descriptor Address */ 73 }; 74 75 struct idmac_desc { 76 __le32 des0; /* Control Descriptor */ 77 #define IDMAC_DES0_DIC BIT(1) 78 #define IDMAC_DES0_LD BIT(2) 79 #define IDMAC_DES0_FD BIT(3) 80 #define IDMAC_DES0_CH BIT(4) 81 #define IDMAC_DES0_ER BIT(5) 82 #define IDMAC_DES0_CES BIT(30) 83 #define IDMAC_DES0_OWN BIT(31) 84 85 __le32 des1; /* Buffer sizes */ 86 #define IDMAC_SET_BUFFER1_SIZE(d, s) \ 87 ((d)->des1 = ((d)->des1 & cpu_to_le32(0x03ffe000)) | (cpu_to_le32((s) & 0x1fff))) 88 89 __le32 des2; /* buffer 1 physical address */ 90 91 __le32 des3; /* buffer 2 physical address */ 92 }; 93 94 /* Each descriptor can transfer up to 4KB of data in chained mode */ 95 #define DW_MCI_DESC_DATA_LENGTH 0x1000 96 97 #if defined(CONFIG_DEBUG_FS) 98 static int dw_mci_req_show(struct seq_file *s, void *v) 99 { 100 struct dw_mci *host = s->private; 101 struct mmc_request *mrq; 102 struct mmc_command *cmd; 103 struct mmc_command *stop; 104 struct mmc_data *data; 105 106 /* Make sure we get a consistent snapshot */ 107 spin_lock_bh(&host->lock); 108 mrq = host->mrq; 109 110 if (mrq) { 111 cmd = mrq->cmd; 112 data = mrq->data; 113 stop = mrq->stop; 114 115 if (cmd) 116 seq_printf(s, 117 "CMD%u(0x%x) flg %x rsp %x %x %x %x err %d\n", 118 cmd->opcode, cmd->arg, cmd->flags, 119 cmd->resp[0], cmd->resp[1], cmd->resp[2], 120 cmd->resp[2], cmd->error); 121 if (data) 122 seq_printf(s, "DATA %u / %u * %u flg %x err %d\n", 123 data->bytes_xfered, data->blocks, 124 data->blksz, data->flags, data->error); 125 if (stop) 126 seq_printf(s, 127 "CMD%u(0x%x) flg %x rsp %x %x %x %x err %d\n", 128 stop->opcode, stop->arg, stop->flags, 129 stop->resp[0], stop->resp[1], stop->resp[2], 130 stop->resp[2], stop->error); 131 } 132 133 spin_unlock_bh(&host->lock); 134 135 return 0; 136 } 137 DEFINE_SHOW_ATTRIBUTE(dw_mci_req); 138 139 static int dw_mci_regs_show(struct seq_file *s, void *v) 140 { 141 struct dw_mci *host = s->private; 142 143 pm_runtime_get_sync(host->dev); 144 145 seq_printf(s, "STATUS:\t0x%08x\n", mci_readl(host, STATUS)); 146 seq_printf(s, "RINTSTS:\t0x%08x\n", mci_readl(host, RINTSTS)); 147 seq_printf(s, "CMD:\t0x%08x\n", mci_readl(host, CMD)); 148 seq_printf(s, "CTRL:\t0x%08x\n", mci_readl(host, CTRL)); 149 seq_printf(s, "INTMASK:\t0x%08x\n", mci_readl(host, INTMASK)); 150 seq_printf(s, "CLKENA:\t0x%08x\n", mci_readl(host, CLKENA)); 151 152 pm_runtime_put_autosuspend(host->dev); 153 154 return 0; 155 } 156 DEFINE_SHOW_ATTRIBUTE(dw_mci_regs); 157 158 static void dw_mci_init_debugfs(struct dw_mci *host) 159 { 160 struct mmc_host *mmc = host->mmc; 161 struct dentry *root; 162 163 root = mmc->debugfs_root; 164 if (!root) 165 return; 166 167 debugfs_create_file("regs", 0400, root, host, &dw_mci_regs_fops); 168 debugfs_create_file("req", 0400, root, host, &dw_mci_req_fops); 169 debugfs_create_u32("state", 0400, root, &host->state); 170 debugfs_create_xul("pending_events", 0400, root, 171 &host->pending_events); 172 debugfs_create_xul("completed_events", 0400, root, 173 &host->completed_events); 174 #ifdef CONFIG_FAULT_INJECTION 175 fault_create_debugfs_attr("fail_data_crc", root, &host->fail_data_crc); 176 #endif 177 } 178 #endif /* defined(CONFIG_DEBUG_FS) */ 179 180 static bool dw_mci_ctrl_reset(struct dw_mci *host, u32 reset) 181 { 182 u32 ctrl; 183 184 ctrl = mci_readl(host, CTRL); 185 ctrl |= reset; 186 mci_writel(host, CTRL, ctrl); 187 188 /* wait till resets clear */ 189 if (readl_poll_timeout_atomic(host->regs + SDMMC_CTRL, ctrl, 190 !(ctrl & reset), 191 1, 500 * USEC_PER_MSEC)) { 192 dev_err(host->dev, 193 "Timeout resetting block (ctrl reset %#x)\n", 194 ctrl & reset); 195 return false; 196 } 197 198 return true; 199 } 200 201 static void dw_mci_wait_while_busy(struct dw_mci *host, u32 cmd_flags) 202 { 203 u32 status; 204 205 /* 206 * Databook says that before issuing a new data transfer command 207 * we need to check to see if the card is busy. Data transfer commands 208 * all have SDMMC_CMD_PRV_DAT_WAIT set, so we'll key off that. 209 * 210 * ...also allow sending for SDMMC_CMD_VOLT_SWITCH where busy is 211 * expected. 212 */ 213 if ((cmd_flags & SDMMC_CMD_PRV_DAT_WAIT) && 214 !(cmd_flags & SDMMC_CMD_VOLT_SWITCH)) { 215 if (readl_poll_timeout_atomic(host->regs + SDMMC_STATUS, 216 status, 217 !(status & SDMMC_STATUS_BUSY), 218 10, 500 * USEC_PER_MSEC)) 219 dev_err(host->dev, "Busy; trying anyway\n"); 220 } 221 } 222 223 static void mci_send_cmd(struct dw_mci *host, u32 cmd, u32 arg) 224 { 225 unsigned int cmd_status = 0; 226 227 mci_writel(host, CMDARG, arg); 228 wmb(); /* drain writebuffer */ 229 dw_mci_wait_while_busy(host, cmd); 230 mci_writel(host, CMD, SDMMC_CMD_START | cmd); 231 232 if (readl_poll_timeout_atomic(host->regs + SDMMC_CMD, cmd_status, 233 !(cmd_status & SDMMC_CMD_START), 234 1, 500 * USEC_PER_MSEC)) 235 dev_err(&host->mmc->class_dev, 236 "Timeout sending command (cmd %#x arg %#x status %#x)\n", 237 cmd, arg, cmd_status); 238 } 239 240 static u32 dw_mci_prepare_command(struct mmc_host *mmc, struct mmc_command *cmd) 241 { 242 struct dw_mci *host = mmc_priv(mmc); 243 u32 cmdr; 244 245 cmd->error = -EINPROGRESS; 246 cmdr = cmd->opcode; 247 248 if (cmd->opcode == MMC_STOP_TRANSMISSION || 249 cmd->opcode == MMC_GO_IDLE_STATE || 250 cmd->opcode == MMC_GO_INACTIVE_STATE || 251 (cmd->opcode == SD_IO_RW_DIRECT && 252 ((cmd->arg >> 9) & 0x1FFFF) == SDIO_CCCR_ABORT)) 253 cmdr |= SDMMC_CMD_STOP; 254 else if (cmd->opcode != MMC_SEND_STATUS && cmd->data) 255 cmdr |= SDMMC_CMD_PRV_DAT_WAIT; 256 257 if (cmd->opcode == SD_SWITCH_VOLTAGE) { 258 u32 clk_en_a; 259 260 /* Special bit makes CMD11 not die */ 261 cmdr |= SDMMC_CMD_VOLT_SWITCH; 262 263 /* Change state to continue to handle CMD11 weirdness */ 264 WARN_ON(host->state != STATE_SENDING_CMD); 265 host->state = STATE_SENDING_CMD11; 266 267 /* 268 * We need to disable low power mode (automatic clock stop) 269 * while doing voltage switch so we don't confuse the card, 270 * since stopping the clock is a specific part of the UHS 271 * voltage change dance. 272 * 273 * Note that low power mode (SDMMC_CLKEN_LOW_PWR) will be 274 * unconditionally turned back on in dw_mci_setup_bus() if it's 275 * ever called with a non-zero clock. That shouldn't happen 276 * until the voltage change is all done. 277 */ 278 clk_en_a = mci_readl(host, CLKENA); 279 clk_en_a &= ~SDMMC_CLKEN_LOW_PWR; 280 mci_writel(host, CLKENA, clk_en_a); 281 mci_send_cmd(host, SDMMC_CMD_UPD_CLK | 282 SDMMC_CMD_PRV_DAT_WAIT, 0); 283 } 284 285 if (cmd->flags & MMC_RSP_PRESENT) { 286 /* We expect a response, so set this bit */ 287 cmdr |= SDMMC_CMD_RESP_EXP; 288 if (cmd->flags & MMC_RSP_136) 289 cmdr |= SDMMC_CMD_RESP_LONG; 290 } 291 292 if (cmd->flags & MMC_RSP_CRC) 293 cmdr |= SDMMC_CMD_RESP_CRC; 294 295 if (cmd->data) { 296 cmdr |= SDMMC_CMD_DAT_EXP; 297 if (cmd->data->flags & MMC_DATA_WRITE) 298 cmdr |= SDMMC_CMD_DAT_WR; 299 } 300 301 if (!test_bit(DW_MMC_CARD_NO_USE_HOLD, &host->flags)) 302 cmdr |= SDMMC_CMD_USE_HOLD_REG; 303 304 return cmdr; 305 } 306 307 static u32 dw_mci_prep_stop_abort(struct dw_mci *host, struct mmc_command *cmd) 308 { 309 struct mmc_command *stop; 310 u32 cmdr; 311 312 if (!cmd->data) 313 return 0; 314 315 stop = &host->stop_abort; 316 cmdr = cmd->opcode; 317 memset(stop, 0, sizeof(struct mmc_command)); 318 319 if (cmdr == MMC_READ_SINGLE_BLOCK || 320 cmdr == MMC_READ_MULTIPLE_BLOCK || 321 cmdr == MMC_WRITE_BLOCK || 322 cmdr == MMC_WRITE_MULTIPLE_BLOCK || 323 mmc_op_tuning(cmdr) || 324 cmdr == MMC_GEN_CMD) { 325 stop->opcode = MMC_STOP_TRANSMISSION; 326 stop->arg = 0; 327 stop->flags = MMC_RSP_R1B | MMC_CMD_AC; 328 } else if (cmdr == SD_IO_RW_EXTENDED) { 329 stop->opcode = SD_IO_RW_DIRECT; 330 stop->arg |= (1 << 31) | (0 << 28) | (SDIO_CCCR_ABORT << 9) | 331 ((cmd->arg >> 28) & 0x7); 332 stop->flags = MMC_RSP_SPI_R5 | MMC_RSP_R5 | MMC_CMD_AC; 333 } else { 334 return 0; 335 } 336 337 cmdr = stop->opcode | SDMMC_CMD_STOP | 338 SDMMC_CMD_RESP_CRC | SDMMC_CMD_RESP_EXP; 339 340 if (!test_bit(DW_MMC_CARD_NO_USE_HOLD, &host->flags)) 341 cmdr |= SDMMC_CMD_USE_HOLD_REG; 342 343 return cmdr; 344 } 345 346 static inline void dw_mci_set_cto(struct dw_mci *host) 347 { 348 unsigned int cto_clks; 349 unsigned int cto_div; 350 unsigned int cto_ms; 351 unsigned long irqflags; 352 353 cto_clks = mci_readl(host, TMOUT) & 0xff; 354 cto_div = (mci_readl(host, CLKDIV) & 0xff) * 2; 355 if (cto_div == 0) 356 cto_div = 1; 357 358 cto_ms = DIV_ROUND_UP_ULL((u64)MSEC_PER_SEC * cto_clks * cto_div, 359 host->bus_hz); 360 361 /* add a bit spare time */ 362 cto_ms += 10; 363 364 /* 365 * The durations we're working with are fairly short so we have to be 366 * extra careful about synchronization here. Specifically in hardware a 367 * command timeout is _at most_ 5.1 ms, so that means we expect an 368 * interrupt (either command done or timeout) to come rather quickly 369 * after the mci_writel. ...but just in case we have a long interrupt 370 * latency let's add a bit of paranoia. 371 * 372 * In general we'll assume that at least an interrupt will be asserted 373 * in hardware by the time the cto_timer runs. ...and if it hasn't 374 * been asserted in hardware by that time then we'll assume it'll never 375 * come. 376 */ 377 spin_lock_irqsave(&host->irq_lock, irqflags); 378 if (!test_bit(EVENT_CMD_COMPLETE, &host->pending_events)) 379 mod_timer(&host->cto_timer, 380 jiffies + msecs_to_jiffies(cto_ms) + 1); 381 spin_unlock_irqrestore(&host->irq_lock, irqflags); 382 } 383 384 static void dw_mci_start_command(struct dw_mci *host, 385 struct mmc_command *cmd, u32 cmd_flags) 386 { 387 host->cmd = cmd; 388 dev_vdbg(host->dev, 389 "start command: ARGR=0x%08x CMDR=0x%08x\n", 390 cmd->arg, cmd_flags); 391 392 mci_writel(host, CMDARG, cmd->arg); 393 wmb(); /* drain writebuffer */ 394 dw_mci_wait_while_busy(host, cmd_flags); 395 396 mci_writel(host, CMD, cmd_flags | SDMMC_CMD_START); 397 398 /* response expected command only */ 399 if (cmd_flags & SDMMC_CMD_RESP_EXP) 400 dw_mci_set_cto(host); 401 } 402 403 static inline void send_stop_abort(struct dw_mci *host, struct mmc_data *data) 404 { 405 struct mmc_command *stop = &host->stop_abort; 406 407 dw_mci_start_command(host, stop, host->stop_cmdr); 408 } 409 410 /* DMA interface functions */ 411 static void dw_mci_stop_dma(struct dw_mci *host) 412 { 413 if (host->using_dma) { 414 host->dma_ops->stop(host); 415 host->dma_ops->cleanup(host); 416 } 417 418 /* Data transfer was stopped by the interrupt handler */ 419 set_bit(EVENT_XFER_COMPLETE, &host->pending_events); 420 } 421 422 static void dw_mci_dma_cleanup(struct dw_mci *host) 423 { 424 struct mmc_data *data = host->data; 425 426 if (data && data->host_cookie == COOKIE_MAPPED) { 427 dma_unmap_sg(host->dev, 428 data->sg, 429 data->sg_len, 430 mmc_get_dma_dir(data)); 431 data->host_cookie = COOKIE_UNMAPPED; 432 } 433 } 434 435 static void dw_mci_idmac_reset(struct dw_mci *host) 436 { 437 u32 bmod = mci_readl(host, BMOD); 438 /* Software reset of DMA */ 439 bmod |= SDMMC_IDMAC_SWRESET; 440 mci_writel(host, BMOD, bmod); 441 } 442 443 static void dw_mci_idmac_stop_dma(struct dw_mci *host) 444 { 445 u32 temp; 446 447 /* Disable and reset the IDMAC interface */ 448 temp = mci_readl(host, CTRL); 449 temp &= ~SDMMC_CTRL_USE_IDMAC; 450 temp |= SDMMC_CTRL_DMA_RESET; 451 mci_writel(host, CTRL, temp); 452 453 /* Stop the IDMAC running */ 454 temp = mci_readl(host, BMOD); 455 temp &= ~(SDMMC_IDMAC_ENABLE | SDMMC_IDMAC_FB); 456 temp |= SDMMC_IDMAC_SWRESET; 457 mci_writel(host, BMOD, temp); 458 } 459 460 static void dw_mci_dmac_complete_dma(void *arg) 461 { 462 struct dw_mci *host = arg; 463 struct mmc_data *data = host->data; 464 465 dev_vdbg(host->dev, "DMA complete\n"); 466 467 if ((host->use_dma == TRANS_MODE_EDMAC) && 468 data && (data->flags & MMC_DATA_READ)) 469 /* Invalidate cache after read */ 470 dma_sync_sg_for_cpu(mmc_dev(host->mmc), 471 data->sg, 472 data->sg_len, 473 DMA_FROM_DEVICE); 474 475 host->dma_ops->cleanup(host); 476 477 /* 478 * If the card was removed, data will be NULL. No point in trying to 479 * send the stop command or waiting for NBUSY in this case. 480 */ 481 if (data) { 482 set_bit(EVENT_XFER_COMPLETE, &host->pending_events); 483 queue_work(system_bh_wq, &host->bh_work); 484 } 485 } 486 487 static int dw_mci_idmac_init(struct dw_mci *host) 488 { 489 int i; 490 491 if (host->dma_64bit_address == 1) { 492 struct idmac_desc_64addr *p; 493 494 host->desc_num = 495 DESC_RING_BUF_SZ / sizeof(struct idmac_desc_64addr); 496 497 /* Forward link the descriptor list */ 498 for (i = 0, p = host->sg_cpu; i < host->desc_num - 1; 499 i++, p++) { 500 p->des6 = (host->sg_dma + 501 (sizeof(struct idmac_desc_64addr) * 502 (i + 1))) & 0xffffffff; 503 504 p->des7 = (u64)(host->sg_dma + 505 (sizeof(struct idmac_desc_64addr) * 506 (i + 1))) >> 32; 507 /* Initialize reserved and buffer size fields to "0" */ 508 p->des0 = 0; 509 p->des1 = 0; 510 p->des2 = 0; 511 p->des3 = 0; 512 } 513 514 /* Set the last descriptor as the end-of-ring descriptor */ 515 p->des6 = host->sg_dma & 0xffffffff; 516 p->des7 = (u64)host->sg_dma >> 32; 517 p->des0 = IDMAC_DES0_ER; 518 519 } else { 520 struct idmac_desc *p; 521 522 host->desc_num = 523 DESC_RING_BUF_SZ / sizeof(struct idmac_desc); 524 525 /* Forward link the descriptor list */ 526 for (i = 0, p = host->sg_cpu; 527 i < host->desc_num - 1; 528 i++, p++) { 529 p->des3 = cpu_to_le32(host->sg_dma + 530 (sizeof(struct idmac_desc) * (i + 1))); 531 p->des0 = 0; 532 p->des1 = 0; 533 } 534 535 /* Set the last descriptor as the end-of-ring descriptor */ 536 p->des3 = cpu_to_le32(host->sg_dma); 537 p->des0 = cpu_to_le32(IDMAC_DES0_ER); 538 } 539 540 dw_mci_idmac_reset(host); 541 542 if (host->dma_64bit_address == 1) { 543 /* Mask out interrupts - get Tx & Rx complete only */ 544 mci_writel(host, IDSTS64, IDMAC_INT_CLR); 545 mci_writel(host, IDINTEN64, SDMMC_IDMAC_INT_NI | 546 SDMMC_IDMAC_INT_RI | SDMMC_IDMAC_INT_TI); 547 548 /* Set the descriptor base address */ 549 mci_writel(host, DBADDRL, host->sg_dma & 0xffffffff); 550 mci_writel(host, DBADDRU, (u64)host->sg_dma >> 32); 551 552 } else { 553 /* Mask out interrupts - get Tx & Rx complete only */ 554 mci_writel(host, IDSTS, IDMAC_INT_CLR); 555 mci_writel(host, IDINTEN, SDMMC_IDMAC_INT_NI | 556 SDMMC_IDMAC_INT_RI | SDMMC_IDMAC_INT_TI); 557 558 /* Set the descriptor base address */ 559 mci_writel(host, DBADDR, host->sg_dma); 560 } 561 562 return 0; 563 } 564 565 static inline int dw_mci_prepare_desc(struct dw_mci *host, struct mmc_data *data, 566 unsigned int sg_len, bool is_64bit) 567 { 568 unsigned int desc_len; 569 struct idmac_desc *desc_first, *desc_last, *desc; 570 struct idmac_desc_64addr *desc64_first, *desc64_last, *desc64; 571 u32 val, des0; 572 int i, err; 573 574 if (is_64bit) 575 desc64_first = desc64_last = desc64 = host->sg_cpu; 576 else 577 desc_first = desc_last = desc = host->sg_cpu; 578 579 for (i = 0; i < sg_len; i++) { 580 unsigned int length = sg_dma_len(&data->sg[i]); 581 582 u64 mem_addr = sg_dma_address(&data->sg[i]); 583 584 while (length > 0) { 585 desc_len = (length <= DW_MCI_DESC_DATA_LENGTH) ? 586 length : DW_MCI_DESC_DATA_LENGTH; 587 588 length -= desc_len; 589 590 /* 591 * Wait for the former clear OWN bit operation 592 * of IDMAC to make sure that this descriptor 593 * isn't still owned by IDMAC as IDMAC's write 594 * ops and CPU's read ops are asynchronous. 595 */ 596 if (is_64bit) 597 err = readl_poll_timeout_atomic(&desc64->des0, val, 598 IDMAC_OWN_CLR64(val), 10, 100 * USEC_PER_MSEC); 599 else 600 err = readl_poll_timeout_atomic(&desc->des0, val, 601 IDMAC_OWN_CLR64(val), 10, 100 * USEC_PER_MSEC); 602 if (err) 603 goto err_own_bit; 604 605 des0 = IDMAC_DES0_OWN | IDMAC_DES0_DIC | IDMAC_DES0_CH; 606 if (is_64bit) 607 desc64->des0 = des0; 608 else 609 desc->des0 = cpu_to_le32(des0); 610 611 /* 612 * 1. Set OWN bit and disable interrupts for this descriptor 613 * 2. Set Buffer length 614 * Set physical address to DMA to/from 615 */ 616 if (is_64bit) { 617 desc64->des0 = IDMAC_DES0_OWN | IDMAC_DES0_DIC | IDMAC_DES0_CH; 618 IDMAC_64ADDR_SET_BUFFER1_SIZE(desc64, desc_len); 619 desc64->des4 = mem_addr & 0xffffffff; 620 desc64->des5 = mem_addr >> 32; 621 } else { 622 IDMAC_SET_BUFFER1_SIZE(desc, desc_len); 623 desc->des2 = cpu_to_le32(mem_addr); 624 } 625 626 /* Update physical address for the next desc */ 627 mem_addr += desc_len; 628 629 /* Save pointer to the last descriptor */ 630 if (is_64bit) { 631 desc64_last = desc64; 632 desc64++; 633 } else { 634 desc_last = desc; 635 desc++; 636 } 637 } 638 } 639 640 /* Set the first descriptor and the last descriptor */ 641 if (is_64bit) { 642 desc64_first->des0 |= IDMAC_DES0_FD; 643 desc64_last->des0 &= ~(IDMAC_DES0_CH | IDMAC_DES0_DIC); 644 desc64_last->des0 |= IDMAC_DES0_LD; 645 } else { 646 desc_first->des0 |= cpu_to_le32(IDMAC_DES0_FD); 647 desc_last->des0 &= cpu_to_le32(~(IDMAC_DES0_CH | IDMAC_DES0_DIC)); 648 desc_last->des0 |= cpu_to_le32(IDMAC_DES0_LD); 649 } 650 651 return 0; 652 err_own_bit: 653 /* restore the descriptor chain as it's polluted */ 654 dev_dbg(host->dev, "descriptor is still owned by IDMAC.\n"); 655 memset(host->sg_cpu, 0, DESC_RING_BUF_SZ); 656 dw_mci_idmac_init(host); 657 return -EINVAL; 658 } 659 660 static int dw_mci_idmac_start_dma(struct dw_mci *host, unsigned int sg_len) 661 { 662 u32 temp; 663 int ret; 664 665 ret = dw_mci_prepare_desc(host, host->data, sg_len, host->dma_64bit_address); 666 if (ret) 667 goto out; 668 669 /* drain writebuffer */ 670 wmb(); 671 672 /* Make sure to reset DMA in case we did PIO before this */ 673 dw_mci_ctrl_reset(host, SDMMC_CTRL_DMA_RESET); 674 dw_mci_idmac_reset(host); 675 676 /* Select IDMAC interface */ 677 temp = mci_readl(host, CTRL); 678 temp |= SDMMC_CTRL_USE_IDMAC; 679 mci_writel(host, CTRL, temp); 680 681 /* drain writebuffer */ 682 wmb(); 683 684 /* Enable the IDMAC */ 685 temp = mci_readl(host, BMOD); 686 temp |= SDMMC_IDMAC_ENABLE | SDMMC_IDMAC_FB; 687 mci_writel(host, BMOD, temp); 688 689 /* Start it running */ 690 mci_writel(host, PLDMND, 1); 691 692 out: 693 return ret; 694 } 695 696 static const struct dw_mci_dma_ops dw_mci_idmac_ops = { 697 .init = dw_mci_idmac_init, 698 .start = dw_mci_idmac_start_dma, 699 .stop = dw_mci_idmac_stop_dma, 700 .complete = dw_mci_dmac_complete_dma, 701 .cleanup = dw_mci_dma_cleanup, 702 }; 703 704 static void dw_mci_edmac_stop_dma(struct dw_mci *host) 705 { 706 dmaengine_terminate_async(host->dms->ch); 707 } 708 709 static int dw_mci_edmac_start_dma(struct dw_mci *host, 710 unsigned int sg_len) 711 { 712 struct dma_slave_config cfg; 713 struct dma_async_tx_descriptor *desc = NULL; 714 struct scatterlist *sgl = host->data->sg; 715 static const u32 mszs[] = {1, 4, 8, 16, 32, 64, 128, 256}; 716 u32 sg_elems = host->data->sg_len; 717 u32 fifoth_val; 718 u32 fifo_offset = host->fifo_reg - host->regs; 719 int ret = 0; 720 721 /* Set external dma config: burst size, burst width */ 722 memset(&cfg, 0, sizeof(cfg)); 723 cfg.dst_addr = host->phy_regs + fifo_offset; 724 cfg.src_addr = cfg.dst_addr; 725 cfg.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; 726 cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; 727 728 /* Match burst msize with external dma config */ 729 fifoth_val = mci_readl(host, FIFOTH); 730 cfg.dst_maxburst = mszs[(fifoth_val >> 28) & 0x7]; 731 cfg.src_maxburst = cfg.dst_maxburst; 732 733 if (host->data->flags & MMC_DATA_WRITE) 734 cfg.direction = DMA_MEM_TO_DEV; 735 else 736 cfg.direction = DMA_DEV_TO_MEM; 737 738 ret = dmaengine_slave_config(host->dms->ch, &cfg); 739 if (ret) { 740 dev_err(host->dev, "Failed to config edmac.\n"); 741 return -EBUSY; 742 } 743 744 desc = dmaengine_prep_slave_sg(host->dms->ch, sgl, 745 sg_len, cfg.direction, 746 DMA_PREP_INTERRUPT | DMA_CTRL_ACK); 747 if (!desc) { 748 dev_err(host->dev, "Can't prepare slave sg.\n"); 749 return -EBUSY; 750 } 751 752 /* Set dw_mci_dmac_complete_dma as callback */ 753 desc->callback = dw_mci_dmac_complete_dma; 754 desc->callback_param = (void *)host; 755 dmaengine_submit(desc); 756 757 /* Flush cache before write */ 758 if (host->data->flags & MMC_DATA_WRITE) 759 dma_sync_sg_for_device(mmc_dev(host->mmc), sgl, 760 sg_elems, DMA_TO_DEVICE); 761 762 dma_async_issue_pending(host->dms->ch); 763 764 return 0; 765 } 766 767 static int dw_mci_edmac_init(struct dw_mci *host) 768 { 769 /* Request external dma channel */ 770 host->dms = kzalloc_obj(struct dw_mci_dma_slave); 771 if (!host->dms) 772 return -ENOMEM; 773 774 host->dms->ch = dma_request_chan(host->dev, "rx-tx"); 775 if (IS_ERR(host->dms->ch)) { 776 int ret = PTR_ERR(host->dms->ch); 777 778 dev_err(host->dev, "Failed to get external DMA channel.\n"); 779 kfree(host->dms); 780 host->dms = NULL; 781 return ret; 782 } 783 784 return 0; 785 } 786 787 static void dw_mci_edmac_exit(struct dw_mci *host) 788 { 789 if (host->dms) { 790 if (host->dms->ch) { 791 dma_release_channel(host->dms->ch); 792 host->dms->ch = NULL; 793 } 794 kfree(host->dms); 795 host->dms = NULL; 796 } 797 } 798 799 static const struct dw_mci_dma_ops dw_mci_edmac_ops = { 800 .init = dw_mci_edmac_init, 801 .exit = dw_mci_edmac_exit, 802 .start = dw_mci_edmac_start_dma, 803 .stop = dw_mci_edmac_stop_dma, 804 .complete = dw_mci_dmac_complete_dma, 805 .cleanup = dw_mci_dma_cleanup, 806 }; 807 808 static int dw_mci_pre_dma_transfer(struct dw_mci *host, 809 struct mmc_data *data, 810 int cookie) 811 { 812 struct scatterlist *sg; 813 unsigned int i, sg_len; 814 815 if (data->host_cookie == COOKIE_PRE_MAPPED) 816 return data->sg_len; 817 818 /* 819 * We don't do DMA on "complex" transfers, i.e. with 820 * non-word-aligned buffers or lengths. Also, we don't bother 821 * with all the DMA setup overhead for short transfers. 822 */ 823 if (data->blocks * data->blksz < host->dma_threshold) 824 return -EINVAL; 825 826 if (data->blksz & 3) 827 return -EINVAL; 828 829 for_each_sg(data->sg, sg, data->sg_len, i) { 830 if (sg->offset & 3 || sg->length & 3) 831 return -EINVAL; 832 } 833 834 sg_len = dma_map_sg(host->dev, 835 data->sg, 836 data->sg_len, 837 mmc_get_dma_dir(data)); 838 if (sg_len == 0) 839 return -EINVAL; 840 841 data->host_cookie = cookie; 842 843 return sg_len; 844 } 845 846 static void dw_mci_pre_req(struct mmc_host *mmc, 847 struct mmc_request *mrq) 848 { 849 struct dw_mci *host = mmc_priv(mmc); 850 struct mmc_data *data = mrq->data; 851 852 if (!host->use_dma || !data) 853 return; 854 855 /* This data might be unmapped at this time */ 856 data->host_cookie = COOKIE_UNMAPPED; 857 858 if (dw_mci_pre_dma_transfer(host, mrq->data, 859 COOKIE_PRE_MAPPED) < 0) 860 data->host_cookie = COOKIE_UNMAPPED; 861 } 862 863 static void dw_mci_post_req(struct mmc_host *mmc, 864 struct mmc_request *mrq, 865 int err) 866 { 867 struct dw_mci *host = mmc_priv(mmc); 868 struct mmc_data *data = mrq->data; 869 870 if (!host->use_dma || !data) 871 return; 872 873 if (data->host_cookie != COOKIE_UNMAPPED) 874 dma_unmap_sg(host->dev, 875 data->sg, 876 data->sg_len, 877 mmc_get_dma_dir(data)); 878 data->host_cookie = COOKIE_UNMAPPED; 879 } 880 881 static int dw_mci_get_cd(struct mmc_host *mmc) 882 { 883 struct dw_mci *host = mmc_priv(mmc); 884 int gpio_cd = mmc_gpio_get_cd(mmc); 885 886 if (mmc->caps & MMC_CAP_NEEDS_POLL) 887 return 1; 888 889 if (!mmc_card_is_removable(mmc)) 890 return 1; 891 892 /* Try slot gpio detection */ 893 if (gpio_cd >= 0) 894 return !!gpio_cd; 895 896 /* Host native card detect */ 897 return !(mci_readl(host, CDETECT) & BIT(0)); 898 } 899 900 static void dw_mci_adjust_fifoth(struct dw_mci *host, struct mmc_data *data) 901 { 902 unsigned int blksz = data->blksz; 903 static const u32 mszs[] = {1, 4, 8, 16, 32, 64, 128, 256}; 904 u32 fifo_width = 1 << host->data_shift; 905 u32 blksz_depth = blksz / fifo_width, fifoth_val; 906 u32 msize = 0, rx_wmark = 1, tx_wmark, tx_wmark_invers; 907 int idx = ARRAY_SIZE(mszs) - 1; 908 909 /* pio should ship this scenario */ 910 if (!host->use_dma) 911 return; 912 913 tx_wmark = (host->fifo_depth) / 2; 914 tx_wmark_invers = host->fifo_depth - tx_wmark; 915 916 /* 917 * MSIZE is '1', 918 * if blksz is not a multiple of the FIFO width 919 */ 920 if (blksz % fifo_width) 921 goto done; 922 923 do { 924 if (!((blksz_depth % mszs[idx]) || 925 (tx_wmark_invers % mszs[idx]))) { 926 msize = idx; 927 rx_wmark = mszs[idx] - 1; 928 break; 929 } 930 } while (--idx > 0); 931 /* 932 * If idx is '0', it won't be tried 933 * Thus, initial values are uesed 934 */ 935 done: 936 fifoth_val = SDMMC_SET_FIFOTH(msize, rx_wmark, tx_wmark); 937 mci_writel(host, FIFOTH, fifoth_val); 938 } 939 940 static void dw_mci_ctrl_thld(struct dw_mci *host, struct mmc_data *data) 941 { 942 unsigned int blksz = data->blksz; 943 u32 blksz_depth, fifo_depth; 944 u16 thld_size; 945 u8 enable; 946 947 /* 948 * CDTHRCTL doesn't exist prior to 240A (in fact that register offset is 949 * in the FIFO region, so we really shouldn't access it). 950 */ 951 if (host->verid < DW_MMC_240A || 952 (host->verid < DW_MMC_280A && data->flags & MMC_DATA_WRITE)) 953 return; 954 955 /* 956 * Card write Threshold is introduced since 2.80a 957 * It's used when HS400 mode is enabled. 958 */ 959 if (data->flags & MMC_DATA_WRITE && 960 host->timing != MMC_TIMING_MMC_HS400) 961 goto disable; 962 963 if (data->flags & MMC_DATA_WRITE) 964 enable = SDMMC_CARD_WR_THR_EN; 965 else 966 enable = SDMMC_CARD_RD_THR_EN; 967 968 if (host->timing != MMC_TIMING_MMC_HS200 && 969 host->timing != MMC_TIMING_UHS_SDR104 && 970 host->timing != MMC_TIMING_MMC_HS400) 971 goto disable; 972 973 blksz_depth = blksz / (1 << host->data_shift); 974 fifo_depth = host->fifo_depth; 975 976 if (blksz_depth > fifo_depth) 977 goto disable; 978 979 /* 980 * If (blksz_depth) >= (fifo_depth >> 1), should be 'thld_size <= blksz' 981 * If (blksz_depth) < (fifo_depth >> 1), should be thld_size = blksz 982 * Currently just choose blksz. 983 */ 984 thld_size = blksz; 985 mci_writel(host, CDTHRCTL, SDMMC_SET_THLD(thld_size, enable)); 986 return; 987 988 disable: 989 mci_writel(host, CDTHRCTL, 0); 990 } 991 992 static int dw_mci_submit_data_dma(struct dw_mci *host, struct mmc_data *data) 993 { 994 unsigned long irqflags; 995 int sg_len; 996 u32 temp; 997 998 host->using_dma = 0; 999 1000 /* If we don't have a channel, we can't do DMA */ 1001 if (!host->use_dma) 1002 return -ENODEV; 1003 1004 sg_len = dw_mci_pre_dma_transfer(host, data, COOKIE_MAPPED); 1005 if (sg_len < 0) { 1006 host->dma_ops->stop(host); 1007 return sg_len; 1008 } 1009 1010 host->using_dma = 1; 1011 1012 if (host->use_dma == TRANS_MODE_IDMAC) 1013 dev_vdbg(host->dev, 1014 "sd sg_cpu: %#lx sg_dma: %#lx sg_len: %d\n", 1015 (unsigned long)host->sg_cpu, 1016 (unsigned long)host->sg_dma, 1017 sg_len); 1018 1019 /* 1020 * Decide the MSIZE and RX/TX Watermark. 1021 * If current block size is same with previous size, 1022 * no need to update fifoth. 1023 */ 1024 if (host->prev_blksz != data->blksz) 1025 dw_mci_adjust_fifoth(host, data); 1026 1027 /* Enable the DMA interface */ 1028 temp = mci_readl(host, CTRL); 1029 temp |= SDMMC_CTRL_DMA_ENABLE; 1030 mci_writel(host, CTRL, temp); 1031 1032 /* Disable RX/TX IRQs, let DMA handle it */ 1033 spin_lock_irqsave(&host->irq_lock, irqflags); 1034 temp = mci_readl(host, INTMASK); 1035 temp &= ~(SDMMC_INT_RXDR | SDMMC_INT_TXDR); 1036 mci_writel(host, INTMASK, temp); 1037 spin_unlock_irqrestore(&host->irq_lock, irqflags); 1038 1039 if (host->dma_ops->start(host, sg_len)) { 1040 host->dma_ops->stop(host); 1041 /* We can't do DMA, try PIO for this one */ 1042 dev_dbg(host->dev, 1043 "%s: fall back to PIO mode for current transfer\n", 1044 __func__); 1045 return -ENODEV; 1046 } 1047 1048 return 0; 1049 } 1050 1051 static void dw_mci_submit_data(struct dw_mci *host, struct mmc_data *data) 1052 { 1053 unsigned long irqflags; 1054 int flags = SG_MITER_ATOMIC; 1055 u32 temp; 1056 1057 data->error = -EINPROGRESS; 1058 1059 WARN_ON(host->data); 1060 host->sg = NULL; 1061 host->data = data; 1062 1063 if (data->flags & MMC_DATA_READ) 1064 host->dir_status = MMC_DATA_READ; 1065 else 1066 host->dir_status = MMC_DATA_WRITE; 1067 1068 dw_mci_ctrl_thld(host, data); 1069 1070 if (dw_mci_submit_data_dma(host, data)) { 1071 if (host->data->flags & MMC_DATA_READ) 1072 flags |= SG_MITER_TO_SG; 1073 else 1074 flags |= SG_MITER_FROM_SG; 1075 1076 sg_miter_start(&host->sg_miter, data->sg, data->sg_len, flags); 1077 host->sg = data->sg; 1078 host->part_buf_start = 0; 1079 host->part_buf_count = 0; 1080 1081 mci_writel(host, RINTSTS, SDMMC_INT_TXDR | SDMMC_INT_RXDR); 1082 1083 spin_lock_irqsave(&host->irq_lock, irqflags); 1084 temp = mci_readl(host, INTMASK); 1085 temp |= SDMMC_INT_TXDR | SDMMC_INT_RXDR; 1086 mci_writel(host, INTMASK, temp); 1087 spin_unlock_irqrestore(&host->irq_lock, irqflags); 1088 1089 temp = mci_readl(host, CTRL); 1090 temp &= ~SDMMC_CTRL_DMA_ENABLE; 1091 mci_writel(host, CTRL, temp); 1092 1093 /* 1094 * Use the initial fifoth_val for PIO mode. If wm_algined 1095 * is set, we set watermark same as data size. 1096 * If next issued data may be transferred by DMA mode, 1097 * prev_blksz should be invalidated. 1098 */ 1099 if (host->wm_aligned) 1100 dw_mci_adjust_fifoth(host, data); 1101 else 1102 mci_writel(host, FIFOTH, host->fifoth_val); 1103 host->prev_blksz = 0; 1104 } else { 1105 /* 1106 * Keep the current block size. 1107 * It will be used to decide whether to update 1108 * fifoth register next time. 1109 */ 1110 host->prev_blksz = data->blksz; 1111 } 1112 } 1113 1114 static void dw_mci_setup_bus(struct dw_mci *host, bool force_clkinit) 1115 { 1116 unsigned int clock = host->clock; 1117 u32 div; 1118 u32 clk_en_a; 1119 u32 sdmmc_cmd_bits = SDMMC_CMD_UPD_CLK | SDMMC_CMD_PRV_DAT_WAIT; 1120 1121 /* We must continue to set bit 28 in CMD until the change is complete */ 1122 if (host->state == STATE_WAITING_CMD11_DONE) 1123 sdmmc_cmd_bits |= SDMMC_CMD_VOLT_SWITCH; 1124 1125 host->mmc->actual_clock = 0; 1126 1127 if (!clock) { 1128 mci_writel(host, CLKENA, 0); 1129 mci_send_cmd(host, sdmmc_cmd_bits, 0); 1130 } else if (clock != host->current_speed || force_clkinit) { 1131 div = host->bus_hz / clock; 1132 if (host->bus_hz % clock && host->bus_hz > clock) 1133 /* 1134 * move the + 1 after the divide to prevent 1135 * over-clocking the card. 1136 */ 1137 div += 1; 1138 1139 div = (host->bus_hz != clock) ? DIV_ROUND_UP(div, 2) : 0; 1140 1141 if ((clock != host->clk_old && 1142 !test_bit(DW_MMC_CARD_NEEDS_POLL, &host->flags)) || 1143 force_clkinit) { 1144 /* Silent the verbose log if calling from PM context */ 1145 if (!force_clkinit) 1146 dev_info(&host->mmc->class_dev, 1147 "Bus speed = %dHz (req %dHz, actual %dHZ div = %d)\n", 1148 host->bus_hz, clock, 1149 div ? ((host->bus_hz / div) >> 1) : 1150 host->bus_hz, div); 1151 1152 /* 1153 * If card is polling, display the message only 1154 * one time at boot time. 1155 */ 1156 if (host->mmc->caps & MMC_CAP_NEEDS_POLL && 1157 host->mmc->f_min == clock) 1158 set_bit(DW_MMC_CARD_NEEDS_POLL, &host->flags); 1159 } 1160 1161 /* disable clock */ 1162 mci_writel(host, CLKENA, 0); 1163 mci_writel(host, CLKSRC, 0); 1164 1165 /* inform CIU */ 1166 mci_send_cmd(host, sdmmc_cmd_bits, 0); 1167 1168 /* set clock to desired speed */ 1169 mci_writel(host, CLKDIV, div); 1170 1171 /* inform CIU */ 1172 mci_send_cmd(host, sdmmc_cmd_bits, 0); 1173 1174 /* enable clock; only low power if no SDIO */ 1175 clk_en_a = SDMMC_CLKEN_ENABLE; 1176 if (!test_bit(DW_MMC_CARD_NO_LOW_PWR, &host->flags)) 1177 clk_en_a |= SDMMC_CLKEN_LOW_PWR; 1178 mci_writel(host, CLKENA, clk_en_a); 1179 1180 /* inform CIU */ 1181 mci_send_cmd(host, sdmmc_cmd_bits, 0); 1182 1183 /* keep the last clock value that was requested from core */ 1184 host->clk_old = clock; 1185 host->mmc->actual_clock = div ? ((host->bus_hz / div) >> 1) : 1186 host->bus_hz; 1187 } 1188 1189 host->current_speed = clock; 1190 1191 /* Set the current bus width */ 1192 mci_writel(host, CTYPE, host->ctype); 1193 } 1194 1195 static void dw_mci_set_data_timeout(struct dw_mci *host, 1196 unsigned int timeout_ns) 1197 { 1198 const struct dw_mci_drv_data *drv_data = host->drv_data; 1199 u32 clk_div, tmout; 1200 u64 tmp; 1201 1202 if (drv_data && drv_data->set_data_timeout) 1203 return drv_data->set_data_timeout(host, timeout_ns); 1204 1205 clk_div = (mci_readl(host, CLKDIV) & 0xFF) * 2; 1206 if (clk_div == 0) 1207 clk_div = 1; 1208 1209 tmp = DIV_ROUND_UP_ULL((u64)timeout_ns * host->bus_hz, NSEC_PER_SEC); 1210 tmp = DIV_ROUND_UP_ULL(tmp, clk_div); 1211 1212 /* TMOUT[7:0] (RESPONSE_TIMEOUT) */ 1213 tmout = 0xFF; /* Set maximum */ 1214 1215 /* TMOUT[31:8] (DATA_TIMEOUT) */ 1216 if (!tmp || tmp > 0xFFFFFF) 1217 tmout |= (0xFFFFFF << 8); 1218 else 1219 tmout |= (tmp & 0xFFFFFF) << 8; 1220 1221 mci_writel(host, TMOUT, tmout); 1222 dev_dbg(host->dev, "timeout_ns: %u => TMOUT[31:8]: %#08x", 1223 timeout_ns, tmout >> 8); 1224 } 1225 1226 static void dw_mci_start_request(struct dw_mci *host, struct mmc_command *cmd) 1227 { 1228 struct mmc_request *mrq; 1229 struct mmc_data *data; 1230 u32 cmdflags; 1231 1232 mrq = host->mrq; 1233 1234 host->mrq = mrq; 1235 1236 host->pending_events = 0; 1237 host->completed_events = 0; 1238 host->cmd_status = 0; 1239 host->data_status = 0; 1240 host->dir_status = 0; 1241 1242 data = cmd->data; 1243 if (data) { 1244 dw_mci_set_data_timeout(host, data->timeout_ns); 1245 mci_writel(host, BYTCNT, data->blksz*data->blocks); 1246 mci_writel(host, BLKSIZ, data->blksz); 1247 } 1248 1249 cmdflags = dw_mci_prepare_command(host->mmc, cmd); 1250 1251 /* this is the first command, send the initialization clock */ 1252 if (test_and_clear_bit(DW_MMC_CARD_NEED_INIT, &host->flags)) 1253 cmdflags |= SDMMC_CMD_INIT; 1254 1255 if (data) { 1256 dw_mci_submit_data(host, data); 1257 wmb(); /* drain writebuffer */ 1258 } 1259 1260 dw_mci_start_command(host, cmd, cmdflags); 1261 1262 if (cmd->opcode == SD_SWITCH_VOLTAGE) { 1263 unsigned long irqflags; 1264 1265 /* 1266 * Databook says to fail after 2ms w/ no response, but evidence 1267 * shows that sometimes the cmd11 interrupt takes over 130ms. 1268 * We'll set to 500ms, plus an extra jiffy just in case jiffies 1269 * is just about to roll over. 1270 * 1271 * We do this whole thing under spinlock and only if the 1272 * command hasn't already completed (indicating the irq 1273 * already ran so we don't want the timeout). 1274 */ 1275 spin_lock_irqsave(&host->irq_lock, irqflags); 1276 if (!test_bit(EVENT_CMD_COMPLETE, &host->pending_events)) 1277 mod_timer(&host->cmd11_timer, 1278 jiffies + msecs_to_jiffies(500) + 1); 1279 spin_unlock_irqrestore(&host->irq_lock, irqflags); 1280 } 1281 1282 host->stop_cmdr = dw_mci_prep_stop_abort(host, cmd); 1283 } 1284 1285 static void dw_mci_request(struct mmc_host *mmc, struct mmc_request *mrq) 1286 { 1287 struct dw_mci *host = mmc_priv(mmc); 1288 struct mmc_command *cmd; 1289 1290 WARN_ON(host->mrq); 1291 1292 /* 1293 * The check for card presence and queueing of the request must be 1294 * atomic, otherwise the card could be removed in between and the 1295 * request wouldn't fail until another card was inserted. 1296 */ 1297 if (!dw_mci_get_cd(mmc)) { 1298 mrq->cmd->error = -ENOMEDIUM; 1299 mmc_request_done(mmc, mrq); 1300 return; 1301 } 1302 1303 spin_lock_bh(&host->lock); 1304 1305 dev_vdbg(&host->mmc->class_dev, "request: state=%d\n", 1306 host->state); 1307 1308 host->mrq = mrq; 1309 if (host->state == STATE_WAITING_CMD11_DONE) { 1310 dev_warn(&host->mmc->class_dev, 1311 "Voltage change didn't complete\n"); 1312 /* 1313 * this case isn't expected to happen, so we can 1314 * either crash here or just try to continue on 1315 * in the closest possible state 1316 */ 1317 host->state = STATE_IDLE; 1318 } 1319 1320 if (host->state == STATE_IDLE) { 1321 host->state = STATE_SENDING_CMD; 1322 cmd = mrq->sbc ? mrq->sbc : mrq->cmd; 1323 dw_mci_start_request(host, cmd); 1324 } 1325 1326 spin_unlock_bh(&host->lock); 1327 } 1328 1329 static void dw_mci_set_ios(struct mmc_host *mmc, struct mmc_ios *ios) 1330 { 1331 struct dw_mci *host = mmc_priv(mmc); 1332 const struct dw_mci_drv_data *drv_data = host->drv_data; 1333 u32 regs; 1334 int ret; 1335 1336 switch (ios->bus_width) { 1337 case MMC_BUS_WIDTH_4: 1338 host->ctype = SDMMC_CTYPE_4BIT; 1339 break; 1340 case MMC_BUS_WIDTH_8: 1341 host->ctype = SDMMC_CTYPE_8BIT; 1342 break; 1343 default: 1344 /* set default 1 bit mode */ 1345 host->ctype = SDMMC_CTYPE_1BIT; 1346 } 1347 1348 regs = mci_readl(host, UHS_REG); 1349 1350 /* DDR mode set */ 1351 if (ios->timing == MMC_TIMING_MMC_DDR52 || 1352 ios->timing == MMC_TIMING_UHS_DDR50 || 1353 ios->timing == MMC_TIMING_MMC_HS400) 1354 regs |= BIT(16); 1355 else 1356 regs &= ~BIT(16); 1357 1358 mci_writel(host, UHS_REG, regs); 1359 host->timing = ios->timing; 1360 1361 /* 1362 * Use mirror of ios->clock to prevent race with mmc 1363 * core ios update when finding the minimum. 1364 */ 1365 host->clock = ios->clock; 1366 1367 if (drv_data && drv_data->set_ios) 1368 drv_data->set_ios(host, ios); 1369 1370 switch (ios->power_mode) { 1371 case MMC_POWER_UP: 1372 ret = mmc_regulator_set_ocr(mmc, mmc->supply.vmmc, ios->vdd); 1373 if (ret) { 1374 dev_err(host->dev, "failed to enable vmmc regulator\n"); 1375 return; 1376 } 1377 set_bit(DW_MMC_CARD_NEED_INIT, &host->flags); 1378 regs = mci_readl(host, PWREN); 1379 regs |= BIT(0); 1380 mci_writel(host, PWREN, regs); 1381 break; 1382 case MMC_POWER_ON: 1383 mmc_regulator_enable_vqmmc(mmc); 1384 /* Adjust clock / bus width after power is up */ 1385 dw_mci_setup_bus(host, false); 1386 1387 break; 1388 case MMC_POWER_OFF: 1389 /* Turn clock off before power goes down */ 1390 dw_mci_setup_bus(host, false); 1391 1392 if (!IS_ERR(mmc->supply.vmmc)) 1393 mmc_regulator_set_ocr(mmc, mmc->supply.vmmc, 0); 1394 1395 mmc_regulator_disable_vqmmc(mmc); 1396 1397 regs = mci_readl(host, PWREN); 1398 regs &= ~BIT(0); 1399 mci_writel(host, PWREN, regs); 1400 /* Reset our state machine after powering off */ 1401 dw_mci_ctrl_reset(host, SDMMC_CTRL_ALL_RESET_FLAGS); 1402 break; 1403 default: 1404 break; 1405 } 1406 1407 if (host->state == STATE_WAITING_CMD11_DONE && ios->clock != 0) 1408 host->state = STATE_IDLE; 1409 } 1410 1411 static int dw_mci_card_busy(struct mmc_host *mmc) 1412 { 1413 struct dw_mci *host = mmc_priv(mmc); 1414 u32 status; 1415 1416 /* 1417 * Check the busy bit which is low when DAT[3:0] 1418 * (the data lines) are 0000 1419 */ 1420 status = mci_readl(host, STATUS); 1421 1422 return !!(status & SDMMC_STATUS_BUSY); 1423 } 1424 1425 static int dw_mci_switch_voltage(struct mmc_host *mmc, struct mmc_ios *ios) 1426 { 1427 struct dw_mci *host = mmc_priv(mmc); 1428 const struct dw_mci_drv_data *drv_data = host->drv_data; 1429 u32 uhs; 1430 u32 v18 = SDMMC_UHS_18V; 1431 int ret; 1432 1433 if (drv_data && drv_data->switch_voltage) 1434 return drv_data->switch_voltage(host, ios); 1435 1436 /* 1437 * Program the voltage. Note that some instances of dw_mmc may use 1438 * the UHS_REG for this. For other instances (like exynos) the UHS_REG 1439 * does no harm but you need to set the regulator directly. Try both. 1440 */ 1441 uhs = mci_readl(host, UHS_REG); 1442 if (ios->signal_voltage == MMC_SIGNAL_VOLTAGE_330) 1443 uhs &= ~v18; 1444 else 1445 uhs |= v18; 1446 1447 if (!IS_ERR(mmc->supply.vqmmc)) { 1448 ret = mmc_regulator_set_vqmmc(mmc, ios); 1449 if (ret < 0) { 1450 dev_dbg(&mmc->class_dev, 1451 "Regulator set error %d - %s V\n", 1452 ret, uhs & v18 ? "1.8" : "3.3"); 1453 return ret; 1454 } 1455 } 1456 mci_writel(host, UHS_REG, uhs); 1457 1458 return 0; 1459 } 1460 1461 static int dw_mci_get_ro(struct mmc_host *mmc) 1462 { 1463 int read_only; 1464 struct dw_mci *host = mmc_priv(mmc); 1465 int gpio_ro = mmc_gpio_get_ro(mmc); 1466 1467 /* Use platform get_ro function, else try on board write protect */ 1468 if (gpio_ro >= 0) 1469 read_only = gpio_ro; 1470 else 1471 read_only = 1472 mci_readl(host, WRTPRT) & BIT(0) ? 1 : 0; 1473 1474 dev_dbg(&mmc->class_dev, "card is %s\n", 1475 read_only ? "read-only" : "read-write"); 1476 1477 return read_only; 1478 } 1479 1480 static void dw_mci_hw_reset(struct mmc_host *mmc) 1481 { 1482 struct dw_mci *host = mmc_priv(mmc); 1483 const struct dw_mci_drv_data *drv_data = host->drv_data; 1484 int reset; 1485 1486 if (host->use_dma == TRANS_MODE_IDMAC) 1487 dw_mci_idmac_reset(host); 1488 1489 if (!dw_mci_ctrl_reset(host, SDMMC_CTRL_DMA_RESET | 1490 SDMMC_CTRL_FIFO_RESET)) 1491 return; 1492 1493 if (drv_data && drv_data->hw_reset) { 1494 drv_data->hw_reset(host); 1495 return; 1496 } 1497 1498 /* 1499 * According to eMMC spec, card reset procedure: 1500 * tRstW >= 1us: RST_n pulse width 1501 * tRSCA >= 200us: RST_n to Command time 1502 * tRSTH >= 1us: RST_n high period 1503 */ 1504 reset = mci_readl(host, RST_N); 1505 reset &= ~SDMMC_RST_HWACTIVE; 1506 mci_writel(host, RST_N, reset); 1507 usleep_range(1, 2); 1508 reset |= SDMMC_RST_HWACTIVE; 1509 mci_writel(host, RST_N, reset); 1510 usleep_range(200, 300); 1511 } 1512 1513 static void dw_mci_prepare_sdio_irq(struct dw_mci *host, bool prepare) 1514 { 1515 const u32 clken_low_pwr = SDMMC_CLKEN_LOW_PWR; 1516 u32 clk_en_a_old; 1517 u32 clk_en_a; 1518 1519 /* 1520 * Low power mode will stop the card clock when idle. According to the 1521 * description of the CLKENA register we should disable low power mode 1522 * for SDIO cards if we need SDIO interrupts to work. 1523 */ 1524 1525 clk_en_a_old = mci_readl(host, CLKENA); 1526 if (prepare) { 1527 set_bit(DW_MMC_CARD_NO_LOW_PWR, &host->flags); 1528 clk_en_a = clk_en_a_old & ~clken_low_pwr; 1529 } else { 1530 clear_bit(DW_MMC_CARD_NO_LOW_PWR, &host->flags); 1531 clk_en_a = clk_en_a_old | clken_low_pwr; 1532 } 1533 1534 if (clk_en_a != clk_en_a_old) { 1535 mci_writel(host, CLKENA, clk_en_a); 1536 mci_send_cmd(host, SDMMC_CMD_UPD_CLK | SDMMC_CMD_PRV_DAT_WAIT, 1537 0); 1538 } 1539 } 1540 1541 static void __dw_mci_enable_sdio_irq(struct dw_mci *host, int enb) 1542 { 1543 unsigned long irqflags; 1544 u32 int_mask; 1545 1546 spin_lock_irqsave(&host->irq_lock, irqflags); 1547 1548 /* Enable/disable Slot Specific SDIO interrupt */ 1549 int_mask = mci_readl(host, INTMASK); 1550 if (enb) 1551 int_mask |= SDMMC_INT_SDIO(host->sdio_irq); 1552 else 1553 int_mask &= ~SDMMC_INT_SDIO(host->sdio_irq); 1554 mci_writel(host, INTMASK, int_mask); 1555 1556 spin_unlock_irqrestore(&host->irq_lock, irqflags); 1557 } 1558 1559 static void dw_mci_enable_sdio_irq(struct mmc_host *mmc, int enb) 1560 { 1561 struct dw_mci *host = mmc_priv(mmc); 1562 1563 dw_mci_prepare_sdio_irq(host, enb); 1564 __dw_mci_enable_sdio_irq(host, enb); 1565 1566 /* Avoid runtime suspending the device when SDIO IRQ is enabled */ 1567 if (enb) 1568 pm_runtime_get_noresume(host->dev); 1569 else 1570 pm_runtime_put_noidle(host->dev); 1571 } 1572 1573 static void dw_mci_ack_sdio_irq(struct mmc_host *mmc) 1574 { 1575 struct dw_mci *host = mmc_priv(mmc); 1576 1577 __dw_mci_enable_sdio_irq(host, 1); 1578 } 1579 1580 static int dw_mci_execute_tuning(struct mmc_host *mmc, u32 opcode) 1581 { 1582 struct dw_mci *host = mmc_priv(mmc); 1583 const struct dw_mci_drv_data *drv_data = host->drv_data; 1584 int err = -EINVAL; 1585 1586 if (drv_data && drv_data->execute_tuning) 1587 err = drv_data->execute_tuning(host, opcode); 1588 return err; 1589 } 1590 1591 static int dw_mci_prepare_hs400_tuning(struct mmc_host *mmc, 1592 struct mmc_ios *ios) 1593 { 1594 struct dw_mci *host = mmc_priv(mmc); 1595 const struct dw_mci_drv_data *drv_data = host->drv_data; 1596 1597 if (drv_data && drv_data->prepare_hs400_tuning) 1598 return drv_data->prepare_hs400_tuning(host, ios); 1599 1600 return 0; 1601 } 1602 1603 static bool dw_mci_reset(struct dw_mci *host) 1604 { 1605 u32 flags = SDMMC_CTRL_RESET | SDMMC_CTRL_FIFO_RESET; 1606 bool ret = false; 1607 u32 status = 0; 1608 1609 /* 1610 * Resetting generates a block interrupt, hence setting 1611 * the scatter-gather pointer to NULL. 1612 */ 1613 if (host->sg) { 1614 sg_miter_stop(&host->sg_miter); 1615 host->sg = NULL; 1616 } 1617 1618 if (host->use_dma) 1619 flags |= SDMMC_CTRL_DMA_RESET; 1620 1621 if (dw_mci_ctrl_reset(host, flags)) { 1622 /* 1623 * In all cases we clear the RAWINTS 1624 * register to clear any interrupts. 1625 */ 1626 mci_writel(host, RINTSTS, 0xFFFFFFFF); 1627 1628 if (!host->use_dma) { 1629 ret = true; 1630 goto ciu_out; 1631 } 1632 1633 /* Wait for dma_req to be cleared */ 1634 if (readl_poll_timeout_atomic(host->regs + SDMMC_STATUS, 1635 status, 1636 !(status & SDMMC_STATUS_DMA_REQ), 1637 1, 500 * USEC_PER_MSEC)) { 1638 dev_err(host->dev, 1639 "%s: Timeout waiting for dma_req to be cleared\n", 1640 __func__); 1641 goto ciu_out; 1642 } 1643 1644 /* when using DMA next we reset the fifo again */ 1645 if (!dw_mci_ctrl_reset(host, SDMMC_CTRL_FIFO_RESET)) 1646 goto ciu_out; 1647 } else { 1648 /* if the controller reset bit did clear, then set clock regs */ 1649 if (!(mci_readl(host, CTRL) & SDMMC_CTRL_RESET)) { 1650 dev_err(host->dev, 1651 "%s: fifo/dma reset bits didn't clear but ciu was reset, doing clock update\n", 1652 __func__); 1653 goto ciu_out; 1654 } 1655 } 1656 1657 if (host->use_dma == TRANS_MODE_IDMAC) 1658 /* It is also required that we reinit idmac */ 1659 dw_mci_idmac_init(host); 1660 1661 ret = true; 1662 1663 ciu_out: 1664 /* After a CTRL reset we need to have CIU set clock registers */ 1665 mci_send_cmd(host, SDMMC_CMD_UPD_CLK, 0); 1666 1667 return ret; 1668 } 1669 1670 static const struct mmc_host_ops dw_mci_ops = { 1671 .request = dw_mci_request, 1672 .pre_req = dw_mci_pre_req, 1673 .post_req = dw_mci_post_req, 1674 .set_ios = dw_mci_set_ios, 1675 .get_ro = dw_mci_get_ro, 1676 .get_cd = dw_mci_get_cd, 1677 .card_hw_reset = dw_mci_hw_reset, 1678 .enable_sdio_irq = dw_mci_enable_sdio_irq, 1679 .ack_sdio_irq = dw_mci_ack_sdio_irq, 1680 .execute_tuning = dw_mci_execute_tuning, 1681 .card_busy = dw_mci_card_busy, 1682 .start_signal_voltage_switch = dw_mci_switch_voltage, 1683 .prepare_hs400_tuning = dw_mci_prepare_hs400_tuning, 1684 }; 1685 1686 #ifdef CONFIG_FAULT_INJECTION 1687 static enum hrtimer_restart dw_mci_fault_timer(struct hrtimer *t) 1688 { 1689 struct dw_mci *host = container_of(t, struct dw_mci, fault_timer); 1690 unsigned long flags; 1691 1692 spin_lock_irqsave(&host->irq_lock, flags); 1693 1694 /* 1695 * Only inject an error if we haven't already got an error or data over 1696 * interrupt. 1697 */ 1698 if (!host->data_status) { 1699 host->data_status = SDMMC_INT_DCRC; 1700 set_bit(EVENT_DATA_ERROR, &host->pending_events); 1701 queue_work(system_bh_wq, &host->bh_work); 1702 } 1703 1704 spin_unlock_irqrestore(&host->irq_lock, flags); 1705 1706 return HRTIMER_NORESTART; 1707 } 1708 1709 static void dw_mci_start_fault_timer(struct dw_mci *host) 1710 { 1711 struct mmc_data *data = host->data; 1712 1713 if (!data || data->blocks <= 1) 1714 return; 1715 1716 if (!should_fail(&host->fail_data_crc, 1)) 1717 return; 1718 1719 /* 1720 * Try to inject the error at random points during the data transfer. 1721 */ 1722 hrtimer_start(&host->fault_timer, 1723 ms_to_ktime(get_random_u32_below(25)), 1724 HRTIMER_MODE_REL); 1725 } 1726 1727 static void dw_mci_stop_fault_timer(struct dw_mci *host) 1728 { 1729 hrtimer_cancel(&host->fault_timer); 1730 } 1731 1732 static void dw_mci_init_fault(struct dw_mci *host) 1733 { 1734 host->fail_data_crc = (struct fault_attr) FAULT_ATTR_INITIALIZER; 1735 1736 hrtimer_setup(&host->fault_timer, dw_mci_fault_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); 1737 } 1738 #else 1739 static void dw_mci_init_fault(struct dw_mci *host) 1740 { 1741 } 1742 1743 static void dw_mci_start_fault_timer(struct dw_mci *host) 1744 { 1745 } 1746 1747 static void dw_mci_stop_fault_timer(struct dw_mci *host) 1748 { 1749 } 1750 #endif 1751 1752 static void dw_mci_request_end(struct dw_mci *host, struct mmc_request *mrq) 1753 __releases(&host->lock) 1754 __acquires(&host->lock) 1755 { 1756 struct mmc_host *prev_mmc = host->mmc; 1757 1758 WARN_ON(host->cmd || host->data); 1759 1760 host->mrq = NULL; 1761 1762 if (host->state == STATE_SENDING_CMD11) 1763 host->state = STATE_WAITING_CMD11_DONE; 1764 else 1765 host->state = STATE_IDLE; 1766 1767 spin_unlock(&host->lock); 1768 mmc_request_done(prev_mmc, mrq); 1769 spin_lock(&host->lock); 1770 } 1771 1772 static int dw_mci_command_complete(struct dw_mci *host, struct mmc_command *cmd) 1773 { 1774 u32 status = host->cmd_status; 1775 1776 host->cmd_status = 0; 1777 1778 /* Read the response from the card (up to 16 bytes) */ 1779 if (cmd->flags & MMC_RSP_PRESENT) { 1780 if (cmd->flags & MMC_RSP_136) { 1781 cmd->resp[3] = mci_readl(host, RESP0); 1782 cmd->resp[2] = mci_readl(host, RESP1); 1783 cmd->resp[1] = mci_readl(host, RESP2); 1784 cmd->resp[0] = mci_readl(host, RESP3); 1785 } else { 1786 cmd->resp[0] = mci_readl(host, RESP0); 1787 cmd->resp[1] = 0; 1788 cmd->resp[2] = 0; 1789 cmd->resp[3] = 0; 1790 } 1791 } 1792 1793 if (status & SDMMC_INT_RTO) 1794 cmd->error = -ETIMEDOUT; 1795 else if ((cmd->flags & MMC_RSP_CRC) && (status & SDMMC_INT_RCRC)) 1796 cmd->error = -EILSEQ; 1797 else if (status & SDMMC_INT_RESP_ERR) 1798 cmd->error = -EIO; 1799 else 1800 cmd->error = 0; 1801 1802 return cmd->error; 1803 } 1804 1805 static int dw_mci_data_complete(struct dw_mci *host, struct mmc_data *data) 1806 { 1807 u32 status = host->data_status; 1808 1809 if (status & DW_MCI_DATA_ERROR_FLAGS) { 1810 if (status & SDMMC_INT_DRTO) { 1811 data->error = -ETIMEDOUT; 1812 } else if (status & SDMMC_INT_DCRC) { 1813 data->error = -EILSEQ; 1814 } else if (status & SDMMC_INT_EBE) { 1815 if (host->dir_status == 1816 MMC_DATA_WRITE) { 1817 /* 1818 * No data CRC status was returned. 1819 * The number of bytes transferred 1820 * will be exaggerated in PIO mode. 1821 */ 1822 data->bytes_xfered = 0; 1823 data->error = -ETIMEDOUT; 1824 } else if (host->dir_status == 1825 MMC_DATA_READ) { 1826 data->error = -EILSEQ; 1827 } 1828 } else { 1829 /* SDMMC_INT_SBE is included */ 1830 data->error = -EILSEQ; 1831 } 1832 1833 dev_dbg(host->dev, "data error, status 0x%08x\n", status); 1834 1835 /* 1836 * After an error, there may be data lingering 1837 * in the FIFO 1838 */ 1839 dw_mci_reset(host); 1840 } else { 1841 data->bytes_xfered = data->blocks * data->blksz; 1842 data->error = 0; 1843 } 1844 1845 return data->error; 1846 } 1847 1848 static void dw_mci_set_drto(struct dw_mci *host) 1849 { 1850 const struct dw_mci_drv_data *drv_data = host->drv_data; 1851 unsigned int drto_clks; 1852 unsigned int drto_div; 1853 unsigned int drto_ms; 1854 unsigned long irqflags; 1855 1856 if (drv_data && drv_data->get_drto_clks) 1857 drto_clks = drv_data->get_drto_clks(host); 1858 else 1859 drto_clks = mci_readl(host, TMOUT) >> 8; 1860 drto_div = (mci_readl(host, CLKDIV) & 0xff) * 2; 1861 if (drto_div == 0) 1862 drto_div = 1; 1863 1864 drto_ms = DIV_ROUND_UP_ULL((u64)MSEC_PER_SEC * drto_clks * drto_div, 1865 host->bus_hz); 1866 1867 dev_dbg(host->dev, "drto_ms: %u\n", drto_ms); 1868 1869 /* add a bit spare time */ 1870 drto_ms += 10; 1871 1872 spin_lock_irqsave(&host->irq_lock, irqflags); 1873 if (!test_bit(EVENT_DATA_COMPLETE, &host->pending_events)) 1874 mod_timer(&host->dto_timer, 1875 jiffies + msecs_to_jiffies(drto_ms)); 1876 spin_unlock_irqrestore(&host->irq_lock, irqflags); 1877 } 1878 1879 static bool dw_mci_clear_pending_cmd_complete(struct dw_mci *host) 1880 { 1881 if (!test_bit(EVENT_CMD_COMPLETE, &host->pending_events)) 1882 return false; 1883 1884 /* 1885 * Really be certain that the timer has stopped. This is a bit of 1886 * paranoia and could only really happen if we had really bad 1887 * interrupt latency and the interrupt routine and timeout were 1888 * running concurrently so that the timer_delete() in the interrupt 1889 * handler couldn't run. 1890 */ 1891 WARN_ON(timer_delete_sync(&host->cto_timer)); 1892 clear_bit(EVENT_CMD_COMPLETE, &host->pending_events); 1893 1894 return true; 1895 } 1896 1897 static bool dw_mci_clear_pending_data_complete(struct dw_mci *host) 1898 { 1899 if (!test_bit(EVENT_DATA_COMPLETE, &host->pending_events)) 1900 return false; 1901 1902 /* Extra paranoia just like dw_mci_clear_pending_cmd_complete() */ 1903 WARN_ON(timer_delete_sync(&host->dto_timer)); 1904 clear_bit(EVENT_DATA_COMPLETE, &host->pending_events); 1905 1906 return true; 1907 } 1908 1909 static void dw_mci_work_func(struct work_struct *t) 1910 { 1911 struct dw_mci *host = from_work(host, t, bh_work); 1912 struct mmc_data *data; 1913 struct mmc_command *cmd; 1914 struct mmc_request *mrq; 1915 enum dw_mci_state state; 1916 enum dw_mci_state prev_state; 1917 unsigned int err; 1918 1919 spin_lock(&host->lock); 1920 1921 state = host->state; 1922 data = host->data; 1923 mrq = host->mrq; 1924 1925 do { 1926 prev_state = state; 1927 1928 switch (state) { 1929 case STATE_IDLE: 1930 case STATE_WAITING_CMD11_DONE: 1931 break; 1932 1933 case STATE_SENDING_CMD11: 1934 case STATE_SENDING_CMD: 1935 if (!dw_mci_clear_pending_cmd_complete(host)) 1936 break; 1937 1938 cmd = host->cmd; 1939 host->cmd = NULL; 1940 set_bit(EVENT_CMD_COMPLETE, &host->completed_events); 1941 err = dw_mci_command_complete(host, cmd); 1942 if (cmd == mrq->sbc && !err) { 1943 dw_mci_start_request(host, mrq->cmd); 1944 goto unlock; 1945 } 1946 1947 if (cmd->data && err) { 1948 /* 1949 * During UHS tuning sequence, sending the stop 1950 * command after the response CRC error would 1951 * throw the system into a confused state 1952 * causing all future tuning phases to report 1953 * failure. 1954 * 1955 * In such case controller will move into a data 1956 * transfer state after a response error or 1957 * response CRC error. Let's let that finish 1958 * before trying to send a stop, so we'll go to 1959 * STATE_SENDING_DATA. 1960 * 1961 * Although letting the data transfer take place 1962 * will waste a bit of time (we already know 1963 * the command was bad), it can't cause any 1964 * errors since it's possible it would have 1965 * taken place anyway if this bh work got 1966 * delayed. Allowing the transfer to take place 1967 * avoids races and keeps things simple. 1968 */ 1969 if (err != -ETIMEDOUT && 1970 host->dir_status == MMC_DATA_READ) { 1971 state = STATE_SENDING_DATA; 1972 continue; 1973 } 1974 1975 send_stop_abort(host, data); 1976 dw_mci_stop_dma(host); 1977 state = STATE_SENDING_STOP; 1978 break; 1979 } 1980 1981 if (!cmd->data || err) { 1982 dw_mci_request_end(host, mrq); 1983 goto unlock; 1984 } 1985 1986 prev_state = state = STATE_SENDING_DATA; 1987 fallthrough; 1988 1989 case STATE_SENDING_DATA: 1990 /* 1991 * We could get a data error and never a transfer 1992 * complete so we'd better check for it here. 1993 * 1994 * Note that we don't really care if we also got a 1995 * transfer complete; stopping the DMA and sending an 1996 * abort won't hurt. 1997 */ 1998 if (test_and_clear_bit(EVENT_DATA_ERROR, 1999 &host->pending_events)) { 2000 if (!(host->data_status & (SDMMC_INT_DRTO | 2001 SDMMC_INT_EBE))) 2002 send_stop_abort(host, data); 2003 dw_mci_stop_dma(host); 2004 state = STATE_DATA_ERROR; 2005 break; 2006 } 2007 2008 if (!test_and_clear_bit(EVENT_XFER_COMPLETE, 2009 &host->pending_events)) { 2010 /* 2011 * If all data-related interrupts don't come 2012 * within the given time in reading data state. 2013 */ 2014 if (host->dir_status == MMC_DATA_READ) 2015 dw_mci_set_drto(host); 2016 break; 2017 } 2018 2019 set_bit(EVENT_XFER_COMPLETE, &host->completed_events); 2020 2021 /* 2022 * Handle an EVENT_DATA_ERROR that might have shown up 2023 * before the transfer completed. This might not have 2024 * been caught by the check above because the interrupt 2025 * could have gone off between the previous check and 2026 * the check for transfer complete. 2027 * 2028 * Technically this ought not be needed assuming we 2029 * get a DATA_COMPLETE eventually (we'll notice the 2030 * error and end the request), but it shouldn't hurt. 2031 * 2032 * This has the advantage of sending the stop command. 2033 */ 2034 if (test_and_clear_bit(EVENT_DATA_ERROR, 2035 &host->pending_events)) { 2036 if (!(host->data_status & (SDMMC_INT_DRTO | 2037 SDMMC_INT_EBE))) 2038 send_stop_abort(host, data); 2039 dw_mci_stop_dma(host); 2040 state = STATE_DATA_ERROR; 2041 break; 2042 } 2043 prev_state = state = STATE_DATA_BUSY; 2044 2045 fallthrough; 2046 2047 case STATE_DATA_BUSY: 2048 if (!dw_mci_clear_pending_data_complete(host)) { 2049 /* 2050 * If data error interrupt comes but data over 2051 * interrupt doesn't come within the given time. 2052 * in reading data state. 2053 */ 2054 if (host->dir_status == MMC_DATA_READ) 2055 dw_mci_set_drto(host); 2056 break; 2057 } 2058 2059 dw_mci_stop_fault_timer(host); 2060 dw_mci_stop_dma(host); 2061 host->data = NULL; 2062 set_bit(EVENT_DATA_COMPLETE, &host->completed_events); 2063 err = dw_mci_data_complete(host, data); 2064 2065 if (!err) { 2066 if (!data->stop || mrq->sbc) { 2067 if (mrq->sbc && data->stop) 2068 data->stop->error = 0; 2069 dw_mci_request_end(host, mrq); 2070 goto unlock; 2071 } 2072 2073 /* stop command for open-ended transfer*/ 2074 if (data->stop) 2075 send_stop_abort(host, data); 2076 } else { 2077 /* 2078 * If we don't have a command complete now we'll 2079 * never get one since we just reset everything; 2080 * better end the request. 2081 * 2082 * If we do have a command complete we'll fall 2083 * through to the SENDING_STOP command and 2084 * everything will be peachy keen. 2085 */ 2086 if (!test_bit(EVENT_CMD_COMPLETE, 2087 &host->pending_events)) { 2088 host->cmd = NULL; 2089 dw_mci_request_end(host, mrq); 2090 goto unlock; 2091 } 2092 } 2093 2094 /* 2095 * If err has non-zero, 2096 * stop-abort command has been already issued. 2097 */ 2098 prev_state = state = STATE_SENDING_STOP; 2099 2100 fallthrough; 2101 2102 case STATE_SENDING_STOP: 2103 if (!dw_mci_clear_pending_cmd_complete(host)) 2104 break; 2105 2106 /* CMD error in data command */ 2107 if (mrq->cmd->error && mrq->data) 2108 dw_mci_reset(host); 2109 2110 dw_mci_stop_fault_timer(host); 2111 host->cmd = NULL; 2112 host->data = NULL; 2113 2114 if (!mrq->sbc && mrq->stop) 2115 dw_mci_command_complete(host, mrq->stop); 2116 else 2117 host->cmd_status = 0; 2118 2119 dw_mci_request_end(host, mrq); 2120 goto unlock; 2121 2122 case STATE_DATA_ERROR: 2123 if (!test_and_clear_bit(EVENT_XFER_COMPLETE, 2124 &host->pending_events)) 2125 break; 2126 2127 state = STATE_DATA_BUSY; 2128 break; 2129 } 2130 } while (state != prev_state); 2131 2132 host->state = state; 2133 unlock: 2134 spin_unlock(&host->lock); 2135 2136 } 2137 2138 /* push final bytes to part_buf, only use during push */ 2139 static void dw_mci_set_part_bytes(struct dw_mci *host, void *buf, int cnt) 2140 { 2141 memcpy((void *)&host->part_buf, buf, cnt); 2142 host->part_buf_count = cnt; 2143 } 2144 2145 /* append bytes to part_buf, only use during push */ 2146 static int dw_mci_push_part_bytes(struct dw_mci *host, void *buf, int cnt) 2147 { 2148 cnt = min(cnt, (1 << host->data_shift) - host->part_buf_count); 2149 memcpy((void *)&host->part_buf + host->part_buf_count, buf, cnt); 2150 host->part_buf_count += cnt; 2151 return cnt; 2152 } 2153 2154 /* pull first bytes from part_buf, only use during pull */ 2155 static int dw_mci_pull_part_bytes(struct dw_mci *host, void *buf, int cnt) 2156 { 2157 cnt = min_t(int, cnt, host->part_buf_count); 2158 if (cnt) { 2159 memcpy(buf, (void *)&host->part_buf + host->part_buf_start, 2160 cnt); 2161 host->part_buf_count -= cnt; 2162 host->part_buf_start += cnt; 2163 } 2164 return cnt; 2165 } 2166 2167 /* pull final bytes from the part_buf, assuming it's just been filled */ 2168 static void dw_mci_pull_final_bytes(struct dw_mci *host, void *buf, int cnt) 2169 { 2170 memcpy(buf, &host->part_buf, cnt); 2171 host->part_buf_start = cnt; 2172 host->part_buf_count = (1 << host->data_shift) - cnt; 2173 } 2174 2175 static void dw_mci_push_data16(struct dw_mci *host, void *buf, int cnt) 2176 { 2177 struct mmc_data *data = host->data; 2178 int init_cnt = cnt; 2179 2180 /* try and push anything in the part_buf */ 2181 if (unlikely(host->part_buf_count)) { 2182 int len = dw_mci_push_part_bytes(host, buf, cnt); 2183 2184 buf += len; 2185 cnt -= len; 2186 if (host->part_buf_count == 2) { 2187 mci_fifo_writew(host->fifo_reg, host->part_buf16); 2188 host->part_buf_count = 0; 2189 } 2190 } 2191 #ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS 2192 if (unlikely((unsigned long)buf & 0x1)) { 2193 while (cnt >= 2) { 2194 u16 aligned_buf[64]; 2195 int len = min(cnt & -2, (int)sizeof(aligned_buf)); 2196 int items = len >> 1; 2197 int i; 2198 /* memcpy from input buffer into aligned buffer */ 2199 memcpy(aligned_buf, buf, len); 2200 buf += len; 2201 cnt -= len; 2202 /* push data from aligned buffer into fifo */ 2203 for (i = 0; i < items; ++i) 2204 mci_fifo_writew(host->fifo_reg, aligned_buf[i]); 2205 } 2206 } else 2207 #endif 2208 { 2209 u16 *pdata = buf; 2210 2211 for (; cnt >= 2; cnt -= 2) 2212 mci_fifo_writew(host->fifo_reg, *pdata++); 2213 buf = pdata; 2214 } 2215 /* put anything remaining in the part_buf */ 2216 if (cnt) { 2217 dw_mci_set_part_bytes(host, buf, cnt); 2218 /* Push data if we have reached the expected data length */ 2219 if ((data->bytes_xfered + init_cnt) == 2220 (data->blksz * data->blocks)) 2221 mci_fifo_writew(host->fifo_reg, host->part_buf16); 2222 } 2223 } 2224 2225 static void dw_mci_pull_data16(struct dw_mci *host, void *buf, int cnt) 2226 { 2227 #ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS 2228 if (unlikely((unsigned long)buf & 0x1)) { 2229 while (cnt >= 2) { 2230 /* pull data from fifo into aligned buffer */ 2231 u16 aligned_buf[64]; 2232 int len = min(cnt & -2, (int)sizeof(aligned_buf)); 2233 int items = len >> 1; 2234 int i; 2235 2236 for (i = 0; i < items; ++i) 2237 aligned_buf[i] = mci_fifo_readw(host->fifo_reg); 2238 /* memcpy from aligned buffer into output buffer */ 2239 memcpy(buf, aligned_buf, len); 2240 buf += len; 2241 cnt -= len; 2242 } 2243 } else 2244 #endif 2245 { 2246 u16 *pdata = buf; 2247 2248 for (; cnt >= 2; cnt -= 2) 2249 *pdata++ = mci_fifo_readw(host->fifo_reg); 2250 buf = pdata; 2251 } 2252 if (cnt) { 2253 host->part_buf16 = mci_fifo_readw(host->fifo_reg); 2254 dw_mci_pull_final_bytes(host, buf, cnt); 2255 } 2256 } 2257 2258 static void dw_mci_push_data32(struct dw_mci *host, void *buf, int cnt) 2259 { 2260 struct mmc_data *data = host->data; 2261 int init_cnt = cnt; 2262 2263 /* try and push anything in the part_buf */ 2264 if (unlikely(host->part_buf_count)) { 2265 int len = dw_mci_push_part_bytes(host, buf, cnt); 2266 2267 buf += len; 2268 cnt -= len; 2269 if (host->part_buf_count == 4) { 2270 mci_fifo_writel(host->fifo_reg, host->part_buf32); 2271 host->part_buf_count = 0; 2272 } 2273 } 2274 #ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS 2275 if (unlikely((unsigned long)buf & 0x3)) { 2276 while (cnt >= 4) { 2277 u32 aligned_buf[32]; 2278 int len = min(cnt & -4, (int)sizeof(aligned_buf)); 2279 int items = len >> 2; 2280 int i; 2281 /* memcpy from input buffer into aligned buffer */ 2282 memcpy(aligned_buf, buf, len); 2283 buf += len; 2284 cnt -= len; 2285 /* push data from aligned buffer into fifo */ 2286 for (i = 0; i < items; ++i) 2287 mci_fifo_writel(host->fifo_reg, aligned_buf[i]); 2288 } 2289 } else 2290 #endif 2291 { 2292 u32 *pdata = buf; 2293 2294 for (; cnt >= 4; cnt -= 4) 2295 mci_fifo_writel(host->fifo_reg, *pdata++); 2296 buf = pdata; 2297 } 2298 /* put anything remaining in the part_buf */ 2299 if (cnt) { 2300 dw_mci_set_part_bytes(host, buf, cnt); 2301 /* Push data if we have reached the expected data length */ 2302 if ((data->bytes_xfered + init_cnt) == 2303 (data->blksz * data->blocks)) 2304 mci_fifo_writel(host->fifo_reg, host->part_buf32); 2305 } 2306 } 2307 2308 static void dw_mci_pull_data32(struct dw_mci *host, void *buf, int cnt) 2309 { 2310 #ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS 2311 if (unlikely((unsigned long)buf & 0x3)) { 2312 while (cnt >= 4) { 2313 /* pull data from fifo into aligned buffer */ 2314 u32 aligned_buf[32]; 2315 int len = min(cnt & -4, (int)sizeof(aligned_buf)); 2316 int items = len >> 2; 2317 int i; 2318 2319 for (i = 0; i < items; ++i) 2320 aligned_buf[i] = mci_fifo_readl(host->fifo_reg); 2321 /* memcpy from aligned buffer into output buffer */ 2322 memcpy(buf, aligned_buf, len); 2323 buf += len; 2324 cnt -= len; 2325 } 2326 } else 2327 #endif 2328 { 2329 u32 *pdata = buf; 2330 2331 for (; cnt >= 4; cnt -= 4) 2332 *pdata++ = mci_fifo_readl(host->fifo_reg); 2333 buf = pdata; 2334 } 2335 if (cnt) { 2336 host->part_buf32 = mci_fifo_readl(host->fifo_reg); 2337 dw_mci_pull_final_bytes(host, buf, cnt); 2338 } 2339 } 2340 2341 static void dw_mci_push_data64(struct dw_mci *host, void *buf, int cnt) 2342 { 2343 struct mmc_data *data = host->data; 2344 int init_cnt = cnt; 2345 2346 /* try and push anything in the part_buf */ 2347 if (unlikely(host->part_buf_count)) { 2348 int len = dw_mci_push_part_bytes(host, buf, cnt); 2349 2350 buf += len; 2351 cnt -= len; 2352 2353 if (host->part_buf_count == 8) { 2354 mci_fifo_writeq(host->fifo_reg, host->part_buf); 2355 host->part_buf_count = 0; 2356 } 2357 } 2358 #ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS 2359 if (unlikely((unsigned long)buf & 0x7)) { 2360 while (cnt >= 8) { 2361 u64 aligned_buf[16]; 2362 int len = min(cnt & -8, (int)sizeof(aligned_buf)); 2363 int items = len >> 3; 2364 int i; 2365 /* memcpy from input buffer into aligned buffer */ 2366 memcpy(aligned_buf, buf, len); 2367 buf += len; 2368 cnt -= len; 2369 /* push data from aligned buffer into fifo */ 2370 for (i = 0; i < items; ++i) 2371 mci_fifo_writeq(host->fifo_reg, aligned_buf[i]); 2372 } 2373 } else 2374 #endif 2375 { 2376 u64 *pdata = buf; 2377 2378 for (; cnt >= 8; cnt -= 8) 2379 mci_fifo_writeq(host->fifo_reg, *pdata++); 2380 buf = pdata; 2381 } 2382 /* put anything remaining in the part_buf */ 2383 if (cnt) { 2384 dw_mci_set_part_bytes(host, buf, cnt); 2385 /* Push data if we have reached the expected data length */ 2386 if ((data->bytes_xfered + init_cnt) == 2387 (data->blksz * data->blocks)) 2388 mci_fifo_writeq(host->fifo_reg, host->part_buf); 2389 } 2390 } 2391 2392 static void dw_mci_pull_data64(struct dw_mci *host, void *buf, int cnt) 2393 { 2394 #ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS 2395 if (unlikely((unsigned long)buf & 0x7)) { 2396 while (cnt >= 8) { 2397 /* pull data from fifo into aligned buffer */ 2398 u64 aligned_buf[16]; 2399 int len = min(cnt & -8, (int)sizeof(aligned_buf)); 2400 int items = len >> 3; 2401 int i; 2402 2403 for (i = 0; i < items; ++i) 2404 aligned_buf[i] = mci_fifo_readq(host->fifo_reg); 2405 2406 /* memcpy from aligned buffer into output buffer */ 2407 memcpy(buf, aligned_buf, len); 2408 buf += len; 2409 cnt -= len; 2410 } 2411 } else 2412 #endif 2413 { 2414 u64 *pdata = buf; 2415 2416 for (; cnt >= 8; cnt -= 8) 2417 *pdata++ = mci_fifo_readq(host->fifo_reg); 2418 buf = pdata; 2419 } 2420 if (cnt) { 2421 host->part_buf = mci_fifo_readq(host->fifo_reg); 2422 dw_mci_pull_final_bytes(host, buf, cnt); 2423 } 2424 } 2425 2426 static void dw_mci_push_data64_32(struct dw_mci *host, void *buf, int cnt) 2427 { 2428 struct mmc_data *data = host->data; 2429 int init_cnt = cnt; 2430 2431 /* try and push anything in the part_buf */ 2432 if (unlikely(host->part_buf_count)) { 2433 int len = dw_mci_push_part_bytes(host, buf, cnt); 2434 2435 buf += len; 2436 cnt -= len; 2437 2438 if (host->part_buf_count == 8) { 2439 mci_fifo_l_writeq(host->fifo_reg, host->part_buf); 2440 host->part_buf_count = 0; 2441 } 2442 } 2443 #ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS 2444 if (unlikely((unsigned long)buf & 0x7)) { 2445 while (cnt >= 8) { 2446 u64 aligned_buf[16]; 2447 int len = min(cnt & -8, (int)sizeof(aligned_buf)); 2448 int items = len >> 3; 2449 int i; 2450 /* memcpy from input buffer into aligned buffer */ 2451 memcpy(aligned_buf, buf, len); 2452 buf += len; 2453 cnt -= len; 2454 /* push data from aligned buffer into fifo */ 2455 for (i = 0; i < items; ++i) 2456 mci_fifo_l_writeq(host->fifo_reg, aligned_buf[i]); 2457 } 2458 } else 2459 #endif 2460 { 2461 u64 *pdata = buf; 2462 2463 for (; cnt >= 8; cnt -= 8) 2464 mci_fifo_l_writeq(host->fifo_reg, *pdata++); 2465 buf = pdata; 2466 } 2467 /* put anything remaining in the part_buf */ 2468 if (cnt) { 2469 dw_mci_set_part_bytes(host, buf, cnt); 2470 /* Push data if we have reached the expected data length */ 2471 if ((data->bytes_xfered + init_cnt) == 2472 (data->blksz * data->blocks)) 2473 mci_fifo_l_writeq(host->fifo_reg, host->part_buf); 2474 } 2475 } 2476 2477 static void dw_mci_pull_data64_32(struct dw_mci *host, void *buf, int cnt) 2478 { 2479 #ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS 2480 if (unlikely((unsigned long)buf & 0x7)) { 2481 while (cnt >= 8) { 2482 /* pull data from fifo into aligned buffer */ 2483 u64 aligned_buf[16]; 2484 int len = min(cnt & -8, (int)sizeof(aligned_buf)); 2485 int items = len >> 3; 2486 int i; 2487 2488 for (i = 0; i < items; ++i) 2489 aligned_buf[i] = mci_fifo_l_readq(host->fifo_reg); 2490 2491 /* memcpy from aligned buffer into output buffer */ 2492 memcpy(buf, aligned_buf, len); 2493 buf += len; 2494 cnt -= len; 2495 } 2496 } else 2497 #endif 2498 { 2499 u64 *pdata = buf; 2500 2501 for (; cnt >= 8; cnt -= 8) 2502 *pdata++ = mci_fifo_l_readq(host->fifo_reg); 2503 buf = pdata; 2504 } 2505 if (cnt) { 2506 host->part_buf = mci_fifo_l_readq(host->fifo_reg); 2507 dw_mci_pull_final_bytes(host, buf, cnt); 2508 } 2509 } 2510 2511 static void dw_mci_pull_data(struct dw_mci *host, void *buf, int cnt) 2512 { 2513 int len; 2514 2515 /* get remaining partial bytes */ 2516 len = dw_mci_pull_part_bytes(host, buf, cnt); 2517 if (unlikely(len == cnt)) 2518 return; 2519 buf += len; 2520 cnt -= len; 2521 2522 /* get the rest of the data */ 2523 host->pull_data(host, buf, cnt); 2524 } 2525 2526 static void dw_mci_read_data_pio(struct dw_mci *host, bool dto) 2527 { 2528 struct sg_mapping_iter *sg_miter = &host->sg_miter; 2529 void *buf; 2530 unsigned int offset; 2531 struct mmc_data *data = host->data; 2532 int shift = host->data_shift; 2533 u32 status; 2534 unsigned int len; 2535 unsigned int remain, fcnt; 2536 2537 do { 2538 if (!sg_miter_next(sg_miter)) 2539 goto done; 2540 2541 host->sg = sg_miter->piter.sg; 2542 buf = sg_miter->addr; 2543 remain = sg_miter->length; 2544 offset = 0; 2545 2546 do { 2547 fcnt = (SDMMC_GET_FCNT(mci_readl(host, STATUS)) 2548 << shift) + host->part_buf_count; 2549 len = min(remain, fcnt); 2550 if (!len) 2551 break; 2552 dw_mci_pull_data(host, (void *)(buf + offset), len); 2553 data->bytes_xfered += len; 2554 offset += len; 2555 remain -= len; 2556 } while (remain); 2557 2558 sg_miter->consumed = offset; 2559 status = mci_readl(host, MINTSTS); 2560 mci_writel(host, RINTSTS, SDMMC_INT_RXDR); 2561 /* if the RXDR is ready read again */ 2562 } while ((status & SDMMC_INT_RXDR) || 2563 (dto && SDMMC_GET_FCNT(mci_readl(host, STATUS)))); 2564 2565 if (!remain) { 2566 if (!sg_miter_next(sg_miter)) 2567 goto done; 2568 sg_miter->consumed = 0; 2569 } 2570 sg_miter_stop(sg_miter); 2571 return; 2572 2573 done: 2574 sg_miter_stop(sg_miter); 2575 host->sg = NULL; 2576 smp_wmb(); /* drain writebuffer */ 2577 set_bit(EVENT_XFER_COMPLETE, &host->pending_events); 2578 } 2579 2580 static void dw_mci_write_data_pio(struct dw_mci *host) 2581 { 2582 struct sg_mapping_iter *sg_miter = &host->sg_miter; 2583 void *buf; 2584 unsigned int offset; 2585 struct mmc_data *data = host->data; 2586 int shift = host->data_shift; 2587 u32 status; 2588 unsigned int len; 2589 unsigned int fifo_depth = host->fifo_depth; 2590 unsigned int remain, fcnt; 2591 2592 do { 2593 if (!sg_miter_next(sg_miter)) 2594 goto done; 2595 2596 host->sg = sg_miter->piter.sg; 2597 buf = sg_miter->addr; 2598 remain = sg_miter->length; 2599 offset = 0; 2600 2601 do { 2602 fcnt = ((fifo_depth - 2603 SDMMC_GET_FCNT(mci_readl(host, STATUS))) 2604 << shift) - host->part_buf_count; 2605 len = min(remain, fcnt); 2606 if (!len) 2607 break; 2608 host->push_data(host, (void *)(buf + offset), len); 2609 data->bytes_xfered += len; 2610 offset += len; 2611 remain -= len; 2612 } while (remain); 2613 2614 sg_miter->consumed = offset; 2615 status = mci_readl(host, MINTSTS); 2616 mci_writel(host, RINTSTS, SDMMC_INT_TXDR); 2617 } while (status & SDMMC_INT_TXDR); /* if TXDR write again */ 2618 2619 if (!remain) { 2620 if (!sg_miter_next(sg_miter)) 2621 goto done; 2622 sg_miter->consumed = 0; 2623 } 2624 sg_miter_stop(sg_miter); 2625 return; 2626 2627 done: 2628 sg_miter_stop(sg_miter); 2629 host->sg = NULL; 2630 smp_wmb(); /* drain writebuffer */ 2631 set_bit(EVENT_XFER_COMPLETE, &host->pending_events); 2632 } 2633 2634 static void dw_mci_cmd_interrupt(struct dw_mci *host, u32 status) 2635 { 2636 timer_delete(&host->cto_timer); 2637 2638 if (!host->cmd_status) 2639 host->cmd_status = status; 2640 2641 smp_wmb(); /* drain writebuffer */ 2642 2643 set_bit(EVENT_CMD_COMPLETE, &host->pending_events); 2644 queue_work(system_bh_wq, &host->bh_work); 2645 2646 dw_mci_start_fault_timer(host); 2647 } 2648 2649 static void dw_mci_handle_cd(struct dw_mci *host) 2650 { 2651 mmc_detect_change(host->mmc, 2652 msecs_to_jiffies(host->detect_delay_ms)); 2653 } 2654 2655 static irqreturn_t dw_mci_interrupt(int irq, void *dev_id) 2656 { 2657 struct dw_mci *host = dev_id; 2658 u32 pending; 2659 2660 pending = mci_readl(host, MINTSTS); /* read-only mask reg */ 2661 2662 if (pending) { 2663 /* Check volt switch first, since it can look like an error */ 2664 if ((host->state == STATE_SENDING_CMD11) && 2665 (pending & SDMMC_INT_VOLT_SWITCH)) { 2666 mci_writel(host, RINTSTS, SDMMC_INT_VOLT_SWITCH); 2667 pending &= ~SDMMC_INT_VOLT_SWITCH; 2668 2669 /* 2670 * Hold the lock; we know cmd11_timer can't be kicked 2671 * off after the lock is released, so safe to delete. 2672 */ 2673 spin_lock(&host->irq_lock); 2674 dw_mci_cmd_interrupt(host, pending); 2675 spin_unlock(&host->irq_lock); 2676 2677 timer_delete(&host->cmd11_timer); 2678 } 2679 2680 if (pending & DW_MCI_CMD_ERROR_FLAGS) { 2681 spin_lock(&host->irq_lock); 2682 2683 timer_delete(&host->cto_timer); 2684 mci_writel(host, RINTSTS, DW_MCI_CMD_ERROR_FLAGS); 2685 host->cmd_status = pending; 2686 smp_wmb(); /* drain writebuffer */ 2687 set_bit(EVENT_CMD_COMPLETE, &host->pending_events); 2688 2689 spin_unlock(&host->irq_lock); 2690 } 2691 2692 if (pending & DW_MCI_DATA_ERROR_FLAGS) { 2693 spin_lock(&host->irq_lock); 2694 2695 if (host->quirks & DW_MMC_QUIRK_EXTENDED_TMOUT) 2696 timer_delete(&host->dto_timer); 2697 2698 /* if there is an error report DATA_ERROR */ 2699 mci_writel(host, RINTSTS, DW_MCI_DATA_ERROR_FLAGS); 2700 host->data_status = pending; 2701 smp_wmb(); /* drain writebuffer */ 2702 set_bit(EVENT_DATA_ERROR, &host->pending_events); 2703 2704 if (host->quirks & DW_MMC_QUIRK_EXTENDED_TMOUT) 2705 /* In case of error, we cannot expect a DTO */ 2706 set_bit(EVENT_DATA_COMPLETE, 2707 &host->pending_events); 2708 2709 queue_work(system_bh_wq, &host->bh_work); 2710 2711 spin_unlock(&host->irq_lock); 2712 } 2713 2714 if (pending & SDMMC_INT_DATA_OVER) { 2715 spin_lock(&host->irq_lock); 2716 2717 timer_delete(&host->dto_timer); 2718 2719 mci_writel(host, RINTSTS, SDMMC_INT_DATA_OVER); 2720 if (!host->data_status) 2721 host->data_status = pending; 2722 smp_wmb(); /* drain writebuffer */ 2723 if (host->dir_status == MMC_DATA_READ) { 2724 if (host->sg != NULL) 2725 dw_mci_read_data_pio(host, true); 2726 } 2727 set_bit(EVENT_DATA_COMPLETE, &host->pending_events); 2728 queue_work(system_bh_wq, &host->bh_work); 2729 2730 spin_unlock(&host->irq_lock); 2731 } 2732 2733 if (pending & SDMMC_INT_RXDR) { 2734 mci_writel(host, RINTSTS, SDMMC_INT_RXDR); 2735 if (host->dir_status == MMC_DATA_READ && host->sg) 2736 dw_mci_read_data_pio(host, false); 2737 } 2738 2739 if (pending & SDMMC_INT_TXDR) { 2740 mci_writel(host, RINTSTS, SDMMC_INT_TXDR); 2741 if (host->dir_status == MMC_DATA_WRITE && host->sg) 2742 dw_mci_write_data_pio(host); 2743 } 2744 2745 if (pending & SDMMC_INT_CMD_DONE) { 2746 spin_lock(&host->irq_lock); 2747 2748 mci_writel(host, RINTSTS, SDMMC_INT_CMD_DONE); 2749 dw_mci_cmd_interrupt(host, pending); 2750 2751 spin_unlock(&host->irq_lock); 2752 } 2753 2754 if (pending & SDMMC_INT_CD) { 2755 mci_writel(host, RINTSTS, SDMMC_INT_CD); 2756 dw_mci_handle_cd(host); 2757 } 2758 2759 if (pending & SDMMC_INT_SDIO(host->sdio_irq)) { 2760 mci_writel(host, RINTSTS, 2761 SDMMC_INT_SDIO(host->sdio_irq)); 2762 __dw_mci_enable_sdio_irq(host, 0); 2763 sdio_signal_irq(host->mmc); 2764 } 2765 2766 } 2767 2768 if (host->use_dma != TRANS_MODE_IDMAC) 2769 return IRQ_HANDLED; 2770 2771 /* Handle IDMA interrupts */ 2772 if (host->dma_64bit_address == 1) { 2773 pending = mci_readl(host, IDSTS64); 2774 if (pending & (SDMMC_IDMAC_INT_TI | SDMMC_IDMAC_INT_RI)) { 2775 mci_writel(host, IDSTS64, SDMMC_IDMAC_INT_TI | 2776 SDMMC_IDMAC_INT_RI); 2777 mci_writel(host, IDSTS64, SDMMC_IDMAC_INT_NI); 2778 if (!test_bit(EVENT_DATA_ERROR, &host->pending_events)) 2779 host->dma_ops->complete((void *)host); 2780 } 2781 } else { 2782 pending = mci_readl(host, IDSTS); 2783 if (pending & (SDMMC_IDMAC_INT_TI | SDMMC_IDMAC_INT_RI)) { 2784 mci_writel(host, IDSTS, SDMMC_IDMAC_INT_TI | 2785 SDMMC_IDMAC_INT_RI); 2786 mci_writel(host, IDSTS, SDMMC_IDMAC_INT_NI); 2787 if (!test_bit(EVENT_DATA_ERROR, &host->pending_events)) 2788 host->dma_ops->complete((void *)host); 2789 } 2790 } 2791 2792 return IRQ_HANDLED; 2793 } 2794 2795 static int dw_mci_init_host_caps(struct dw_mci *host) 2796 { 2797 const struct dw_mci_drv_data *drv_data = host->drv_data; 2798 struct mmc_host *mmc = host->mmc; 2799 int ctrl_id; 2800 2801 if (drv_data) 2802 mmc->caps |= drv_data->common_caps; 2803 2804 if (host->dev->of_node) 2805 ctrl_id = mmc->index; 2806 else 2807 ctrl_id = to_platform_device(host->dev)->id; 2808 2809 if (drv_data && drv_data->caps) { 2810 if (ctrl_id >= drv_data->num_caps) { 2811 dev_err(host->dev, "invalid controller id %d\n", 2812 ctrl_id); 2813 return -EINVAL; 2814 } 2815 mmc->caps |= drv_data->caps[ctrl_id]; 2816 } 2817 2818 /* if host has set a minimum_freq, we should respect it */ 2819 if (host->minimum_speed) 2820 mmc->f_min = host->minimum_speed; 2821 else 2822 mmc->f_min = DW_MCI_FREQ_MIN; 2823 2824 if (!mmc->f_max) 2825 mmc->f_max = DW_MCI_FREQ_MAX; 2826 2827 /* Process SDIO IRQs through the sdio_irq_work. */ 2828 if (mmc->caps & MMC_CAP_SDIO_IRQ) 2829 mmc->caps2 |= MMC_CAP2_SDIO_IRQ_NOTHREAD; 2830 2831 return 0; 2832 } 2833 2834 static int dw_mci_init_host(struct dw_mci *host) 2835 { 2836 struct mmc_host *mmc = host->mmc; 2837 int ret; 2838 2839 mmc->ops = &dw_mci_ops; 2840 2841 /*if there are external regulators, get them*/ 2842 ret = mmc_regulator_get_supply(mmc); 2843 if (ret) 2844 return ret; 2845 2846 if (!mmc->ocr_avail) 2847 mmc->ocr_avail = MMC_VDD_32_33 | MMC_VDD_33_34; 2848 2849 ret = mmc_of_parse(mmc); 2850 if (ret) 2851 return ret; 2852 2853 mmc_of_parse_clk_phase(host->dev, &host->phase_map); 2854 2855 ret = dw_mci_init_host_caps(host); 2856 if (ret) 2857 return ret; 2858 2859 /* Useful defaults if platform data is unset. */ 2860 if (host->use_dma == TRANS_MODE_IDMAC) { 2861 mmc->max_segs = host->desc_num; 2862 mmc->max_blk_size = 65535; 2863 mmc->max_seg_size = 0x1000; 2864 mmc->max_req_size = mmc->max_seg_size * host->desc_num; 2865 mmc->max_blk_count = mmc->max_req_size / 512; 2866 } else if (host->use_dma == TRANS_MODE_EDMAC) { 2867 mmc->max_segs = 64; 2868 mmc->max_blk_size = 65535; 2869 mmc->max_blk_count = 65535; 2870 mmc->max_req_size = 2871 mmc->max_blk_size * mmc->max_blk_count; 2872 mmc->max_seg_size = mmc->max_req_size; 2873 } else { 2874 /* TRANS_MODE_PIO */ 2875 mmc->max_segs = 64; 2876 mmc->max_blk_size = 65535; /* BLKSIZ is 16 bits */ 2877 mmc->max_blk_count = 512; 2878 mmc->max_req_size = mmc->max_blk_size * 2879 mmc->max_blk_count; 2880 mmc->max_seg_size = mmc->max_req_size; 2881 } 2882 2883 if (mmc->caps & MMC_CAP_NEEDS_POLL) 2884 dev_info(&mmc->class_dev, "card is polling.\n"); 2885 else if (!mmc_card_is_removable(mmc)) 2886 dev_info(&mmc->class_dev, "card is non-removable.\n"); 2887 2888 dw_mci_get_cd(mmc); 2889 2890 ret = mmc_add_host(mmc); 2891 if (ret) 2892 return ret; 2893 2894 #if defined(CONFIG_DEBUG_FS) 2895 dw_mci_init_debugfs(host); 2896 #endif 2897 2898 return 0; 2899 } 2900 2901 static void dw_mci_cleanup_host(struct dw_mci *host) 2902 { 2903 /* Debugfs stuff is cleaned up by mmc core */ 2904 mmc_remove_host(host->mmc); 2905 } 2906 2907 static void dw_mci_init_dma(struct dw_mci *host) 2908 { 2909 int addr_config; 2910 struct device *dev = host->dev; 2911 2912 /* 2913 * Check tansfer mode from HCON[17:16] 2914 * Clear the ambiguous description of dw_mmc databook: 2915 * 2b'00: No DMA Interface -> Actually means using Internal DMA block 2916 * 2b'01: DesignWare DMA Interface -> Synopsys DW-DMA block 2917 * 2b'10: Generic DMA Interface -> non-Synopsys generic DMA block 2918 * 2b'11: Non DW DMA Interface -> pio only 2919 * Compared to DesignWare DMA Interface, Generic DMA Interface has a 2920 * simpler request/acknowledge handshake mechanism and both of them 2921 * are regarded as external dma master for dw_mmc. 2922 */ 2923 host->use_dma = SDMMC_GET_TRANS_MODE(mci_readl(host, HCON)); 2924 if (host->use_dma == DMA_INTERFACE_IDMA) { 2925 host->use_dma = TRANS_MODE_IDMAC; 2926 } else if (host->use_dma == DMA_INTERFACE_DWDMA || 2927 host->use_dma == DMA_INTERFACE_GDMA) { 2928 host->use_dma = TRANS_MODE_EDMAC; 2929 } else { 2930 goto no_dma; 2931 } 2932 2933 /* Determine which DMA interface to use */ 2934 if (host->use_dma == TRANS_MODE_IDMAC) { 2935 /* 2936 * Check ADDR_CONFIG bit in HCON to find 2937 * IDMAC address bus width 2938 */ 2939 addr_config = SDMMC_GET_ADDR_CONFIG(mci_readl(host, HCON)); 2940 2941 if (addr_config == 1) { 2942 /* host supports IDMAC in 64-bit address mode */ 2943 host->dma_64bit_address = 1; 2944 dev_info(host->dev, 2945 "IDMAC supports 64-bit address mode.\n"); 2946 if (dma_set_mask_and_coherent(host->dev, DMA_BIT_MASK(64))) 2947 dev_info(host->dev, "Fail to set 64-bit DMA mask"); 2948 } else { 2949 /* host supports IDMAC in 32-bit address mode */ 2950 host->dma_64bit_address = 0; 2951 dev_info(host->dev, 2952 "IDMAC supports 32-bit address mode.\n"); 2953 } 2954 2955 /* Alloc memory for sg translation */ 2956 host->sg_cpu = dmam_alloc_coherent(host->dev, 2957 DESC_RING_BUF_SZ, 2958 &host->sg_dma, GFP_KERNEL); 2959 if (!host->sg_cpu) { 2960 dev_err(host->dev, 2961 "%s: could not alloc DMA memory\n", 2962 __func__); 2963 goto no_dma; 2964 } 2965 2966 host->dma_ops = &dw_mci_idmac_ops; 2967 dev_info(host->dev, "Using internal DMA controller.\n"); 2968 } else { 2969 /* TRANS_MODE_EDMAC: check dma bindings again */ 2970 if ((device_property_string_array_count(dev, "dma-names") < 0) || 2971 !device_property_present(dev, "dmas")) { 2972 goto no_dma; 2973 } 2974 host->dma_ops = &dw_mci_edmac_ops; 2975 dev_info(host->dev, "Using external DMA controller.\n"); 2976 } 2977 2978 if (host->dma_ops->init && host->dma_ops->start && 2979 host->dma_ops->stop && host->dma_ops->cleanup) { 2980 if (host->dma_ops->init(host)) { 2981 dev_err(host->dev, "%s: Unable to initialize DMA Controller.\n", 2982 __func__); 2983 goto no_dma; 2984 } 2985 } else { 2986 dev_err(host->dev, "DMA initialization not found.\n"); 2987 goto no_dma; 2988 } 2989 2990 return; 2991 2992 no_dma: 2993 dev_info(host->dev, "Using PIO mode.\n"); 2994 host->use_dma = TRANS_MODE_PIO; 2995 } 2996 2997 static void dw_mci_cmd11_timer(struct timer_list *t) 2998 { 2999 struct dw_mci *host = timer_container_of(host, t, cmd11_timer); 3000 3001 if (host->state != STATE_SENDING_CMD11) { 3002 dev_warn(host->dev, "Unexpected CMD11 timeout\n"); 3003 return; 3004 } 3005 3006 host->cmd_status = SDMMC_INT_RTO; 3007 set_bit(EVENT_CMD_COMPLETE, &host->pending_events); 3008 queue_work(system_bh_wq, &host->bh_work); 3009 } 3010 3011 static void dw_mci_cto_timer(struct timer_list *t) 3012 { 3013 struct dw_mci *host = timer_container_of(host, t, cto_timer); 3014 unsigned long irqflags; 3015 u32 pending; 3016 3017 spin_lock_irqsave(&host->irq_lock, irqflags); 3018 3019 /* 3020 * If somehow we have very bad interrupt latency it's remotely possible 3021 * that the timer could fire while the interrupt is still pending or 3022 * while the interrupt is midway through running. Let's be paranoid 3023 * and detect those two cases. Note that this is paranoia is somewhat 3024 * justified because in this function we don't actually cancel the 3025 * pending command in the controller--we just assume it will never come. 3026 */ 3027 pending = mci_readl(host, MINTSTS); /* read-only mask reg */ 3028 if (pending & (DW_MCI_CMD_ERROR_FLAGS | SDMMC_INT_CMD_DONE)) { 3029 /* The interrupt should fire; no need to act but we can warn */ 3030 dev_warn(host->dev, "Unexpected interrupt latency\n"); 3031 goto exit; 3032 } 3033 if (test_bit(EVENT_CMD_COMPLETE, &host->pending_events)) { 3034 /* Presumably interrupt handler couldn't delete the timer */ 3035 dev_warn(host->dev, "CTO timeout when already completed\n"); 3036 goto exit; 3037 } 3038 3039 /* 3040 * Continued paranoia to make sure we're in the state we expect. 3041 * This paranoia isn't really justified but it seems good to be safe. 3042 */ 3043 switch (host->state) { 3044 case STATE_SENDING_CMD11: 3045 case STATE_SENDING_CMD: 3046 case STATE_SENDING_STOP: 3047 /* 3048 * If CMD_DONE interrupt does NOT come in sending command 3049 * state, we should notify the driver to terminate current 3050 * transfer and report a command timeout to the core. 3051 */ 3052 host->cmd_status = SDMMC_INT_RTO; 3053 set_bit(EVENT_CMD_COMPLETE, &host->pending_events); 3054 queue_work(system_bh_wq, &host->bh_work); 3055 break; 3056 default: 3057 dev_warn(host->dev, "Unexpected command timeout, state %d\n", 3058 host->state); 3059 break; 3060 } 3061 3062 exit: 3063 spin_unlock_irqrestore(&host->irq_lock, irqflags); 3064 } 3065 3066 static void dw_mci_dto_timer(struct timer_list *t) 3067 { 3068 struct dw_mci *host = timer_container_of(host, t, dto_timer); 3069 unsigned long irqflags; 3070 u32 pending; 3071 3072 spin_lock_irqsave(&host->irq_lock, irqflags); 3073 3074 /* 3075 * The DTO timer is much longer than the CTO timer, so it's even less 3076 * likely that we'll these cases, but it pays to be paranoid. 3077 */ 3078 pending = mci_readl(host, MINTSTS); /* read-only mask reg */ 3079 if (pending & SDMMC_INT_DATA_OVER) { 3080 /* The interrupt should fire; no need to act but we can warn */ 3081 dev_warn(host->dev, "Unexpected data interrupt latency\n"); 3082 goto exit; 3083 } 3084 if (test_bit(EVENT_DATA_COMPLETE, &host->pending_events)) { 3085 /* Presumably interrupt handler couldn't delete the timer */ 3086 dev_warn(host->dev, "DTO timeout when already completed\n"); 3087 goto exit; 3088 } 3089 3090 /* 3091 * Continued paranoia to make sure we're in the state we expect. 3092 * This paranoia isn't really justified but it seems good to be safe. 3093 */ 3094 switch (host->state) { 3095 case STATE_SENDING_DATA: 3096 case STATE_DATA_BUSY: 3097 /* 3098 * If DTO interrupt does NOT come in sending data state, 3099 * we should notify the driver to terminate current transfer 3100 * and report a data timeout to the core. 3101 */ 3102 host->data_status = SDMMC_INT_DRTO; 3103 set_bit(EVENT_DATA_ERROR, &host->pending_events); 3104 set_bit(EVENT_DATA_COMPLETE, &host->pending_events); 3105 queue_work(system_bh_wq, &host->bh_work); 3106 break; 3107 default: 3108 dev_warn(host->dev, "Unexpected data timeout, state %d\n", 3109 host->state); 3110 break; 3111 } 3112 3113 exit: 3114 spin_unlock_irqrestore(&host->irq_lock, irqflags); 3115 } 3116 3117 static int dw_mci_parse_dt(struct dw_mci *host) 3118 { 3119 struct device *dev = host->dev; 3120 const struct dw_mci_drv_data *drv_data = host->drv_data; 3121 int ret; 3122 u32 clock_frequency; 3123 3124 /* find reset controller when exist */ 3125 host->rstc = devm_reset_control_get_optional_exclusive(dev, "reset"); 3126 if (IS_ERR(host->rstc)) 3127 return PTR_ERR(host->rstc); 3128 3129 if (!host->fifo_depth && device_property_read_u32(dev, "fifo-depth", &host->fifo_depth)) 3130 dev_info(dev, 3131 "fifo-depth property not found, using value of FIFOTH register as default\n"); 3132 3133 if (!host->detect_delay_ms) 3134 device_property_read_u32(dev, "card-detect-delay", 3135 &host->detect_delay_ms); 3136 3137 if (!host->data_addr_override) 3138 device_property_read_u32(dev, "data-addr", &host->data_addr_override); 3139 3140 if (device_property_present(dev, "fifo-watermark-aligned")) 3141 host->wm_aligned = true; 3142 3143 if (!host->bus_hz && !device_property_read_u32(dev, "clock-frequency", &clock_frequency)) 3144 host->bus_hz = clock_frequency; 3145 3146 if (drv_data && drv_data->parse_dt) { 3147 ret = drv_data->parse_dt(host); 3148 if (ret) 3149 return ret; 3150 } 3151 3152 return 0; 3153 } 3154 3155 static void dw_mci_enable_cd(struct dw_mci *host) 3156 { 3157 unsigned long irqflags; 3158 u32 temp; 3159 3160 /* 3161 * No need for CD if host has a non-error GPIO 3162 * as well as broken card detection is found. 3163 */ 3164 if (host->mmc->caps & MMC_CAP_NEEDS_POLL) 3165 return; 3166 3167 if (mmc_gpio_get_cd(host->mmc) < 0) { 3168 spin_lock_irqsave(&host->irq_lock, irqflags); 3169 temp = mci_readl(host, INTMASK); 3170 temp |= SDMMC_INT_CD; 3171 mci_writel(host, INTMASK, temp); 3172 spin_unlock_irqrestore(&host->irq_lock, irqflags); 3173 } 3174 } 3175 3176 struct dw_mci *dw_mci_alloc_host(struct device *dev) 3177 { 3178 struct mmc_host *mmc; 3179 struct dw_mci *host; 3180 3181 mmc = devm_mmc_alloc_host(dev, sizeof(struct dw_mci)); 3182 if (!mmc) 3183 return ERR_PTR(-ENOMEM); 3184 3185 host = mmc_priv(mmc); 3186 host->mmc = mmc; 3187 host->dev = dev; 3188 host->dma_threshold = 16; 3189 3190 return host; 3191 } 3192 EXPORT_SYMBOL(dw_mci_alloc_host); 3193 3194 int dw_mci_probe(struct dw_mci *host) 3195 { 3196 const struct dw_mci_drv_data *drv_data = host->drv_data; 3197 int width, i, ret = 0; 3198 u32 fifo_size; 3199 3200 ret = dw_mci_parse_dt(host); 3201 if (ret) 3202 return dev_err_probe(host->dev, ret, "parse dt failed\n"); 3203 3204 host->biu_clk = devm_clk_get(host->dev, "biu"); 3205 if (IS_ERR(host->biu_clk)) { 3206 dev_dbg(host->dev, "biu clock not available\n"); 3207 ret = PTR_ERR(host->biu_clk); 3208 if (ret == -EPROBE_DEFER) 3209 return ret; 3210 3211 } else { 3212 ret = clk_prepare_enable(host->biu_clk); 3213 if (ret) { 3214 dev_err(host->dev, "failed to enable biu clock\n"); 3215 return ret; 3216 } 3217 } 3218 3219 host->ciu_clk = devm_clk_get(host->dev, "ciu"); 3220 if (IS_ERR(host->ciu_clk)) { 3221 dev_dbg(host->dev, "ciu clock not available\n"); 3222 ret = PTR_ERR(host->ciu_clk); 3223 if (ret == -EPROBE_DEFER) 3224 goto err_clk_biu; 3225 } else { 3226 ret = clk_prepare_enable(host->ciu_clk); 3227 if (ret) { 3228 dev_err(host->dev, "failed to enable ciu clock\n"); 3229 goto err_clk_biu; 3230 } 3231 3232 if (host->bus_hz) { 3233 ret = clk_set_rate(host->ciu_clk, host->bus_hz); 3234 if (ret) 3235 dev_warn(host->dev, 3236 "Unable to set bus rate to %uHz\n", 3237 host->bus_hz); 3238 } 3239 host->bus_hz = clk_get_rate(host->ciu_clk); 3240 } 3241 3242 if (!host->bus_hz) { 3243 dev_err(host->dev, 3244 "Platform data must supply bus speed\n"); 3245 ret = -ENODEV; 3246 goto err_clk_ciu; 3247 } 3248 3249 if (host->rstc) { 3250 reset_control_assert(host->rstc); 3251 usleep_range(10, 50); 3252 reset_control_deassert(host->rstc); 3253 } 3254 3255 if (drv_data && drv_data->init) { 3256 ret = drv_data->init(host); 3257 if (ret) { 3258 dev_err(host->dev, 3259 "implementation specific init failed\n"); 3260 goto err_clk_ciu; 3261 } 3262 } 3263 3264 timer_setup(&host->cmd11_timer, dw_mci_cmd11_timer, 0); 3265 timer_setup(&host->cto_timer, dw_mci_cto_timer, 0); 3266 timer_setup(&host->dto_timer, dw_mci_dto_timer, 0); 3267 3268 spin_lock_init(&host->lock); 3269 spin_lock_init(&host->irq_lock); 3270 3271 dw_mci_init_fault(host); 3272 3273 /* 3274 * Get the host data width - this assumes that HCON has been set with 3275 * the correct values. 3276 */ 3277 i = SDMMC_GET_HDATA_WIDTH(mci_readl(host, HCON)); 3278 if (!i) { 3279 host->push_data = dw_mci_push_data16; 3280 host->pull_data = dw_mci_pull_data16; 3281 width = 16; 3282 host->data_shift = 1; 3283 } else if (i == 2) { 3284 if ((host->quirks & DW_MMC_QUIRK_FIFO64_32)) { 3285 host->push_data = dw_mci_push_data64_32; 3286 host->pull_data = dw_mci_pull_data64_32; 3287 } else { 3288 host->push_data = dw_mci_push_data64; 3289 host->pull_data = dw_mci_pull_data64; 3290 } 3291 width = 64; 3292 host->data_shift = 3; 3293 } else { 3294 /* Check for a reserved value, and warn if it is */ 3295 WARN((i != 1), 3296 "HCON reports a reserved host data width!\n" 3297 "Defaulting to 32-bit access.\n"); 3298 host->push_data = dw_mci_push_data32; 3299 host->pull_data = dw_mci_pull_data32; 3300 width = 32; 3301 host->data_shift = 2; 3302 } 3303 3304 /* Reset all blocks */ 3305 if (!dw_mci_ctrl_reset(host, SDMMC_CTRL_ALL_RESET_FLAGS)) { 3306 ret = -ENODEV; 3307 goto err_clk_ciu; 3308 } 3309 3310 dw_mci_init_dma(host); 3311 3312 /* Clear the interrupts for the host controller */ 3313 mci_writel(host, RINTSTS, 0xFFFFFFFF); 3314 mci_writel(host, INTMASK, 0); /* disable all mmc interrupt first */ 3315 3316 /* Put in max timeout */ 3317 mci_writel(host, TMOUT, 0xFFFFFFFF); 3318 3319 /* 3320 * FIFO threshold settings RxMark = fifo_size / 2 - 1, 3321 * Tx Mark = fifo_size / 2 DMA Size = 8 3322 */ 3323 if (!host->fifo_depth) { 3324 /* 3325 * Power-on value of RX_WMark is FIFO_DEPTH-1, but this may 3326 * have been overwritten by the bootloader, just like we're 3327 * about to do, so if you know the value for your hardware, you 3328 * should put it in the platform data. 3329 */ 3330 fifo_size = mci_readl(host, FIFOTH); 3331 fifo_size = 1 + ((fifo_size >> 16) & 0xfff); 3332 } else { 3333 fifo_size = host->fifo_depth; 3334 } 3335 host->fifo_depth = fifo_size; 3336 host->fifoth_val = 3337 SDMMC_SET_FIFOTH(0x2, fifo_size / 2 - 1, fifo_size / 2); 3338 mci_writel(host, FIFOTH, host->fifoth_val); 3339 3340 /* disable clock to CIU */ 3341 mci_writel(host, CLKENA, 0); 3342 mci_writel(host, CLKSRC, 0); 3343 3344 /* 3345 * In 2.40a spec, Data offset is changed. 3346 * Need to check the version-id and set data-offset for DATA register. 3347 */ 3348 host->verid = SDMMC_GET_VERID(mci_readl(host, VERID)); 3349 dev_info(host->dev, "Version ID is %04x\n", host->verid); 3350 3351 if (host->data_addr_override) 3352 host->fifo_reg = host->regs + host->data_addr_override; 3353 else if (host->verid < DW_MMC_240A) 3354 host->fifo_reg = host->regs + DATA_OFFSET; 3355 else 3356 host->fifo_reg = host->regs + DATA_240A_OFFSET; 3357 3358 INIT_WORK(&host->bh_work, dw_mci_work_func); 3359 ret = devm_request_irq(host->dev, host->irq, dw_mci_interrupt, 3360 host->irq_flags, "dw-mci", host); 3361 if (ret) 3362 goto err_dmaunmap; 3363 3364 /* 3365 * Enable interrupts for command done, data over, data empty, 3366 * receive ready and error such as transmit, receive timeout, crc error 3367 */ 3368 mci_writel(host, INTMASK, SDMMC_INT_CMD_DONE | SDMMC_INT_DATA_OVER | 3369 SDMMC_INT_TXDR | SDMMC_INT_RXDR | 3370 DW_MCI_ERROR_FLAGS); 3371 /* Enable mci interrupt */ 3372 mci_writel(host, CTRL, SDMMC_CTRL_INT_ENABLE); 3373 3374 dev_info(host->dev, 3375 "DW MMC controller at irq %d,%d bit host data width,%u deep fifo\n", 3376 host->irq, width, fifo_size); 3377 3378 ret = dw_mci_init_host(host); 3379 if (ret) { 3380 dev_dbg(host->dev, "host init failed\n"); 3381 goto err_dmaunmap; 3382 } 3383 3384 /* Now that host is setup, we can enable card detect */ 3385 dw_mci_enable_cd(host); 3386 3387 return 0; 3388 3389 err_dmaunmap: 3390 if (host->use_dma && host->dma_ops->exit) 3391 host->dma_ops->exit(host); 3392 3393 reset_control_assert(host->rstc); 3394 3395 err_clk_ciu: 3396 clk_disable_unprepare(host->ciu_clk); 3397 3398 err_clk_biu: 3399 clk_disable_unprepare(host->biu_clk); 3400 3401 return ret; 3402 } 3403 EXPORT_SYMBOL(dw_mci_probe); 3404 3405 void dw_mci_remove(struct dw_mci *host) 3406 { 3407 dev_dbg(host->dev, "remove host\n"); 3408 dw_mci_cleanup_host(host); 3409 3410 mci_writel(host, RINTSTS, 0xFFFFFFFF); 3411 mci_writel(host, INTMASK, 0); /* disable all mmc interrupt first */ 3412 3413 /* disable clock to CIU */ 3414 mci_writel(host, CLKENA, 0); 3415 mci_writel(host, CLKSRC, 0); 3416 3417 if (host->use_dma && host->dma_ops->exit) 3418 host->dma_ops->exit(host); 3419 3420 reset_control_assert(host->rstc); 3421 3422 clk_disable_unprepare(host->ciu_clk); 3423 clk_disable_unprepare(host->biu_clk); 3424 } 3425 EXPORT_SYMBOL(dw_mci_remove); 3426 3427 int dw_mci_runtime_suspend(struct device *dev) 3428 { 3429 struct dw_mci *host = dev_get_drvdata(dev); 3430 3431 if (host->use_dma && host->dma_ops->exit) 3432 host->dma_ops->exit(host); 3433 3434 clk_disable_unprepare(host->ciu_clk); 3435 3436 if (mmc_host_can_gpio_cd(host->mmc) || 3437 !mmc_card_is_removable(host->mmc)) 3438 clk_disable_unprepare(host->biu_clk); 3439 3440 return 0; 3441 } 3442 EXPORT_SYMBOL(dw_mci_runtime_suspend); 3443 3444 int dw_mci_runtime_resume(struct device *dev) 3445 { 3446 int ret = 0; 3447 struct dw_mci *host = dev_get_drvdata(dev); 3448 3449 if (mmc_host_can_gpio_cd(host->mmc) || 3450 !mmc_card_is_removable(host->mmc)) { 3451 ret = clk_prepare_enable(host->biu_clk); 3452 if (ret) 3453 return ret; 3454 } 3455 3456 ret = clk_prepare_enable(host->ciu_clk); 3457 if (ret) 3458 goto err; 3459 3460 if (!dw_mci_ctrl_reset(host, SDMMC_CTRL_ALL_RESET_FLAGS)) { 3461 clk_disable_unprepare(host->ciu_clk); 3462 ret = -ENODEV; 3463 goto err; 3464 } 3465 3466 if (host->use_dma && host->dma_ops->init) { 3467 ret = host->dma_ops->init(host); 3468 if (ret) 3469 return ret; 3470 } 3471 3472 /* 3473 * Restore the initial value at FIFOTH register 3474 * And Invalidate the prev_blksz with zero 3475 */ 3476 mci_writel(host, FIFOTH, host->fifoth_val); 3477 host->prev_blksz = 0; 3478 3479 /* Put in max timeout */ 3480 mci_writel(host, TMOUT, 0xFFFFFFFF); 3481 3482 mci_writel(host, RINTSTS, 0xFFFFFFFF); 3483 mci_writel(host, INTMASK, SDMMC_INT_CMD_DONE | SDMMC_INT_DATA_OVER | 3484 SDMMC_INT_TXDR | SDMMC_INT_RXDR | 3485 DW_MCI_ERROR_FLAGS); 3486 mci_writel(host, CTRL, SDMMC_CTRL_INT_ENABLE); 3487 3488 3489 if (host->mmc->pm_flags & MMC_PM_KEEP_POWER) 3490 dw_mci_set_ios(host->mmc, &host->mmc->ios); 3491 3492 /* Force setup bus to guarantee available clock output */ 3493 dw_mci_setup_bus(host, true); 3494 3495 /* Re-enable SDIO interrupts. */ 3496 if (sdio_irq_claimed(host->mmc)) 3497 __dw_mci_enable_sdio_irq(host, 1); 3498 3499 /* Now that host is setup, we can enable card detect */ 3500 dw_mci_enable_cd(host); 3501 3502 return 0; 3503 3504 err: 3505 if (mmc_host_can_gpio_cd(host->mmc) || 3506 !mmc_card_is_removable(host->mmc)) 3507 clk_disable_unprepare(host->biu_clk); 3508 3509 return ret; 3510 } 3511 EXPORT_SYMBOL(dw_mci_runtime_resume); 3512 3513 const struct dev_pm_ops dw_mci_pmops = { 3514 SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, pm_runtime_force_resume) 3515 RUNTIME_PM_OPS(dw_mci_runtime_suspend, dw_mci_runtime_resume, NULL) 3516 }; 3517 EXPORT_SYMBOL_GPL(dw_mci_pmops); 3518 3519 MODULE_DESCRIPTION("DW Multimedia Card Interface driver"); 3520 MODULE_AUTHOR("NXP Semiconductor VietNam"); 3521 MODULE_AUTHOR("Imagination Technologies Ltd"); 3522 MODULE_LICENSE("GPL v2"); 3523