1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Synopsys DesignWare Multimedia Card Interface driver 4 * (Based on NXP driver for lpc 31xx) 5 * 6 * Copyright (C) 2009 NXP Semiconductors 7 * Copyright (C) 2009, 2010 Imagination Technologies Ltd. 8 */ 9 10 #include <linux/bitops.h> 11 #include <linux/clk.h> 12 #include <linux/debugfs.h> 13 #include <linux/delay.h> 14 #include <linux/device.h> 15 #include <linux/dma-mapping.h> 16 #include <linux/err.h> 17 #include <linux/interrupt.h> 18 #include <linux/iopoll.h> 19 #include <linux/irq.h> 20 #include <linux/ktime.h> 21 #include <linux/mmc/card.h> 22 #include <linux/mmc/host.h> 23 #include <linux/mmc/mmc.h> 24 #include <linux/mmc/sd.h> 25 #include <linux/mmc/sdio.h> 26 #include <linux/mmc/slot-gpio.h> 27 #include <linux/module.h> 28 #include <linux/of.h> 29 #include <linux/platform_device.h> 30 #include <linux/pm_runtime.h> 31 #include <linux/regulator/consumer.h> 32 33 #include "dw_mmc.h" 34 35 /* Common flag combinations */ 36 #define DW_MCI_DATA_ERROR_FLAGS (SDMMC_INT_DRTO | SDMMC_INT_DCRC | \ 37 SDMMC_INT_HTO | SDMMC_INT_SBE | \ 38 SDMMC_INT_EBE | SDMMC_INT_HLE) 39 #define DW_MCI_CMD_ERROR_FLAGS (SDMMC_INT_RTO | SDMMC_INT_RCRC | \ 40 SDMMC_INT_RESP_ERR | SDMMC_INT_HLE) 41 #define DW_MCI_ERROR_FLAGS (DW_MCI_DATA_ERROR_FLAGS | \ 42 DW_MCI_CMD_ERROR_FLAGS) 43 #define DW_MCI_DMA_THRESHOLD 16 44 45 #define DW_MCI_FREQ_MAX 200000000 /* unit: HZ */ 46 #define DW_MCI_FREQ_MIN 100000 /* unit: HZ */ 47 48 #define IDMAC_INT_CLR (SDMMC_IDMAC_INT_AI | SDMMC_IDMAC_INT_NI | \ 49 SDMMC_IDMAC_INT_CES | SDMMC_IDMAC_INT_DU | \ 50 SDMMC_IDMAC_INT_FBE | SDMMC_IDMAC_INT_RI | \ 51 SDMMC_IDMAC_INT_TI) 52 53 #define DESC_RING_BUF_SZ PAGE_SIZE 54 55 struct idmac_desc_64addr { 56 u32 des0; /* Control Descriptor */ 57 #define IDMAC_OWN_CLR64(x) \ 58 !((x) & cpu_to_le32(IDMAC_DES0_OWN)) 59 60 u32 des1; /* Reserved */ 61 62 u32 des2; /*Buffer sizes */ 63 #define IDMAC_64ADDR_SET_BUFFER1_SIZE(d, s) \ 64 ((d)->des2 = ((d)->des2 & cpu_to_le32(0x03ffe000)) | \ 65 ((cpu_to_le32(s)) & cpu_to_le32(0x1fff))) 66 67 u32 des3; /* Reserved */ 68 69 u32 des4; /* Lower 32-bits of Buffer Address Pointer 1*/ 70 u32 des5; /* Upper 32-bits of Buffer Address Pointer 1*/ 71 72 u32 des6; /* Lower 32-bits of Next Descriptor Address */ 73 u32 des7; /* Upper 32-bits of Next Descriptor Address */ 74 }; 75 76 struct idmac_desc { 77 __le32 des0; /* Control Descriptor */ 78 #define IDMAC_DES0_DIC BIT(1) 79 #define IDMAC_DES0_LD BIT(2) 80 #define IDMAC_DES0_FD BIT(3) 81 #define IDMAC_DES0_CH BIT(4) 82 #define IDMAC_DES0_ER BIT(5) 83 #define IDMAC_DES0_CES BIT(30) 84 #define IDMAC_DES0_OWN BIT(31) 85 86 __le32 des1; /* Buffer sizes */ 87 #define IDMAC_SET_BUFFER1_SIZE(d, s) \ 88 ((d)->des1 = ((d)->des1 & cpu_to_le32(0x03ffe000)) | (cpu_to_le32((s) & 0x1fff))) 89 90 __le32 des2; /* buffer 1 physical address */ 91 92 __le32 des3; /* buffer 2 physical address */ 93 }; 94 95 /* Each descriptor can transfer up to 4KB of data in chained mode */ 96 #define DW_MCI_DESC_DATA_LENGTH 0x1000 97 98 #if defined(CONFIG_DEBUG_FS) 99 static int dw_mci_req_show(struct seq_file *s, void *v) 100 { 101 struct dw_mci *host = s->private; 102 struct mmc_request *mrq; 103 struct mmc_command *cmd; 104 struct mmc_command *stop; 105 struct mmc_data *data; 106 107 /* Make sure we get a consistent snapshot */ 108 spin_lock_bh(&host->lock); 109 mrq = host->mrq; 110 111 if (mrq) { 112 cmd = mrq->cmd; 113 data = mrq->data; 114 stop = mrq->stop; 115 116 if (cmd) 117 seq_printf(s, 118 "CMD%u(0x%x) flg %x rsp %x %x %x %x err %d\n", 119 cmd->opcode, cmd->arg, cmd->flags, 120 cmd->resp[0], cmd->resp[1], cmd->resp[2], 121 cmd->resp[2], cmd->error); 122 if (data) 123 seq_printf(s, "DATA %u / %u * %u flg %x err %d\n", 124 data->bytes_xfered, data->blocks, 125 data->blksz, data->flags, data->error); 126 if (stop) 127 seq_printf(s, 128 "CMD%u(0x%x) flg %x rsp %x %x %x %x err %d\n", 129 stop->opcode, stop->arg, stop->flags, 130 stop->resp[0], stop->resp[1], stop->resp[2], 131 stop->resp[2], stop->error); 132 } 133 134 spin_unlock_bh(&host->lock); 135 136 return 0; 137 } 138 DEFINE_SHOW_ATTRIBUTE(dw_mci_req); 139 140 static int dw_mci_regs_show(struct seq_file *s, void *v) 141 { 142 struct dw_mci *host = s->private; 143 144 pm_runtime_get_sync(host->dev); 145 146 seq_printf(s, "STATUS:\t0x%08x\n", mci_readl(host, STATUS)); 147 seq_printf(s, "RINTSTS:\t0x%08x\n", mci_readl(host, RINTSTS)); 148 seq_printf(s, "CMD:\t0x%08x\n", mci_readl(host, CMD)); 149 seq_printf(s, "CTRL:\t0x%08x\n", mci_readl(host, CTRL)); 150 seq_printf(s, "INTMASK:\t0x%08x\n", mci_readl(host, INTMASK)); 151 seq_printf(s, "CLKENA:\t0x%08x\n", mci_readl(host, CLKENA)); 152 153 pm_runtime_put_autosuspend(host->dev); 154 155 return 0; 156 } 157 DEFINE_SHOW_ATTRIBUTE(dw_mci_regs); 158 159 static void dw_mci_init_debugfs(struct dw_mci *host) 160 { 161 struct mmc_host *mmc = host->mmc; 162 struct dentry *root; 163 164 root = mmc->debugfs_root; 165 if (!root) 166 return; 167 168 debugfs_create_file("regs", 0400, root, host, &dw_mci_regs_fops); 169 debugfs_create_file("req", 0400, root, host, &dw_mci_req_fops); 170 debugfs_create_u32("state", 0400, root, &host->state); 171 debugfs_create_xul("pending_events", 0400, root, 172 &host->pending_events); 173 debugfs_create_xul("completed_events", 0400, root, 174 &host->completed_events); 175 #ifdef CONFIG_FAULT_INJECTION 176 fault_create_debugfs_attr("fail_data_crc", root, &host->fail_data_crc); 177 #endif 178 } 179 #endif /* defined(CONFIG_DEBUG_FS) */ 180 181 static bool dw_mci_ctrl_reset(struct dw_mci *host, u32 reset) 182 { 183 u32 ctrl; 184 185 ctrl = mci_readl(host, CTRL); 186 ctrl |= reset; 187 mci_writel(host, CTRL, ctrl); 188 189 /* wait till resets clear */ 190 if (readl_poll_timeout_atomic(host->regs + SDMMC_CTRL, ctrl, 191 !(ctrl & reset), 192 1, 500 * USEC_PER_MSEC)) { 193 dev_err(host->dev, 194 "Timeout resetting block (ctrl reset %#x)\n", 195 ctrl & reset); 196 return false; 197 } 198 199 return true; 200 } 201 202 static void dw_mci_wait_while_busy(struct dw_mci *host, u32 cmd_flags) 203 { 204 u32 status; 205 206 /* 207 * Databook says that before issuing a new data transfer command 208 * we need to check to see if the card is busy. Data transfer commands 209 * all have SDMMC_CMD_PRV_DAT_WAIT set, so we'll key off that. 210 * 211 * ...also allow sending for SDMMC_CMD_VOLT_SWITCH where busy is 212 * expected. 213 */ 214 if ((cmd_flags & SDMMC_CMD_PRV_DAT_WAIT) && 215 !(cmd_flags & SDMMC_CMD_VOLT_SWITCH)) { 216 if (readl_poll_timeout_atomic(host->regs + SDMMC_STATUS, 217 status, 218 !(status & SDMMC_STATUS_BUSY), 219 10, 500 * USEC_PER_MSEC)) 220 dev_err(host->dev, "Busy; trying anyway\n"); 221 } 222 } 223 224 static void mci_send_cmd(struct dw_mci *host, u32 cmd, u32 arg) 225 { 226 unsigned int cmd_status = 0; 227 228 mci_writel(host, CMDARG, arg); 229 wmb(); /* drain writebuffer */ 230 dw_mci_wait_while_busy(host, cmd); 231 mci_writel(host, CMD, SDMMC_CMD_START | cmd); 232 233 if (readl_poll_timeout_atomic(host->regs + SDMMC_CMD, cmd_status, 234 !(cmd_status & SDMMC_CMD_START), 235 1, 500 * USEC_PER_MSEC)) 236 dev_err(&host->mmc->class_dev, 237 "Timeout sending command (cmd %#x arg %#x status %#x)\n", 238 cmd, arg, cmd_status); 239 } 240 241 static u32 dw_mci_prepare_command(struct mmc_host *mmc, struct mmc_command *cmd) 242 { 243 struct dw_mci *host = mmc_priv(mmc); 244 u32 cmdr; 245 246 cmd->error = -EINPROGRESS; 247 cmdr = cmd->opcode; 248 249 if (cmd->opcode == MMC_STOP_TRANSMISSION || 250 cmd->opcode == MMC_GO_IDLE_STATE || 251 cmd->opcode == MMC_GO_INACTIVE_STATE || 252 (cmd->opcode == SD_IO_RW_DIRECT && 253 ((cmd->arg >> 9) & 0x1FFFF) == SDIO_CCCR_ABORT)) 254 cmdr |= SDMMC_CMD_STOP; 255 else if (cmd->opcode != MMC_SEND_STATUS && cmd->data) 256 cmdr |= SDMMC_CMD_PRV_DAT_WAIT; 257 258 if (cmd->opcode == SD_SWITCH_VOLTAGE) { 259 u32 clk_en_a; 260 261 /* Special bit makes CMD11 not die */ 262 cmdr |= SDMMC_CMD_VOLT_SWITCH; 263 264 /* Change state to continue to handle CMD11 weirdness */ 265 WARN_ON(host->state != STATE_SENDING_CMD); 266 host->state = STATE_SENDING_CMD11; 267 268 /* 269 * We need to disable low power mode (automatic clock stop) 270 * while doing voltage switch so we don't confuse the card, 271 * since stopping the clock is a specific part of the UHS 272 * voltage change dance. 273 * 274 * Note that low power mode (SDMMC_CLKEN_LOW_PWR) will be 275 * unconditionally turned back on in dw_mci_setup_bus() if it's 276 * ever called with a non-zero clock. That shouldn't happen 277 * until the voltage change is all done. 278 */ 279 clk_en_a = mci_readl(host, CLKENA); 280 clk_en_a &= ~SDMMC_CLKEN_LOW_PWR; 281 mci_writel(host, CLKENA, clk_en_a); 282 mci_send_cmd(host, SDMMC_CMD_UPD_CLK | 283 SDMMC_CMD_PRV_DAT_WAIT, 0); 284 } 285 286 if (cmd->flags & MMC_RSP_PRESENT) { 287 /* We expect a response, so set this bit */ 288 cmdr |= SDMMC_CMD_RESP_EXP; 289 if (cmd->flags & MMC_RSP_136) 290 cmdr |= SDMMC_CMD_RESP_LONG; 291 } 292 293 if (cmd->flags & MMC_RSP_CRC) 294 cmdr |= SDMMC_CMD_RESP_CRC; 295 296 if (cmd->data) { 297 cmdr |= SDMMC_CMD_DAT_EXP; 298 if (cmd->data->flags & MMC_DATA_WRITE) 299 cmdr |= SDMMC_CMD_DAT_WR; 300 } 301 302 if (!test_bit(DW_MMC_CARD_NO_USE_HOLD, &host->flags)) 303 cmdr |= SDMMC_CMD_USE_HOLD_REG; 304 305 return cmdr; 306 } 307 308 static u32 dw_mci_prep_stop_abort(struct dw_mci *host, struct mmc_command *cmd) 309 { 310 struct mmc_command *stop; 311 u32 cmdr; 312 313 if (!cmd->data) 314 return 0; 315 316 stop = &host->stop_abort; 317 cmdr = cmd->opcode; 318 memset(stop, 0, sizeof(struct mmc_command)); 319 320 if (cmdr == MMC_READ_SINGLE_BLOCK || 321 cmdr == MMC_READ_MULTIPLE_BLOCK || 322 cmdr == MMC_WRITE_BLOCK || 323 cmdr == MMC_WRITE_MULTIPLE_BLOCK || 324 mmc_op_tuning(cmdr) || 325 cmdr == MMC_GEN_CMD) { 326 stop->opcode = MMC_STOP_TRANSMISSION; 327 stop->arg = 0; 328 stop->flags = MMC_RSP_R1B | MMC_CMD_AC; 329 } else if (cmdr == SD_IO_RW_EXTENDED) { 330 stop->opcode = SD_IO_RW_DIRECT; 331 stop->arg |= (1 << 31) | (0 << 28) | (SDIO_CCCR_ABORT << 9) | 332 ((cmd->arg >> 28) & 0x7); 333 stop->flags = MMC_RSP_SPI_R5 | MMC_RSP_R5 | MMC_CMD_AC; 334 } else { 335 return 0; 336 } 337 338 cmdr = stop->opcode | SDMMC_CMD_STOP | 339 SDMMC_CMD_RESP_CRC | SDMMC_CMD_RESP_EXP; 340 341 if (!test_bit(DW_MMC_CARD_NO_USE_HOLD, &host->flags)) 342 cmdr |= SDMMC_CMD_USE_HOLD_REG; 343 344 return cmdr; 345 } 346 347 static inline void dw_mci_set_cto(struct dw_mci *host) 348 { 349 unsigned int cto_clks; 350 unsigned int cto_div; 351 unsigned int cto_ms; 352 unsigned long irqflags; 353 354 cto_clks = mci_readl(host, TMOUT) & 0xff; 355 cto_div = (mci_readl(host, CLKDIV) & 0xff) * 2; 356 if (cto_div == 0) 357 cto_div = 1; 358 359 cto_ms = DIV_ROUND_UP_ULL((u64)MSEC_PER_SEC * cto_clks * cto_div, 360 host->bus_hz); 361 362 /* add a bit spare time */ 363 cto_ms += 10; 364 365 /* 366 * The durations we're working with are fairly short so we have to be 367 * extra careful about synchronization here. Specifically in hardware a 368 * command timeout is _at most_ 5.1 ms, so that means we expect an 369 * interrupt (either command done or timeout) to come rather quickly 370 * after the mci_writel. ...but just in case we have a long interrupt 371 * latency let's add a bit of paranoia. 372 * 373 * In general we'll assume that at least an interrupt will be asserted 374 * in hardware by the time the cto_timer runs. ...and if it hasn't 375 * been asserted in hardware by that time then we'll assume it'll never 376 * come. 377 */ 378 spin_lock_irqsave(&host->irq_lock, irqflags); 379 if (!test_bit(EVENT_CMD_COMPLETE, &host->pending_events)) 380 mod_timer(&host->cto_timer, 381 jiffies + msecs_to_jiffies(cto_ms) + 1); 382 spin_unlock_irqrestore(&host->irq_lock, irqflags); 383 } 384 385 static void dw_mci_start_command(struct dw_mci *host, 386 struct mmc_command *cmd, u32 cmd_flags) 387 { 388 host->cmd = cmd; 389 dev_vdbg(host->dev, 390 "start command: ARGR=0x%08x CMDR=0x%08x\n", 391 cmd->arg, cmd_flags); 392 393 mci_writel(host, CMDARG, cmd->arg); 394 wmb(); /* drain writebuffer */ 395 dw_mci_wait_while_busy(host, cmd_flags); 396 397 mci_writel(host, CMD, cmd_flags | SDMMC_CMD_START); 398 399 /* response expected command only */ 400 if (cmd_flags & SDMMC_CMD_RESP_EXP) 401 dw_mci_set_cto(host); 402 } 403 404 static inline void send_stop_abort(struct dw_mci *host, struct mmc_data *data) 405 { 406 struct mmc_command *stop = &host->stop_abort; 407 408 dw_mci_start_command(host, stop, host->stop_cmdr); 409 } 410 411 /* DMA interface functions */ 412 static void dw_mci_stop_dma(struct dw_mci *host) 413 { 414 if (host->using_dma) { 415 host->dma_ops->stop(host); 416 host->dma_ops->cleanup(host); 417 } 418 419 /* Data transfer was stopped by the interrupt handler */ 420 set_bit(EVENT_XFER_COMPLETE, &host->pending_events); 421 } 422 423 static void dw_mci_dma_cleanup(struct dw_mci *host) 424 { 425 struct mmc_data *data = host->data; 426 427 if (data && data->host_cookie == COOKIE_MAPPED) { 428 dma_unmap_sg(host->dev, 429 data->sg, 430 data->sg_len, 431 mmc_get_dma_dir(data)); 432 data->host_cookie = COOKIE_UNMAPPED; 433 } 434 } 435 436 static void dw_mci_idmac_reset(struct dw_mci *host) 437 { 438 u32 bmod = mci_readl(host, BMOD); 439 /* Software reset of DMA */ 440 bmod |= SDMMC_IDMAC_SWRESET; 441 mci_writel(host, BMOD, bmod); 442 } 443 444 static void dw_mci_idmac_stop_dma(struct dw_mci *host) 445 { 446 u32 temp; 447 448 /* Disable and reset the IDMAC interface */ 449 temp = mci_readl(host, CTRL); 450 temp &= ~SDMMC_CTRL_USE_IDMAC; 451 temp |= SDMMC_CTRL_DMA_RESET; 452 mci_writel(host, CTRL, temp); 453 454 /* Stop the IDMAC running */ 455 temp = mci_readl(host, BMOD); 456 temp &= ~(SDMMC_IDMAC_ENABLE | SDMMC_IDMAC_FB); 457 temp |= SDMMC_IDMAC_SWRESET; 458 mci_writel(host, BMOD, temp); 459 } 460 461 static void dw_mci_dmac_complete_dma(void *arg) 462 { 463 struct dw_mci *host = arg; 464 struct mmc_data *data = host->data; 465 466 dev_vdbg(host->dev, "DMA complete\n"); 467 468 if ((host->use_dma == TRANS_MODE_EDMAC) && 469 data && (data->flags & MMC_DATA_READ)) 470 /* Invalidate cache after read */ 471 dma_sync_sg_for_cpu(mmc_dev(host->mmc), 472 data->sg, 473 data->sg_len, 474 DMA_FROM_DEVICE); 475 476 host->dma_ops->cleanup(host); 477 478 /* 479 * If the card was removed, data will be NULL. No point in trying to 480 * send the stop command or waiting for NBUSY in this case. 481 */ 482 if (data) { 483 set_bit(EVENT_XFER_COMPLETE, &host->pending_events); 484 queue_work(system_bh_wq, &host->bh_work); 485 } 486 } 487 488 static int dw_mci_idmac_init(struct dw_mci *host) 489 { 490 int i; 491 492 if (host->dma_64bit_address == 1) { 493 struct idmac_desc_64addr *p; 494 /* Number of descriptors in the ring buffer */ 495 host->ring_size = 496 DESC_RING_BUF_SZ / sizeof(struct idmac_desc_64addr); 497 498 /* Forward link the descriptor list */ 499 for (i = 0, p = host->sg_cpu; i < host->ring_size - 1; 500 i++, p++) { 501 p->des6 = (host->sg_dma + 502 (sizeof(struct idmac_desc_64addr) * 503 (i + 1))) & 0xffffffff; 504 505 p->des7 = (u64)(host->sg_dma + 506 (sizeof(struct idmac_desc_64addr) * 507 (i + 1))) >> 32; 508 /* Initialize reserved and buffer size fields to "0" */ 509 p->des0 = 0; 510 p->des1 = 0; 511 p->des2 = 0; 512 p->des3 = 0; 513 } 514 515 /* Set the last descriptor as the end-of-ring descriptor */ 516 p->des6 = host->sg_dma & 0xffffffff; 517 p->des7 = (u64)host->sg_dma >> 32; 518 p->des0 = IDMAC_DES0_ER; 519 520 } else { 521 struct idmac_desc *p; 522 /* Number of descriptors in the ring buffer */ 523 host->ring_size = 524 DESC_RING_BUF_SZ / sizeof(struct idmac_desc); 525 526 /* Forward link the descriptor list */ 527 for (i = 0, p = host->sg_cpu; 528 i < host->ring_size - 1; 529 i++, p++) { 530 p->des3 = cpu_to_le32(host->sg_dma + 531 (sizeof(struct idmac_desc) * (i + 1))); 532 p->des0 = 0; 533 p->des1 = 0; 534 } 535 536 /* Set the last descriptor as the end-of-ring descriptor */ 537 p->des3 = cpu_to_le32(host->sg_dma); 538 p->des0 = cpu_to_le32(IDMAC_DES0_ER); 539 } 540 541 dw_mci_idmac_reset(host); 542 543 if (host->dma_64bit_address == 1) { 544 /* Mask out interrupts - get Tx & Rx complete only */ 545 mci_writel(host, IDSTS64, IDMAC_INT_CLR); 546 mci_writel(host, IDINTEN64, SDMMC_IDMAC_INT_NI | 547 SDMMC_IDMAC_INT_RI | SDMMC_IDMAC_INT_TI); 548 549 /* Set the descriptor base address */ 550 mci_writel(host, DBADDRL, host->sg_dma & 0xffffffff); 551 mci_writel(host, DBADDRU, (u64)host->sg_dma >> 32); 552 553 } else { 554 /* Mask out interrupts - get Tx & Rx complete only */ 555 mci_writel(host, IDSTS, IDMAC_INT_CLR); 556 mci_writel(host, IDINTEN, SDMMC_IDMAC_INT_NI | 557 SDMMC_IDMAC_INT_RI | SDMMC_IDMAC_INT_TI); 558 559 /* Set the descriptor base address */ 560 mci_writel(host, DBADDR, host->sg_dma); 561 } 562 563 return 0; 564 } 565 566 static inline int dw_mci_prepare_desc(struct dw_mci *host, struct mmc_data *data, 567 unsigned int sg_len, bool is_64bit) 568 { 569 unsigned int desc_len; 570 struct idmac_desc *desc_first, *desc_last, *desc; 571 struct idmac_desc_64addr *desc64_first, *desc64_last, *desc64; 572 u32 val, des0; 573 int i, err; 574 575 if (is_64bit) 576 desc64_first = desc64_last = desc64 = host->sg_cpu; 577 else 578 desc_first = desc_last = desc = host->sg_cpu; 579 580 for (i = 0; i < sg_len; i++) { 581 unsigned int length = sg_dma_len(&data->sg[i]); 582 583 u64 mem_addr = sg_dma_address(&data->sg[i]); 584 585 while (length > 0) { 586 desc_len = (length <= DW_MCI_DESC_DATA_LENGTH) ? 587 length : DW_MCI_DESC_DATA_LENGTH; 588 589 length -= desc_len; 590 591 /* 592 * Wait for the former clear OWN bit operation 593 * of IDMAC to make sure that this descriptor 594 * isn't still owned by IDMAC as IDMAC's write 595 * ops and CPU's read ops are asynchronous. 596 */ 597 if (is_64bit) 598 err = readl_poll_timeout_atomic(&desc64->des0, val, 599 IDMAC_OWN_CLR64(val), 10, 100 * USEC_PER_MSEC); 600 else 601 err = readl_poll_timeout_atomic(&desc->des0, val, 602 IDMAC_OWN_CLR64(val), 10, 100 * USEC_PER_MSEC); 603 if (err) 604 goto err_own_bit; 605 606 des0 = IDMAC_DES0_OWN | IDMAC_DES0_DIC | IDMAC_DES0_CH; 607 if (is_64bit) 608 desc64->des0 = des0; 609 else 610 desc->des0 = cpu_to_le32(des0); 611 612 /* 613 * 1. Set OWN bit and disable interrupts for this descriptor 614 * 2. Set Buffer length 615 * Set physical address to DMA to/from 616 */ 617 if (is_64bit) { 618 desc64->des0 = IDMAC_DES0_OWN | IDMAC_DES0_DIC | IDMAC_DES0_CH; 619 IDMAC_64ADDR_SET_BUFFER1_SIZE(desc64, desc_len); 620 desc64->des4 = mem_addr & 0xffffffff; 621 desc64->des5 = mem_addr >> 32; 622 } else { 623 IDMAC_SET_BUFFER1_SIZE(desc, desc_len); 624 desc->des2 = cpu_to_le32(mem_addr); 625 } 626 627 /* Update physical address for the next desc */ 628 mem_addr += desc_len; 629 630 /* Save pointer to the last descriptor */ 631 if (is_64bit) { 632 desc64_last = desc64; 633 desc64++; 634 } else { 635 desc_last = desc; 636 desc++; 637 } 638 } 639 } 640 641 /* Set the first descriptor and the last descriptor */ 642 if (is_64bit) { 643 desc64_first->des0 |= IDMAC_DES0_FD; 644 desc64_last->des0 &= ~(IDMAC_DES0_CH | IDMAC_DES0_DIC); 645 desc64_last->des0 |= IDMAC_DES0_LD; 646 } else { 647 desc_first->des0 |= cpu_to_le32(IDMAC_DES0_FD); 648 desc_last->des0 &= cpu_to_le32(~(IDMAC_DES0_CH | IDMAC_DES0_DIC)); 649 desc_last->des0 |= cpu_to_le32(IDMAC_DES0_LD); 650 } 651 652 return 0; 653 err_own_bit: 654 /* restore the descriptor chain as it's polluted */ 655 dev_dbg(host->dev, "descriptor is still owned by IDMAC.\n"); 656 memset(host->sg_cpu, 0, DESC_RING_BUF_SZ); 657 dw_mci_idmac_init(host); 658 return -EINVAL; 659 } 660 661 static int dw_mci_idmac_start_dma(struct dw_mci *host, unsigned int sg_len) 662 { 663 u32 temp; 664 int ret; 665 666 ret = dw_mci_prepare_desc(host, host->data, sg_len, host->dma_64bit_address); 667 if (ret) 668 goto out; 669 670 /* drain writebuffer */ 671 wmb(); 672 673 /* Make sure to reset DMA in case we did PIO before this */ 674 dw_mci_ctrl_reset(host, SDMMC_CTRL_DMA_RESET); 675 dw_mci_idmac_reset(host); 676 677 /* Select IDMAC interface */ 678 temp = mci_readl(host, CTRL); 679 temp |= SDMMC_CTRL_USE_IDMAC; 680 mci_writel(host, CTRL, temp); 681 682 /* drain writebuffer */ 683 wmb(); 684 685 /* Enable the IDMAC */ 686 temp = mci_readl(host, BMOD); 687 temp |= SDMMC_IDMAC_ENABLE | SDMMC_IDMAC_FB; 688 mci_writel(host, BMOD, temp); 689 690 /* Start it running */ 691 mci_writel(host, PLDMND, 1); 692 693 out: 694 return ret; 695 } 696 697 static const struct dw_mci_dma_ops dw_mci_idmac_ops = { 698 .init = dw_mci_idmac_init, 699 .start = dw_mci_idmac_start_dma, 700 .stop = dw_mci_idmac_stop_dma, 701 .complete = dw_mci_dmac_complete_dma, 702 .cleanup = dw_mci_dma_cleanup, 703 }; 704 705 static void dw_mci_edmac_stop_dma(struct dw_mci *host) 706 { 707 dmaengine_terminate_async(host->dms->ch); 708 } 709 710 static int dw_mci_edmac_start_dma(struct dw_mci *host, 711 unsigned int sg_len) 712 { 713 struct dma_slave_config cfg; 714 struct dma_async_tx_descriptor *desc = NULL; 715 struct scatterlist *sgl = host->data->sg; 716 static const u32 mszs[] = {1, 4, 8, 16, 32, 64, 128, 256}; 717 u32 sg_elems = host->data->sg_len; 718 u32 fifoth_val; 719 u32 fifo_offset = host->fifo_reg - host->regs; 720 int ret = 0; 721 722 /* Set external dma config: burst size, burst width */ 723 memset(&cfg, 0, sizeof(cfg)); 724 cfg.dst_addr = host->phy_regs + fifo_offset; 725 cfg.src_addr = cfg.dst_addr; 726 cfg.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; 727 cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; 728 729 /* Match burst msize with external dma config */ 730 fifoth_val = mci_readl(host, FIFOTH); 731 cfg.dst_maxburst = mszs[(fifoth_val >> 28) & 0x7]; 732 cfg.src_maxburst = cfg.dst_maxburst; 733 734 if (host->data->flags & MMC_DATA_WRITE) 735 cfg.direction = DMA_MEM_TO_DEV; 736 else 737 cfg.direction = DMA_DEV_TO_MEM; 738 739 ret = dmaengine_slave_config(host->dms->ch, &cfg); 740 if (ret) { 741 dev_err(host->dev, "Failed to config edmac.\n"); 742 return -EBUSY; 743 } 744 745 desc = dmaengine_prep_slave_sg(host->dms->ch, sgl, 746 sg_len, cfg.direction, 747 DMA_PREP_INTERRUPT | DMA_CTRL_ACK); 748 if (!desc) { 749 dev_err(host->dev, "Can't prepare slave sg.\n"); 750 return -EBUSY; 751 } 752 753 /* Set dw_mci_dmac_complete_dma as callback */ 754 desc->callback = dw_mci_dmac_complete_dma; 755 desc->callback_param = (void *)host; 756 dmaengine_submit(desc); 757 758 /* Flush cache before write */ 759 if (host->data->flags & MMC_DATA_WRITE) 760 dma_sync_sg_for_device(mmc_dev(host->mmc), sgl, 761 sg_elems, DMA_TO_DEVICE); 762 763 dma_async_issue_pending(host->dms->ch); 764 765 return 0; 766 } 767 768 static int dw_mci_edmac_init(struct dw_mci *host) 769 { 770 /* Request external dma channel */ 771 host->dms = kzalloc_obj(struct dw_mci_dma_slave); 772 if (!host->dms) 773 return -ENOMEM; 774 775 host->dms->ch = dma_request_chan(host->dev, "rx-tx"); 776 if (IS_ERR(host->dms->ch)) { 777 int ret = PTR_ERR(host->dms->ch); 778 779 dev_err(host->dev, "Failed to get external DMA channel.\n"); 780 kfree(host->dms); 781 host->dms = NULL; 782 return ret; 783 } 784 785 return 0; 786 } 787 788 static void dw_mci_edmac_exit(struct dw_mci *host) 789 { 790 if (host->dms) { 791 if (host->dms->ch) { 792 dma_release_channel(host->dms->ch); 793 host->dms->ch = NULL; 794 } 795 kfree(host->dms); 796 host->dms = NULL; 797 } 798 } 799 800 static const struct dw_mci_dma_ops dw_mci_edmac_ops = { 801 .init = dw_mci_edmac_init, 802 .exit = dw_mci_edmac_exit, 803 .start = dw_mci_edmac_start_dma, 804 .stop = dw_mci_edmac_stop_dma, 805 .complete = dw_mci_dmac_complete_dma, 806 .cleanup = dw_mci_dma_cleanup, 807 }; 808 809 static int dw_mci_pre_dma_transfer(struct dw_mci *host, 810 struct mmc_data *data, 811 int cookie) 812 { 813 struct scatterlist *sg; 814 unsigned int i, sg_len; 815 816 if (data->host_cookie == COOKIE_PRE_MAPPED) 817 return data->sg_len; 818 819 /* 820 * We don't do DMA on "complex" transfers, i.e. with 821 * non-word-aligned buffers or lengths. Also, we don't bother 822 * with all the DMA setup overhead for short transfers. 823 */ 824 if (data->blocks * data->blksz < DW_MCI_DMA_THRESHOLD) 825 return -EINVAL; 826 827 if (data->blksz & 3) 828 return -EINVAL; 829 830 for_each_sg(data->sg, sg, data->sg_len, i) { 831 if (sg->offset & 3 || sg->length & 3) 832 return -EINVAL; 833 } 834 835 sg_len = dma_map_sg(host->dev, 836 data->sg, 837 data->sg_len, 838 mmc_get_dma_dir(data)); 839 if (sg_len == 0) 840 return -EINVAL; 841 842 data->host_cookie = cookie; 843 844 return sg_len; 845 } 846 847 static void dw_mci_pre_req(struct mmc_host *mmc, 848 struct mmc_request *mrq) 849 { 850 struct dw_mci *host = mmc_priv(mmc); 851 struct mmc_data *data = mrq->data; 852 853 if (!host->use_dma || !data) 854 return; 855 856 /* This data might be unmapped at this time */ 857 data->host_cookie = COOKIE_UNMAPPED; 858 859 if (dw_mci_pre_dma_transfer(host, mrq->data, 860 COOKIE_PRE_MAPPED) < 0) 861 data->host_cookie = COOKIE_UNMAPPED; 862 } 863 864 static void dw_mci_post_req(struct mmc_host *mmc, 865 struct mmc_request *mrq, 866 int err) 867 { 868 struct dw_mci *host = mmc_priv(mmc); 869 struct mmc_data *data = mrq->data; 870 871 if (!host->use_dma || !data) 872 return; 873 874 if (data->host_cookie != COOKIE_UNMAPPED) 875 dma_unmap_sg(host->dev, 876 data->sg, 877 data->sg_len, 878 mmc_get_dma_dir(data)); 879 data->host_cookie = COOKIE_UNMAPPED; 880 } 881 882 static int dw_mci_get_cd(struct mmc_host *mmc) 883 { 884 struct dw_mci *host = mmc_priv(mmc); 885 int gpio_cd = mmc_gpio_get_cd(mmc); 886 887 if (mmc->caps & MMC_CAP_NEEDS_POLL) 888 return 1; 889 890 if (!mmc_card_is_removable(mmc)) 891 return 1; 892 893 /* Try slot gpio detection */ 894 if (gpio_cd >= 0) 895 return !!gpio_cd; 896 897 /* Host native card detect */ 898 return !(mci_readl(host, CDETECT) & BIT(0)); 899 } 900 901 static void dw_mci_adjust_fifoth(struct dw_mci *host, struct mmc_data *data) 902 { 903 unsigned int blksz = data->blksz; 904 static const u32 mszs[] = {1, 4, 8, 16, 32, 64, 128, 256}; 905 u32 fifo_width = 1 << host->data_shift; 906 u32 blksz_depth = blksz / fifo_width, fifoth_val; 907 u32 msize = 0, rx_wmark = 1, tx_wmark, tx_wmark_invers; 908 int idx = ARRAY_SIZE(mszs) - 1; 909 910 /* pio should ship this scenario */ 911 if (!host->use_dma) 912 return; 913 914 tx_wmark = (host->fifo_depth) / 2; 915 tx_wmark_invers = host->fifo_depth - tx_wmark; 916 917 /* 918 * MSIZE is '1', 919 * if blksz is not a multiple of the FIFO width 920 */ 921 if (blksz % fifo_width) 922 goto done; 923 924 do { 925 if (!((blksz_depth % mszs[idx]) || 926 (tx_wmark_invers % mszs[idx]))) { 927 msize = idx; 928 rx_wmark = mszs[idx] - 1; 929 break; 930 } 931 } while (--idx > 0); 932 /* 933 * If idx is '0', it won't be tried 934 * Thus, initial values are uesed 935 */ 936 done: 937 fifoth_val = SDMMC_SET_FIFOTH(msize, rx_wmark, tx_wmark); 938 mci_writel(host, FIFOTH, fifoth_val); 939 } 940 941 static void dw_mci_ctrl_thld(struct dw_mci *host, struct mmc_data *data) 942 { 943 unsigned int blksz = data->blksz; 944 u32 blksz_depth, fifo_depth; 945 u16 thld_size; 946 u8 enable; 947 948 /* 949 * CDTHRCTL doesn't exist prior to 240A (in fact that register offset is 950 * in the FIFO region, so we really shouldn't access it). 951 */ 952 if (host->verid < DW_MMC_240A || 953 (host->verid < DW_MMC_280A && data->flags & MMC_DATA_WRITE)) 954 return; 955 956 /* 957 * Card write Threshold is introduced since 2.80a 958 * It's used when HS400 mode is enabled. 959 */ 960 if (data->flags & MMC_DATA_WRITE && 961 host->timing != MMC_TIMING_MMC_HS400) 962 goto disable; 963 964 if (data->flags & MMC_DATA_WRITE) 965 enable = SDMMC_CARD_WR_THR_EN; 966 else 967 enable = SDMMC_CARD_RD_THR_EN; 968 969 if (host->timing != MMC_TIMING_MMC_HS200 && 970 host->timing != MMC_TIMING_UHS_SDR104 && 971 host->timing != MMC_TIMING_MMC_HS400) 972 goto disable; 973 974 blksz_depth = blksz / (1 << host->data_shift); 975 fifo_depth = host->fifo_depth; 976 977 if (blksz_depth > fifo_depth) 978 goto disable; 979 980 /* 981 * If (blksz_depth) >= (fifo_depth >> 1), should be 'thld_size <= blksz' 982 * If (blksz_depth) < (fifo_depth >> 1), should be thld_size = blksz 983 * Currently just choose blksz. 984 */ 985 thld_size = blksz; 986 mci_writel(host, CDTHRCTL, SDMMC_SET_THLD(thld_size, enable)); 987 return; 988 989 disable: 990 mci_writel(host, CDTHRCTL, 0); 991 } 992 993 static int dw_mci_submit_data_dma(struct dw_mci *host, struct mmc_data *data) 994 { 995 unsigned long irqflags; 996 int sg_len; 997 u32 temp; 998 999 host->using_dma = 0; 1000 1001 /* If we don't have a channel, we can't do DMA */ 1002 if (!host->use_dma) 1003 return -ENODEV; 1004 1005 sg_len = dw_mci_pre_dma_transfer(host, data, COOKIE_MAPPED); 1006 if (sg_len < 0) { 1007 host->dma_ops->stop(host); 1008 return sg_len; 1009 } 1010 1011 host->using_dma = 1; 1012 1013 if (host->use_dma == TRANS_MODE_IDMAC) 1014 dev_vdbg(host->dev, 1015 "sd sg_cpu: %#lx sg_dma: %#lx sg_len: %d\n", 1016 (unsigned long)host->sg_cpu, 1017 (unsigned long)host->sg_dma, 1018 sg_len); 1019 1020 /* 1021 * Decide the MSIZE and RX/TX Watermark. 1022 * If current block size is same with previous size, 1023 * no need to update fifoth. 1024 */ 1025 if (host->prev_blksz != data->blksz) 1026 dw_mci_adjust_fifoth(host, data); 1027 1028 /* Enable the DMA interface */ 1029 temp = mci_readl(host, CTRL); 1030 temp |= SDMMC_CTRL_DMA_ENABLE; 1031 mci_writel(host, CTRL, temp); 1032 1033 /* Disable RX/TX IRQs, let DMA handle it */ 1034 spin_lock_irqsave(&host->irq_lock, irqflags); 1035 temp = mci_readl(host, INTMASK); 1036 temp &= ~(SDMMC_INT_RXDR | SDMMC_INT_TXDR); 1037 mci_writel(host, INTMASK, temp); 1038 spin_unlock_irqrestore(&host->irq_lock, irqflags); 1039 1040 if (host->dma_ops->start(host, sg_len)) { 1041 host->dma_ops->stop(host); 1042 /* We can't do DMA, try PIO for this one */ 1043 dev_dbg(host->dev, 1044 "%s: fall back to PIO mode for current transfer\n", 1045 __func__); 1046 return -ENODEV; 1047 } 1048 1049 return 0; 1050 } 1051 1052 static void dw_mci_submit_data(struct dw_mci *host, struct mmc_data *data) 1053 { 1054 unsigned long irqflags; 1055 int flags = SG_MITER_ATOMIC; 1056 u32 temp; 1057 1058 data->error = -EINPROGRESS; 1059 1060 WARN_ON(host->data); 1061 host->sg = NULL; 1062 host->data = data; 1063 1064 if (data->flags & MMC_DATA_READ) 1065 host->dir_status = MMC_DATA_READ; 1066 else 1067 host->dir_status = MMC_DATA_WRITE; 1068 1069 dw_mci_ctrl_thld(host, data); 1070 1071 if (dw_mci_submit_data_dma(host, data)) { 1072 if (host->data->flags & MMC_DATA_READ) 1073 flags |= SG_MITER_TO_SG; 1074 else 1075 flags |= SG_MITER_FROM_SG; 1076 1077 sg_miter_start(&host->sg_miter, data->sg, data->sg_len, flags); 1078 host->sg = data->sg; 1079 host->part_buf_start = 0; 1080 host->part_buf_count = 0; 1081 1082 mci_writel(host, RINTSTS, SDMMC_INT_TXDR | SDMMC_INT_RXDR); 1083 1084 spin_lock_irqsave(&host->irq_lock, irqflags); 1085 temp = mci_readl(host, INTMASK); 1086 temp |= SDMMC_INT_TXDR | SDMMC_INT_RXDR; 1087 mci_writel(host, INTMASK, temp); 1088 spin_unlock_irqrestore(&host->irq_lock, irqflags); 1089 1090 temp = mci_readl(host, CTRL); 1091 temp &= ~SDMMC_CTRL_DMA_ENABLE; 1092 mci_writel(host, CTRL, temp); 1093 1094 /* 1095 * Use the initial fifoth_val for PIO mode. If wm_algined 1096 * is set, we set watermark same as data size. 1097 * If next issued data may be transferred by DMA mode, 1098 * prev_blksz should be invalidated. 1099 */ 1100 if (host->wm_aligned) 1101 dw_mci_adjust_fifoth(host, data); 1102 else 1103 mci_writel(host, FIFOTH, host->fifoth_val); 1104 host->prev_blksz = 0; 1105 } else { 1106 /* 1107 * Keep the current block size. 1108 * It will be used to decide whether to update 1109 * fifoth register next time. 1110 */ 1111 host->prev_blksz = data->blksz; 1112 } 1113 } 1114 1115 static void dw_mci_setup_bus(struct dw_mci *host, bool force_clkinit) 1116 { 1117 unsigned int clock = host->clock; 1118 u32 div; 1119 u32 clk_en_a; 1120 u32 sdmmc_cmd_bits = SDMMC_CMD_UPD_CLK | SDMMC_CMD_PRV_DAT_WAIT; 1121 1122 /* We must continue to set bit 28 in CMD until the change is complete */ 1123 if (host->state == STATE_WAITING_CMD11_DONE) 1124 sdmmc_cmd_bits |= SDMMC_CMD_VOLT_SWITCH; 1125 1126 host->mmc->actual_clock = 0; 1127 1128 if (!clock) { 1129 mci_writel(host, CLKENA, 0); 1130 mci_send_cmd(host, sdmmc_cmd_bits, 0); 1131 } else if (clock != host->current_speed || force_clkinit) { 1132 div = host->bus_hz / clock; 1133 if (host->bus_hz % clock && host->bus_hz > clock) 1134 /* 1135 * move the + 1 after the divide to prevent 1136 * over-clocking the card. 1137 */ 1138 div += 1; 1139 1140 div = (host->bus_hz != clock) ? DIV_ROUND_UP(div, 2) : 0; 1141 1142 if ((clock != host->clk_old && 1143 !test_bit(DW_MMC_CARD_NEEDS_POLL, &host->flags)) || 1144 force_clkinit) { 1145 /* Silent the verbose log if calling from PM context */ 1146 if (!force_clkinit) 1147 dev_info(&host->mmc->class_dev, 1148 "Bus speed = %dHz (req %dHz, actual %dHZ div = %d)\n", 1149 host->bus_hz, clock, 1150 div ? ((host->bus_hz / div) >> 1) : 1151 host->bus_hz, div); 1152 1153 /* 1154 * If card is polling, display the message only 1155 * one time at boot time. 1156 */ 1157 if (host->mmc->caps & MMC_CAP_NEEDS_POLL && 1158 host->mmc->f_min == clock) 1159 set_bit(DW_MMC_CARD_NEEDS_POLL, &host->flags); 1160 } 1161 1162 /* disable clock */ 1163 mci_writel(host, CLKENA, 0); 1164 mci_writel(host, CLKSRC, 0); 1165 1166 /* inform CIU */ 1167 mci_send_cmd(host, sdmmc_cmd_bits, 0); 1168 1169 /* set clock to desired speed */ 1170 mci_writel(host, CLKDIV, div); 1171 1172 /* inform CIU */ 1173 mci_send_cmd(host, sdmmc_cmd_bits, 0); 1174 1175 /* enable clock; only low power if no SDIO */ 1176 clk_en_a = SDMMC_CLKEN_ENABLE; 1177 if (!test_bit(DW_MMC_CARD_NO_LOW_PWR, &host->flags)) 1178 clk_en_a |= SDMMC_CLKEN_LOW_PWR; 1179 mci_writel(host, CLKENA, clk_en_a); 1180 1181 /* inform CIU */ 1182 mci_send_cmd(host, sdmmc_cmd_bits, 0); 1183 1184 /* keep the last clock value that was requested from core */ 1185 host->clk_old = clock; 1186 host->mmc->actual_clock = div ? ((host->bus_hz / div) >> 1) : 1187 host->bus_hz; 1188 } 1189 1190 host->current_speed = clock; 1191 1192 /* Set the current bus width */ 1193 mci_writel(host, CTYPE, host->ctype); 1194 } 1195 1196 static void dw_mci_set_data_timeout(struct dw_mci *host, 1197 unsigned int timeout_ns) 1198 { 1199 const struct dw_mci_drv_data *drv_data = host->drv_data; 1200 u32 clk_div, tmout; 1201 u64 tmp; 1202 1203 if (drv_data && drv_data->set_data_timeout) 1204 return drv_data->set_data_timeout(host, timeout_ns); 1205 1206 clk_div = (mci_readl(host, CLKDIV) & 0xFF) * 2; 1207 if (clk_div == 0) 1208 clk_div = 1; 1209 1210 tmp = DIV_ROUND_UP_ULL((u64)timeout_ns * host->bus_hz, NSEC_PER_SEC); 1211 tmp = DIV_ROUND_UP_ULL(tmp, clk_div); 1212 1213 /* TMOUT[7:0] (RESPONSE_TIMEOUT) */ 1214 tmout = 0xFF; /* Set maximum */ 1215 1216 /* TMOUT[31:8] (DATA_TIMEOUT) */ 1217 if (!tmp || tmp > 0xFFFFFF) 1218 tmout |= (0xFFFFFF << 8); 1219 else 1220 tmout |= (tmp & 0xFFFFFF) << 8; 1221 1222 mci_writel(host, TMOUT, tmout); 1223 dev_dbg(host->dev, "timeout_ns: %u => TMOUT[31:8]: %#08x", 1224 timeout_ns, tmout >> 8); 1225 } 1226 1227 static void dw_mci_start_request(struct dw_mci *host, struct mmc_command *cmd) 1228 { 1229 struct mmc_request *mrq; 1230 struct mmc_data *data; 1231 u32 cmdflags; 1232 1233 mrq = host->mrq; 1234 1235 host->mrq = mrq; 1236 1237 host->pending_events = 0; 1238 host->completed_events = 0; 1239 host->cmd_status = 0; 1240 host->data_status = 0; 1241 host->dir_status = 0; 1242 1243 data = cmd->data; 1244 if (data) { 1245 dw_mci_set_data_timeout(host, data->timeout_ns); 1246 mci_writel(host, BYTCNT, data->blksz*data->blocks); 1247 mci_writel(host, BLKSIZ, data->blksz); 1248 } 1249 1250 cmdflags = dw_mci_prepare_command(host->mmc, cmd); 1251 1252 /* this is the first command, send the initialization clock */ 1253 if (test_and_clear_bit(DW_MMC_CARD_NEED_INIT, &host->flags)) 1254 cmdflags |= SDMMC_CMD_INIT; 1255 1256 if (data) { 1257 dw_mci_submit_data(host, data); 1258 wmb(); /* drain writebuffer */ 1259 } 1260 1261 dw_mci_start_command(host, cmd, cmdflags); 1262 1263 if (cmd->opcode == SD_SWITCH_VOLTAGE) { 1264 unsigned long irqflags; 1265 1266 /* 1267 * Databook says to fail after 2ms w/ no response, but evidence 1268 * shows that sometimes the cmd11 interrupt takes over 130ms. 1269 * We'll set to 500ms, plus an extra jiffy just in case jiffies 1270 * is just about to roll over. 1271 * 1272 * We do this whole thing under spinlock and only if the 1273 * command hasn't already completed (indicating the irq 1274 * already ran so we don't want the timeout). 1275 */ 1276 spin_lock_irqsave(&host->irq_lock, irqflags); 1277 if (!test_bit(EVENT_CMD_COMPLETE, &host->pending_events)) 1278 mod_timer(&host->cmd11_timer, 1279 jiffies + msecs_to_jiffies(500) + 1); 1280 spin_unlock_irqrestore(&host->irq_lock, irqflags); 1281 } 1282 1283 host->stop_cmdr = dw_mci_prep_stop_abort(host, cmd); 1284 } 1285 1286 static void dw_mci_request(struct mmc_host *mmc, struct mmc_request *mrq) 1287 { 1288 struct dw_mci *host = mmc_priv(mmc); 1289 struct mmc_command *cmd; 1290 1291 WARN_ON(host->mrq); 1292 1293 /* 1294 * The check for card presence and queueing of the request must be 1295 * atomic, otherwise the card could be removed in between and the 1296 * request wouldn't fail until another card was inserted. 1297 */ 1298 if (!dw_mci_get_cd(mmc)) { 1299 mrq->cmd->error = -ENOMEDIUM; 1300 mmc_request_done(mmc, mrq); 1301 return; 1302 } 1303 1304 spin_lock_bh(&host->lock); 1305 1306 dev_vdbg(&host->mmc->class_dev, "request: state=%d\n", 1307 host->state); 1308 1309 host->mrq = mrq; 1310 if (host->state == STATE_WAITING_CMD11_DONE) { 1311 dev_warn(&host->mmc->class_dev, 1312 "Voltage change didn't complete\n"); 1313 /* 1314 * this case isn't expected to happen, so we can 1315 * either crash here or just try to continue on 1316 * in the closest possible state 1317 */ 1318 host->state = STATE_IDLE; 1319 } 1320 1321 if (host->state == STATE_IDLE) { 1322 host->state = STATE_SENDING_CMD; 1323 cmd = mrq->sbc ? mrq->sbc : mrq->cmd; 1324 dw_mci_start_request(host, cmd); 1325 } 1326 1327 spin_unlock_bh(&host->lock); 1328 } 1329 1330 static void dw_mci_set_ios(struct mmc_host *mmc, struct mmc_ios *ios) 1331 { 1332 struct dw_mci *host = mmc_priv(mmc); 1333 const struct dw_mci_drv_data *drv_data = host->drv_data; 1334 u32 regs; 1335 int ret; 1336 1337 switch (ios->bus_width) { 1338 case MMC_BUS_WIDTH_4: 1339 host->ctype = SDMMC_CTYPE_4BIT; 1340 break; 1341 case MMC_BUS_WIDTH_8: 1342 host->ctype = SDMMC_CTYPE_8BIT; 1343 break; 1344 default: 1345 /* set default 1 bit mode */ 1346 host->ctype = SDMMC_CTYPE_1BIT; 1347 } 1348 1349 regs = mci_readl(host, UHS_REG); 1350 1351 /* DDR mode set */ 1352 if (ios->timing == MMC_TIMING_MMC_DDR52 || 1353 ios->timing == MMC_TIMING_UHS_DDR50 || 1354 ios->timing == MMC_TIMING_MMC_HS400) 1355 regs |= BIT(16); 1356 else 1357 regs &= ~BIT(16); 1358 1359 mci_writel(host, UHS_REG, regs); 1360 host->timing = ios->timing; 1361 1362 /* 1363 * Use mirror of ios->clock to prevent race with mmc 1364 * core ios update when finding the minimum. 1365 */ 1366 host->clock = ios->clock; 1367 1368 if (drv_data && drv_data->set_ios) 1369 drv_data->set_ios(host, ios); 1370 1371 switch (ios->power_mode) { 1372 case MMC_POWER_UP: 1373 ret = mmc_regulator_set_ocr(mmc, mmc->supply.vmmc, ios->vdd); 1374 if (ret) { 1375 dev_err(host->dev, "failed to enable vmmc regulator\n"); 1376 return; 1377 } 1378 set_bit(DW_MMC_CARD_NEED_INIT, &host->flags); 1379 regs = mci_readl(host, PWREN); 1380 regs |= BIT(0); 1381 mci_writel(host, PWREN, regs); 1382 break; 1383 case MMC_POWER_ON: 1384 mmc_regulator_enable_vqmmc(mmc); 1385 /* Adjust clock / bus width after power is up */ 1386 dw_mci_setup_bus(host, false); 1387 1388 break; 1389 case MMC_POWER_OFF: 1390 /* Turn clock off before power goes down */ 1391 dw_mci_setup_bus(host, false); 1392 1393 if (!IS_ERR(mmc->supply.vmmc)) 1394 mmc_regulator_set_ocr(mmc, mmc->supply.vmmc, 0); 1395 1396 mmc_regulator_disable_vqmmc(mmc); 1397 1398 regs = mci_readl(host, PWREN); 1399 regs &= ~BIT(0); 1400 mci_writel(host, PWREN, regs); 1401 /* Reset our state machine after powering off */ 1402 dw_mci_ctrl_reset(host, SDMMC_CTRL_ALL_RESET_FLAGS); 1403 break; 1404 default: 1405 break; 1406 } 1407 1408 if (host->state == STATE_WAITING_CMD11_DONE && ios->clock != 0) 1409 host->state = STATE_IDLE; 1410 } 1411 1412 static int dw_mci_card_busy(struct mmc_host *mmc) 1413 { 1414 struct dw_mci *host = mmc_priv(mmc); 1415 u32 status; 1416 1417 /* 1418 * Check the busy bit which is low when DAT[3:0] 1419 * (the data lines) are 0000 1420 */ 1421 status = mci_readl(host, STATUS); 1422 1423 return !!(status & SDMMC_STATUS_BUSY); 1424 } 1425 1426 static int dw_mci_switch_voltage(struct mmc_host *mmc, struct mmc_ios *ios) 1427 { 1428 struct dw_mci *host = mmc_priv(mmc); 1429 const struct dw_mci_drv_data *drv_data = host->drv_data; 1430 u32 uhs; 1431 u32 v18 = SDMMC_UHS_18V; 1432 int ret; 1433 1434 if (drv_data && drv_data->switch_voltage) 1435 return drv_data->switch_voltage(host, ios); 1436 1437 /* 1438 * Program the voltage. Note that some instances of dw_mmc may use 1439 * the UHS_REG for this. For other instances (like exynos) the UHS_REG 1440 * does no harm but you need to set the regulator directly. Try both. 1441 */ 1442 uhs = mci_readl(host, UHS_REG); 1443 if (ios->signal_voltage == MMC_SIGNAL_VOLTAGE_330) 1444 uhs &= ~v18; 1445 else 1446 uhs |= v18; 1447 1448 if (!IS_ERR(mmc->supply.vqmmc)) { 1449 ret = mmc_regulator_set_vqmmc(mmc, ios); 1450 if (ret < 0) { 1451 dev_dbg(&mmc->class_dev, 1452 "Regulator set error %d - %s V\n", 1453 ret, uhs & v18 ? "1.8" : "3.3"); 1454 return ret; 1455 } 1456 } 1457 mci_writel(host, UHS_REG, uhs); 1458 1459 return 0; 1460 } 1461 1462 static int dw_mci_get_ro(struct mmc_host *mmc) 1463 { 1464 int read_only; 1465 struct dw_mci *host = mmc_priv(mmc); 1466 int gpio_ro = mmc_gpio_get_ro(mmc); 1467 1468 /* Use platform get_ro function, else try on board write protect */ 1469 if (gpio_ro >= 0) 1470 read_only = gpio_ro; 1471 else 1472 read_only = 1473 mci_readl(host, WRTPRT) & BIT(0) ? 1 : 0; 1474 1475 dev_dbg(&mmc->class_dev, "card is %s\n", 1476 read_only ? "read-only" : "read-write"); 1477 1478 return read_only; 1479 } 1480 1481 static void dw_mci_hw_reset(struct mmc_host *mmc) 1482 { 1483 struct dw_mci *host = mmc_priv(mmc); 1484 const struct dw_mci_drv_data *drv_data = host->drv_data; 1485 int reset; 1486 1487 if (host->use_dma == TRANS_MODE_IDMAC) 1488 dw_mci_idmac_reset(host); 1489 1490 if (!dw_mci_ctrl_reset(host, SDMMC_CTRL_DMA_RESET | 1491 SDMMC_CTRL_FIFO_RESET)) 1492 return; 1493 1494 if (drv_data && drv_data->hw_reset) { 1495 drv_data->hw_reset(host); 1496 return; 1497 } 1498 1499 /* 1500 * According to eMMC spec, card reset procedure: 1501 * tRstW >= 1us: RST_n pulse width 1502 * tRSCA >= 200us: RST_n to Command time 1503 * tRSTH >= 1us: RST_n high period 1504 */ 1505 reset = mci_readl(host, RST_N); 1506 reset &= ~SDMMC_RST_HWACTIVE; 1507 mci_writel(host, RST_N, reset); 1508 usleep_range(1, 2); 1509 reset |= SDMMC_RST_HWACTIVE; 1510 mci_writel(host, RST_N, reset); 1511 usleep_range(200, 300); 1512 } 1513 1514 static void dw_mci_prepare_sdio_irq(struct dw_mci *host, bool prepare) 1515 { 1516 const u32 clken_low_pwr = SDMMC_CLKEN_LOW_PWR; 1517 u32 clk_en_a_old; 1518 u32 clk_en_a; 1519 1520 /* 1521 * Low power mode will stop the card clock when idle. According to the 1522 * description of the CLKENA register we should disable low power mode 1523 * for SDIO cards if we need SDIO interrupts to work. 1524 */ 1525 1526 clk_en_a_old = mci_readl(host, CLKENA); 1527 if (prepare) { 1528 set_bit(DW_MMC_CARD_NO_LOW_PWR, &host->flags); 1529 clk_en_a = clk_en_a_old & ~clken_low_pwr; 1530 } else { 1531 clear_bit(DW_MMC_CARD_NO_LOW_PWR, &host->flags); 1532 clk_en_a = clk_en_a_old | clken_low_pwr; 1533 } 1534 1535 if (clk_en_a != clk_en_a_old) { 1536 mci_writel(host, CLKENA, clk_en_a); 1537 mci_send_cmd(host, SDMMC_CMD_UPD_CLK | SDMMC_CMD_PRV_DAT_WAIT, 1538 0); 1539 } 1540 } 1541 1542 static void __dw_mci_enable_sdio_irq(struct dw_mci *host, int enb) 1543 { 1544 unsigned long irqflags; 1545 u32 int_mask; 1546 1547 spin_lock_irqsave(&host->irq_lock, irqflags); 1548 1549 /* Enable/disable Slot Specific SDIO interrupt */ 1550 int_mask = mci_readl(host, INTMASK); 1551 if (enb) 1552 int_mask |= SDMMC_INT_SDIO(host->sdio_irq); 1553 else 1554 int_mask &= ~SDMMC_INT_SDIO(host->sdio_irq); 1555 mci_writel(host, INTMASK, int_mask); 1556 1557 spin_unlock_irqrestore(&host->irq_lock, irqflags); 1558 } 1559 1560 static void dw_mci_enable_sdio_irq(struct mmc_host *mmc, int enb) 1561 { 1562 struct dw_mci *host = mmc_priv(mmc); 1563 1564 dw_mci_prepare_sdio_irq(host, enb); 1565 __dw_mci_enable_sdio_irq(host, enb); 1566 1567 /* Avoid runtime suspending the device when SDIO IRQ is enabled */ 1568 if (enb) 1569 pm_runtime_get_noresume(host->dev); 1570 else 1571 pm_runtime_put_noidle(host->dev); 1572 } 1573 1574 static void dw_mci_ack_sdio_irq(struct mmc_host *mmc) 1575 { 1576 struct dw_mci *host = mmc_priv(mmc); 1577 1578 __dw_mci_enable_sdio_irq(host, 1); 1579 } 1580 1581 static int dw_mci_execute_tuning(struct mmc_host *mmc, u32 opcode) 1582 { 1583 struct dw_mci *host = mmc_priv(mmc); 1584 const struct dw_mci_drv_data *drv_data = host->drv_data; 1585 int err = -EINVAL; 1586 1587 if (drv_data && drv_data->execute_tuning) 1588 err = drv_data->execute_tuning(host, opcode); 1589 return err; 1590 } 1591 1592 static int dw_mci_prepare_hs400_tuning(struct mmc_host *mmc, 1593 struct mmc_ios *ios) 1594 { 1595 struct dw_mci *host = mmc_priv(mmc); 1596 const struct dw_mci_drv_data *drv_data = host->drv_data; 1597 1598 if (drv_data && drv_data->prepare_hs400_tuning) 1599 return drv_data->prepare_hs400_tuning(host, ios); 1600 1601 return 0; 1602 } 1603 1604 static bool dw_mci_reset(struct dw_mci *host) 1605 { 1606 u32 flags = SDMMC_CTRL_RESET | SDMMC_CTRL_FIFO_RESET; 1607 bool ret = false; 1608 u32 status = 0; 1609 1610 /* 1611 * Resetting generates a block interrupt, hence setting 1612 * the scatter-gather pointer to NULL. 1613 */ 1614 if (host->sg) { 1615 sg_miter_stop(&host->sg_miter); 1616 host->sg = NULL; 1617 } 1618 1619 if (host->use_dma) 1620 flags |= SDMMC_CTRL_DMA_RESET; 1621 1622 if (dw_mci_ctrl_reset(host, flags)) { 1623 /* 1624 * In all cases we clear the RAWINTS 1625 * register to clear any interrupts. 1626 */ 1627 mci_writel(host, RINTSTS, 0xFFFFFFFF); 1628 1629 if (!host->use_dma) { 1630 ret = true; 1631 goto ciu_out; 1632 } 1633 1634 /* Wait for dma_req to be cleared */ 1635 if (readl_poll_timeout_atomic(host->regs + SDMMC_STATUS, 1636 status, 1637 !(status & SDMMC_STATUS_DMA_REQ), 1638 1, 500 * USEC_PER_MSEC)) { 1639 dev_err(host->dev, 1640 "%s: Timeout waiting for dma_req to be cleared\n", 1641 __func__); 1642 goto ciu_out; 1643 } 1644 1645 /* when using DMA next we reset the fifo again */ 1646 if (!dw_mci_ctrl_reset(host, SDMMC_CTRL_FIFO_RESET)) 1647 goto ciu_out; 1648 } else { 1649 /* if the controller reset bit did clear, then set clock regs */ 1650 if (!(mci_readl(host, CTRL) & SDMMC_CTRL_RESET)) { 1651 dev_err(host->dev, 1652 "%s: fifo/dma reset bits didn't clear but ciu was reset, doing clock update\n", 1653 __func__); 1654 goto ciu_out; 1655 } 1656 } 1657 1658 if (host->use_dma == TRANS_MODE_IDMAC) 1659 /* It is also required that we reinit idmac */ 1660 dw_mci_idmac_init(host); 1661 1662 ret = true; 1663 1664 ciu_out: 1665 /* After a CTRL reset we need to have CIU set clock registers */ 1666 mci_send_cmd(host, SDMMC_CMD_UPD_CLK, 0); 1667 1668 return ret; 1669 } 1670 1671 static const struct mmc_host_ops dw_mci_ops = { 1672 .request = dw_mci_request, 1673 .pre_req = dw_mci_pre_req, 1674 .post_req = dw_mci_post_req, 1675 .set_ios = dw_mci_set_ios, 1676 .get_ro = dw_mci_get_ro, 1677 .get_cd = dw_mci_get_cd, 1678 .card_hw_reset = dw_mci_hw_reset, 1679 .enable_sdio_irq = dw_mci_enable_sdio_irq, 1680 .ack_sdio_irq = dw_mci_ack_sdio_irq, 1681 .execute_tuning = dw_mci_execute_tuning, 1682 .card_busy = dw_mci_card_busy, 1683 .start_signal_voltage_switch = dw_mci_switch_voltage, 1684 .prepare_hs400_tuning = dw_mci_prepare_hs400_tuning, 1685 }; 1686 1687 #ifdef CONFIG_FAULT_INJECTION 1688 static enum hrtimer_restart dw_mci_fault_timer(struct hrtimer *t) 1689 { 1690 struct dw_mci *host = container_of(t, struct dw_mci, fault_timer); 1691 unsigned long flags; 1692 1693 spin_lock_irqsave(&host->irq_lock, flags); 1694 1695 /* 1696 * Only inject an error if we haven't already got an error or data over 1697 * interrupt. 1698 */ 1699 if (!host->data_status) { 1700 host->data_status = SDMMC_INT_DCRC; 1701 set_bit(EVENT_DATA_ERROR, &host->pending_events); 1702 queue_work(system_bh_wq, &host->bh_work); 1703 } 1704 1705 spin_unlock_irqrestore(&host->irq_lock, flags); 1706 1707 return HRTIMER_NORESTART; 1708 } 1709 1710 static void dw_mci_start_fault_timer(struct dw_mci *host) 1711 { 1712 struct mmc_data *data = host->data; 1713 1714 if (!data || data->blocks <= 1) 1715 return; 1716 1717 if (!should_fail(&host->fail_data_crc, 1)) 1718 return; 1719 1720 /* 1721 * Try to inject the error at random points during the data transfer. 1722 */ 1723 hrtimer_start(&host->fault_timer, 1724 ms_to_ktime(get_random_u32_below(25)), 1725 HRTIMER_MODE_REL); 1726 } 1727 1728 static void dw_mci_stop_fault_timer(struct dw_mci *host) 1729 { 1730 hrtimer_cancel(&host->fault_timer); 1731 } 1732 1733 static void dw_mci_init_fault(struct dw_mci *host) 1734 { 1735 host->fail_data_crc = (struct fault_attr) FAULT_ATTR_INITIALIZER; 1736 1737 hrtimer_setup(&host->fault_timer, dw_mci_fault_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); 1738 } 1739 #else 1740 static void dw_mci_init_fault(struct dw_mci *host) 1741 { 1742 } 1743 1744 static void dw_mci_start_fault_timer(struct dw_mci *host) 1745 { 1746 } 1747 1748 static void dw_mci_stop_fault_timer(struct dw_mci *host) 1749 { 1750 } 1751 #endif 1752 1753 static void dw_mci_request_end(struct dw_mci *host, struct mmc_request *mrq) 1754 __releases(&host->lock) 1755 __acquires(&host->lock) 1756 { 1757 struct mmc_host *prev_mmc = host->mmc; 1758 1759 WARN_ON(host->cmd || host->data); 1760 1761 host->mrq = NULL; 1762 1763 if (host->state == STATE_SENDING_CMD11) 1764 host->state = STATE_WAITING_CMD11_DONE; 1765 else 1766 host->state = STATE_IDLE; 1767 1768 spin_unlock(&host->lock); 1769 mmc_request_done(prev_mmc, mrq); 1770 spin_lock(&host->lock); 1771 } 1772 1773 static int dw_mci_command_complete(struct dw_mci *host, struct mmc_command *cmd) 1774 { 1775 u32 status = host->cmd_status; 1776 1777 host->cmd_status = 0; 1778 1779 /* Read the response from the card (up to 16 bytes) */ 1780 if (cmd->flags & MMC_RSP_PRESENT) { 1781 if (cmd->flags & MMC_RSP_136) { 1782 cmd->resp[3] = mci_readl(host, RESP0); 1783 cmd->resp[2] = mci_readl(host, RESP1); 1784 cmd->resp[1] = mci_readl(host, RESP2); 1785 cmd->resp[0] = mci_readl(host, RESP3); 1786 } else { 1787 cmd->resp[0] = mci_readl(host, RESP0); 1788 cmd->resp[1] = 0; 1789 cmd->resp[2] = 0; 1790 cmd->resp[3] = 0; 1791 } 1792 } 1793 1794 if (status & SDMMC_INT_RTO) 1795 cmd->error = -ETIMEDOUT; 1796 else if ((cmd->flags & MMC_RSP_CRC) && (status & SDMMC_INT_RCRC)) 1797 cmd->error = -EILSEQ; 1798 else if (status & SDMMC_INT_RESP_ERR) 1799 cmd->error = -EIO; 1800 else 1801 cmd->error = 0; 1802 1803 return cmd->error; 1804 } 1805 1806 static int dw_mci_data_complete(struct dw_mci *host, struct mmc_data *data) 1807 { 1808 u32 status = host->data_status; 1809 1810 if (status & DW_MCI_DATA_ERROR_FLAGS) { 1811 if (status & SDMMC_INT_DRTO) { 1812 data->error = -ETIMEDOUT; 1813 } else if (status & SDMMC_INT_DCRC) { 1814 data->error = -EILSEQ; 1815 } else if (status & SDMMC_INT_EBE) { 1816 if (host->dir_status == 1817 MMC_DATA_WRITE) { 1818 /* 1819 * No data CRC status was returned. 1820 * The number of bytes transferred 1821 * will be exaggerated in PIO mode. 1822 */ 1823 data->bytes_xfered = 0; 1824 data->error = -ETIMEDOUT; 1825 } else if (host->dir_status == 1826 MMC_DATA_READ) { 1827 data->error = -EILSEQ; 1828 } 1829 } else { 1830 /* SDMMC_INT_SBE is included */ 1831 data->error = -EILSEQ; 1832 } 1833 1834 dev_dbg(host->dev, "data error, status 0x%08x\n", status); 1835 1836 /* 1837 * After an error, there may be data lingering 1838 * in the FIFO 1839 */ 1840 dw_mci_reset(host); 1841 } else { 1842 data->bytes_xfered = data->blocks * data->blksz; 1843 data->error = 0; 1844 } 1845 1846 return data->error; 1847 } 1848 1849 static void dw_mci_set_drto(struct dw_mci *host) 1850 { 1851 const struct dw_mci_drv_data *drv_data = host->drv_data; 1852 unsigned int drto_clks; 1853 unsigned int drto_div; 1854 unsigned int drto_ms; 1855 unsigned long irqflags; 1856 1857 if (drv_data && drv_data->get_drto_clks) 1858 drto_clks = drv_data->get_drto_clks(host); 1859 else 1860 drto_clks = mci_readl(host, TMOUT) >> 8; 1861 drto_div = (mci_readl(host, CLKDIV) & 0xff) * 2; 1862 if (drto_div == 0) 1863 drto_div = 1; 1864 1865 drto_ms = DIV_ROUND_UP_ULL((u64)MSEC_PER_SEC * drto_clks * drto_div, 1866 host->bus_hz); 1867 1868 dev_dbg(host->dev, "drto_ms: %u\n", drto_ms); 1869 1870 /* add a bit spare time */ 1871 drto_ms += 10; 1872 1873 spin_lock_irqsave(&host->irq_lock, irqflags); 1874 if (!test_bit(EVENT_DATA_COMPLETE, &host->pending_events)) 1875 mod_timer(&host->dto_timer, 1876 jiffies + msecs_to_jiffies(drto_ms)); 1877 spin_unlock_irqrestore(&host->irq_lock, irqflags); 1878 } 1879 1880 static bool dw_mci_clear_pending_cmd_complete(struct dw_mci *host) 1881 { 1882 if (!test_bit(EVENT_CMD_COMPLETE, &host->pending_events)) 1883 return false; 1884 1885 /* 1886 * Really be certain that the timer has stopped. This is a bit of 1887 * paranoia and could only really happen if we had really bad 1888 * interrupt latency and the interrupt routine and timeout were 1889 * running concurrently so that the timer_delete() in the interrupt 1890 * handler couldn't run. 1891 */ 1892 WARN_ON(timer_delete_sync(&host->cto_timer)); 1893 clear_bit(EVENT_CMD_COMPLETE, &host->pending_events); 1894 1895 return true; 1896 } 1897 1898 static bool dw_mci_clear_pending_data_complete(struct dw_mci *host) 1899 { 1900 if (!test_bit(EVENT_DATA_COMPLETE, &host->pending_events)) 1901 return false; 1902 1903 /* Extra paranoia just like dw_mci_clear_pending_cmd_complete() */ 1904 WARN_ON(timer_delete_sync(&host->dto_timer)); 1905 clear_bit(EVENT_DATA_COMPLETE, &host->pending_events); 1906 1907 return true; 1908 } 1909 1910 static void dw_mci_work_func(struct work_struct *t) 1911 { 1912 struct dw_mci *host = from_work(host, t, bh_work); 1913 struct mmc_data *data; 1914 struct mmc_command *cmd; 1915 struct mmc_request *mrq; 1916 enum dw_mci_state state; 1917 enum dw_mci_state prev_state; 1918 unsigned int err; 1919 1920 spin_lock(&host->lock); 1921 1922 state = host->state; 1923 data = host->data; 1924 mrq = host->mrq; 1925 1926 do { 1927 prev_state = state; 1928 1929 switch (state) { 1930 case STATE_IDLE: 1931 case STATE_WAITING_CMD11_DONE: 1932 break; 1933 1934 case STATE_SENDING_CMD11: 1935 case STATE_SENDING_CMD: 1936 if (!dw_mci_clear_pending_cmd_complete(host)) 1937 break; 1938 1939 cmd = host->cmd; 1940 host->cmd = NULL; 1941 set_bit(EVENT_CMD_COMPLETE, &host->completed_events); 1942 err = dw_mci_command_complete(host, cmd); 1943 if (cmd == mrq->sbc && !err) { 1944 dw_mci_start_request(host, mrq->cmd); 1945 goto unlock; 1946 } 1947 1948 if (cmd->data && err) { 1949 /* 1950 * During UHS tuning sequence, sending the stop 1951 * command after the response CRC error would 1952 * throw the system into a confused state 1953 * causing all future tuning phases to report 1954 * failure. 1955 * 1956 * In such case controller will move into a data 1957 * transfer state after a response error or 1958 * response CRC error. Let's let that finish 1959 * before trying to send a stop, so we'll go to 1960 * STATE_SENDING_DATA. 1961 * 1962 * Although letting the data transfer take place 1963 * will waste a bit of time (we already know 1964 * the command was bad), it can't cause any 1965 * errors since it's possible it would have 1966 * taken place anyway if this bh work got 1967 * delayed. Allowing the transfer to take place 1968 * avoids races and keeps things simple. 1969 */ 1970 if (err != -ETIMEDOUT && 1971 host->dir_status == MMC_DATA_READ) { 1972 state = STATE_SENDING_DATA; 1973 continue; 1974 } 1975 1976 send_stop_abort(host, data); 1977 dw_mci_stop_dma(host); 1978 state = STATE_SENDING_STOP; 1979 break; 1980 } 1981 1982 if (!cmd->data || err) { 1983 dw_mci_request_end(host, mrq); 1984 goto unlock; 1985 } 1986 1987 prev_state = state = STATE_SENDING_DATA; 1988 fallthrough; 1989 1990 case STATE_SENDING_DATA: 1991 /* 1992 * We could get a data error and never a transfer 1993 * complete so we'd better check for it here. 1994 * 1995 * Note that we don't really care if we also got a 1996 * transfer complete; stopping the DMA and sending an 1997 * abort won't hurt. 1998 */ 1999 if (test_and_clear_bit(EVENT_DATA_ERROR, 2000 &host->pending_events)) { 2001 if (!(host->data_status & (SDMMC_INT_DRTO | 2002 SDMMC_INT_EBE))) 2003 send_stop_abort(host, data); 2004 dw_mci_stop_dma(host); 2005 state = STATE_DATA_ERROR; 2006 break; 2007 } 2008 2009 if (!test_and_clear_bit(EVENT_XFER_COMPLETE, 2010 &host->pending_events)) { 2011 /* 2012 * If all data-related interrupts don't come 2013 * within the given time in reading data state. 2014 */ 2015 if (host->dir_status == MMC_DATA_READ) 2016 dw_mci_set_drto(host); 2017 break; 2018 } 2019 2020 set_bit(EVENT_XFER_COMPLETE, &host->completed_events); 2021 2022 /* 2023 * Handle an EVENT_DATA_ERROR that might have shown up 2024 * before the transfer completed. This might not have 2025 * been caught by the check above because the interrupt 2026 * could have gone off between the previous check and 2027 * the check for transfer complete. 2028 * 2029 * Technically this ought not be needed assuming we 2030 * get a DATA_COMPLETE eventually (we'll notice the 2031 * error and end the request), but it shouldn't hurt. 2032 * 2033 * This has the advantage of sending the stop command. 2034 */ 2035 if (test_and_clear_bit(EVENT_DATA_ERROR, 2036 &host->pending_events)) { 2037 if (!(host->data_status & (SDMMC_INT_DRTO | 2038 SDMMC_INT_EBE))) 2039 send_stop_abort(host, data); 2040 dw_mci_stop_dma(host); 2041 state = STATE_DATA_ERROR; 2042 break; 2043 } 2044 prev_state = state = STATE_DATA_BUSY; 2045 2046 fallthrough; 2047 2048 case STATE_DATA_BUSY: 2049 if (!dw_mci_clear_pending_data_complete(host)) { 2050 /* 2051 * If data error interrupt comes but data over 2052 * interrupt doesn't come within the given time. 2053 * in reading data state. 2054 */ 2055 if (host->dir_status == MMC_DATA_READ) 2056 dw_mci_set_drto(host); 2057 break; 2058 } 2059 2060 dw_mci_stop_fault_timer(host); 2061 host->data = NULL; 2062 set_bit(EVENT_DATA_COMPLETE, &host->completed_events); 2063 err = dw_mci_data_complete(host, data); 2064 2065 if (!err) { 2066 if (!data->stop || mrq->sbc) { 2067 if (mrq->sbc && data->stop) 2068 data->stop->error = 0; 2069 dw_mci_request_end(host, mrq); 2070 goto unlock; 2071 } 2072 2073 /* stop command for open-ended transfer*/ 2074 if (data->stop) 2075 send_stop_abort(host, data); 2076 } else { 2077 /* 2078 * If we don't have a command complete now we'll 2079 * never get one since we just reset everything; 2080 * better end the request. 2081 * 2082 * If we do have a command complete we'll fall 2083 * through to the SENDING_STOP command and 2084 * everything will be peachy keen. 2085 */ 2086 if (!test_bit(EVENT_CMD_COMPLETE, 2087 &host->pending_events)) { 2088 host->cmd = NULL; 2089 dw_mci_request_end(host, mrq); 2090 goto unlock; 2091 } 2092 } 2093 2094 /* 2095 * If err has non-zero, 2096 * stop-abort command has been already issued. 2097 */ 2098 prev_state = state = STATE_SENDING_STOP; 2099 2100 fallthrough; 2101 2102 case STATE_SENDING_STOP: 2103 if (!dw_mci_clear_pending_cmd_complete(host)) 2104 break; 2105 2106 /* CMD error in data command */ 2107 if (mrq->cmd->error && mrq->data) 2108 dw_mci_reset(host); 2109 2110 dw_mci_stop_fault_timer(host); 2111 host->cmd = NULL; 2112 host->data = NULL; 2113 2114 if (!mrq->sbc && mrq->stop) 2115 dw_mci_command_complete(host, mrq->stop); 2116 else 2117 host->cmd_status = 0; 2118 2119 dw_mci_request_end(host, mrq); 2120 goto unlock; 2121 2122 case STATE_DATA_ERROR: 2123 if (!test_and_clear_bit(EVENT_XFER_COMPLETE, 2124 &host->pending_events)) 2125 break; 2126 2127 state = STATE_DATA_BUSY; 2128 break; 2129 } 2130 } while (state != prev_state); 2131 2132 host->state = state; 2133 unlock: 2134 spin_unlock(&host->lock); 2135 2136 } 2137 2138 /* push final bytes to part_buf, only use during push */ 2139 static void dw_mci_set_part_bytes(struct dw_mci *host, void *buf, int cnt) 2140 { 2141 memcpy((void *)&host->part_buf, buf, cnt); 2142 host->part_buf_count = cnt; 2143 } 2144 2145 /* append bytes to part_buf, only use during push */ 2146 static int dw_mci_push_part_bytes(struct dw_mci *host, void *buf, int cnt) 2147 { 2148 cnt = min(cnt, (1 << host->data_shift) - host->part_buf_count); 2149 memcpy((void *)&host->part_buf + host->part_buf_count, buf, cnt); 2150 host->part_buf_count += cnt; 2151 return cnt; 2152 } 2153 2154 /* pull first bytes from part_buf, only use during pull */ 2155 static int dw_mci_pull_part_bytes(struct dw_mci *host, void *buf, int cnt) 2156 { 2157 cnt = min_t(int, cnt, host->part_buf_count); 2158 if (cnt) { 2159 memcpy(buf, (void *)&host->part_buf + host->part_buf_start, 2160 cnt); 2161 host->part_buf_count -= cnt; 2162 host->part_buf_start += cnt; 2163 } 2164 return cnt; 2165 } 2166 2167 /* pull final bytes from the part_buf, assuming it's just been filled */ 2168 static void dw_mci_pull_final_bytes(struct dw_mci *host, void *buf, int cnt) 2169 { 2170 memcpy(buf, &host->part_buf, cnt); 2171 host->part_buf_start = cnt; 2172 host->part_buf_count = (1 << host->data_shift) - cnt; 2173 } 2174 2175 static void dw_mci_push_data16(struct dw_mci *host, void *buf, int cnt) 2176 { 2177 struct mmc_data *data = host->data; 2178 int init_cnt = cnt; 2179 2180 /* try and push anything in the part_buf */ 2181 if (unlikely(host->part_buf_count)) { 2182 int len = dw_mci_push_part_bytes(host, buf, cnt); 2183 2184 buf += len; 2185 cnt -= len; 2186 if (host->part_buf_count == 2) { 2187 mci_fifo_writew(host->fifo_reg, host->part_buf16); 2188 host->part_buf_count = 0; 2189 } 2190 } 2191 #ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS 2192 if (unlikely((unsigned long)buf & 0x1)) { 2193 while (cnt >= 2) { 2194 u16 aligned_buf[64]; 2195 int len = min(cnt & -2, (int)sizeof(aligned_buf)); 2196 int items = len >> 1; 2197 int i; 2198 /* memcpy from input buffer into aligned buffer */ 2199 memcpy(aligned_buf, buf, len); 2200 buf += len; 2201 cnt -= len; 2202 /* push data from aligned buffer into fifo */ 2203 for (i = 0; i < items; ++i) 2204 mci_fifo_writew(host->fifo_reg, aligned_buf[i]); 2205 } 2206 } else 2207 #endif 2208 { 2209 u16 *pdata = buf; 2210 2211 for (; cnt >= 2; cnt -= 2) 2212 mci_fifo_writew(host->fifo_reg, *pdata++); 2213 buf = pdata; 2214 } 2215 /* put anything remaining in the part_buf */ 2216 if (cnt) { 2217 dw_mci_set_part_bytes(host, buf, cnt); 2218 /* Push data if we have reached the expected data length */ 2219 if ((data->bytes_xfered + init_cnt) == 2220 (data->blksz * data->blocks)) 2221 mci_fifo_writew(host->fifo_reg, host->part_buf16); 2222 } 2223 } 2224 2225 static void dw_mci_pull_data16(struct dw_mci *host, void *buf, int cnt) 2226 { 2227 #ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS 2228 if (unlikely((unsigned long)buf & 0x1)) { 2229 while (cnt >= 2) { 2230 /* pull data from fifo into aligned buffer */ 2231 u16 aligned_buf[64]; 2232 int len = min(cnt & -2, (int)sizeof(aligned_buf)); 2233 int items = len >> 1; 2234 int i; 2235 2236 for (i = 0; i < items; ++i) 2237 aligned_buf[i] = mci_fifo_readw(host->fifo_reg); 2238 /* memcpy from aligned buffer into output buffer */ 2239 memcpy(buf, aligned_buf, len); 2240 buf += len; 2241 cnt -= len; 2242 } 2243 } else 2244 #endif 2245 { 2246 u16 *pdata = buf; 2247 2248 for (; cnt >= 2; cnt -= 2) 2249 *pdata++ = mci_fifo_readw(host->fifo_reg); 2250 buf = pdata; 2251 } 2252 if (cnt) { 2253 host->part_buf16 = mci_fifo_readw(host->fifo_reg); 2254 dw_mci_pull_final_bytes(host, buf, cnt); 2255 } 2256 } 2257 2258 static void dw_mci_push_data32(struct dw_mci *host, void *buf, int cnt) 2259 { 2260 struct mmc_data *data = host->data; 2261 int init_cnt = cnt; 2262 2263 /* try and push anything in the part_buf */ 2264 if (unlikely(host->part_buf_count)) { 2265 int len = dw_mci_push_part_bytes(host, buf, cnt); 2266 2267 buf += len; 2268 cnt -= len; 2269 if (host->part_buf_count == 4) { 2270 mci_fifo_writel(host->fifo_reg, host->part_buf32); 2271 host->part_buf_count = 0; 2272 } 2273 } 2274 #ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS 2275 if (unlikely((unsigned long)buf & 0x3)) { 2276 while (cnt >= 4) { 2277 u32 aligned_buf[32]; 2278 int len = min(cnt & -4, (int)sizeof(aligned_buf)); 2279 int items = len >> 2; 2280 int i; 2281 /* memcpy from input buffer into aligned buffer */ 2282 memcpy(aligned_buf, buf, len); 2283 buf += len; 2284 cnt -= len; 2285 /* push data from aligned buffer into fifo */ 2286 for (i = 0; i < items; ++i) 2287 mci_fifo_writel(host->fifo_reg, aligned_buf[i]); 2288 } 2289 } else 2290 #endif 2291 { 2292 u32 *pdata = buf; 2293 2294 for (; cnt >= 4; cnt -= 4) 2295 mci_fifo_writel(host->fifo_reg, *pdata++); 2296 buf = pdata; 2297 } 2298 /* put anything remaining in the part_buf */ 2299 if (cnt) { 2300 dw_mci_set_part_bytes(host, buf, cnt); 2301 /* Push data if we have reached the expected data length */ 2302 if ((data->bytes_xfered + init_cnt) == 2303 (data->blksz * data->blocks)) 2304 mci_fifo_writel(host->fifo_reg, host->part_buf32); 2305 } 2306 } 2307 2308 static void dw_mci_pull_data32(struct dw_mci *host, void *buf, int cnt) 2309 { 2310 #ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS 2311 if (unlikely((unsigned long)buf & 0x3)) { 2312 while (cnt >= 4) { 2313 /* pull data from fifo into aligned buffer */ 2314 u32 aligned_buf[32]; 2315 int len = min(cnt & -4, (int)sizeof(aligned_buf)); 2316 int items = len >> 2; 2317 int i; 2318 2319 for (i = 0; i < items; ++i) 2320 aligned_buf[i] = mci_fifo_readl(host->fifo_reg); 2321 /* memcpy from aligned buffer into output buffer */ 2322 memcpy(buf, aligned_buf, len); 2323 buf += len; 2324 cnt -= len; 2325 } 2326 } else 2327 #endif 2328 { 2329 u32 *pdata = buf; 2330 2331 for (; cnt >= 4; cnt -= 4) 2332 *pdata++ = mci_fifo_readl(host->fifo_reg); 2333 buf = pdata; 2334 } 2335 if (cnt) { 2336 host->part_buf32 = mci_fifo_readl(host->fifo_reg); 2337 dw_mci_pull_final_bytes(host, buf, cnt); 2338 } 2339 } 2340 2341 static void dw_mci_push_data64(struct dw_mci *host, void *buf, int cnt) 2342 { 2343 struct mmc_data *data = host->data; 2344 int init_cnt = cnt; 2345 2346 /* try and push anything in the part_buf */ 2347 if (unlikely(host->part_buf_count)) { 2348 int len = dw_mci_push_part_bytes(host, buf, cnt); 2349 2350 buf += len; 2351 cnt -= len; 2352 2353 if (host->part_buf_count == 8) { 2354 mci_fifo_writeq(host->fifo_reg, host->part_buf); 2355 host->part_buf_count = 0; 2356 } 2357 } 2358 #ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS 2359 if (unlikely((unsigned long)buf & 0x7)) { 2360 while (cnt >= 8) { 2361 u64 aligned_buf[16]; 2362 int len = min(cnt & -8, (int)sizeof(aligned_buf)); 2363 int items = len >> 3; 2364 int i; 2365 /* memcpy from input buffer into aligned buffer */ 2366 memcpy(aligned_buf, buf, len); 2367 buf += len; 2368 cnt -= len; 2369 /* push data from aligned buffer into fifo */ 2370 for (i = 0; i < items; ++i) 2371 mci_fifo_writeq(host->fifo_reg, aligned_buf[i]); 2372 } 2373 } else 2374 #endif 2375 { 2376 u64 *pdata = buf; 2377 2378 for (; cnt >= 8; cnt -= 8) 2379 mci_fifo_writeq(host->fifo_reg, *pdata++); 2380 buf = pdata; 2381 } 2382 /* put anything remaining in the part_buf */ 2383 if (cnt) { 2384 dw_mci_set_part_bytes(host, buf, cnt); 2385 /* Push data if we have reached the expected data length */ 2386 if ((data->bytes_xfered + init_cnt) == 2387 (data->blksz * data->blocks)) 2388 mci_fifo_writeq(host->fifo_reg, host->part_buf); 2389 } 2390 } 2391 2392 static void dw_mci_pull_data64(struct dw_mci *host, void *buf, int cnt) 2393 { 2394 #ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS 2395 if (unlikely((unsigned long)buf & 0x7)) { 2396 while (cnt >= 8) { 2397 /* pull data from fifo into aligned buffer */ 2398 u64 aligned_buf[16]; 2399 int len = min(cnt & -8, (int)sizeof(aligned_buf)); 2400 int items = len >> 3; 2401 int i; 2402 2403 for (i = 0; i < items; ++i) 2404 aligned_buf[i] = mci_fifo_readq(host->fifo_reg); 2405 2406 /* memcpy from aligned buffer into output buffer */ 2407 memcpy(buf, aligned_buf, len); 2408 buf += len; 2409 cnt -= len; 2410 } 2411 } else 2412 #endif 2413 { 2414 u64 *pdata = buf; 2415 2416 for (; cnt >= 8; cnt -= 8) 2417 *pdata++ = mci_fifo_readq(host->fifo_reg); 2418 buf = pdata; 2419 } 2420 if (cnt) { 2421 host->part_buf = mci_fifo_readq(host->fifo_reg); 2422 dw_mci_pull_final_bytes(host, buf, cnt); 2423 } 2424 } 2425 2426 static void dw_mci_push_data64_32(struct dw_mci *host, void *buf, int cnt) 2427 { 2428 struct mmc_data *data = host->data; 2429 int init_cnt = cnt; 2430 2431 /* try and push anything in the part_buf */ 2432 if (unlikely(host->part_buf_count)) { 2433 int len = dw_mci_push_part_bytes(host, buf, cnt); 2434 2435 buf += len; 2436 cnt -= len; 2437 2438 if (host->part_buf_count == 8) { 2439 mci_fifo_l_writeq(host->fifo_reg, host->part_buf); 2440 host->part_buf_count = 0; 2441 } 2442 } 2443 #ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS 2444 if (unlikely((unsigned long)buf & 0x7)) { 2445 while (cnt >= 8) { 2446 u64 aligned_buf[16]; 2447 int len = min(cnt & -8, (int)sizeof(aligned_buf)); 2448 int items = len >> 3; 2449 int i; 2450 /* memcpy from input buffer into aligned buffer */ 2451 memcpy(aligned_buf, buf, len); 2452 buf += len; 2453 cnt -= len; 2454 /* push data from aligned buffer into fifo */ 2455 for (i = 0; i < items; ++i) 2456 mci_fifo_l_writeq(host->fifo_reg, aligned_buf[i]); 2457 } 2458 } else 2459 #endif 2460 { 2461 u64 *pdata = buf; 2462 2463 for (; cnt >= 8; cnt -= 8) 2464 mci_fifo_l_writeq(host->fifo_reg, *pdata++); 2465 buf = pdata; 2466 } 2467 /* put anything remaining in the part_buf */ 2468 if (cnt) { 2469 dw_mci_set_part_bytes(host, buf, cnt); 2470 /* Push data if we have reached the expected data length */ 2471 if ((data->bytes_xfered + init_cnt) == 2472 (data->blksz * data->blocks)) 2473 mci_fifo_l_writeq(host->fifo_reg, host->part_buf); 2474 } 2475 } 2476 2477 static void dw_mci_pull_data64_32(struct dw_mci *host, void *buf, int cnt) 2478 { 2479 #ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS 2480 if (unlikely((unsigned long)buf & 0x7)) { 2481 while (cnt >= 8) { 2482 /* pull data from fifo into aligned buffer */ 2483 u64 aligned_buf[16]; 2484 int len = min(cnt & -8, (int)sizeof(aligned_buf)); 2485 int items = len >> 3; 2486 int i; 2487 2488 for (i = 0; i < items; ++i) 2489 aligned_buf[i] = mci_fifo_l_readq(host->fifo_reg); 2490 2491 /* memcpy from aligned buffer into output buffer */ 2492 memcpy(buf, aligned_buf, len); 2493 buf += len; 2494 cnt -= len; 2495 } 2496 } else 2497 #endif 2498 { 2499 u64 *pdata = buf; 2500 2501 for (; cnt >= 8; cnt -= 8) 2502 *pdata++ = mci_fifo_l_readq(host->fifo_reg); 2503 buf = pdata; 2504 } 2505 if (cnt) { 2506 host->part_buf = mci_fifo_l_readq(host->fifo_reg); 2507 dw_mci_pull_final_bytes(host, buf, cnt); 2508 } 2509 } 2510 2511 static void dw_mci_pull_data(struct dw_mci *host, void *buf, int cnt) 2512 { 2513 int len; 2514 2515 /* get remaining partial bytes */ 2516 len = dw_mci_pull_part_bytes(host, buf, cnt); 2517 if (unlikely(len == cnt)) 2518 return; 2519 buf += len; 2520 cnt -= len; 2521 2522 /* get the rest of the data */ 2523 host->pull_data(host, buf, cnt); 2524 } 2525 2526 static void dw_mci_read_data_pio(struct dw_mci *host, bool dto) 2527 { 2528 struct sg_mapping_iter *sg_miter = &host->sg_miter; 2529 void *buf; 2530 unsigned int offset; 2531 struct mmc_data *data = host->data; 2532 int shift = host->data_shift; 2533 u32 status; 2534 unsigned int len; 2535 unsigned int remain, fcnt; 2536 2537 do { 2538 if (!sg_miter_next(sg_miter)) 2539 goto done; 2540 2541 host->sg = sg_miter->piter.sg; 2542 buf = sg_miter->addr; 2543 remain = sg_miter->length; 2544 offset = 0; 2545 2546 do { 2547 fcnt = (SDMMC_GET_FCNT(mci_readl(host, STATUS)) 2548 << shift) + host->part_buf_count; 2549 len = min(remain, fcnt); 2550 if (!len) 2551 break; 2552 dw_mci_pull_data(host, (void *)(buf + offset), len); 2553 data->bytes_xfered += len; 2554 offset += len; 2555 remain -= len; 2556 } while (remain); 2557 2558 sg_miter->consumed = offset; 2559 status = mci_readl(host, MINTSTS); 2560 mci_writel(host, RINTSTS, SDMMC_INT_RXDR); 2561 /* if the RXDR is ready read again */ 2562 } while ((status & SDMMC_INT_RXDR) || 2563 (dto && SDMMC_GET_FCNT(mci_readl(host, STATUS)))); 2564 2565 if (!remain) { 2566 if (!sg_miter_next(sg_miter)) 2567 goto done; 2568 sg_miter->consumed = 0; 2569 } 2570 sg_miter_stop(sg_miter); 2571 return; 2572 2573 done: 2574 sg_miter_stop(sg_miter); 2575 host->sg = NULL; 2576 smp_wmb(); /* drain writebuffer */ 2577 set_bit(EVENT_XFER_COMPLETE, &host->pending_events); 2578 } 2579 2580 static void dw_mci_write_data_pio(struct dw_mci *host) 2581 { 2582 struct sg_mapping_iter *sg_miter = &host->sg_miter; 2583 void *buf; 2584 unsigned int offset; 2585 struct mmc_data *data = host->data; 2586 int shift = host->data_shift; 2587 u32 status; 2588 unsigned int len; 2589 unsigned int fifo_depth = host->fifo_depth; 2590 unsigned int remain, fcnt; 2591 2592 do { 2593 if (!sg_miter_next(sg_miter)) 2594 goto done; 2595 2596 host->sg = sg_miter->piter.sg; 2597 buf = sg_miter->addr; 2598 remain = sg_miter->length; 2599 offset = 0; 2600 2601 do { 2602 fcnt = ((fifo_depth - 2603 SDMMC_GET_FCNT(mci_readl(host, STATUS))) 2604 << shift) - host->part_buf_count; 2605 len = min(remain, fcnt); 2606 if (!len) 2607 break; 2608 host->push_data(host, (void *)(buf + offset), len); 2609 data->bytes_xfered += len; 2610 offset += len; 2611 remain -= len; 2612 } while (remain); 2613 2614 sg_miter->consumed = offset; 2615 status = mci_readl(host, MINTSTS); 2616 mci_writel(host, RINTSTS, SDMMC_INT_TXDR); 2617 } while (status & SDMMC_INT_TXDR); /* if TXDR write again */ 2618 2619 if (!remain) { 2620 if (!sg_miter_next(sg_miter)) 2621 goto done; 2622 sg_miter->consumed = 0; 2623 } 2624 sg_miter_stop(sg_miter); 2625 return; 2626 2627 done: 2628 sg_miter_stop(sg_miter); 2629 host->sg = NULL; 2630 smp_wmb(); /* drain writebuffer */ 2631 set_bit(EVENT_XFER_COMPLETE, &host->pending_events); 2632 } 2633 2634 static void dw_mci_cmd_interrupt(struct dw_mci *host, u32 status) 2635 { 2636 timer_delete(&host->cto_timer); 2637 2638 if (!host->cmd_status) 2639 host->cmd_status = status; 2640 2641 smp_wmb(); /* drain writebuffer */ 2642 2643 set_bit(EVENT_CMD_COMPLETE, &host->pending_events); 2644 queue_work(system_bh_wq, &host->bh_work); 2645 2646 dw_mci_start_fault_timer(host); 2647 } 2648 2649 static void dw_mci_handle_cd(struct dw_mci *host) 2650 { 2651 mmc_detect_change(host->mmc, 2652 msecs_to_jiffies(host->detect_delay_ms)); 2653 } 2654 2655 static irqreturn_t dw_mci_interrupt(int irq, void *dev_id) 2656 { 2657 struct dw_mci *host = dev_id; 2658 u32 pending; 2659 2660 pending = mci_readl(host, MINTSTS); /* read-only mask reg */ 2661 2662 if (pending) { 2663 /* Check volt switch first, since it can look like an error */ 2664 if ((host->state == STATE_SENDING_CMD11) && 2665 (pending & SDMMC_INT_VOLT_SWITCH)) { 2666 mci_writel(host, RINTSTS, SDMMC_INT_VOLT_SWITCH); 2667 pending &= ~SDMMC_INT_VOLT_SWITCH; 2668 2669 /* 2670 * Hold the lock; we know cmd11_timer can't be kicked 2671 * off after the lock is released, so safe to delete. 2672 */ 2673 spin_lock(&host->irq_lock); 2674 dw_mci_cmd_interrupt(host, pending); 2675 spin_unlock(&host->irq_lock); 2676 2677 timer_delete(&host->cmd11_timer); 2678 } 2679 2680 if (pending & DW_MCI_CMD_ERROR_FLAGS) { 2681 spin_lock(&host->irq_lock); 2682 2683 timer_delete(&host->cto_timer); 2684 mci_writel(host, RINTSTS, DW_MCI_CMD_ERROR_FLAGS); 2685 host->cmd_status = pending; 2686 smp_wmb(); /* drain writebuffer */ 2687 set_bit(EVENT_CMD_COMPLETE, &host->pending_events); 2688 2689 spin_unlock(&host->irq_lock); 2690 } 2691 2692 if (pending & DW_MCI_DATA_ERROR_FLAGS) { 2693 spin_lock(&host->irq_lock); 2694 2695 if (host->quirks & DW_MMC_QUIRK_EXTENDED_TMOUT) 2696 timer_delete(&host->dto_timer); 2697 2698 /* if there is an error report DATA_ERROR */ 2699 mci_writel(host, RINTSTS, DW_MCI_DATA_ERROR_FLAGS); 2700 host->data_status = pending; 2701 smp_wmb(); /* drain writebuffer */ 2702 set_bit(EVENT_DATA_ERROR, &host->pending_events); 2703 2704 if (host->quirks & DW_MMC_QUIRK_EXTENDED_TMOUT) 2705 /* In case of error, we cannot expect a DTO */ 2706 set_bit(EVENT_DATA_COMPLETE, 2707 &host->pending_events); 2708 2709 queue_work(system_bh_wq, &host->bh_work); 2710 2711 spin_unlock(&host->irq_lock); 2712 } 2713 2714 if (pending & SDMMC_INT_DATA_OVER) { 2715 spin_lock(&host->irq_lock); 2716 2717 timer_delete(&host->dto_timer); 2718 2719 mci_writel(host, RINTSTS, SDMMC_INT_DATA_OVER); 2720 if (!host->data_status) 2721 host->data_status = pending; 2722 smp_wmb(); /* drain writebuffer */ 2723 if (host->dir_status == MMC_DATA_READ) { 2724 if (host->sg != NULL) 2725 dw_mci_read_data_pio(host, true); 2726 } 2727 set_bit(EVENT_DATA_COMPLETE, &host->pending_events); 2728 queue_work(system_bh_wq, &host->bh_work); 2729 2730 spin_unlock(&host->irq_lock); 2731 } 2732 2733 if (pending & SDMMC_INT_RXDR) { 2734 mci_writel(host, RINTSTS, SDMMC_INT_RXDR); 2735 if (host->dir_status == MMC_DATA_READ && host->sg) 2736 dw_mci_read_data_pio(host, false); 2737 } 2738 2739 if (pending & SDMMC_INT_TXDR) { 2740 mci_writel(host, RINTSTS, SDMMC_INT_TXDR); 2741 if (host->dir_status == MMC_DATA_WRITE && host->sg) 2742 dw_mci_write_data_pio(host); 2743 } 2744 2745 if (pending & SDMMC_INT_CMD_DONE) { 2746 spin_lock(&host->irq_lock); 2747 2748 mci_writel(host, RINTSTS, SDMMC_INT_CMD_DONE); 2749 dw_mci_cmd_interrupt(host, pending); 2750 2751 spin_unlock(&host->irq_lock); 2752 } 2753 2754 if (pending & SDMMC_INT_CD) { 2755 mci_writel(host, RINTSTS, SDMMC_INT_CD); 2756 dw_mci_handle_cd(host); 2757 } 2758 2759 if (pending & SDMMC_INT_SDIO(host->sdio_irq)) { 2760 mci_writel(host, RINTSTS, 2761 SDMMC_INT_SDIO(host->sdio_irq)); 2762 __dw_mci_enable_sdio_irq(host, 0); 2763 sdio_signal_irq(host->mmc); 2764 } 2765 2766 } 2767 2768 if (host->use_dma != TRANS_MODE_IDMAC) 2769 return IRQ_HANDLED; 2770 2771 /* Handle IDMA interrupts */ 2772 if (host->dma_64bit_address == 1) { 2773 pending = mci_readl(host, IDSTS64); 2774 if (pending & (SDMMC_IDMAC_INT_TI | SDMMC_IDMAC_INT_RI)) { 2775 mci_writel(host, IDSTS64, SDMMC_IDMAC_INT_TI | 2776 SDMMC_IDMAC_INT_RI); 2777 mci_writel(host, IDSTS64, SDMMC_IDMAC_INT_NI); 2778 if (!test_bit(EVENT_DATA_ERROR, &host->pending_events)) 2779 host->dma_ops->complete((void *)host); 2780 } 2781 } else { 2782 pending = mci_readl(host, IDSTS); 2783 if (pending & (SDMMC_IDMAC_INT_TI | SDMMC_IDMAC_INT_RI)) { 2784 mci_writel(host, IDSTS, SDMMC_IDMAC_INT_TI | 2785 SDMMC_IDMAC_INT_RI); 2786 mci_writel(host, IDSTS, SDMMC_IDMAC_INT_NI); 2787 if (!test_bit(EVENT_DATA_ERROR, &host->pending_events)) 2788 host->dma_ops->complete((void *)host); 2789 } 2790 } 2791 2792 return IRQ_HANDLED; 2793 } 2794 2795 static int dw_mci_init_host_caps(struct dw_mci *host) 2796 { 2797 const struct dw_mci_drv_data *drv_data = host->drv_data; 2798 struct mmc_host *mmc = host->mmc; 2799 int ctrl_id; 2800 2801 if (drv_data) 2802 mmc->caps |= drv_data->common_caps; 2803 2804 if (host->dev->of_node) 2805 ctrl_id = mmc->index; 2806 else 2807 ctrl_id = to_platform_device(host->dev)->id; 2808 2809 if (drv_data && drv_data->caps) { 2810 if (ctrl_id >= drv_data->num_caps) { 2811 dev_err(host->dev, "invalid controller id %d\n", 2812 ctrl_id); 2813 return -EINVAL; 2814 } 2815 mmc->caps |= drv_data->caps[ctrl_id]; 2816 } 2817 2818 /* if host has set a minimum_freq, we should respect it */ 2819 if (host->minimum_speed) 2820 mmc->f_min = host->minimum_speed; 2821 else 2822 mmc->f_min = DW_MCI_FREQ_MIN; 2823 2824 if (!mmc->f_max) 2825 mmc->f_max = DW_MCI_FREQ_MAX; 2826 2827 /* Process SDIO IRQs through the sdio_irq_work. */ 2828 if (mmc->caps & MMC_CAP_SDIO_IRQ) 2829 mmc->caps2 |= MMC_CAP2_SDIO_IRQ_NOTHREAD; 2830 2831 return 0; 2832 } 2833 2834 static int dw_mci_init_host(struct dw_mci *host) 2835 { 2836 struct mmc_host *mmc = host->mmc; 2837 int ret; 2838 2839 mmc->ops = &dw_mci_ops; 2840 2841 /*if there are external regulators, get them*/ 2842 ret = mmc_regulator_get_supply(mmc); 2843 if (ret) 2844 return ret; 2845 2846 if (!mmc->ocr_avail) 2847 mmc->ocr_avail = MMC_VDD_32_33 | MMC_VDD_33_34; 2848 2849 ret = mmc_of_parse(mmc); 2850 if (ret) 2851 return ret; 2852 2853 mmc_of_parse_clk_phase(host->dev, &host->phase_map); 2854 2855 ret = dw_mci_init_host_caps(host); 2856 if (ret) 2857 return ret; 2858 2859 /* Useful defaults if platform data is unset. */ 2860 if (host->use_dma == TRANS_MODE_IDMAC) { 2861 mmc->max_segs = host->ring_size; 2862 mmc->max_blk_size = 65535; 2863 mmc->max_seg_size = 0x1000; 2864 mmc->max_req_size = mmc->max_seg_size * host->ring_size; 2865 mmc->max_blk_count = mmc->max_req_size / 512; 2866 } else if (host->use_dma == TRANS_MODE_EDMAC) { 2867 mmc->max_segs = 64; 2868 mmc->max_blk_size = 65535; 2869 mmc->max_blk_count = 65535; 2870 mmc->max_req_size = 2871 mmc->max_blk_size * mmc->max_blk_count; 2872 mmc->max_seg_size = mmc->max_req_size; 2873 } else { 2874 /* TRANS_MODE_PIO */ 2875 mmc->max_segs = 64; 2876 mmc->max_blk_size = 65535; /* BLKSIZ is 16 bits */ 2877 mmc->max_blk_count = 512; 2878 mmc->max_req_size = mmc->max_blk_size * 2879 mmc->max_blk_count; 2880 mmc->max_seg_size = mmc->max_req_size; 2881 } 2882 2883 if (mmc->caps & MMC_CAP_NEEDS_POLL) 2884 dev_info(&mmc->class_dev, "card is polling.\n"); 2885 else if (!mmc_card_is_removable(mmc)) 2886 dev_info(&mmc->class_dev, "card is non-removable.\n"); 2887 2888 dw_mci_get_cd(mmc); 2889 2890 ret = mmc_add_host(mmc); 2891 if (ret) 2892 return ret; 2893 2894 #if defined(CONFIG_DEBUG_FS) 2895 dw_mci_init_debugfs(host); 2896 #endif 2897 2898 return 0; 2899 } 2900 2901 static void dw_mci_cleanup_host(struct dw_mci *host) 2902 { 2903 /* Debugfs stuff is cleaned up by mmc core */ 2904 mmc_remove_host(host->mmc); 2905 } 2906 2907 static void dw_mci_init_dma(struct dw_mci *host) 2908 { 2909 int addr_config; 2910 struct device *dev = host->dev; 2911 2912 /* 2913 * Check tansfer mode from HCON[17:16] 2914 * Clear the ambiguous description of dw_mmc databook: 2915 * 2b'00: No DMA Interface -> Actually means using Internal DMA block 2916 * 2b'01: DesignWare DMA Interface -> Synopsys DW-DMA block 2917 * 2b'10: Generic DMA Interface -> non-Synopsys generic DMA block 2918 * 2b'11: Non DW DMA Interface -> pio only 2919 * Compared to DesignWare DMA Interface, Generic DMA Interface has a 2920 * simpler request/acknowledge handshake mechanism and both of them 2921 * are regarded as external dma master for dw_mmc. 2922 */ 2923 host->use_dma = SDMMC_GET_TRANS_MODE(mci_readl(host, HCON)); 2924 if (host->use_dma == DMA_INTERFACE_IDMA) { 2925 host->use_dma = TRANS_MODE_IDMAC; 2926 } else if (host->use_dma == DMA_INTERFACE_DWDMA || 2927 host->use_dma == DMA_INTERFACE_GDMA) { 2928 host->use_dma = TRANS_MODE_EDMAC; 2929 } else { 2930 goto no_dma; 2931 } 2932 2933 /* Determine which DMA interface to use */ 2934 if (host->use_dma == TRANS_MODE_IDMAC) { 2935 /* 2936 * Check ADDR_CONFIG bit in HCON to find 2937 * IDMAC address bus width 2938 */ 2939 addr_config = SDMMC_GET_ADDR_CONFIG(mci_readl(host, HCON)); 2940 2941 if (addr_config == 1) { 2942 /* host supports IDMAC in 64-bit address mode */ 2943 host->dma_64bit_address = 1; 2944 dev_info(host->dev, 2945 "IDMAC supports 64-bit address mode.\n"); 2946 if (dma_set_mask_and_coherent(host->dev, DMA_BIT_MASK(64))) 2947 dev_info(host->dev, "Fail to set 64-bit DMA mask"); 2948 } else { 2949 /* host supports IDMAC in 32-bit address mode */ 2950 host->dma_64bit_address = 0; 2951 dev_info(host->dev, 2952 "IDMAC supports 32-bit address mode.\n"); 2953 } 2954 2955 /* Alloc memory for sg translation */ 2956 host->sg_cpu = dmam_alloc_coherent(host->dev, 2957 DESC_RING_BUF_SZ, 2958 &host->sg_dma, GFP_KERNEL); 2959 if (!host->sg_cpu) { 2960 dev_err(host->dev, 2961 "%s: could not alloc DMA memory\n", 2962 __func__); 2963 goto no_dma; 2964 } 2965 2966 host->dma_ops = &dw_mci_idmac_ops; 2967 dev_info(host->dev, "Using internal DMA controller.\n"); 2968 } else { 2969 /* TRANS_MODE_EDMAC: check dma bindings again */ 2970 if ((device_property_string_array_count(dev, "dma-names") < 0) || 2971 !device_property_present(dev, "dmas")) { 2972 goto no_dma; 2973 } 2974 host->dma_ops = &dw_mci_edmac_ops; 2975 dev_info(host->dev, "Using external DMA controller.\n"); 2976 } 2977 2978 if (host->dma_ops->init && host->dma_ops->start && 2979 host->dma_ops->stop && host->dma_ops->cleanup) { 2980 if (host->dma_ops->init(host)) { 2981 dev_err(host->dev, "%s: Unable to initialize DMA Controller.\n", 2982 __func__); 2983 goto no_dma; 2984 } 2985 } else { 2986 dev_err(host->dev, "DMA initialization not found.\n"); 2987 goto no_dma; 2988 } 2989 2990 return; 2991 2992 no_dma: 2993 dev_info(host->dev, "Using PIO mode.\n"); 2994 host->use_dma = TRANS_MODE_PIO; 2995 } 2996 2997 static void dw_mci_cmd11_timer(struct timer_list *t) 2998 { 2999 struct dw_mci *host = timer_container_of(host, t, cmd11_timer); 3000 3001 if (host->state != STATE_SENDING_CMD11) { 3002 dev_warn(host->dev, "Unexpected CMD11 timeout\n"); 3003 return; 3004 } 3005 3006 host->cmd_status = SDMMC_INT_RTO; 3007 set_bit(EVENT_CMD_COMPLETE, &host->pending_events); 3008 queue_work(system_bh_wq, &host->bh_work); 3009 } 3010 3011 static void dw_mci_cto_timer(struct timer_list *t) 3012 { 3013 struct dw_mci *host = timer_container_of(host, t, cto_timer); 3014 unsigned long irqflags; 3015 u32 pending; 3016 3017 spin_lock_irqsave(&host->irq_lock, irqflags); 3018 3019 /* 3020 * If somehow we have very bad interrupt latency it's remotely possible 3021 * that the timer could fire while the interrupt is still pending or 3022 * while the interrupt is midway through running. Let's be paranoid 3023 * and detect those two cases. Note that this is paranoia is somewhat 3024 * justified because in this function we don't actually cancel the 3025 * pending command in the controller--we just assume it will never come. 3026 */ 3027 pending = mci_readl(host, MINTSTS); /* read-only mask reg */ 3028 if (pending & (DW_MCI_CMD_ERROR_FLAGS | SDMMC_INT_CMD_DONE)) { 3029 /* The interrupt should fire; no need to act but we can warn */ 3030 dev_warn(host->dev, "Unexpected interrupt latency\n"); 3031 goto exit; 3032 } 3033 if (test_bit(EVENT_CMD_COMPLETE, &host->pending_events)) { 3034 /* Presumably interrupt handler couldn't delete the timer */ 3035 dev_warn(host->dev, "CTO timeout when already completed\n"); 3036 goto exit; 3037 } 3038 3039 /* 3040 * Continued paranoia to make sure we're in the state we expect. 3041 * This paranoia isn't really justified but it seems good to be safe. 3042 */ 3043 switch (host->state) { 3044 case STATE_SENDING_CMD11: 3045 case STATE_SENDING_CMD: 3046 case STATE_SENDING_STOP: 3047 /* 3048 * If CMD_DONE interrupt does NOT come in sending command 3049 * state, we should notify the driver to terminate current 3050 * transfer and report a command timeout to the core. 3051 */ 3052 host->cmd_status = SDMMC_INT_RTO; 3053 set_bit(EVENT_CMD_COMPLETE, &host->pending_events); 3054 queue_work(system_bh_wq, &host->bh_work); 3055 break; 3056 default: 3057 dev_warn(host->dev, "Unexpected command timeout, state %d\n", 3058 host->state); 3059 break; 3060 } 3061 3062 exit: 3063 spin_unlock_irqrestore(&host->irq_lock, irqflags); 3064 } 3065 3066 static void dw_mci_dto_timer(struct timer_list *t) 3067 { 3068 struct dw_mci *host = timer_container_of(host, t, dto_timer); 3069 unsigned long irqflags; 3070 u32 pending; 3071 3072 spin_lock_irqsave(&host->irq_lock, irqflags); 3073 3074 /* 3075 * The DTO timer is much longer than the CTO timer, so it's even less 3076 * likely that we'll these cases, but it pays to be paranoid. 3077 */ 3078 pending = mci_readl(host, MINTSTS); /* read-only mask reg */ 3079 if (pending & SDMMC_INT_DATA_OVER) { 3080 /* The interrupt should fire; no need to act but we can warn */ 3081 dev_warn(host->dev, "Unexpected data interrupt latency\n"); 3082 goto exit; 3083 } 3084 if (test_bit(EVENT_DATA_COMPLETE, &host->pending_events)) { 3085 /* Presumably interrupt handler couldn't delete the timer */ 3086 dev_warn(host->dev, "DTO timeout when already completed\n"); 3087 goto exit; 3088 } 3089 3090 /* 3091 * Continued paranoia to make sure we're in the state we expect. 3092 * This paranoia isn't really justified but it seems good to be safe. 3093 */ 3094 switch (host->state) { 3095 case STATE_SENDING_DATA: 3096 case STATE_DATA_BUSY: 3097 /* 3098 * If DTO interrupt does NOT come in sending data state, 3099 * we should notify the driver to terminate current transfer 3100 * and report a data timeout to the core. 3101 */ 3102 host->data_status = SDMMC_INT_DRTO; 3103 set_bit(EVENT_DATA_ERROR, &host->pending_events); 3104 set_bit(EVENT_DATA_COMPLETE, &host->pending_events); 3105 queue_work(system_bh_wq, &host->bh_work); 3106 break; 3107 default: 3108 dev_warn(host->dev, "Unexpected data timeout, state %d\n", 3109 host->state); 3110 break; 3111 } 3112 3113 exit: 3114 spin_unlock_irqrestore(&host->irq_lock, irqflags); 3115 } 3116 3117 static int dw_mci_parse_dt(struct dw_mci *host) 3118 { 3119 struct device *dev = host->dev; 3120 const struct dw_mci_drv_data *drv_data = host->drv_data; 3121 int ret; 3122 u32 clock_frequency; 3123 3124 /* find reset controller when exist */ 3125 host->rstc = devm_reset_control_get_optional_exclusive(dev, "reset"); 3126 if (IS_ERR(host->rstc)) 3127 return PTR_ERR(host->rstc); 3128 3129 if (!host->fifo_depth && device_property_read_u32(dev, "fifo-depth", &host->fifo_depth)) 3130 dev_info(dev, 3131 "fifo-depth property not found, using value of FIFOTH register as default\n"); 3132 3133 if (!host->detect_delay_ms) 3134 device_property_read_u32(dev, "card-detect-delay", 3135 &host->detect_delay_ms); 3136 3137 if (!host->data_addr_override) 3138 device_property_read_u32(dev, "data-addr", &host->data_addr_override); 3139 3140 if (device_property_present(dev, "fifo-watermark-aligned")) 3141 host->wm_aligned = true; 3142 3143 if (!host->bus_hz && !device_property_read_u32(dev, "clock-frequency", &clock_frequency)) 3144 host->bus_hz = clock_frequency; 3145 3146 if (drv_data && drv_data->parse_dt) { 3147 ret = drv_data->parse_dt(host); 3148 if (ret) 3149 return ret; 3150 } 3151 3152 return 0; 3153 } 3154 3155 static void dw_mci_enable_cd(struct dw_mci *host) 3156 { 3157 unsigned long irqflags; 3158 u32 temp; 3159 3160 /* 3161 * No need for CD if host has a non-error GPIO 3162 * as well as broken card detection is found. 3163 */ 3164 if (host->mmc->caps & MMC_CAP_NEEDS_POLL) 3165 return; 3166 3167 if (mmc_gpio_get_cd(host->mmc) < 0) { 3168 spin_lock_irqsave(&host->irq_lock, irqflags); 3169 temp = mci_readl(host, INTMASK); 3170 temp |= SDMMC_INT_CD; 3171 mci_writel(host, INTMASK, temp); 3172 spin_unlock_irqrestore(&host->irq_lock, irqflags); 3173 } 3174 } 3175 3176 struct dw_mci *dw_mci_alloc_host(struct device *dev) 3177 { 3178 struct mmc_host *mmc; 3179 struct dw_mci *host; 3180 3181 mmc = devm_mmc_alloc_host(dev, sizeof(struct dw_mci)); 3182 if (!mmc) 3183 return ERR_PTR(-ENOMEM); 3184 3185 host = mmc_priv(mmc); 3186 host->mmc = mmc; 3187 host->dev = dev; 3188 3189 return host; 3190 } 3191 EXPORT_SYMBOL(dw_mci_alloc_host); 3192 3193 int dw_mci_probe(struct dw_mci *host) 3194 { 3195 const struct dw_mci_drv_data *drv_data = host->drv_data; 3196 int width, i, ret = 0; 3197 u32 fifo_size; 3198 3199 ret = dw_mci_parse_dt(host); 3200 if (ret) 3201 return dev_err_probe(host->dev, ret, "parse dt failed\n"); 3202 3203 host->biu_clk = devm_clk_get(host->dev, "biu"); 3204 if (IS_ERR(host->biu_clk)) { 3205 dev_dbg(host->dev, "biu clock not available\n"); 3206 ret = PTR_ERR(host->biu_clk); 3207 if (ret == -EPROBE_DEFER) 3208 return ret; 3209 3210 } else { 3211 ret = clk_prepare_enable(host->biu_clk); 3212 if (ret) { 3213 dev_err(host->dev, "failed to enable biu clock\n"); 3214 return ret; 3215 } 3216 } 3217 3218 host->ciu_clk = devm_clk_get(host->dev, "ciu"); 3219 if (IS_ERR(host->ciu_clk)) { 3220 dev_dbg(host->dev, "ciu clock not available\n"); 3221 ret = PTR_ERR(host->ciu_clk); 3222 if (ret == -EPROBE_DEFER) 3223 goto err_clk_biu; 3224 } else { 3225 ret = clk_prepare_enable(host->ciu_clk); 3226 if (ret) { 3227 dev_err(host->dev, "failed to enable ciu clock\n"); 3228 goto err_clk_biu; 3229 } 3230 3231 if (host->bus_hz) { 3232 ret = clk_set_rate(host->ciu_clk, host->bus_hz); 3233 if (ret) 3234 dev_warn(host->dev, 3235 "Unable to set bus rate to %uHz\n", 3236 host->bus_hz); 3237 } 3238 host->bus_hz = clk_get_rate(host->ciu_clk); 3239 } 3240 3241 if (!host->bus_hz) { 3242 dev_err(host->dev, 3243 "Platform data must supply bus speed\n"); 3244 ret = -ENODEV; 3245 goto err_clk_ciu; 3246 } 3247 3248 if (host->rstc) { 3249 reset_control_assert(host->rstc); 3250 usleep_range(10, 50); 3251 reset_control_deassert(host->rstc); 3252 } 3253 3254 if (drv_data && drv_data->init) { 3255 ret = drv_data->init(host); 3256 if (ret) { 3257 dev_err(host->dev, 3258 "implementation specific init failed\n"); 3259 goto err_clk_ciu; 3260 } 3261 } 3262 3263 timer_setup(&host->cmd11_timer, dw_mci_cmd11_timer, 0); 3264 timer_setup(&host->cto_timer, dw_mci_cto_timer, 0); 3265 timer_setup(&host->dto_timer, dw_mci_dto_timer, 0); 3266 3267 spin_lock_init(&host->lock); 3268 spin_lock_init(&host->irq_lock); 3269 3270 dw_mci_init_fault(host); 3271 3272 /* 3273 * Get the host data width - this assumes that HCON has been set with 3274 * the correct values. 3275 */ 3276 i = SDMMC_GET_HDATA_WIDTH(mci_readl(host, HCON)); 3277 if (!i) { 3278 host->push_data = dw_mci_push_data16; 3279 host->pull_data = dw_mci_pull_data16; 3280 width = 16; 3281 host->data_shift = 1; 3282 } else if (i == 2) { 3283 if ((host->quirks & DW_MMC_QUIRK_FIFO64_32)) { 3284 host->push_data = dw_mci_push_data64_32; 3285 host->pull_data = dw_mci_pull_data64_32; 3286 } else { 3287 host->push_data = dw_mci_push_data64; 3288 host->pull_data = dw_mci_pull_data64; 3289 } 3290 width = 64; 3291 host->data_shift = 3; 3292 } else { 3293 /* Check for a reserved value, and warn if it is */ 3294 WARN((i != 1), 3295 "HCON reports a reserved host data width!\n" 3296 "Defaulting to 32-bit access.\n"); 3297 host->push_data = dw_mci_push_data32; 3298 host->pull_data = dw_mci_pull_data32; 3299 width = 32; 3300 host->data_shift = 2; 3301 } 3302 3303 /* Reset all blocks */ 3304 if (!dw_mci_ctrl_reset(host, SDMMC_CTRL_ALL_RESET_FLAGS)) { 3305 ret = -ENODEV; 3306 goto err_clk_ciu; 3307 } 3308 3309 dw_mci_init_dma(host); 3310 3311 /* Clear the interrupts for the host controller */ 3312 mci_writel(host, RINTSTS, 0xFFFFFFFF); 3313 mci_writel(host, INTMASK, 0); /* disable all mmc interrupt first */ 3314 3315 /* Put in max timeout */ 3316 mci_writel(host, TMOUT, 0xFFFFFFFF); 3317 3318 /* 3319 * FIFO threshold settings RxMark = fifo_size / 2 - 1, 3320 * Tx Mark = fifo_size / 2 DMA Size = 8 3321 */ 3322 if (!host->fifo_depth) { 3323 /* 3324 * Power-on value of RX_WMark is FIFO_DEPTH-1, but this may 3325 * have been overwritten by the bootloader, just like we're 3326 * about to do, so if you know the value for your hardware, you 3327 * should put it in the platform data. 3328 */ 3329 fifo_size = mci_readl(host, FIFOTH); 3330 fifo_size = 1 + ((fifo_size >> 16) & 0xfff); 3331 } else { 3332 fifo_size = host->fifo_depth; 3333 } 3334 host->fifo_depth = fifo_size; 3335 host->fifoth_val = 3336 SDMMC_SET_FIFOTH(0x2, fifo_size / 2 - 1, fifo_size / 2); 3337 mci_writel(host, FIFOTH, host->fifoth_val); 3338 3339 /* disable clock to CIU */ 3340 mci_writel(host, CLKENA, 0); 3341 mci_writel(host, CLKSRC, 0); 3342 3343 /* 3344 * In 2.40a spec, Data offset is changed. 3345 * Need to check the version-id and set data-offset for DATA register. 3346 */ 3347 host->verid = SDMMC_GET_VERID(mci_readl(host, VERID)); 3348 dev_info(host->dev, "Version ID is %04x\n", host->verid); 3349 3350 if (host->data_addr_override) 3351 host->fifo_reg = host->regs + host->data_addr_override; 3352 else if (host->verid < DW_MMC_240A) 3353 host->fifo_reg = host->regs + DATA_OFFSET; 3354 else 3355 host->fifo_reg = host->regs + DATA_240A_OFFSET; 3356 3357 INIT_WORK(&host->bh_work, dw_mci_work_func); 3358 ret = devm_request_irq(host->dev, host->irq, dw_mci_interrupt, 3359 host->irq_flags, "dw-mci", host); 3360 if (ret) 3361 goto err_dmaunmap; 3362 3363 /* 3364 * Enable interrupts for command done, data over, data empty, 3365 * receive ready and error such as transmit, receive timeout, crc error 3366 */ 3367 mci_writel(host, INTMASK, SDMMC_INT_CMD_DONE | SDMMC_INT_DATA_OVER | 3368 SDMMC_INT_TXDR | SDMMC_INT_RXDR | 3369 DW_MCI_ERROR_FLAGS); 3370 /* Enable mci interrupt */ 3371 mci_writel(host, CTRL, SDMMC_CTRL_INT_ENABLE); 3372 3373 dev_info(host->dev, 3374 "DW MMC controller at irq %d,%d bit host data width,%u deep fifo\n", 3375 host->irq, width, fifo_size); 3376 3377 ret = dw_mci_init_host(host); 3378 if (ret) { 3379 dev_dbg(host->dev, "host init failed\n"); 3380 goto err_dmaunmap; 3381 } 3382 3383 /* Now that host is setup, we can enable card detect */ 3384 dw_mci_enable_cd(host); 3385 3386 return 0; 3387 3388 err_dmaunmap: 3389 if (host->use_dma && host->dma_ops->exit) 3390 host->dma_ops->exit(host); 3391 3392 reset_control_assert(host->rstc); 3393 3394 err_clk_ciu: 3395 clk_disable_unprepare(host->ciu_clk); 3396 3397 err_clk_biu: 3398 clk_disable_unprepare(host->biu_clk); 3399 3400 return ret; 3401 } 3402 EXPORT_SYMBOL(dw_mci_probe); 3403 3404 void dw_mci_remove(struct dw_mci *host) 3405 { 3406 dev_dbg(host->dev, "remove host\n"); 3407 dw_mci_cleanup_host(host); 3408 3409 mci_writel(host, RINTSTS, 0xFFFFFFFF); 3410 mci_writel(host, INTMASK, 0); /* disable all mmc interrupt first */ 3411 3412 /* disable clock to CIU */ 3413 mci_writel(host, CLKENA, 0); 3414 mci_writel(host, CLKSRC, 0); 3415 3416 if (host->use_dma && host->dma_ops->exit) 3417 host->dma_ops->exit(host); 3418 3419 reset_control_assert(host->rstc); 3420 3421 clk_disable_unprepare(host->ciu_clk); 3422 clk_disable_unprepare(host->biu_clk); 3423 } 3424 EXPORT_SYMBOL(dw_mci_remove); 3425 3426 int dw_mci_runtime_suspend(struct device *dev) 3427 { 3428 struct dw_mci *host = dev_get_drvdata(dev); 3429 3430 if (host->use_dma && host->dma_ops->exit) 3431 host->dma_ops->exit(host); 3432 3433 clk_disable_unprepare(host->ciu_clk); 3434 3435 if (mmc_host_can_gpio_cd(host->mmc) || 3436 !mmc_card_is_removable(host->mmc)) 3437 clk_disable_unprepare(host->biu_clk); 3438 3439 return 0; 3440 } 3441 EXPORT_SYMBOL(dw_mci_runtime_suspend); 3442 3443 int dw_mci_runtime_resume(struct device *dev) 3444 { 3445 int ret = 0; 3446 struct dw_mci *host = dev_get_drvdata(dev); 3447 3448 if (mmc_host_can_gpio_cd(host->mmc) || 3449 !mmc_card_is_removable(host->mmc)) { 3450 ret = clk_prepare_enable(host->biu_clk); 3451 if (ret) 3452 return ret; 3453 } 3454 3455 ret = clk_prepare_enable(host->ciu_clk); 3456 if (ret) 3457 goto err; 3458 3459 if (!dw_mci_ctrl_reset(host, SDMMC_CTRL_ALL_RESET_FLAGS)) { 3460 clk_disable_unprepare(host->ciu_clk); 3461 ret = -ENODEV; 3462 goto err; 3463 } 3464 3465 if (host->use_dma && host->dma_ops->init) { 3466 ret = host->dma_ops->init(host); 3467 if (ret) 3468 return ret; 3469 } 3470 3471 /* 3472 * Restore the initial value at FIFOTH register 3473 * And Invalidate the prev_blksz with zero 3474 */ 3475 mci_writel(host, FIFOTH, host->fifoth_val); 3476 host->prev_blksz = 0; 3477 3478 /* Put in max timeout */ 3479 mci_writel(host, TMOUT, 0xFFFFFFFF); 3480 3481 mci_writel(host, RINTSTS, 0xFFFFFFFF); 3482 mci_writel(host, INTMASK, SDMMC_INT_CMD_DONE | SDMMC_INT_DATA_OVER | 3483 SDMMC_INT_TXDR | SDMMC_INT_RXDR | 3484 DW_MCI_ERROR_FLAGS); 3485 mci_writel(host, CTRL, SDMMC_CTRL_INT_ENABLE); 3486 3487 3488 if (host->mmc->pm_flags & MMC_PM_KEEP_POWER) 3489 dw_mci_set_ios(host->mmc, &host->mmc->ios); 3490 3491 /* Force setup bus to guarantee available clock output */ 3492 dw_mci_setup_bus(host, true); 3493 3494 /* Re-enable SDIO interrupts. */ 3495 if (sdio_irq_claimed(host->mmc)) 3496 __dw_mci_enable_sdio_irq(host, 1); 3497 3498 /* Now that host is setup, we can enable card detect */ 3499 dw_mci_enable_cd(host); 3500 3501 return 0; 3502 3503 err: 3504 if (mmc_host_can_gpio_cd(host->mmc) || 3505 !mmc_card_is_removable(host->mmc)) 3506 clk_disable_unprepare(host->biu_clk); 3507 3508 return ret; 3509 } 3510 EXPORT_SYMBOL(dw_mci_runtime_resume); 3511 3512 const struct dev_pm_ops dw_mci_pmops = { 3513 SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, pm_runtime_force_resume) 3514 RUNTIME_PM_OPS(dw_mci_runtime_suspend, dw_mci_runtime_resume, NULL) 3515 }; 3516 EXPORT_SYMBOL_GPL(dw_mci_pmops); 3517 3518 MODULE_DESCRIPTION("DW Multimedia Card Interface driver"); 3519 MODULE_AUTHOR("NXP Semiconductor VietNam"); 3520 MODULE_AUTHOR("Imagination Technologies Ltd"); 3521 MODULE_LICENSE("GPL v2"); 3522