1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Synopsys DesignWare Multimedia Card Interface driver 4 * (Based on NXP driver for lpc 31xx) 5 * 6 * Copyright (C) 2009 NXP Semiconductors 7 * Copyright (C) 2009, 2010 Imagination Technologies Ltd. 8 */ 9 10 #include <linux/bitops.h> 11 #include <linux/clk.h> 12 #include <linux/debugfs.h> 13 #include <linux/delay.h> 14 #include <linux/device.h> 15 #include <linux/dma-mapping.h> 16 #include <linux/err.h> 17 #include <linux/interrupt.h> 18 #include <linux/iopoll.h> 19 #include <linux/irq.h> 20 #include <linux/ktime.h> 21 #include <linux/mmc/card.h> 22 #include <linux/mmc/host.h> 23 #include <linux/mmc/mmc.h> 24 #include <linux/mmc/sd.h> 25 #include <linux/mmc/sdio.h> 26 #include <linux/mmc/slot-gpio.h> 27 #include <linux/module.h> 28 #include <linux/of.h> 29 #include <linux/platform_device.h> 30 #include <linux/pm_runtime.h> 31 #include <linux/regulator/consumer.h> 32 33 #include "dw_mmc.h" 34 35 /* Common flag combinations */ 36 #define DW_MCI_DATA_ERROR_FLAGS (SDMMC_INT_DRTO | SDMMC_INT_DCRC | \ 37 SDMMC_INT_HTO | SDMMC_INT_SBE | \ 38 SDMMC_INT_EBE | SDMMC_INT_HLE) 39 #define DW_MCI_CMD_ERROR_FLAGS (SDMMC_INT_RTO | SDMMC_INT_RCRC | \ 40 SDMMC_INT_RESP_ERR | SDMMC_INT_HLE) 41 #define DW_MCI_ERROR_FLAGS (DW_MCI_DATA_ERROR_FLAGS | \ 42 DW_MCI_CMD_ERROR_FLAGS) 43 44 #define DW_MCI_FREQ_MAX 200000000 /* unit: HZ */ 45 #define DW_MCI_FREQ_MIN 100000 /* unit: HZ */ 46 47 /* Event sets guarded by the central watchdog, see below. */ 48 #define DW_MCI_WD_CMD_EVENTS BIT(EVENT_CMD_COMPLETE) 49 #define DW_MCI_WD_DATA_EVENTS (BIT(EVENT_DATA_ERROR) | \ 50 BIT(EVENT_DATA_COMPLETE)) 51 /* Extra watch rounds granted while the completion IRQ is merely late. */ 52 #define DW_MCI_WD_INFLIGHT_GRACE_MS 10 53 54 /* 55 * Central watchdog 56 * ================ 57 * 58 * A single hrtimer guards the legs of a request (command, data, voltage 59 * switch) against the hardware going silent: whenever a completion event 60 * that was expected within a bounded time does not arrive in time, the 61 * callback synthesizes it so that the request state machine can error out. 62 * 63 * @wd_events holds the pending_events bits still being waited for and 64 * @wd_states the host->state values for which the watch is valid; both 65 * are protected by irq_lock. Every producer of the guarded events posts 66 * them under irq_lock as well, so a callback holding irq_lock can never 67 * miss an already-delivered event. A stale callback which raced past 68 * all checks therefore degrades to at most one redundant state-machine 69 * run instead of completing a foreign leg. Arming replaces whatever 70 * watch was outstanding before: command and data legs are strictly 71 * serial, so consecutive arms on the same path collapse onto the latest 72 * deadline. 73 * 74 * On expiry, before declaring a timeout, the callback double-checks that 75 * the completion interrupt is not already latched in hardware; if it is, 76 * only the interrupt latency is to blame and the watch is granted extra 77 * rounds instead of failing a transfer that is about to complete. 78 */ 79 static void dw_mci_wd_deliver(struct dw_mci *host, unsigned long events) 80 { 81 host->wd_events &= ~events; 82 if (!host->wd_events) 83 hrtimer_try_to_cancel(&host->wd_timer); 84 } 85 86 /* 87 * Arm a new watch for @events valid while host->state is one of @states, 88 * expiring after @ms milliseconds. @already lists bits whose delivery 89 * makes the watch pointless before it is even armed: arming is skipped 90 * when any of them has been posted concurrently. Must be called with 91 * irq_lock held. 92 */ 93 static void dw_mci_wd_arm(struct dw_mci *host, unsigned long ms, 94 unsigned long already, unsigned long events, 95 unsigned long states) 96 { 97 lockdep_assert_held(&host->irq_lock); 98 99 if (host->pending_events & already) { 100 /* 101 * The event was delivered concurrently; no need for a 102 * watch anymore. 103 */ 104 host->wd_events = 0; 105 hrtimer_try_to_cancel(&host->wd_timer); 106 return; 107 } 108 109 host->wd_events = events; 110 host->wd_states = states; 111 host->wd_deadline_ns = ktime_get_ns() + (u64)ms * NSEC_PER_MSEC; 112 hrtimer_start(&host->wd_timer, ms_to_ktime(ms), HRTIMER_MODE_REL); 113 } 114 115 /* Called from hardirq context on watchdog expiry. */ 116 static enum hrtimer_restart dw_mci_watchdog_fn(struct hrtimer *t) 117 { 118 struct dw_mci *host = container_of(t, struct dw_mci, wd_timer); 119 unsigned long irqflags; 120 u32 pending; 121 122 spin_lock_irqsave(&host->irq_lock, irqflags); 123 124 /* 125 * Last line of defense against a false timeout: when the completion 126 * interrupt is already latched in MINTSTS but its handler has not 127 * been scheduled yet, the leg is about to finish normally. Warn 128 * about the latency and keep watching instead of failing the 129 * transfer; delivery by the real interrupt will stop the watch. 130 * The retired per-timer callbacks went passive here and relied on 131 * the latched interrupt arriving; rearming bounds the damage if 132 * that assumption is ever broken. 133 */ 134 if (host->wd_events) { 135 pending = mci_readl(host, MINTSTS); /* read-only mask reg */ 136 if ((host->wd_events == DW_MCI_WD_CMD_EVENTS && 137 (pending & (DW_MCI_CMD_ERROR_FLAGS | 138 SDMMC_INT_CMD_DONE))) || 139 (host->wd_events == DW_MCI_WD_DATA_EVENTS && 140 (pending & (SDMMC_INT_DATA_OVER | 141 DW_MCI_DATA_ERROR_FLAGS)))) { 142 dev_warn(host->dev, 143 "Interrupt latency, state %d (watched %#lx)\n", 144 host->state, host->wd_events); 145 hrtimer_forward_now(&host->wd_timer, 146 ms_to_ktime(DW_MCI_WD_INFLIGHT_GRACE_MS)); 147 spin_unlock_irqrestore(&host->irq_lock, irqflags); 148 return HRTIMER_RESTART; 149 } 150 } 151 152 if (host->wd_events == DW_MCI_WD_DATA_EVENTS) { 153 dev_warn(host->dev, 154 "Data timeout, state %d\n", host->state); 155 host->data_status = SDMMC_INT_DRTO; 156 set_bit(EVENT_DATA_ERROR, &host->pending_events); 157 set_bit(EVENT_DATA_COMPLETE, &host->pending_events); 158 } else if (host->wd_events) { 159 dev_warn(host->dev, 160 "Command timeout, state %d (watched %#lx)\n", 161 host->state, host->wd_events); 162 host->cmd_status = SDMMC_INT_RTO; 163 set_bit(EVENT_CMD_COMPLETE, &host->pending_events); 164 } 165 166 if (host->wd_events) 167 queue_work(system_bh_wq, &host->bh_work); 168 169 host->wd_events = 0; 170 host->wd_states = 0; 171 host->wd_deadline_ns = 0; 172 173 spin_unlock_irqrestore(&host->irq_lock, irqflags); 174 175 return HRTIMER_NORESTART; 176 } 177 178 #define IDMAC_INT_CLR (SDMMC_IDMAC_INT_AI | SDMMC_IDMAC_INT_NI | \ 179 SDMMC_IDMAC_INT_CES | SDMMC_IDMAC_INT_DU | \ 180 SDMMC_IDMAC_INT_FBE | SDMMC_IDMAC_INT_RI | \ 181 SDMMC_IDMAC_INT_TI) 182 183 #define DESC_RING_BUF_SZ PAGE_SIZE 184 185 struct idmac_desc_64addr { 186 u32 des0; /* Control Descriptor */ 187 #define IDMAC_OWN_CLR64(x) \ 188 !((x) & cpu_to_le32(IDMAC_DES0_OWN)) 189 190 u32 des1; /* Reserved */ 191 192 u32 des2; /*Buffer sizes */ 193 #define IDMAC_64ADDR_SET_BUFFER1_SIZE(d, s) \ 194 ((d)->des2 = ((d)->des2 & cpu_to_le32(0x03ffe000)) | \ 195 ((cpu_to_le32(s)) & cpu_to_le32(0x1fff))) 196 197 u32 des3; /* Reserved */ 198 199 u32 des4; /* Lower 32-bits of Buffer Address Pointer 1*/ 200 u32 des5; /* Upper 32-bits of Buffer Address Pointer 1*/ 201 202 u32 des6; /* Lower 32-bits of Next Descriptor Address */ 203 u32 des7; /* Upper 32-bits of Next Descriptor Address */ 204 }; 205 206 struct idmac_desc { 207 __le32 des0; /* Control Descriptor */ 208 #define IDMAC_DES0_DIC BIT(1) 209 #define IDMAC_DES0_LD BIT(2) 210 #define IDMAC_DES0_FD BIT(3) 211 #define IDMAC_DES0_CH BIT(4) 212 #define IDMAC_DES0_ER BIT(5) 213 #define IDMAC_DES0_CES BIT(30) 214 #define IDMAC_DES0_OWN BIT(31) 215 216 __le32 des1; /* Buffer sizes */ 217 #define IDMAC_SET_BUFFER1_SIZE(d, s) \ 218 ((d)->des1 = ((d)->des1 & cpu_to_le32(0x03ffe000)) | (cpu_to_le32((s) & 0x1fff))) 219 220 __le32 des2; /* buffer 1 physical address */ 221 222 __le32 des3; /* buffer 2 physical address */ 223 }; 224 225 /* Each descriptor can transfer up to 4KB of data in chained mode */ 226 #define DW_MCI_DESC_DATA_LENGTH 0x1000 227 228 #if defined(CONFIG_DEBUG_FS) 229 static int dw_mci_req_show(struct seq_file *s, void *v) 230 { 231 struct dw_mci *host = s->private; 232 struct mmc_request *mrq; 233 struct mmc_command *cmd; 234 struct mmc_command *stop; 235 struct mmc_data *data; 236 237 /* Make sure we get a consistent snapshot */ 238 spin_lock_bh(&host->lock); 239 mrq = host->mrq; 240 241 if (mrq) { 242 cmd = mrq->cmd; 243 data = mrq->data; 244 stop = mrq->stop; 245 246 if (cmd) 247 seq_printf(s, 248 "CMD%u(0x%x) flg %x rsp %x %x %x %x err %d\n", 249 cmd->opcode, cmd->arg, cmd->flags, 250 cmd->resp[0], cmd->resp[1], cmd->resp[2], 251 cmd->resp[2], cmd->error); 252 if (data) 253 seq_printf(s, "DATA %u / %u * %u flg %x err %d\n", 254 data->bytes_xfered, data->blocks, 255 data->blksz, data->flags, data->error); 256 if (stop) 257 seq_printf(s, 258 "CMD%u(0x%x) flg %x rsp %x %x %x %x err %d\n", 259 stop->opcode, stop->arg, stop->flags, 260 stop->resp[0], stop->resp[1], stop->resp[2], 261 stop->resp[2], stop->error); 262 } 263 264 spin_unlock_bh(&host->lock); 265 266 return 0; 267 } 268 DEFINE_SHOW_ATTRIBUTE(dw_mci_req); 269 270 static int dw_mci_regs_show(struct seq_file *s, void *v) 271 { 272 struct dw_mci *host = s->private; 273 274 pm_runtime_get_sync(host->dev); 275 276 seq_printf(s, "STATUS:\t0x%08x\n", mci_readl(host, STATUS)); 277 seq_printf(s, "RINTSTS:\t0x%08x\n", mci_readl(host, RINTSTS)); 278 seq_printf(s, "CMD:\t0x%08x\n", mci_readl(host, CMD)); 279 seq_printf(s, "CTRL:\t0x%08x\n", mci_readl(host, CTRL)); 280 seq_printf(s, "INTMASK:\t0x%08x\n", mci_readl(host, INTMASK)); 281 seq_printf(s, "CLKENA:\t0x%08x\n", mci_readl(host, CLKENA)); 282 283 pm_runtime_put_autosuspend(host->dev); 284 285 return 0; 286 } 287 DEFINE_SHOW_ATTRIBUTE(dw_mci_regs); 288 289 static void dw_mci_init_debugfs(struct dw_mci *host) 290 { 291 struct mmc_host *mmc = host->mmc; 292 struct dentry *root; 293 294 root = mmc->debugfs_root; 295 if (!root) 296 return; 297 298 debugfs_create_file("regs", 0400, root, host, &dw_mci_regs_fops); 299 debugfs_create_file("req", 0400, root, host, &dw_mci_req_fops); 300 debugfs_create_u32("state", 0400, root, &host->state); 301 debugfs_create_xul("pending_events", 0400, root, 302 &host->pending_events); 303 debugfs_create_xul("completed_events", 0400, root, 304 &host->completed_events); 305 debugfs_create_x64("wd_deadline_ns", 0400, root, 306 &host->wd_deadline_ns); 307 debugfs_create_xul("wd_events", 0400, root, &host->wd_events); 308 debugfs_create_xul("wd_states", 0400, root, &host->wd_states); 309 #ifdef CONFIG_FAULT_INJECTION 310 fault_create_debugfs_attr("fail_data_crc", root, &host->fail_data_crc); 311 #endif 312 } 313 #endif /* defined(CONFIG_DEBUG_FS) */ 314 315 static bool dw_mci_ctrl_reset(struct dw_mci *host, u32 reset) 316 { 317 u32 ctrl; 318 319 ctrl = mci_readl(host, CTRL); 320 ctrl |= reset; 321 mci_writel(host, CTRL, ctrl); 322 323 /* wait till resets clear */ 324 if (readl_poll_timeout_atomic(host->regs + SDMMC_CTRL, ctrl, 325 !(ctrl & reset), 326 1, 500 * USEC_PER_MSEC)) { 327 dev_err(host->dev, 328 "Timeout resetting block (ctrl reset %#x)\n", 329 ctrl & reset); 330 return false; 331 } 332 333 return true; 334 } 335 336 static void dw_mci_wait_while_busy(struct dw_mci *host, u32 cmd_flags) 337 { 338 u32 status; 339 340 /* 341 * Databook says that before issuing a new data transfer command 342 * we need to check to see if the card is busy. Data transfer commands 343 * all have SDMMC_CMD_PRV_DAT_WAIT set, so we'll key off that. 344 * 345 * ...also allow sending for SDMMC_CMD_VOLT_SWITCH where busy is 346 * expected. 347 */ 348 if ((cmd_flags & SDMMC_CMD_PRV_DAT_WAIT) && 349 !(cmd_flags & SDMMC_CMD_VOLT_SWITCH)) { 350 if (readl_poll_timeout_atomic(host->regs + SDMMC_STATUS, 351 status, 352 !(status & SDMMC_STATUS_BUSY), 353 10, 500 * USEC_PER_MSEC)) 354 dev_err(host->dev, "Busy; trying anyway\n"); 355 } 356 } 357 358 static void mci_send_cmd(struct dw_mci *host, u32 cmd, u32 arg) 359 { 360 unsigned int cmd_status = 0; 361 362 mci_writel(host, CMDARG, arg); 363 wmb(); /* drain writebuffer */ 364 dw_mci_wait_while_busy(host, cmd); 365 mci_writel(host, CMD, SDMMC_CMD_START | cmd); 366 367 if (readl_poll_timeout_atomic(host->regs + SDMMC_CMD, cmd_status, 368 !(cmd_status & SDMMC_CMD_START), 369 1, 500 * USEC_PER_MSEC)) 370 dev_err(&host->mmc->class_dev, 371 "Timeout sending command (cmd %#x arg %#x status %#x)\n", 372 cmd, arg, cmd_status); 373 } 374 375 static u32 dw_mci_prepare_command(struct mmc_host *mmc, struct mmc_command *cmd) 376 { 377 struct dw_mci *host = mmc_priv(mmc); 378 u32 cmdr; 379 380 cmd->error = -EINPROGRESS; 381 cmdr = cmd->opcode; 382 383 if (cmd->opcode == MMC_STOP_TRANSMISSION || 384 cmd->opcode == MMC_GO_IDLE_STATE || 385 cmd->opcode == MMC_GO_INACTIVE_STATE || 386 (cmd->opcode == SD_IO_RW_DIRECT && 387 ((cmd->arg >> 9) & 0x1FFFF) == SDIO_CCCR_ABORT)) 388 cmdr |= SDMMC_CMD_STOP; 389 else if (cmd->opcode != MMC_SEND_STATUS && cmd->data) 390 cmdr |= SDMMC_CMD_PRV_DAT_WAIT; 391 392 if (cmd->opcode == SD_SWITCH_VOLTAGE) { 393 u32 clk_en_a; 394 395 /* Special bit makes CMD11 not die */ 396 cmdr |= SDMMC_CMD_VOLT_SWITCH; 397 398 /* Change state to continue to handle CMD11 weirdness */ 399 WARN_ON(host->state != STATE_SENDING_CMD); 400 host->state = STATE_SENDING_CMD11; 401 402 /* 403 * We need to disable low power mode (automatic clock stop) 404 * while doing voltage switch so we don't confuse the card, 405 * since stopping the clock is a specific part of the UHS 406 * voltage change dance. 407 * 408 * Note that low power mode (SDMMC_CLKEN_LOW_PWR) will be 409 * unconditionally turned back on in dw_mci_setup_bus() if it's 410 * ever called with a non-zero clock. That shouldn't happen 411 * until the voltage change is all done. 412 */ 413 clk_en_a = mci_readl(host, CLKENA); 414 clk_en_a &= ~SDMMC_CLKEN_LOW_PWR; 415 mci_writel(host, CLKENA, clk_en_a); 416 mci_send_cmd(host, SDMMC_CMD_UPD_CLK | 417 SDMMC_CMD_PRV_DAT_WAIT, 0); 418 } 419 420 if (cmd->flags & MMC_RSP_PRESENT) { 421 /* We expect a response, so set this bit */ 422 cmdr |= SDMMC_CMD_RESP_EXP; 423 if (cmd->flags & MMC_RSP_136) 424 cmdr |= SDMMC_CMD_RESP_LONG; 425 } 426 427 if (cmd->flags & MMC_RSP_CRC) 428 cmdr |= SDMMC_CMD_RESP_CRC; 429 430 if (cmd->data) { 431 cmdr |= SDMMC_CMD_DAT_EXP; 432 if (cmd->data->flags & MMC_DATA_WRITE) 433 cmdr |= SDMMC_CMD_DAT_WR; 434 } 435 436 if (!test_bit(DW_MMC_CARD_NO_USE_HOLD, &host->flags)) 437 cmdr |= SDMMC_CMD_USE_HOLD_REG; 438 439 return cmdr; 440 } 441 442 static u32 dw_mci_prep_stop_abort(struct dw_mci *host, struct mmc_command *cmd) 443 { 444 struct mmc_command *stop; 445 u32 cmdr; 446 447 if (!cmd->data) 448 return 0; 449 450 stop = &host->stop_abort; 451 cmdr = cmd->opcode; 452 memset(stop, 0, sizeof(struct mmc_command)); 453 454 if (cmdr == MMC_READ_SINGLE_BLOCK || 455 cmdr == MMC_READ_MULTIPLE_BLOCK || 456 cmdr == MMC_WRITE_BLOCK || 457 cmdr == MMC_WRITE_MULTIPLE_BLOCK || 458 mmc_op_tuning(cmdr) || 459 cmdr == MMC_GEN_CMD) { 460 stop->opcode = MMC_STOP_TRANSMISSION; 461 stop->arg = 0; 462 stop->flags = MMC_RSP_R1B | MMC_CMD_AC; 463 } else if (cmdr == SD_IO_RW_EXTENDED) { 464 stop->opcode = SD_IO_RW_DIRECT; 465 stop->arg |= (1 << 31) | (0 << 28) | (SDIO_CCCR_ABORT << 9) | 466 ((cmd->arg >> 28) & 0x7); 467 stop->flags = MMC_RSP_SPI_R5 | MMC_RSP_R5 | MMC_CMD_AC; 468 } else { 469 return 0; 470 } 471 472 cmdr = stop->opcode | SDMMC_CMD_STOP | 473 SDMMC_CMD_RESP_CRC | SDMMC_CMD_RESP_EXP; 474 475 if (!test_bit(DW_MMC_CARD_NO_USE_HOLD, &host->flags)) 476 cmdr |= SDMMC_CMD_USE_HOLD_REG; 477 478 return cmdr; 479 } 480 481 static inline void dw_mci_set_cto(struct dw_mci *host) 482 { 483 unsigned int cto_clks; 484 unsigned int cto_div; 485 unsigned int cto_ms; 486 unsigned long irqflags; 487 488 cto_clks = mci_readl(host, TMOUT) & 0xff; 489 cto_div = (mci_readl(host, CLKDIV) & 0xff) * 2; 490 if (cto_div == 0) 491 cto_div = 1; 492 493 cto_ms = DIV_ROUND_UP_ULL((u64)MSEC_PER_SEC * cto_clks * cto_div, 494 host->bus_hz); 495 496 /* add a bit spare time */ 497 cto_ms += 10; 498 499 /* 500 * The durations we're working with are fairly short so we have to be 501 * extra careful about synchronization here. Specifically in hardware a 502 * command timeout is _at most_ 5.1 ms, so that means we expect an 503 * interrupt (either command done or timeout) to come rather quickly 504 * after the mci_writel. 505 * 506 * dw_mci_wd_arm() skips arming when the event was already delivered, 507 * and the watchdog callback rechecks under the same lock before 508 * declaring a timeout, so both orderings of the race are covered. 509 */ 510 spin_lock_irqsave(&host->irq_lock, irqflags); 511 dw_mci_wd_arm(host, cto_ms, DW_MCI_WD_CMD_EVENTS, 512 DW_MCI_WD_CMD_EVENTS, 513 BIT(STATE_SENDING_CMD) | BIT(STATE_SENDING_STOP) | 514 BIT(STATE_SENDING_CMD11)); 515 spin_unlock_irqrestore(&host->irq_lock, irqflags); 516 } 517 518 static void dw_mci_start_command(struct dw_mci *host, 519 struct mmc_command *cmd, u32 cmd_flags) 520 { 521 host->cmd = cmd; 522 dev_vdbg(host->dev, 523 "start command: ARGR=0x%08x CMDR=0x%08x\n", 524 cmd->arg, cmd_flags); 525 526 mci_writel(host, CMDARG, cmd->arg); 527 wmb(); /* drain writebuffer */ 528 dw_mci_wait_while_busy(host, cmd_flags); 529 530 mci_writel(host, CMD, cmd_flags | SDMMC_CMD_START); 531 532 /* response expected command only */ 533 if (cmd_flags & SDMMC_CMD_RESP_EXP) 534 dw_mci_set_cto(host); 535 } 536 537 static inline void send_stop_abort(struct dw_mci *host, struct mmc_data *data) 538 { 539 struct mmc_command *stop = &host->stop_abort; 540 541 dw_mci_start_command(host, stop, host->stop_cmdr); 542 } 543 544 /* DMA interface functions */ 545 static void dw_mci_stop_dma(struct dw_mci *host) 546 { 547 if (host->using_dma) { 548 host->dma_ops->stop(host); 549 host->dma_ops->cleanup(host); 550 } 551 552 /* Data transfer was stopped by the interrupt handler */ 553 set_bit(EVENT_XFER_COMPLETE, &host->pending_events); 554 } 555 556 static void dw_mci_dma_cleanup(struct dw_mci *host) 557 { 558 struct mmc_data *data = host->data; 559 560 if (data && data->host_cookie == COOKIE_MAPPED) { 561 dma_unmap_sg(host->dev, 562 data->sg, 563 data->sg_len, 564 mmc_get_dma_dir(data)); 565 data->host_cookie = COOKIE_UNMAPPED; 566 } 567 } 568 569 static void dw_mci_idmac_reset(struct dw_mci *host) 570 { 571 u32 bmod = mci_readl(host, BMOD); 572 /* Software reset of DMA */ 573 bmod |= SDMMC_IDMAC_SWRESET; 574 mci_writel(host, BMOD, bmod); 575 } 576 577 static void dw_mci_idmac_stop_dma(struct dw_mci *host) 578 { 579 u32 temp; 580 581 /* Disable and reset the IDMAC interface */ 582 temp = mci_readl(host, CTRL); 583 temp &= ~SDMMC_CTRL_USE_IDMAC; 584 temp |= SDMMC_CTRL_DMA_RESET; 585 mci_writel(host, CTRL, temp); 586 587 /* Stop the IDMAC running */ 588 temp = mci_readl(host, BMOD); 589 temp &= ~(SDMMC_IDMAC_ENABLE | SDMMC_IDMAC_FB); 590 temp |= SDMMC_IDMAC_SWRESET; 591 mci_writel(host, BMOD, temp); 592 } 593 594 static void dw_mci_dmac_complete_dma(void *arg) 595 { 596 struct dw_mci *host = arg; 597 struct mmc_data *data = host->data; 598 599 dev_vdbg(host->dev, "DMA complete\n"); 600 601 if ((host->use_dma == TRANS_MODE_EDMAC) && 602 data && (data->flags & MMC_DATA_READ)) 603 /* Invalidate cache after read */ 604 dma_sync_sg_for_cpu(mmc_dev(host->mmc), 605 data->sg, 606 data->sg_len, 607 DMA_FROM_DEVICE); 608 609 host->dma_ops->cleanup(host); 610 611 /* 612 * If the card was removed, data will be NULL. No point in trying to 613 * send the stop command or waiting for NBUSY in this case. 614 */ 615 if (data) { 616 set_bit(EVENT_XFER_COMPLETE, &host->pending_events); 617 queue_work(system_bh_wq, &host->bh_work); 618 } 619 } 620 621 static int dw_mci_idmac_init(struct dw_mci *host) 622 { 623 int i; 624 625 if (host->dma_64bit_address == 1) { 626 struct idmac_desc_64addr *p; 627 628 host->desc_num = 629 DESC_RING_BUF_SZ / sizeof(struct idmac_desc_64addr); 630 631 /* Forward link the descriptor list */ 632 for (i = 0, p = host->sg_cpu; i < host->desc_num - 1; 633 i++, p++) { 634 p->des6 = (host->sg_dma + 635 (sizeof(struct idmac_desc_64addr) * 636 (i + 1))) & 0xffffffff; 637 638 p->des7 = (u64)(host->sg_dma + 639 (sizeof(struct idmac_desc_64addr) * 640 (i + 1))) >> 32; 641 /* Initialize reserved and buffer size fields to "0" */ 642 p->des0 = 0; 643 p->des1 = 0; 644 p->des2 = 0; 645 p->des3 = 0; 646 } 647 648 /* Set the last descriptor as the end-of-ring descriptor */ 649 p->des6 = host->sg_dma & 0xffffffff; 650 p->des7 = (u64)host->sg_dma >> 32; 651 p->des0 = IDMAC_DES0_ER; 652 653 } else { 654 struct idmac_desc *p; 655 656 host->desc_num = 657 DESC_RING_BUF_SZ / sizeof(struct idmac_desc); 658 659 /* Forward link the descriptor list */ 660 for (i = 0, p = host->sg_cpu; 661 i < host->desc_num - 1; 662 i++, p++) { 663 p->des3 = cpu_to_le32(host->sg_dma + 664 (sizeof(struct idmac_desc) * (i + 1))); 665 p->des0 = 0; 666 p->des1 = 0; 667 } 668 669 /* Set the last descriptor as the end-of-ring descriptor */ 670 p->des3 = cpu_to_le32(host->sg_dma); 671 p->des0 = cpu_to_le32(IDMAC_DES0_ER); 672 } 673 674 dw_mci_idmac_reset(host); 675 676 if (host->dma_64bit_address == 1) { 677 /* Mask out interrupts - get Tx & Rx complete only */ 678 mci_writel(host, IDSTS64, IDMAC_INT_CLR); 679 mci_writel(host, IDINTEN64, SDMMC_IDMAC_INT_NI | 680 SDMMC_IDMAC_INT_RI | SDMMC_IDMAC_INT_TI); 681 682 /* Set the descriptor base address */ 683 mci_writel(host, DBADDRL, host->sg_dma & 0xffffffff); 684 mci_writel(host, DBADDRU, (u64)host->sg_dma >> 32); 685 686 } else { 687 /* Mask out interrupts - get Tx & Rx complete only */ 688 mci_writel(host, IDSTS, IDMAC_INT_CLR); 689 mci_writel(host, IDINTEN, SDMMC_IDMAC_INT_NI | 690 SDMMC_IDMAC_INT_RI | SDMMC_IDMAC_INT_TI); 691 692 /* Set the descriptor base address */ 693 mci_writel(host, DBADDR, host->sg_dma); 694 } 695 696 return 0; 697 } 698 699 static inline int dw_mci_prepare_desc(struct dw_mci *host, struct mmc_data *data, 700 unsigned int sg_len, bool is_64bit) 701 { 702 unsigned int desc_len; 703 struct idmac_desc *desc_first, *desc_last, *desc; 704 struct idmac_desc_64addr *desc64_first, *desc64_last, *desc64; 705 u32 val, des0; 706 int i, err; 707 708 if (is_64bit) 709 desc64_first = desc64_last = desc64 = host->sg_cpu; 710 else 711 desc_first = desc_last = desc = host->sg_cpu; 712 713 for (i = 0; i < sg_len; i++) { 714 unsigned int length = sg_dma_len(&data->sg[i]); 715 716 u64 mem_addr = sg_dma_address(&data->sg[i]); 717 718 while (length > 0) { 719 desc_len = (length <= DW_MCI_DESC_DATA_LENGTH) ? 720 length : DW_MCI_DESC_DATA_LENGTH; 721 722 length -= desc_len; 723 724 /* 725 * Wait for the former clear OWN bit operation 726 * of IDMAC to make sure that this descriptor 727 * isn't still owned by IDMAC as IDMAC's write 728 * ops and CPU's read ops are asynchronous. 729 */ 730 if (is_64bit) 731 err = readl_poll_timeout_atomic(&desc64->des0, val, 732 IDMAC_OWN_CLR64(val), 10, 100 * USEC_PER_MSEC); 733 else 734 err = readl_poll_timeout_atomic(&desc->des0, val, 735 IDMAC_OWN_CLR64(val), 10, 100 * USEC_PER_MSEC); 736 if (err) 737 goto err_own_bit; 738 739 des0 = IDMAC_DES0_OWN | IDMAC_DES0_DIC | IDMAC_DES0_CH; 740 if (is_64bit) 741 desc64->des0 = des0; 742 else 743 desc->des0 = cpu_to_le32(des0); 744 745 /* 746 * 1. Set OWN bit and disable interrupts for this descriptor 747 * 2. Set Buffer length 748 * Set physical address to DMA to/from 749 */ 750 if (is_64bit) { 751 desc64->des0 = IDMAC_DES0_OWN | IDMAC_DES0_DIC | IDMAC_DES0_CH; 752 IDMAC_64ADDR_SET_BUFFER1_SIZE(desc64, desc_len); 753 desc64->des4 = mem_addr & 0xffffffff; 754 desc64->des5 = mem_addr >> 32; 755 } else { 756 IDMAC_SET_BUFFER1_SIZE(desc, desc_len); 757 desc->des2 = cpu_to_le32(mem_addr); 758 } 759 760 /* Update physical address for the next desc */ 761 mem_addr += desc_len; 762 763 /* Save pointer to the last descriptor */ 764 if (is_64bit) { 765 desc64_last = desc64; 766 desc64++; 767 } else { 768 desc_last = desc; 769 desc++; 770 } 771 } 772 } 773 774 /* Set the first descriptor and the last descriptor */ 775 if (is_64bit) { 776 desc64_first->des0 |= IDMAC_DES0_FD; 777 desc64_last->des0 &= ~(IDMAC_DES0_CH | IDMAC_DES0_DIC); 778 desc64_last->des0 |= IDMAC_DES0_LD; 779 } else { 780 desc_first->des0 |= cpu_to_le32(IDMAC_DES0_FD); 781 desc_last->des0 &= cpu_to_le32(~(IDMAC_DES0_CH | IDMAC_DES0_DIC)); 782 desc_last->des0 |= cpu_to_le32(IDMAC_DES0_LD); 783 } 784 785 return 0; 786 err_own_bit: 787 /* restore the descriptor chain as it's polluted */ 788 dev_dbg(host->dev, "descriptor is still owned by IDMAC.\n"); 789 memset(host->sg_cpu, 0, DESC_RING_BUF_SZ); 790 dw_mci_idmac_init(host); 791 return -EINVAL; 792 } 793 794 static int dw_mci_idmac_start_dma(struct dw_mci *host, unsigned int sg_len) 795 { 796 u32 temp; 797 int ret; 798 799 ret = dw_mci_prepare_desc(host, host->data, sg_len, host->dma_64bit_address); 800 if (ret) 801 goto out; 802 803 /* drain writebuffer */ 804 wmb(); 805 806 /* Make sure to reset DMA in case we did PIO before this */ 807 dw_mci_ctrl_reset(host, SDMMC_CTRL_DMA_RESET); 808 dw_mci_idmac_reset(host); 809 810 /* Select IDMAC interface */ 811 temp = mci_readl(host, CTRL); 812 temp |= SDMMC_CTRL_USE_IDMAC; 813 mci_writel(host, CTRL, temp); 814 815 /* drain writebuffer */ 816 wmb(); 817 818 /* Enable the IDMAC */ 819 temp = mci_readl(host, BMOD); 820 temp |= SDMMC_IDMAC_ENABLE | SDMMC_IDMAC_FB; 821 mci_writel(host, BMOD, temp); 822 823 /* Start it running */ 824 mci_writel(host, PLDMND, 1); 825 826 out: 827 return ret; 828 } 829 830 static const struct dw_mci_dma_ops dw_mci_idmac_ops = { 831 .init = dw_mci_idmac_init, 832 .start = dw_mci_idmac_start_dma, 833 .stop = dw_mci_idmac_stop_dma, 834 .complete = dw_mci_dmac_complete_dma, 835 .cleanup = dw_mci_dma_cleanup, 836 }; 837 838 static void dw_mci_edmac_stop_dma(struct dw_mci *host) 839 { 840 dmaengine_terminate_async(host->dms->ch); 841 } 842 843 static int dw_mci_edmac_start_dma(struct dw_mci *host, 844 unsigned int sg_len) 845 { 846 struct dma_slave_config cfg; 847 struct dma_async_tx_descriptor *desc = NULL; 848 struct scatterlist *sgl = host->data->sg; 849 static const u32 mszs[] = {1, 4, 8, 16, 32, 64, 128, 256}; 850 u32 sg_elems = host->data->sg_len; 851 u32 fifoth_val; 852 u32 fifo_offset = host->fifo_reg - host->regs; 853 int ret = 0; 854 855 /* Set external dma config: burst size, burst width */ 856 memset(&cfg, 0, sizeof(cfg)); 857 cfg.dst_addr = host->phy_regs + fifo_offset; 858 cfg.src_addr = cfg.dst_addr; 859 cfg.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; 860 cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; 861 862 /* Match burst msize with external dma config */ 863 fifoth_val = mci_readl(host, FIFOTH); 864 cfg.dst_maxburst = mszs[(fifoth_val >> 28) & 0x7]; 865 cfg.src_maxburst = cfg.dst_maxburst; 866 867 if (host->data->flags & MMC_DATA_WRITE) 868 cfg.direction = DMA_MEM_TO_DEV; 869 else 870 cfg.direction = DMA_DEV_TO_MEM; 871 872 ret = dmaengine_slave_config(host->dms->ch, &cfg); 873 if (ret) { 874 dev_err(host->dev, "Failed to config edmac.\n"); 875 return -EBUSY; 876 } 877 878 desc = dmaengine_prep_slave_sg(host->dms->ch, sgl, 879 sg_len, cfg.direction, 880 DMA_PREP_INTERRUPT | DMA_CTRL_ACK); 881 if (!desc) { 882 dev_err(host->dev, "Can't prepare slave sg.\n"); 883 return -EBUSY; 884 } 885 886 /* Set dw_mci_dmac_complete_dma as callback */ 887 desc->callback = dw_mci_dmac_complete_dma; 888 desc->callback_param = (void *)host; 889 dmaengine_submit(desc); 890 891 /* Flush cache before write */ 892 if (host->data->flags & MMC_DATA_WRITE) 893 dma_sync_sg_for_device(mmc_dev(host->mmc), sgl, 894 sg_elems, DMA_TO_DEVICE); 895 896 dma_async_issue_pending(host->dms->ch); 897 898 return 0; 899 } 900 901 static int dw_mci_edmac_init(struct dw_mci *host) 902 { 903 /* Request external dma channel */ 904 host->dms = kzalloc_obj(struct dw_mci_dma_slave); 905 if (!host->dms) 906 return -ENOMEM; 907 908 host->dms->ch = dma_request_chan(host->dev, "rx-tx"); 909 if (IS_ERR(host->dms->ch)) { 910 int ret = PTR_ERR(host->dms->ch); 911 912 dev_err(host->dev, "Failed to get external DMA channel.\n"); 913 kfree(host->dms); 914 host->dms = NULL; 915 return ret; 916 } 917 918 return 0; 919 } 920 921 static void dw_mci_edmac_exit(struct dw_mci *host) 922 { 923 if (host->dms) { 924 if (host->dms->ch) { 925 dma_release_channel(host->dms->ch); 926 host->dms->ch = NULL; 927 } 928 kfree(host->dms); 929 host->dms = NULL; 930 } 931 } 932 933 static const struct dw_mci_dma_ops dw_mci_edmac_ops = { 934 .init = dw_mci_edmac_init, 935 .exit = dw_mci_edmac_exit, 936 .start = dw_mci_edmac_start_dma, 937 .stop = dw_mci_edmac_stop_dma, 938 .complete = dw_mci_dmac_complete_dma, 939 .cleanup = dw_mci_dma_cleanup, 940 }; 941 942 static int dw_mci_pre_dma_transfer(struct dw_mci *host, 943 struct mmc_data *data, 944 int cookie) 945 { 946 struct scatterlist *sg; 947 unsigned int i, sg_len; 948 949 if (data->host_cookie == COOKIE_PRE_MAPPED) 950 return data->sg_len; 951 952 /* 953 * We don't do DMA on "complex" transfers, i.e. with 954 * non-word-aligned buffers or lengths. Also, we don't bother 955 * with all the DMA setup overhead for short transfers. 956 */ 957 if (data->blocks * data->blksz < host->dma_threshold) 958 return -EINVAL; 959 960 if (data->blksz & 3) 961 return -EINVAL; 962 963 for_each_sg(data->sg, sg, data->sg_len, i) { 964 if (sg->offset & 3 || sg->length & 3) 965 return -EINVAL; 966 } 967 968 sg_len = dma_map_sg(host->dev, 969 data->sg, 970 data->sg_len, 971 mmc_get_dma_dir(data)); 972 if (sg_len == 0) 973 return -EINVAL; 974 975 data->host_cookie = cookie; 976 977 return sg_len; 978 } 979 980 static void dw_mci_pre_req(struct mmc_host *mmc, 981 struct mmc_request *mrq) 982 { 983 struct dw_mci *host = mmc_priv(mmc); 984 struct mmc_data *data = mrq->data; 985 986 if (!host->use_dma || !data) 987 return; 988 989 /* This data might be unmapped at this time */ 990 data->host_cookie = COOKIE_UNMAPPED; 991 992 if (dw_mci_pre_dma_transfer(host, mrq->data, 993 COOKIE_PRE_MAPPED) < 0) 994 data->host_cookie = COOKIE_UNMAPPED; 995 } 996 997 static void dw_mci_post_req(struct mmc_host *mmc, 998 struct mmc_request *mrq, 999 int err) 1000 { 1001 struct dw_mci *host = mmc_priv(mmc); 1002 struct mmc_data *data = mrq->data; 1003 1004 if (!host->use_dma || !data) 1005 return; 1006 1007 if (data->host_cookie != COOKIE_UNMAPPED) 1008 dma_unmap_sg(host->dev, 1009 data->sg, 1010 data->sg_len, 1011 mmc_get_dma_dir(data)); 1012 data->host_cookie = COOKIE_UNMAPPED; 1013 } 1014 1015 static int dw_mci_get_cd(struct mmc_host *mmc) 1016 { 1017 struct dw_mci *host = mmc_priv(mmc); 1018 int gpio_cd = mmc_gpio_get_cd(mmc); 1019 1020 if (mmc->caps & MMC_CAP_NEEDS_POLL) 1021 return 1; 1022 1023 if (!mmc_card_is_removable(mmc)) 1024 return 1; 1025 1026 /* Try slot gpio detection */ 1027 if (gpio_cd >= 0) 1028 return !!gpio_cd; 1029 1030 /* Host native card detect */ 1031 return !(mci_readl(host, CDETECT) & BIT(0)); 1032 } 1033 1034 static void dw_mci_adjust_fifoth(struct dw_mci *host, struct mmc_data *data) 1035 { 1036 unsigned int blksz = data->blksz; 1037 static const u32 mszs[] = {1, 4, 8, 16, 32, 64, 128, 256}; 1038 u32 fifo_width = 1 << host->data_shift; 1039 u32 blksz_depth = blksz / fifo_width, fifoth_val; 1040 u32 msize = 0, rx_wmark = 1, tx_wmark, tx_wmark_invers; 1041 int idx = ARRAY_SIZE(mszs) - 1; 1042 1043 /* pio should ship this scenario */ 1044 if (!host->use_dma) 1045 return; 1046 1047 tx_wmark = (host->fifo_depth) / 2; 1048 tx_wmark_invers = host->fifo_depth - tx_wmark; 1049 1050 /* 1051 * MSIZE is '1', 1052 * if blksz is not a multiple of the FIFO width 1053 */ 1054 if (blksz % fifo_width) 1055 goto done; 1056 1057 do { 1058 if (!((blksz_depth % mszs[idx]) || 1059 (tx_wmark_invers % mszs[idx]))) { 1060 msize = idx; 1061 rx_wmark = mszs[idx] - 1; 1062 break; 1063 } 1064 } while (--idx > 0); 1065 /* 1066 * If idx is '0', it won't be tried 1067 * Thus, initial values are uesed 1068 */ 1069 done: 1070 fifoth_val = SDMMC_SET_FIFOTH(msize, rx_wmark, tx_wmark); 1071 mci_writel(host, FIFOTH, fifoth_val); 1072 } 1073 1074 static void dw_mci_ctrl_thld(struct dw_mci *host, struct mmc_data *data) 1075 { 1076 unsigned int blksz = data->blksz; 1077 u32 blksz_depth, fifo_depth; 1078 u16 thld_size; 1079 u8 enable; 1080 1081 /* 1082 * CDTHRCTL doesn't exist prior to 240A (in fact that register offset is 1083 * in the FIFO region, so we really shouldn't access it). 1084 */ 1085 if (host->verid < DW_MMC_240A || 1086 (host->verid < DW_MMC_280A && data->flags & MMC_DATA_WRITE)) 1087 return; 1088 1089 /* 1090 * Card write Threshold is introduced since 2.80a 1091 * It's used when HS400 mode is enabled. 1092 */ 1093 if (data->flags & MMC_DATA_WRITE && 1094 host->timing != MMC_TIMING_MMC_HS400) 1095 goto disable; 1096 1097 if (data->flags & MMC_DATA_WRITE) 1098 enable = SDMMC_CARD_WR_THR_EN; 1099 else 1100 enable = SDMMC_CARD_RD_THR_EN; 1101 1102 if (host->timing != MMC_TIMING_MMC_HS200 && 1103 host->timing != MMC_TIMING_UHS_SDR104 && 1104 host->timing != MMC_TIMING_MMC_HS400) 1105 goto disable; 1106 1107 blksz_depth = blksz / (1 << host->data_shift); 1108 fifo_depth = host->fifo_depth; 1109 1110 if (blksz_depth > fifo_depth) 1111 goto disable; 1112 1113 /* 1114 * If (blksz_depth) >= (fifo_depth >> 1), should be 'thld_size <= blksz' 1115 * If (blksz_depth) < (fifo_depth >> 1), should be thld_size = blksz 1116 * Currently just choose blksz. 1117 */ 1118 thld_size = blksz; 1119 mci_writel(host, CDTHRCTL, SDMMC_SET_THLD(thld_size, enable)); 1120 return; 1121 1122 disable: 1123 mci_writel(host, CDTHRCTL, 0); 1124 } 1125 1126 static int dw_mci_submit_data_dma(struct dw_mci *host, struct mmc_data *data) 1127 { 1128 unsigned long irqflags; 1129 int sg_len; 1130 u32 temp; 1131 1132 host->using_dma = 0; 1133 1134 /* If we don't have a channel, we can't do DMA */ 1135 if (!host->use_dma) 1136 return -ENODEV; 1137 1138 sg_len = dw_mci_pre_dma_transfer(host, data, COOKIE_MAPPED); 1139 if (sg_len < 0) { 1140 host->dma_ops->stop(host); 1141 return sg_len; 1142 } 1143 1144 host->using_dma = 1; 1145 1146 if (host->use_dma == TRANS_MODE_IDMAC) 1147 dev_vdbg(host->dev, 1148 "sd sg_cpu: %#lx sg_dma: %#lx sg_len: %d\n", 1149 (unsigned long)host->sg_cpu, 1150 (unsigned long)host->sg_dma, 1151 sg_len); 1152 1153 /* 1154 * Decide the MSIZE and RX/TX Watermark. 1155 * If current block size is same with previous size, 1156 * no need to update fifoth. 1157 */ 1158 if (host->prev_blksz != data->blksz) 1159 dw_mci_adjust_fifoth(host, data); 1160 1161 /* Enable the DMA interface */ 1162 temp = mci_readl(host, CTRL); 1163 temp |= SDMMC_CTRL_DMA_ENABLE; 1164 mci_writel(host, CTRL, temp); 1165 1166 /* Disable RX/TX IRQs, let DMA handle it */ 1167 spin_lock_irqsave(&host->irq_lock, irqflags); 1168 temp = mci_readl(host, INTMASK); 1169 temp &= ~(SDMMC_INT_RXDR | SDMMC_INT_TXDR); 1170 mci_writel(host, INTMASK, temp); 1171 spin_unlock_irqrestore(&host->irq_lock, irqflags); 1172 1173 if (host->dma_ops->start(host, sg_len)) { 1174 host->dma_ops->stop(host); 1175 /* We can't do DMA, try PIO for this one */ 1176 dev_dbg(host->dev, 1177 "%s: fall back to PIO mode for current transfer\n", 1178 __func__); 1179 return -ENODEV; 1180 } 1181 1182 return 0; 1183 } 1184 1185 static void dw_mci_submit_data(struct dw_mci *host, struct mmc_data *data) 1186 { 1187 unsigned long irqflags; 1188 int flags = SG_MITER_ATOMIC; 1189 u32 temp; 1190 1191 data->error = -EINPROGRESS; 1192 1193 WARN_ON(host->data); 1194 host->sg = NULL; 1195 host->data = data; 1196 1197 if (data->flags & MMC_DATA_READ) 1198 host->dir_status = MMC_DATA_READ; 1199 else 1200 host->dir_status = MMC_DATA_WRITE; 1201 1202 dw_mci_ctrl_thld(host, data); 1203 1204 if (dw_mci_submit_data_dma(host, data)) { 1205 if (host->data->flags & MMC_DATA_READ) 1206 flags |= SG_MITER_TO_SG; 1207 else 1208 flags |= SG_MITER_FROM_SG; 1209 1210 sg_miter_start(&host->sg_miter, data->sg, data->sg_len, flags); 1211 host->sg = data->sg; 1212 host->part_buf_start = 0; 1213 host->part_buf_count = 0; 1214 1215 mci_writel(host, RINTSTS, SDMMC_INT_TXDR | SDMMC_INT_RXDR); 1216 1217 spin_lock_irqsave(&host->irq_lock, irqflags); 1218 temp = mci_readl(host, INTMASK); 1219 temp |= SDMMC_INT_TXDR | SDMMC_INT_RXDR; 1220 mci_writel(host, INTMASK, temp); 1221 spin_unlock_irqrestore(&host->irq_lock, irqflags); 1222 1223 temp = mci_readl(host, CTRL); 1224 temp &= ~SDMMC_CTRL_DMA_ENABLE; 1225 mci_writel(host, CTRL, temp); 1226 1227 /* 1228 * Use the initial fifoth_val for PIO mode. If wm_algined 1229 * is set, we set watermark same as data size. 1230 * If next issued data may be transferred by DMA mode, 1231 * prev_blksz should be invalidated. 1232 */ 1233 if (host->wm_aligned) 1234 dw_mci_adjust_fifoth(host, data); 1235 else 1236 mci_writel(host, FIFOTH, host->fifoth_val); 1237 host->prev_blksz = 0; 1238 } else { 1239 /* 1240 * Keep the current block size. 1241 * It will be used to decide whether to update 1242 * fifoth register next time. 1243 */ 1244 host->prev_blksz = data->blksz; 1245 } 1246 } 1247 1248 static void dw_mci_setup_bus(struct dw_mci *host, bool force_clkinit) 1249 { 1250 unsigned int clock = host->clock; 1251 u32 div; 1252 u32 clk_en_a; 1253 u32 sdmmc_cmd_bits = SDMMC_CMD_UPD_CLK | SDMMC_CMD_PRV_DAT_WAIT; 1254 1255 /* We must continue to set bit 28 in CMD until the change is complete */ 1256 if (host->state == STATE_WAITING_CMD11_DONE) 1257 sdmmc_cmd_bits |= SDMMC_CMD_VOLT_SWITCH; 1258 1259 host->mmc->actual_clock = 0; 1260 1261 if (!clock) { 1262 mci_writel(host, CLKENA, 0); 1263 mci_send_cmd(host, sdmmc_cmd_bits, 0); 1264 } else if (clock != host->current_speed || force_clkinit) { 1265 div = host->bus_hz / clock; 1266 if (host->bus_hz % clock && host->bus_hz > clock) 1267 /* 1268 * move the + 1 after the divide to prevent 1269 * over-clocking the card. 1270 */ 1271 div += 1; 1272 1273 div = (host->bus_hz != clock) ? DIV_ROUND_UP(div, 2) : 0; 1274 1275 if ((clock != host->clk_old && 1276 !test_bit(DW_MMC_CARD_NEEDS_POLL, &host->flags)) || 1277 force_clkinit) { 1278 /* Silent the verbose log if calling from PM context */ 1279 if (!force_clkinit) 1280 dev_info(&host->mmc->class_dev, 1281 "Bus speed = %dHz (req %dHz, actual %dHZ div = %d)\n", 1282 host->bus_hz, clock, 1283 div ? ((host->bus_hz / div) >> 1) : 1284 host->bus_hz, div); 1285 1286 /* 1287 * If card is polling, display the message only 1288 * one time at boot time. 1289 */ 1290 if (host->mmc->caps & MMC_CAP_NEEDS_POLL && 1291 host->mmc->f_min == clock) 1292 set_bit(DW_MMC_CARD_NEEDS_POLL, &host->flags); 1293 } 1294 1295 /* disable clock */ 1296 mci_writel(host, CLKENA, 0); 1297 mci_writel(host, CLKSRC, 0); 1298 1299 /* inform CIU */ 1300 mci_send_cmd(host, sdmmc_cmd_bits, 0); 1301 1302 /* set clock to desired speed */ 1303 mci_writel(host, CLKDIV, div); 1304 1305 /* inform CIU */ 1306 mci_send_cmd(host, sdmmc_cmd_bits, 0); 1307 1308 /* enable clock; only low power if no SDIO */ 1309 clk_en_a = SDMMC_CLKEN_ENABLE; 1310 if (!test_bit(DW_MMC_CARD_NO_LOW_PWR, &host->flags)) 1311 clk_en_a |= SDMMC_CLKEN_LOW_PWR; 1312 mci_writel(host, CLKENA, clk_en_a); 1313 1314 /* inform CIU */ 1315 mci_send_cmd(host, sdmmc_cmd_bits, 0); 1316 1317 /* keep the last clock value that was requested from core */ 1318 host->clk_old = clock; 1319 host->mmc->actual_clock = div ? ((host->bus_hz / div) >> 1) : 1320 host->bus_hz; 1321 } 1322 1323 host->current_speed = clock; 1324 1325 /* Set the current bus width */ 1326 mci_writel(host, CTYPE, host->ctype); 1327 } 1328 1329 static void dw_mci_set_data_timeout(struct dw_mci *host, 1330 unsigned int timeout_ns) 1331 { 1332 const struct dw_mci_drv_data *drv_data = host->drv_data; 1333 u32 clk_div, tmout; 1334 u64 tmp; 1335 1336 if (drv_data && drv_data->set_data_timeout) 1337 return drv_data->set_data_timeout(host, timeout_ns); 1338 1339 clk_div = (mci_readl(host, CLKDIV) & 0xFF) * 2; 1340 if (clk_div == 0) 1341 clk_div = 1; 1342 1343 tmp = DIV_ROUND_UP_ULL((u64)timeout_ns * host->bus_hz, NSEC_PER_SEC); 1344 tmp = DIV_ROUND_UP_ULL(tmp, clk_div); 1345 1346 /* TMOUT[7:0] (RESPONSE_TIMEOUT) */ 1347 tmout = 0xFF; /* Set maximum */ 1348 1349 /* TMOUT[31:8] (DATA_TIMEOUT) */ 1350 if (!tmp || tmp > 0xFFFFFF) 1351 tmout |= (0xFFFFFF << 8); 1352 else 1353 tmout |= (tmp & 0xFFFFFF) << 8; 1354 1355 mci_writel(host, TMOUT, tmout); 1356 dev_dbg(host->dev, "timeout_ns: %u => TMOUT[31:8]: %#08x", 1357 timeout_ns, tmout >> 8); 1358 } 1359 1360 static void dw_mci_start_request(struct dw_mci *host, struct mmc_command *cmd) 1361 { 1362 struct mmc_request *mrq; 1363 struct mmc_data *data; 1364 u32 cmdflags; 1365 1366 mrq = host->mrq; 1367 1368 host->mrq = mrq; 1369 1370 host->pending_events = 0; 1371 host->completed_events = 0; 1372 host->cmd_status = 0; 1373 host->data_status = 0; 1374 host->dir_status = 0; 1375 1376 data = cmd->data; 1377 if (data) { 1378 dw_mci_set_data_timeout(host, data->timeout_ns); 1379 mci_writel(host, BYTCNT, data->blksz*data->blocks); 1380 mci_writel(host, BLKSIZ, data->blksz); 1381 } 1382 1383 cmdflags = dw_mci_prepare_command(host->mmc, cmd); 1384 1385 /* this is the first command, send the initialization clock */ 1386 if (test_and_clear_bit(DW_MMC_CARD_NEED_INIT, &host->flags)) 1387 cmdflags |= SDMMC_CMD_INIT; 1388 1389 if (data) { 1390 dw_mci_submit_data(host, data); 1391 wmb(); /* drain writebuffer */ 1392 } 1393 1394 dw_mci_start_command(host, cmd, cmdflags); 1395 1396 if (cmd->opcode == SD_SWITCH_VOLTAGE) { 1397 unsigned long irqflags; 1398 1399 /* 1400 * Databook says to fail after 2ms w/ no response, but evidence 1401 * shows that sometimes the cmd11 interrupt takes over 130ms. 1402 * We'll set to 500ms. 1403 * 1404 * We do this whole thing under spinlock and only if the 1405 * command hasn't already completed (indicating the irq 1406 * already ran so we don't want the timeout). 1407 */ 1408 spin_lock_irqsave(&host->irq_lock, irqflags); 1409 dw_mci_wd_arm(host, 500, DW_MCI_WD_CMD_EVENTS, 1410 DW_MCI_WD_CMD_EVENTS, 1411 BIT(STATE_SENDING_CMD11)); 1412 spin_unlock_irqrestore(&host->irq_lock, irqflags); 1413 } 1414 1415 host->stop_cmdr = dw_mci_prep_stop_abort(host, cmd); 1416 } 1417 1418 static void dw_mci_request(struct mmc_host *mmc, struct mmc_request *mrq) 1419 { 1420 struct dw_mci *host = mmc_priv(mmc); 1421 struct mmc_command *cmd; 1422 1423 WARN_ON(host->mrq); 1424 1425 /* 1426 * The check for card presence and queueing of the request must be 1427 * atomic, otherwise the card could be removed in between and the 1428 * request wouldn't fail until another card was inserted. 1429 */ 1430 if (!dw_mci_get_cd(mmc)) { 1431 mrq->cmd->error = -ENOMEDIUM; 1432 mmc_request_done(mmc, mrq); 1433 return; 1434 } 1435 1436 spin_lock_bh(&host->lock); 1437 1438 dev_vdbg(&host->mmc->class_dev, "request: state=%d\n", 1439 host->state); 1440 1441 host->mrq = mrq; 1442 if (host->state == STATE_WAITING_CMD11_DONE) { 1443 dev_warn(&host->mmc->class_dev, 1444 "Voltage change didn't complete\n"); 1445 /* 1446 * this case isn't expected to happen, so we can 1447 * either crash here or just try to continue on 1448 * in the closest possible state 1449 */ 1450 host->state = STATE_IDLE; 1451 } 1452 1453 if (host->state == STATE_IDLE) { 1454 host->state = STATE_SENDING_CMD; 1455 cmd = mrq->sbc ? mrq->sbc : mrq->cmd; 1456 dw_mci_start_request(host, cmd); 1457 } 1458 1459 spin_unlock_bh(&host->lock); 1460 } 1461 1462 static void dw_mci_set_ios(struct mmc_host *mmc, struct mmc_ios *ios) 1463 { 1464 struct dw_mci *host = mmc_priv(mmc); 1465 const struct dw_mci_drv_data *drv_data = host->drv_data; 1466 u32 regs; 1467 int ret; 1468 1469 switch (ios->bus_width) { 1470 case MMC_BUS_WIDTH_4: 1471 host->ctype = SDMMC_CTYPE_4BIT; 1472 break; 1473 case MMC_BUS_WIDTH_8: 1474 host->ctype = SDMMC_CTYPE_8BIT; 1475 break; 1476 default: 1477 /* set default 1 bit mode */ 1478 host->ctype = SDMMC_CTYPE_1BIT; 1479 } 1480 1481 regs = mci_readl(host, UHS_REG); 1482 1483 /* DDR mode set */ 1484 if (ios->timing == MMC_TIMING_MMC_DDR52 || 1485 ios->timing == MMC_TIMING_UHS_DDR50 || 1486 ios->timing == MMC_TIMING_MMC_HS400) 1487 regs |= BIT(16); 1488 else 1489 regs &= ~BIT(16); 1490 1491 mci_writel(host, UHS_REG, regs); 1492 host->timing = ios->timing; 1493 1494 /* 1495 * Use mirror of ios->clock to prevent race with mmc 1496 * core ios update when finding the minimum. 1497 */ 1498 host->clock = ios->clock; 1499 1500 if (drv_data && drv_data->set_ios) 1501 drv_data->set_ios(host, ios); 1502 1503 switch (ios->power_mode) { 1504 case MMC_POWER_UP: 1505 ret = mmc_regulator_set_ocr(mmc, mmc->supply.vmmc, ios->vdd); 1506 if (ret) { 1507 dev_err(host->dev, "failed to enable vmmc regulator\n"); 1508 return; 1509 } 1510 set_bit(DW_MMC_CARD_NEED_INIT, &host->flags); 1511 regs = mci_readl(host, PWREN); 1512 regs |= BIT(0); 1513 mci_writel(host, PWREN, regs); 1514 break; 1515 case MMC_POWER_ON: 1516 mmc_regulator_enable_vqmmc(mmc); 1517 /* Adjust clock / bus width after power is up */ 1518 dw_mci_setup_bus(host, false); 1519 1520 break; 1521 case MMC_POWER_OFF: 1522 /* Turn clock off before power goes down */ 1523 dw_mci_setup_bus(host, false); 1524 1525 if (!IS_ERR(mmc->supply.vmmc)) 1526 mmc_regulator_set_ocr(mmc, mmc->supply.vmmc, 0); 1527 1528 mmc_regulator_disable_vqmmc(mmc); 1529 1530 regs = mci_readl(host, PWREN); 1531 regs &= ~BIT(0); 1532 mci_writel(host, PWREN, regs); 1533 /* Reset our state machine after powering off */ 1534 dw_mci_ctrl_reset(host, SDMMC_CTRL_ALL_RESET_FLAGS); 1535 break; 1536 default: 1537 break; 1538 } 1539 1540 if (host->state == STATE_WAITING_CMD11_DONE && ios->clock != 0) 1541 host->state = STATE_IDLE; 1542 } 1543 1544 static int dw_mci_card_busy(struct mmc_host *mmc) 1545 { 1546 struct dw_mci *host = mmc_priv(mmc); 1547 u32 status; 1548 1549 /* 1550 * Check the busy bit which is low when DAT[3:0] 1551 * (the data lines) are 0000 1552 */ 1553 status = mci_readl(host, STATUS); 1554 1555 return !!(status & SDMMC_STATUS_BUSY); 1556 } 1557 1558 static int dw_mci_switch_voltage(struct mmc_host *mmc, struct mmc_ios *ios) 1559 { 1560 struct dw_mci *host = mmc_priv(mmc); 1561 const struct dw_mci_drv_data *drv_data = host->drv_data; 1562 u32 uhs; 1563 u32 v18 = SDMMC_UHS_18V; 1564 int ret; 1565 1566 if (drv_data && drv_data->switch_voltage) 1567 return drv_data->switch_voltage(host, ios); 1568 1569 /* 1570 * Program the voltage. Note that some instances of dw_mmc may use 1571 * the UHS_REG for this. For other instances (like exynos) the UHS_REG 1572 * does no harm but you need to set the regulator directly. Try both. 1573 */ 1574 uhs = mci_readl(host, UHS_REG); 1575 if (ios->signal_voltage == MMC_SIGNAL_VOLTAGE_330) 1576 uhs &= ~v18; 1577 else 1578 uhs |= v18; 1579 1580 if (!IS_ERR(mmc->supply.vqmmc)) { 1581 ret = mmc_regulator_set_vqmmc(mmc, ios); 1582 if (ret < 0) { 1583 dev_dbg(&mmc->class_dev, 1584 "Regulator set error %d - %s V\n", 1585 ret, uhs & v18 ? "1.8" : "3.3"); 1586 return ret; 1587 } 1588 } 1589 mci_writel(host, UHS_REG, uhs); 1590 1591 return 0; 1592 } 1593 1594 static int dw_mci_get_ro(struct mmc_host *mmc) 1595 { 1596 int read_only; 1597 struct dw_mci *host = mmc_priv(mmc); 1598 int gpio_ro = mmc_gpio_get_ro(mmc); 1599 1600 /* Use platform get_ro function, else try on board write protect */ 1601 if (gpio_ro >= 0) 1602 read_only = gpio_ro; 1603 else 1604 read_only = 1605 mci_readl(host, WRTPRT) & BIT(0) ? 1 : 0; 1606 1607 dev_dbg(&mmc->class_dev, "card is %s\n", 1608 read_only ? "read-only" : "read-write"); 1609 1610 return read_only; 1611 } 1612 1613 static void dw_mci_hw_reset(struct mmc_host *mmc) 1614 { 1615 struct dw_mci *host = mmc_priv(mmc); 1616 const struct dw_mci_drv_data *drv_data = host->drv_data; 1617 int reset; 1618 1619 if (host->use_dma == TRANS_MODE_IDMAC) 1620 dw_mci_idmac_reset(host); 1621 1622 if (!dw_mci_ctrl_reset(host, SDMMC_CTRL_DMA_RESET | 1623 SDMMC_CTRL_FIFO_RESET)) 1624 return; 1625 1626 if (drv_data && drv_data->hw_reset) { 1627 drv_data->hw_reset(host); 1628 return; 1629 } 1630 1631 /* 1632 * According to eMMC spec, card reset procedure: 1633 * tRstW >= 1us: RST_n pulse width 1634 * tRSCA >= 200us: RST_n to Command time 1635 * tRSTH >= 1us: RST_n high period 1636 */ 1637 reset = mci_readl(host, RST_N); 1638 reset &= ~SDMMC_RST_HWACTIVE; 1639 mci_writel(host, RST_N, reset); 1640 usleep_range(1, 2); 1641 reset |= SDMMC_RST_HWACTIVE; 1642 mci_writel(host, RST_N, reset); 1643 usleep_range(200, 300); 1644 } 1645 1646 static void dw_mci_prepare_sdio_irq(struct dw_mci *host, bool prepare) 1647 { 1648 const u32 clken_low_pwr = SDMMC_CLKEN_LOW_PWR; 1649 u32 clk_en_a_old; 1650 u32 clk_en_a; 1651 1652 /* 1653 * Low power mode will stop the card clock when idle. According to the 1654 * description of the CLKENA register we should disable low power mode 1655 * for SDIO cards if we need SDIO interrupts to work. 1656 */ 1657 1658 clk_en_a_old = mci_readl(host, CLKENA); 1659 if (prepare) { 1660 set_bit(DW_MMC_CARD_NO_LOW_PWR, &host->flags); 1661 clk_en_a = clk_en_a_old & ~clken_low_pwr; 1662 } else { 1663 clear_bit(DW_MMC_CARD_NO_LOW_PWR, &host->flags); 1664 clk_en_a = clk_en_a_old | clken_low_pwr; 1665 } 1666 1667 if (clk_en_a != clk_en_a_old) { 1668 mci_writel(host, CLKENA, clk_en_a); 1669 mci_send_cmd(host, SDMMC_CMD_UPD_CLK | SDMMC_CMD_PRV_DAT_WAIT, 1670 0); 1671 } 1672 } 1673 1674 static void __dw_mci_enable_sdio_irq(struct dw_mci *host, int enb) 1675 { 1676 unsigned long irqflags; 1677 u32 int_mask; 1678 1679 spin_lock_irqsave(&host->irq_lock, irqflags); 1680 1681 /* Enable/disable Slot Specific SDIO interrupt */ 1682 int_mask = mci_readl(host, INTMASK); 1683 if (enb) 1684 int_mask |= SDMMC_INT_SDIO(host->sdio_irq); 1685 else 1686 int_mask &= ~SDMMC_INT_SDIO(host->sdio_irq); 1687 mci_writel(host, INTMASK, int_mask); 1688 1689 spin_unlock_irqrestore(&host->irq_lock, irqflags); 1690 } 1691 1692 static void dw_mci_enable_sdio_irq(struct mmc_host *mmc, int enb) 1693 { 1694 struct dw_mci *host = mmc_priv(mmc); 1695 1696 dw_mci_prepare_sdio_irq(host, enb); 1697 __dw_mci_enable_sdio_irq(host, enb); 1698 1699 /* Avoid runtime suspending the device when SDIO IRQ is enabled */ 1700 if (enb) 1701 pm_runtime_get_noresume(host->dev); 1702 else 1703 pm_runtime_put_noidle(host->dev); 1704 } 1705 1706 static void dw_mci_ack_sdio_irq(struct mmc_host *mmc) 1707 { 1708 struct dw_mci *host = mmc_priv(mmc); 1709 1710 __dw_mci_enable_sdio_irq(host, 1); 1711 } 1712 1713 static int dw_mci_execute_tuning(struct mmc_host *mmc, u32 opcode) 1714 { 1715 struct dw_mci *host = mmc_priv(mmc); 1716 const struct dw_mci_drv_data *drv_data = host->drv_data; 1717 int err = -EINVAL; 1718 1719 if (drv_data && drv_data->execute_tuning) 1720 err = drv_data->execute_tuning(host, opcode); 1721 return err; 1722 } 1723 1724 static int dw_mci_prepare_hs400_tuning(struct mmc_host *mmc, 1725 struct mmc_ios *ios) 1726 { 1727 struct dw_mci *host = mmc_priv(mmc); 1728 const struct dw_mci_drv_data *drv_data = host->drv_data; 1729 1730 if (drv_data && drv_data->prepare_hs400_tuning) 1731 return drv_data->prepare_hs400_tuning(host, ios); 1732 1733 return 0; 1734 } 1735 1736 static bool dw_mci_reset(struct dw_mci *host) 1737 { 1738 u32 flags = SDMMC_CTRL_RESET | SDMMC_CTRL_FIFO_RESET; 1739 bool ret = false; 1740 u32 status = 0; 1741 1742 /* 1743 * Resetting generates a block interrupt, hence setting 1744 * the scatter-gather pointer to NULL. 1745 */ 1746 if (host->sg) { 1747 sg_miter_stop(&host->sg_miter); 1748 host->sg = NULL; 1749 } 1750 1751 if (host->use_dma) 1752 flags |= SDMMC_CTRL_DMA_RESET; 1753 1754 if (dw_mci_ctrl_reset(host, flags)) { 1755 /* 1756 * In all cases we clear the RAWINTS 1757 * register to clear any interrupts. 1758 */ 1759 mci_writel(host, RINTSTS, 0xFFFFFFFF); 1760 1761 if (!host->use_dma) { 1762 ret = true; 1763 goto ciu_out; 1764 } 1765 1766 /* Wait for dma_req to be cleared */ 1767 if (readl_poll_timeout_atomic(host->regs + SDMMC_STATUS, 1768 status, 1769 !(status & SDMMC_STATUS_DMA_REQ), 1770 1, 500 * USEC_PER_MSEC)) { 1771 dev_err(host->dev, 1772 "%s: Timeout waiting for dma_req to be cleared\n", 1773 __func__); 1774 goto ciu_out; 1775 } 1776 1777 /* when using DMA next we reset the fifo again */ 1778 if (!dw_mci_ctrl_reset(host, SDMMC_CTRL_FIFO_RESET)) 1779 goto ciu_out; 1780 } else { 1781 /* if the controller reset bit did clear, then set clock regs */ 1782 if (!(mci_readl(host, CTRL) & SDMMC_CTRL_RESET)) { 1783 dev_err(host->dev, 1784 "%s: fifo/dma reset bits didn't clear but ciu was reset, doing clock update\n", 1785 __func__); 1786 goto ciu_out; 1787 } 1788 } 1789 1790 if (host->use_dma == TRANS_MODE_IDMAC) 1791 /* It is also required that we reinit idmac */ 1792 dw_mci_idmac_init(host); 1793 1794 ret = true; 1795 1796 ciu_out: 1797 /* After a CTRL reset we need to have CIU set clock registers */ 1798 mci_send_cmd(host, SDMMC_CMD_UPD_CLK, 0); 1799 1800 return ret; 1801 } 1802 1803 static const struct mmc_host_ops dw_mci_ops = { 1804 .request = dw_mci_request, 1805 .pre_req = dw_mci_pre_req, 1806 .post_req = dw_mci_post_req, 1807 .set_ios = dw_mci_set_ios, 1808 .get_ro = dw_mci_get_ro, 1809 .get_cd = dw_mci_get_cd, 1810 .card_hw_reset = dw_mci_hw_reset, 1811 .enable_sdio_irq = dw_mci_enable_sdio_irq, 1812 .ack_sdio_irq = dw_mci_ack_sdio_irq, 1813 .execute_tuning = dw_mci_execute_tuning, 1814 .card_busy = dw_mci_card_busy, 1815 .start_signal_voltage_switch = dw_mci_switch_voltage, 1816 .prepare_hs400_tuning = dw_mci_prepare_hs400_tuning, 1817 }; 1818 1819 #ifdef CONFIG_FAULT_INJECTION 1820 static enum hrtimer_restart dw_mci_fault_timer(struct hrtimer *t) 1821 { 1822 struct dw_mci *host = container_of(t, struct dw_mci, fault_timer); 1823 unsigned long flags; 1824 1825 spin_lock_irqsave(&host->irq_lock, flags); 1826 1827 /* 1828 * Only inject an error if we haven't already got an error or data over 1829 * interrupt. 1830 */ 1831 if (!host->data_status) { 1832 host->data_status = SDMMC_INT_DCRC; 1833 set_bit(EVENT_DATA_ERROR, &host->pending_events); 1834 queue_work(system_bh_wq, &host->bh_work); 1835 } 1836 1837 spin_unlock_irqrestore(&host->irq_lock, flags); 1838 1839 return HRTIMER_NORESTART; 1840 } 1841 1842 static void dw_mci_start_fault_timer(struct dw_mci *host) 1843 { 1844 struct mmc_data *data = host->data; 1845 1846 if (!data || data->blocks <= 1) 1847 return; 1848 1849 if (!should_fail(&host->fail_data_crc, 1)) 1850 return; 1851 1852 /* 1853 * Try to inject the error at random points during the data transfer. 1854 */ 1855 hrtimer_start(&host->fault_timer, 1856 ms_to_ktime(get_random_u32_below(25)), 1857 HRTIMER_MODE_REL); 1858 } 1859 1860 static void dw_mci_stop_fault_timer(struct dw_mci *host) 1861 { 1862 hrtimer_cancel(&host->fault_timer); 1863 } 1864 1865 static void dw_mci_init_fault(struct dw_mci *host) 1866 { 1867 host->fail_data_crc = (struct fault_attr) FAULT_ATTR_INITIALIZER; 1868 1869 hrtimer_setup(&host->fault_timer, dw_mci_fault_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); 1870 } 1871 #else 1872 static void dw_mci_init_fault(struct dw_mci *host) 1873 { 1874 } 1875 1876 static void dw_mci_start_fault_timer(struct dw_mci *host) 1877 { 1878 } 1879 1880 static void dw_mci_stop_fault_timer(struct dw_mci *host) 1881 { 1882 } 1883 #endif 1884 1885 static void dw_mci_request_end(struct dw_mci *host, struct mmc_request *mrq) 1886 __releases(&host->lock) 1887 __acquires(&host->lock) 1888 { 1889 struct mmc_host *prev_mmc = host->mmc; 1890 1891 WARN_ON(host->cmd || host->data); 1892 1893 host->mrq = NULL; 1894 1895 if (host->state == STATE_SENDING_CMD11) 1896 host->state = STATE_WAITING_CMD11_DONE; 1897 else 1898 host->state = STATE_IDLE; 1899 1900 spin_unlock(&host->lock); 1901 mmc_request_done(prev_mmc, mrq); 1902 spin_lock(&host->lock); 1903 } 1904 1905 static int dw_mci_command_complete(struct dw_mci *host, struct mmc_command *cmd) 1906 { 1907 u32 status = host->cmd_status; 1908 1909 host->cmd_status = 0; 1910 1911 /* Read the response from the card (up to 16 bytes) */ 1912 if (cmd->flags & MMC_RSP_PRESENT) { 1913 if (cmd->flags & MMC_RSP_136) { 1914 cmd->resp[3] = mci_readl(host, RESP0); 1915 cmd->resp[2] = mci_readl(host, RESP1); 1916 cmd->resp[1] = mci_readl(host, RESP2); 1917 cmd->resp[0] = mci_readl(host, RESP3); 1918 } else { 1919 cmd->resp[0] = mci_readl(host, RESP0); 1920 cmd->resp[1] = 0; 1921 cmd->resp[2] = 0; 1922 cmd->resp[3] = 0; 1923 } 1924 } 1925 1926 if (status & SDMMC_INT_RTO) 1927 cmd->error = -ETIMEDOUT; 1928 else if ((cmd->flags & MMC_RSP_CRC) && (status & SDMMC_INT_RCRC)) 1929 cmd->error = -EILSEQ; 1930 else if (status & SDMMC_INT_RESP_ERR) 1931 cmd->error = -EIO; 1932 else 1933 cmd->error = 0; 1934 1935 return cmd->error; 1936 } 1937 1938 static int dw_mci_data_complete(struct dw_mci *host, struct mmc_data *data) 1939 { 1940 u32 status = host->data_status; 1941 1942 if (status & DW_MCI_DATA_ERROR_FLAGS) { 1943 if (status & SDMMC_INT_DRTO) { 1944 data->error = -ETIMEDOUT; 1945 } else if (status & SDMMC_INT_DCRC) { 1946 data->error = -EILSEQ; 1947 } else if (status & SDMMC_INT_EBE) { 1948 if (host->dir_status == 1949 MMC_DATA_WRITE) { 1950 /* 1951 * No data CRC status was returned. 1952 * The number of bytes transferred 1953 * will be exaggerated in PIO mode. 1954 */ 1955 data->bytes_xfered = 0; 1956 data->error = -ETIMEDOUT; 1957 } else if (host->dir_status == 1958 MMC_DATA_READ) { 1959 data->error = -EILSEQ; 1960 } 1961 } else { 1962 /* SDMMC_INT_SBE is included */ 1963 data->error = -EILSEQ; 1964 } 1965 1966 dev_dbg(host->dev, "data error, status 0x%08x\n", status); 1967 1968 /* 1969 * After an error, there may be data lingering 1970 * in the FIFO 1971 */ 1972 dw_mci_reset(host); 1973 } else { 1974 data->bytes_xfered = data->blocks * data->blksz; 1975 data->error = 0; 1976 } 1977 1978 return data->error; 1979 } 1980 1981 static void dw_mci_set_drto(struct dw_mci *host) 1982 { 1983 const struct dw_mci_drv_data *drv_data = host->drv_data; 1984 unsigned int drto_clks; 1985 unsigned int drto_div; 1986 unsigned int drto_ms; 1987 unsigned long irqflags; 1988 1989 if (drv_data && drv_data->get_drto_clks) 1990 drto_clks = drv_data->get_drto_clks(host); 1991 else 1992 drto_clks = mci_readl(host, TMOUT) >> 8; 1993 drto_div = (mci_readl(host, CLKDIV) & 0xff) * 2; 1994 if (drto_div == 0) 1995 drto_div = 1; 1996 1997 drto_ms = DIV_ROUND_UP_ULL((u64)MSEC_PER_SEC * drto_clks * drto_div, 1998 host->bus_hz); 1999 2000 dev_dbg(host->dev, "drto_ms: %u\n", drto_ms); 2001 2002 /* add a bit spare time */ 2003 drto_ms += 10; 2004 2005 spin_lock_irqsave(&host->irq_lock, irqflags); 2006 dw_mci_wd_arm(host, drto_ms, BIT(EVENT_DATA_COMPLETE), 2007 DW_MCI_WD_DATA_EVENTS, 2008 BIT(STATE_SENDING_DATA) | BIT(STATE_DATA_BUSY)); 2009 spin_unlock_irqrestore(&host->irq_lock, irqflags); 2010 } 2011 2012 static bool dw_mci_clear_pending_cmd_complete(struct dw_mci *host) 2013 { 2014 if (!test_bit(EVENT_CMD_COMPLETE, &host->pending_events)) 2015 return false; 2016 2017 clear_bit(EVENT_CMD_COMPLETE, &host->pending_events); 2018 2019 return true; 2020 } 2021 2022 static bool dw_mci_clear_pending_data_complete(struct dw_mci *host) 2023 { 2024 if (!test_bit(EVENT_DATA_COMPLETE, &host->pending_events)) 2025 return false; 2026 2027 clear_bit(EVENT_DATA_COMPLETE, &host->pending_events); 2028 2029 return true; 2030 } 2031 2032 static void dw_mci_work_func(struct work_struct *t) 2033 { 2034 struct dw_mci *host = from_work(host, t, bh_work); 2035 struct mmc_data *data; 2036 struct mmc_command *cmd; 2037 struct mmc_request *mrq; 2038 enum dw_mci_state state; 2039 enum dw_mci_state prev_state; 2040 unsigned int err; 2041 2042 spin_lock(&host->lock); 2043 2044 state = host->state; 2045 data = host->data; 2046 mrq = host->mrq; 2047 2048 do { 2049 prev_state = state; 2050 2051 switch (state) { 2052 case STATE_IDLE: 2053 case STATE_WAITING_CMD11_DONE: 2054 break; 2055 2056 case STATE_SENDING_CMD11: 2057 case STATE_SENDING_CMD: 2058 if (!dw_mci_clear_pending_cmd_complete(host)) 2059 break; 2060 2061 cmd = host->cmd; 2062 host->cmd = NULL; 2063 set_bit(EVENT_CMD_COMPLETE, &host->completed_events); 2064 err = dw_mci_command_complete(host, cmd); 2065 if (cmd == mrq->sbc && !err) { 2066 dw_mci_start_request(host, mrq->cmd); 2067 goto unlock; 2068 } 2069 2070 if (cmd->data && err) { 2071 /* 2072 * During UHS tuning sequence, sending the stop 2073 * command after the response CRC error would 2074 * throw the system into a confused state 2075 * causing all future tuning phases to report 2076 * failure. 2077 * 2078 * In such case controller will move into a data 2079 * transfer state after a response error or 2080 * response CRC error. Let's let that finish 2081 * before trying to send a stop, so we'll go to 2082 * STATE_SENDING_DATA. 2083 * 2084 * Although letting the data transfer take place 2085 * will waste a bit of time (we already know 2086 * the command was bad), it can't cause any 2087 * errors since it's possible it would have 2088 * taken place anyway if this bh work got 2089 * delayed. Allowing the transfer to take place 2090 * avoids races and keeps things simple. 2091 */ 2092 if (err != -ETIMEDOUT && 2093 host->dir_status == MMC_DATA_READ) { 2094 state = STATE_SENDING_DATA; 2095 continue; 2096 } 2097 2098 send_stop_abort(host, data); 2099 dw_mci_stop_dma(host); 2100 state = STATE_SENDING_STOP; 2101 break; 2102 } 2103 2104 if (!cmd->data || err) { 2105 dw_mci_request_end(host, mrq); 2106 goto unlock; 2107 } 2108 2109 prev_state = state = STATE_SENDING_DATA; 2110 fallthrough; 2111 2112 case STATE_SENDING_DATA: 2113 /* 2114 * We could get a data error and never a transfer 2115 * complete so we'd better check for it here. 2116 * 2117 * Note that we don't really care if we also got a 2118 * transfer complete; stopping the DMA and sending an 2119 * abort won't hurt. 2120 */ 2121 if (test_and_clear_bit(EVENT_DATA_ERROR, 2122 &host->pending_events)) { 2123 if (!(host->data_status & (SDMMC_INT_DRTO | 2124 SDMMC_INT_EBE))) 2125 send_stop_abort(host, data); 2126 dw_mci_stop_dma(host); 2127 state = STATE_DATA_ERROR; 2128 break; 2129 } 2130 2131 if (!test_and_clear_bit(EVENT_XFER_COMPLETE, 2132 &host->pending_events)) { 2133 /* 2134 * If all data-related interrupts don't come 2135 * within the given time in reading data state. 2136 */ 2137 if (host->dir_status == MMC_DATA_READ) 2138 dw_mci_set_drto(host); 2139 break; 2140 } 2141 2142 set_bit(EVENT_XFER_COMPLETE, &host->completed_events); 2143 2144 /* 2145 * Handle an EVENT_DATA_ERROR that might have shown up 2146 * before the transfer completed. This might not have 2147 * been caught by the check above because the interrupt 2148 * could have gone off between the previous check and 2149 * the check for transfer complete. 2150 * 2151 * Technically this ought not be needed assuming we 2152 * get a DATA_COMPLETE eventually (we'll notice the 2153 * error and end the request), but it shouldn't hurt. 2154 * 2155 * This has the advantage of sending the stop command. 2156 */ 2157 if (test_and_clear_bit(EVENT_DATA_ERROR, 2158 &host->pending_events)) { 2159 if (!(host->data_status & (SDMMC_INT_DRTO | 2160 SDMMC_INT_EBE))) 2161 send_stop_abort(host, data); 2162 dw_mci_stop_dma(host); 2163 state = STATE_DATA_ERROR; 2164 break; 2165 } 2166 prev_state = state = STATE_DATA_BUSY; 2167 2168 fallthrough; 2169 2170 case STATE_DATA_BUSY: 2171 if (!dw_mci_clear_pending_data_complete(host)) { 2172 /* 2173 * If data error interrupt comes but data over 2174 * interrupt doesn't come within the given time. 2175 * in reading data state. 2176 */ 2177 if (host->dir_status == MMC_DATA_READ) 2178 dw_mci_set_drto(host); 2179 break; 2180 } 2181 2182 dw_mci_stop_fault_timer(host); 2183 dw_mci_stop_dma(host); 2184 host->data = NULL; 2185 set_bit(EVENT_DATA_COMPLETE, &host->completed_events); 2186 err = dw_mci_data_complete(host, data); 2187 2188 if (!err) { 2189 if (!data->stop || mrq->sbc) { 2190 if (mrq->sbc && data->stop) 2191 data->stop->error = 0; 2192 dw_mci_request_end(host, mrq); 2193 goto unlock; 2194 } 2195 2196 /* stop command for open-ended transfer*/ 2197 if (data->stop) 2198 send_stop_abort(host, data); 2199 } else { 2200 /* 2201 * If we don't have a command complete now we'll 2202 * never get one since we just reset everything; 2203 * better end the request. 2204 * 2205 * If we do have a command complete we'll fall 2206 * through to the SENDING_STOP command and 2207 * everything will be peachy keen. 2208 */ 2209 if (!test_bit(EVENT_CMD_COMPLETE, 2210 &host->pending_events)) { 2211 host->cmd = NULL; 2212 dw_mci_request_end(host, mrq); 2213 goto unlock; 2214 } 2215 } 2216 2217 /* 2218 * If err has non-zero, 2219 * stop-abort command has been already issued. 2220 */ 2221 prev_state = state = STATE_SENDING_STOP; 2222 2223 fallthrough; 2224 2225 case STATE_SENDING_STOP: 2226 if (!dw_mci_clear_pending_cmd_complete(host)) 2227 break; 2228 2229 /* CMD error in data command */ 2230 if (mrq->cmd->error && mrq->data) 2231 dw_mci_reset(host); 2232 2233 dw_mci_stop_fault_timer(host); 2234 host->cmd = NULL; 2235 host->data = NULL; 2236 2237 if (!mrq->sbc && mrq->stop) 2238 dw_mci_command_complete(host, mrq->stop); 2239 else 2240 host->cmd_status = 0; 2241 2242 dw_mci_request_end(host, mrq); 2243 goto unlock; 2244 2245 case STATE_DATA_ERROR: 2246 if (!test_and_clear_bit(EVENT_XFER_COMPLETE, 2247 &host->pending_events)) 2248 break; 2249 2250 state = STATE_DATA_BUSY; 2251 break; 2252 } 2253 } while (state != prev_state); 2254 2255 host->state = state; 2256 unlock: 2257 spin_unlock(&host->lock); 2258 2259 } 2260 2261 /* push final bytes to part_buf, only use during push */ 2262 static void dw_mci_set_part_bytes(struct dw_mci *host, void *buf, int cnt) 2263 { 2264 memcpy((void *)&host->part_buf, buf, cnt); 2265 host->part_buf_count = cnt; 2266 } 2267 2268 /* append bytes to part_buf, only use during push */ 2269 static int dw_mci_push_part_bytes(struct dw_mci *host, void *buf, int cnt) 2270 { 2271 cnt = min(cnt, (1 << host->data_shift) - host->part_buf_count); 2272 memcpy((void *)&host->part_buf + host->part_buf_count, buf, cnt); 2273 host->part_buf_count += cnt; 2274 return cnt; 2275 } 2276 2277 /* pull first bytes from part_buf, only use during pull */ 2278 static int dw_mci_pull_part_bytes(struct dw_mci *host, void *buf, int cnt) 2279 { 2280 cnt = min_t(int, cnt, host->part_buf_count); 2281 if (cnt) { 2282 memcpy(buf, (void *)&host->part_buf + host->part_buf_start, 2283 cnt); 2284 host->part_buf_count -= cnt; 2285 host->part_buf_start += cnt; 2286 } 2287 return cnt; 2288 } 2289 2290 /* pull final bytes from the part_buf, assuming it's just been filled */ 2291 static void dw_mci_pull_final_bytes(struct dw_mci *host, void *buf, int cnt) 2292 { 2293 memcpy(buf, &host->part_buf, cnt); 2294 host->part_buf_start = cnt; 2295 host->part_buf_count = (1 << host->data_shift) - cnt; 2296 } 2297 2298 static void dw_mci_push_data16(struct dw_mci *host, void *buf, int cnt) 2299 { 2300 struct mmc_data *data = host->data; 2301 int init_cnt = cnt; 2302 2303 /* try and push anything in the part_buf */ 2304 if (unlikely(host->part_buf_count)) { 2305 int len = dw_mci_push_part_bytes(host, buf, cnt); 2306 2307 buf += len; 2308 cnt -= len; 2309 if (host->part_buf_count == 2) { 2310 mci_fifo_writew(host->fifo_reg, host->part_buf16); 2311 host->part_buf_count = 0; 2312 } 2313 } 2314 #ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS 2315 if (unlikely((unsigned long)buf & 0x1)) { 2316 while (cnt >= 2) { 2317 u16 aligned_buf[64]; 2318 int len = min(cnt & -2, (int)sizeof(aligned_buf)); 2319 int items = len >> 1; 2320 int i; 2321 /* memcpy from input buffer into aligned buffer */ 2322 memcpy(aligned_buf, buf, len); 2323 buf += len; 2324 cnt -= len; 2325 /* push data from aligned buffer into fifo */ 2326 for (i = 0; i < items; ++i) 2327 mci_fifo_writew(host->fifo_reg, aligned_buf[i]); 2328 } 2329 } else 2330 #endif 2331 { 2332 u16 *pdata = buf; 2333 2334 for (; cnt >= 2; cnt -= 2) 2335 mci_fifo_writew(host->fifo_reg, *pdata++); 2336 buf = pdata; 2337 } 2338 /* put anything remaining in the part_buf */ 2339 if (cnt) { 2340 dw_mci_set_part_bytes(host, buf, cnt); 2341 /* Push data if we have reached the expected data length */ 2342 if ((data->bytes_xfered + init_cnt) == 2343 (data->blksz * data->blocks)) 2344 mci_fifo_writew(host->fifo_reg, host->part_buf16); 2345 } 2346 } 2347 2348 static void dw_mci_pull_data16(struct dw_mci *host, void *buf, int cnt) 2349 { 2350 #ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS 2351 if (unlikely((unsigned long)buf & 0x1)) { 2352 while (cnt >= 2) { 2353 /* pull data from fifo into aligned buffer */ 2354 u16 aligned_buf[64]; 2355 int len = min(cnt & -2, (int)sizeof(aligned_buf)); 2356 int items = len >> 1; 2357 int i; 2358 2359 for (i = 0; i < items; ++i) 2360 aligned_buf[i] = mci_fifo_readw(host->fifo_reg); 2361 /* memcpy from aligned buffer into output buffer */ 2362 memcpy(buf, aligned_buf, len); 2363 buf += len; 2364 cnt -= len; 2365 } 2366 } else 2367 #endif 2368 { 2369 u16 *pdata = buf; 2370 2371 for (; cnt >= 2; cnt -= 2) 2372 *pdata++ = mci_fifo_readw(host->fifo_reg); 2373 buf = pdata; 2374 } 2375 if (cnt) { 2376 host->part_buf16 = mci_fifo_readw(host->fifo_reg); 2377 dw_mci_pull_final_bytes(host, buf, cnt); 2378 } 2379 } 2380 2381 static void dw_mci_push_data32(struct dw_mci *host, void *buf, int cnt) 2382 { 2383 struct mmc_data *data = host->data; 2384 int init_cnt = cnt; 2385 2386 /* try and push anything in the part_buf */ 2387 if (unlikely(host->part_buf_count)) { 2388 int len = dw_mci_push_part_bytes(host, buf, cnt); 2389 2390 buf += len; 2391 cnt -= len; 2392 if (host->part_buf_count == 4) { 2393 mci_fifo_writel(host->fifo_reg, host->part_buf32); 2394 host->part_buf_count = 0; 2395 } 2396 } 2397 #ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS 2398 if (unlikely((unsigned long)buf & 0x3)) { 2399 while (cnt >= 4) { 2400 u32 aligned_buf[32]; 2401 int len = min(cnt & -4, (int)sizeof(aligned_buf)); 2402 int items = len >> 2; 2403 int i; 2404 /* memcpy from input buffer into aligned buffer */ 2405 memcpy(aligned_buf, buf, len); 2406 buf += len; 2407 cnt -= len; 2408 /* push data from aligned buffer into fifo */ 2409 for (i = 0; i < items; ++i) 2410 mci_fifo_writel(host->fifo_reg, aligned_buf[i]); 2411 } 2412 } else 2413 #endif 2414 { 2415 u32 *pdata = buf; 2416 2417 for (; cnt >= 4; cnt -= 4) 2418 mci_fifo_writel(host->fifo_reg, *pdata++); 2419 buf = pdata; 2420 } 2421 /* put anything remaining in the part_buf */ 2422 if (cnt) { 2423 dw_mci_set_part_bytes(host, buf, cnt); 2424 /* Push data if we have reached the expected data length */ 2425 if ((data->bytes_xfered + init_cnt) == 2426 (data->blksz * data->blocks)) 2427 mci_fifo_writel(host->fifo_reg, host->part_buf32); 2428 } 2429 } 2430 2431 static void dw_mci_pull_data32(struct dw_mci *host, void *buf, int cnt) 2432 { 2433 #ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS 2434 if (unlikely((unsigned long)buf & 0x3)) { 2435 while (cnt >= 4) { 2436 /* pull data from fifo into aligned buffer */ 2437 u32 aligned_buf[32]; 2438 int len = min(cnt & -4, (int)sizeof(aligned_buf)); 2439 int items = len >> 2; 2440 int i; 2441 2442 for (i = 0; i < items; ++i) 2443 aligned_buf[i] = mci_fifo_readl(host->fifo_reg); 2444 /* memcpy from aligned buffer into output buffer */ 2445 memcpy(buf, aligned_buf, len); 2446 buf += len; 2447 cnt -= len; 2448 } 2449 } else 2450 #endif 2451 { 2452 u32 *pdata = buf; 2453 2454 for (; cnt >= 4; cnt -= 4) 2455 *pdata++ = mci_fifo_readl(host->fifo_reg); 2456 buf = pdata; 2457 } 2458 if (cnt) { 2459 host->part_buf32 = mci_fifo_readl(host->fifo_reg); 2460 dw_mci_pull_final_bytes(host, buf, cnt); 2461 } 2462 } 2463 2464 static void dw_mci_push_data64(struct dw_mci *host, void *buf, int cnt) 2465 { 2466 struct mmc_data *data = host->data; 2467 int init_cnt = cnt; 2468 2469 /* try and push anything in the part_buf */ 2470 if (unlikely(host->part_buf_count)) { 2471 int len = dw_mci_push_part_bytes(host, buf, cnt); 2472 2473 buf += len; 2474 cnt -= len; 2475 2476 if (host->part_buf_count == 8) { 2477 mci_fifo_writeq(host->fifo_reg, host->part_buf); 2478 host->part_buf_count = 0; 2479 } 2480 } 2481 #ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS 2482 if (unlikely((unsigned long)buf & 0x7)) { 2483 while (cnt >= 8) { 2484 u64 aligned_buf[16]; 2485 int len = min(cnt & -8, (int)sizeof(aligned_buf)); 2486 int items = len >> 3; 2487 int i; 2488 /* memcpy from input buffer into aligned buffer */ 2489 memcpy(aligned_buf, buf, len); 2490 buf += len; 2491 cnt -= len; 2492 /* push data from aligned buffer into fifo */ 2493 for (i = 0; i < items; ++i) 2494 mci_fifo_writeq(host->fifo_reg, aligned_buf[i]); 2495 } 2496 } else 2497 #endif 2498 { 2499 u64 *pdata = buf; 2500 2501 for (; cnt >= 8; cnt -= 8) 2502 mci_fifo_writeq(host->fifo_reg, *pdata++); 2503 buf = pdata; 2504 } 2505 /* put anything remaining in the part_buf */ 2506 if (cnt) { 2507 dw_mci_set_part_bytes(host, buf, cnt); 2508 /* Push data if we have reached the expected data length */ 2509 if ((data->bytes_xfered + init_cnt) == 2510 (data->blksz * data->blocks)) 2511 mci_fifo_writeq(host->fifo_reg, host->part_buf); 2512 } 2513 } 2514 2515 static void dw_mci_pull_data64(struct dw_mci *host, void *buf, int cnt) 2516 { 2517 #ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS 2518 if (unlikely((unsigned long)buf & 0x7)) { 2519 while (cnt >= 8) { 2520 /* pull data from fifo into aligned buffer */ 2521 u64 aligned_buf[16]; 2522 int len = min(cnt & -8, (int)sizeof(aligned_buf)); 2523 int items = len >> 3; 2524 int i; 2525 2526 for (i = 0; i < items; ++i) 2527 aligned_buf[i] = mci_fifo_readq(host->fifo_reg); 2528 2529 /* memcpy from aligned buffer into output buffer */ 2530 memcpy(buf, aligned_buf, len); 2531 buf += len; 2532 cnt -= len; 2533 } 2534 } else 2535 #endif 2536 { 2537 u64 *pdata = buf; 2538 2539 for (; cnt >= 8; cnt -= 8) 2540 *pdata++ = mci_fifo_readq(host->fifo_reg); 2541 buf = pdata; 2542 } 2543 if (cnt) { 2544 host->part_buf = mci_fifo_readq(host->fifo_reg); 2545 dw_mci_pull_final_bytes(host, buf, cnt); 2546 } 2547 } 2548 2549 static void dw_mci_push_data64_32(struct dw_mci *host, void *buf, int cnt) 2550 { 2551 struct mmc_data *data = host->data; 2552 int init_cnt = cnt; 2553 2554 /* try and push anything in the part_buf */ 2555 if (unlikely(host->part_buf_count)) { 2556 int len = dw_mci_push_part_bytes(host, buf, cnt); 2557 2558 buf += len; 2559 cnt -= len; 2560 2561 if (host->part_buf_count == 8) { 2562 mci_fifo_l_writeq(host->fifo_reg, host->part_buf); 2563 host->part_buf_count = 0; 2564 } 2565 } 2566 #ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS 2567 if (unlikely((unsigned long)buf & 0x7)) { 2568 while (cnt >= 8) { 2569 u64 aligned_buf[16]; 2570 int len = min(cnt & -8, (int)sizeof(aligned_buf)); 2571 int items = len >> 3; 2572 int i; 2573 /* memcpy from input buffer into aligned buffer */ 2574 memcpy(aligned_buf, buf, len); 2575 buf += len; 2576 cnt -= len; 2577 /* push data from aligned buffer into fifo */ 2578 for (i = 0; i < items; ++i) 2579 mci_fifo_l_writeq(host->fifo_reg, aligned_buf[i]); 2580 } 2581 } else 2582 #endif 2583 { 2584 u64 *pdata = buf; 2585 2586 for (; cnt >= 8; cnt -= 8) 2587 mci_fifo_l_writeq(host->fifo_reg, *pdata++); 2588 buf = pdata; 2589 } 2590 /* put anything remaining in the part_buf */ 2591 if (cnt) { 2592 dw_mci_set_part_bytes(host, buf, cnt); 2593 /* Push data if we have reached the expected data length */ 2594 if ((data->bytes_xfered + init_cnt) == 2595 (data->blksz * data->blocks)) 2596 mci_fifo_l_writeq(host->fifo_reg, host->part_buf); 2597 } 2598 } 2599 2600 static void dw_mci_pull_data64_32(struct dw_mci *host, void *buf, int cnt) 2601 { 2602 #ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS 2603 if (unlikely((unsigned long)buf & 0x7)) { 2604 while (cnt >= 8) { 2605 /* pull data from fifo into aligned buffer */ 2606 u64 aligned_buf[16]; 2607 int len = min(cnt & -8, (int)sizeof(aligned_buf)); 2608 int items = len >> 3; 2609 int i; 2610 2611 for (i = 0; i < items; ++i) 2612 aligned_buf[i] = mci_fifo_l_readq(host->fifo_reg); 2613 2614 /* memcpy from aligned buffer into output buffer */ 2615 memcpy(buf, aligned_buf, len); 2616 buf += len; 2617 cnt -= len; 2618 } 2619 } else 2620 #endif 2621 { 2622 u64 *pdata = buf; 2623 2624 for (; cnt >= 8; cnt -= 8) 2625 *pdata++ = mci_fifo_l_readq(host->fifo_reg); 2626 buf = pdata; 2627 } 2628 if (cnt) { 2629 host->part_buf = mci_fifo_l_readq(host->fifo_reg); 2630 dw_mci_pull_final_bytes(host, buf, cnt); 2631 } 2632 } 2633 2634 static void dw_mci_pull_data(struct dw_mci *host, void *buf, int cnt) 2635 { 2636 int len; 2637 2638 /* get remaining partial bytes */ 2639 len = dw_mci_pull_part_bytes(host, buf, cnt); 2640 if (unlikely(len == cnt)) 2641 return; 2642 buf += len; 2643 cnt -= len; 2644 2645 /* get the rest of the data */ 2646 host->pull_data(host, buf, cnt); 2647 } 2648 2649 static void dw_mci_read_data_pio(struct dw_mci *host, bool dto) 2650 { 2651 struct sg_mapping_iter *sg_miter = &host->sg_miter; 2652 void *buf; 2653 unsigned int offset; 2654 struct mmc_data *data = host->data; 2655 int shift = host->data_shift; 2656 u32 status; 2657 unsigned int len; 2658 unsigned int remain, fcnt; 2659 2660 do { 2661 if (!sg_miter_next(sg_miter)) 2662 goto done; 2663 2664 host->sg = sg_miter->piter.sg; 2665 buf = sg_miter->addr; 2666 remain = sg_miter->length; 2667 offset = 0; 2668 2669 do { 2670 fcnt = (SDMMC_GET_FCNT(mci_readl(host, STATUS)) 2671 << shift) + host->part_buf_count; 2672 len = min(remain, fcnt); 2673 if (!len) 2674 break; 2675 dw_mci_pull_data(host, (void *)(buf + offset), len); 2676 data->bytes_xfered += len; 2677 offset += len; 2678 remain -= len; 2679 } while (remain); 2680 2681 sg_miter->consumed = offset; 2682 status = mci_readl(host, MINTSTS); 2683 mci_writel(host, RINTSTS, SDMMC_INT_RXDR); 2684 /* if the RXDR is ready read again */ 2685 } while ((status & SDMMC_INT_RXDR) || 2686 (dto && SDMMC_GET_FCNT(mci_readl(host, STATUS)))); 2687 2688 if (!remain) { 2689 if (!sg_miter_next(sg_miter)) 2690 goto done; 2691 sg_miter->consumed = 0; 2692 } 2693 sg_miter_stop(sg_miter); 2694 return; 2695 2696 done: 2697 sg_miter_stop(sg_miter); 2698 host->sg = NULL; 2699 smp_wmb(); /* drain writebuffer */ 2700 set_bit(EVENT_XFER_COMPLETE, &host->pending_events); 2701 } 2702 2703 static void dw_mci_write_data_pio(struct dw_mci *host) 2704 { 2705 struct sg_mapping_iter *sg_miter = &host->sg_miter; 2706 void *buf; 2707 unsigned int offset; 2708 struct mmc_data *data = host->data; 2709 int shift = host->data_shift; 2710 u32 status; 2711 unsigned int len; 2712 unsigned int fifo_depth = host->fifo_depth; 2713 unsigned int remain, fcnt; 2714 2715 do { 2716 if (!sg_miter_next(sg_miter)) 2717 goto done; 2718 2719 host->sg = sg_miter->piter.sg; 2720 buf = sg_miter->addr; 2721 remain = sg_miter->length; 2722 offset = 0; 2723 2724 do { 2725 fcnt = ((fifo_depth - 2726 SDMMC_GET_FCNT(mci_readl(host, STATUS))) 2727 << shift) - host->part_buf_count; 2728 len = min(remain, fcnt); 2729 if (!len) 2730 break; 2731 host->push_data(host, (void *)(buf + offset), len); 2732 data->bytes_xfered += len; 2733 offset += len; 2734 remain -= len; 2735 } while (remain); 2736 2737 sg_miter->consumed = offset; 2738 status = mci_readl(host, MINTSTS); 2739 mci_writel(host, RINTSTS, SDMMC_INT_TXDR); 2740 } while (status & SDMMC_INT_TXDR); /* if TXDR write again */ 2741 2742 if (!remain) { 2743 if (!sg_miter_next(sg_miter)) 2744 goto done; 2745 sg_miter->consumed = 0; 2746 } 2747 sg_miter_stop(sg_miter); 2748 return; 2749 2750 done: 2751 sg_miter_stop(sg_miter); 2752 host->sg = NULL; 2753 smp_wmb(); /* drain writebuffer */ 2754 set_bit(EVENT_XFER_COMPLETE, &host->pending_events); 2755 } 2756 2757 static void dw_mci_cmd_interrupt(struct dw_mci *host, u32 status) 2758 { 2759 dw_mci_wd_deliver(host, BIT(EVENT_CMD_COMPLETE)); 2760 2761 if (!host->cmd_status) 2762 host->cmd_status = status; 2763 2764 smp_wmb(); /* drain writebuffer */ 2765 2766 set_bit(EVENT_CMD_COMPLETE, &host->pending_events); 2767 queue_work(system_bh_wq, &host->bh_work); 2768 2769 dw_mci_start_fault_timer(host); 2770 } 2771 2772 static void dw_mci_handle_cd(struct dw_mci *host) 2773 { 2774 mmc_detect_change(host->mmc, 2775 msecs_to_jiffies(host->detect_delay_ms)); 2776 } 2777 2778 static irqreturn_t dw_mci_interrupt(int irq, void *dev_id) 2779 { 2780 struct dw_mci *host = dev_id; 2781 u32 pending; 2782 2783 pending = mci_readl(host, MINTSTS); /* read-only mask reg */ 2784 2785 if (pending) { 2786 /* Check volt switch first, since it can look like an error */ 2787 if ((host->state == STATE_SENDING_CMD11) && 2788 (pending & SDMMC_INT_VOLT_SWITCH)) { 2789 mci_writel(host, RINTSTS, SDMMC_INT_VOLT_SWITCH); 2790 pending &= ~SDMMC_INT_VOLT_SWITCH; 2791 2792 spin_lock(&host->irq_lock); 2793 dw_mci_cmd_interrupt(host, pending); 2794 spin_unlock(&host->irq_lock); 2795 } 2796 2797 if (pending & DW_MCI_CMD_ERROR_FLAGS) { 2798 spin_lock(&host->irq_lock); 2799 2800 dw_mci_wd_deliver(host, BIT(EVENT_CMD_COMPLETE)); 2801 mci_writel(host, RINTSTS, DW_MCI_CMD_ERROR_FLAGS); 2802 host->cmd_status = pending; 2803 smp_wmb(); /* drain writebuffer */ 2804 set_bit(EVENT_CMD_COMPLETE, &host->pending_events); 2805 2806 spin_unlock(&host->irq_lock); 2807 } 2808 2809 if (pending & DW_MCI_DATA_ERROR_FLAGS) { 2810 spin_lock(&host->irq_lock); 2811 2812 if (host->quirks & DW_MMC_QUIRK_EXTENDED_TMOUT) 2813 dw_mci_wd_deliver(host, DW_MCI_WD_DATA_EVENTS); 2814 2815 /* if there is an error report DATA_ERROR */ 2816 mci_writel(host, RINTSTS, DW_MCI_DATA_ERROR_FLAGS); 2817 host->data_status = pending; 2818 smp_wmb(); /* drain writebuffer */ 2819 set_bit(EVENT_DATA_ERROR, &host->pending_events); 2820 2821 if (host->quirks & DW_MMC_QUIRK_EXTENDED_TMOUT) 2822 /* In case of error, we cannot expect a DTO */ 2823 set_bit(EVENT_DATA_COMPLETE, 2824 &host->pending_events); 2825 2826 queue_work(system_bh_wq, &host->bh_work); 2827 2828 spin_unlock(&host->irq_lock); 2829 } 2830 2831 if (pending & SDMMC_INT_DATA_OVER) { 2832 spin_lock(&host->irq_lock); 2833 2834 dw_mci_wd_deliver(host, DW_MCI_WD_DATA_EVENTS); 2835 2836 mci_writel(host, RINTSTS, SDMMC_INT_DATA_OVER); 2837 if (!host->data_status) 2838 host->data_status = pending; 2839 smp_wmb(); /* drain writebuffer */ 2840 if (host->dir_status == MMC_DATA_READ) { 2841 if (host->sg != NULL) 2842 dw_mci_read_data_pio(host, true); 2843 } 2844 set_bit(EVENT_DATA_COMPLETE, &host->pending_events); 2845 queue_work(system_bh_wq, &host->bh_work); 2846 2847 spin_unlock(&host->irq_lock); 2848 } 2849 2850 if (pending & SDMMC_INT_RXDR) { 2851 mci_writel(host, RINTSTS, SDMMC_INT_RXDR); 2852 if (host->dir_status == MMC_DATA_READ && host->sg) 2853 dw_mci_read_data_pio(host, false); 2854 } 2855 2856 if (pending & SDMMC_INT_TXDR) { 2857 mci_writel(host, RINTSTS, SDMMC_INT_TXDR); 2858 if (host->dir_status == MMC_DATA_WRITE && host->sg) 2859 dw_mci_write_data_pio(host); 2860 } 2861 2862 if (pending & SDMMC_INT_CMD_DONE) { 2863 spin_lock(&host->irq_lock); 2864 2865 mci_writel(host, RINTSTS, SDMMC_INT_CMD_DONE); 2866 dw_mci_cmd_interrupt(host, pending); 2867 2868 spin_unlock(&host->irq_lock); 2869 } 2870 2871 if (pending & SDMMC_INT_CD) { 2872 mci_writel(host, RINTSTS, SDMMC_INT_CD); 2873 dw_mci_handle_cd(host); 2874 } 2875 2876 if (pending & SDMMC_INT_SDIO(host->sdio_irq)) { 2877 mci_writel(host, RINTSTS, 2878 SDMMC_INT_SDIO(host->sdio_irq)); 2879 __dw_mci_enable_sdio_irq(host, 0); 2880 sdio_signal_irq(host->mmc); 2881 } 2882 2883 } 2884 2885 if (host->use_dma != TRANS_MODE_IDMAC) 2886 return IRQ_HANDLED; 2887 2888 /* Handle IDMA interrupts */ 2889 if (host->dma_64bit_address == 1) { 2890 pending = mci_readl(host, IDSTS64); 2891 if (pending & (SDMMC_IDMAC_INT_TI | SDMMC_IDMAC_INT_RI)) { 2892 mci_writel(host, IDSTS64, SDMMC_IDMAC_INT_TI | 2893 SDMMC_IDMAC_INT_RI); 2894 mci_writel(host, IDSTS64, SDMMC_IDMAC_INT_NI); 2895 if (!test_bit(EVENT_DATA_ERROR, &host->pending_events)) 2896 host->dma_ops->complete((void *)host); 2897 } 2898 } else { 2899 pending = mci_readl(host, IDSTS); 2900 if (pending & (SDMMC_IDMAC_INT_TI | SDMMC_IDMAC_INT_RI)) { 2901 mci_writel(host, IDSTS, SDMMC_IDMAC_INT_TI | 2902 SDMMC_IDMAC_INT_RI); 2903 mci_writel(host, IDSTS, SDMMC_IDMAC_INT_NI); 2904 if (!test_bit(EVENT_DATA_ERROR, &host->pending_events)) 2905 host->dma_ops->complete((void *)host); 2906 } 2907 } 2908 2909 return IRQ_HANDLED; 2910 } 2911 2912 static int dw_mci_init_host_caps(struct dw_mci *host) 2913 { 2914 const struct dw_mci_drv_data *drv_data = host->drv_data; 2915 struct mmc_host *mmc = host->mmc; 2916 int ctrl_id; 2917 2918 if (drv_data) 2919 mmc->caps |= drv_data->common_caps; 2920 2921 if (host->dev->of_node) 2922 ctrl_id = mmc->index; 2923 else 2924 ctrl_id = to_platform_device(host->dev)->id; 2925 2926 if (drv_data && drv_data->caps) { 2927 if (ctrl_id >= drv_data->num_caps) { 2928 dev_err(host->dev, "invalid controller id %d\n", 2929 ctrl_id); 2930 return -EINVAL; 2931 } 2932 mmc->caps |= drv_data->caps[ctrl_id]; 2933 } 2934 2935 /* if host has set a minimum_freq, we should respect it */ 2936 if (host->minimum_speed) 2937 mmc->f_min = host->minimum_speed; 2938 else 2939 mmc->f_min = DW_MCI_FREQ_MIN; 2940 2941 if (!mmc->f_max) 2942 mmc->f_max = DW_MCI_FREQ_MAX; 2943 2944 /* Process SDIO IRQs through the sdio_irq_work. */ 2945 if (mmc->caps & MMC_CAP_SDIO_IRQ) 2946 mmc->caps2 |= MMC_CAP2_SDIO_IRQ_NOTHREAD; 2947 2948 return 0; 2949 } 2950 2951 static int dw_mci_init_host(struct dw_mci *host) 2952 { 2953 struct mmc_host *mmc = host->mmc; 2954 int ret; 2955 2956 mmc->ops = &dw_mci_ops; 2957 2958 /*if there are external regulators, get them*/ 2959 ret = mmc_regulator_get_supply(mmc); 2960 if (ret) 2961 return ret; 2962 2963 if (!mmc->ocr_avail) 2964 mmc->ocr_avail = MMC_VDD_32_33 | MMC_VDD_33_34; 2965 2966 ret = mmc_of_parse(mmc); 2967 if (ret) 2968 return ret; 2969 2970 mmc_of_parse_clk_phase(host->dev, &host->phase_map); 2971 2972 ret = dw_mci_init_host_caps(host); 2973 if (ret) 2974 return ret; 2975 2976 /* Useful defaults if platform data is unset. */ 2977 if (host->use_dma == TRANS_MODE_IDMAC) { 2978 mmc->max_segs = host->desc_num; 2979 mmc->max_blk_size = 65535; 2980 mmc->max_seg_size = 0x1000; 2981 mmc->max_req_size = mmc->max_seg_size * host->desc_num; 2982 mmc->max_blk_count = mmc->max_req_size / 512; 2983 } else if (host->use_dma == TRANS_MODE_EDMAC) { 2984 mmc->max_segs = 64; 2985 mmc->max_blk_size = 65535; 2986 mmc->max_blk_count = 65535; 2987 mmc->max_req_size = 2988 mmc->max_blk_size * mmc->max_blk_count; 2989 mmc->max_seg_size = mmc->max_req_size; 2990 } else { 2991 /* TRANS_MODE_PIO */ 2992 mmc->max_segs = 64; 2993 mmc->max_blk_size = 65535; /* BLKSIZ is 16 bits */ 2994 mmc->max_blk_count = 512; 2995 mmc->max_req_size = mmc->max_blk_size * 2996 mmc->max_blk_count; 2997 mmc->max_seg_size = mmc->max_req_size; 2998 } 2999 3000 if (mmc->caps & MMC_CAP_NEEDS_POLL) 3001 dev_info(&mmc->class_dev, "card is polling.\n"); 3002 else if (!mmc_card_is_removable(mmc)) 3003 dev_info(&mmc->class_dev, "card is non-removable.\n"); 3004 3005 dw_mci_get_cd(mmc); 3006 3007 ret = mmc_add_host(mmc); 3008 if (ret) 3009 return ret; 3010 3011 #if defined(CONFIG_DEBUG_FS) 3012 dw_mci_init_debugfs(host); 3013 #endif 3014 3015 return 0; 3016 } 3017 3018 static void dw_mci_cleanup_host(struct dw_mci *host) 3019 { 3020 /* Debugfs stuff is cleaned up by mmc core */ 3021 mmc_remove_host(host->mmc); 3022 } 3023 3024 static void dw_mci_init_dma(struct dw_mci *host) 3025 { 3026 int addr_config; 3027 struct device *dev = host->dev; 3028 3029 /* 3030 * Check tansfer mode from HCON[17:16] 3031 * Clear the ambiguous description of dw_mmc databook: 3032 * 2b'00: No DMA Interface -> Actually means using Internal DMA block 3033 * 2b'01: DesignWare DMA Interface -> Synopsys DW-DMA block 3034 * 2b'10: Generic DMA Interface -> non-Synopsys generic DMA block 3035 * 2b'11: Non DW DMA Interface -> pio only 3036 * Compared to DesignWare DMA Interface, Generic DMA Interface has a 3037 * simpler request/acknowledge handshake mechanism and both of them 3038 * are regarded as external dma master for dw_mmc. 3039 */ 3040 host->use_dma = SDMMC_GET_TRANS_MODE(mci_readl(host, HCON)); 3041 if (host->use_dma == DMA_INTERFACE_IDMA) { 3042 host->use_dma = TRANS_MODE_IDMAC; 3043 } else if (host->use_dma == DMA_INTERFACE_DWDMA || 3044 host->use_dma == DMA_INTERFACE_GDMA) { 3045 host->use_dma = TRANS_MODE_EDMAC; 3046 } else { 3047 goto no_dma; 3048 } 3049 3050 /* Determine which DMA interface to use */ 3051 if (host->use_dma == TRANS_MODE_IDMAC) { 3052 /* 3053 * Check ADDR_CONFIG bit in HCON to find 3054 * IDMAC address bus width 3055 */ 3056 addr_config = SDMMC_GET_ADDR_CONFIG(mci_readl(host, HCON)); 3057 3058 if (addr_config == 1) { 3059 /* host supports IDMAC in 64-bit address mode */ 3060 host->dma_64bit_address = 1; 3061 dev_info(host->dev, 3062 "IDMAC supports 64-bit address mode.\n"); 3063 if (dma_set_mask_and_coherent(host->dev, DMA_BIT_MASK(64))) 3064 dev_info(host->dev, "Fail to set 64-bit DMA mask"); 3065 } else { 3066 /* host supports IDMAC in 32-bit address mode */ 3067 host->dma_64bit_address = 0; 3068 dev_info(host->dev, 3069 "IDMAC supports 32-bit address mode.\n"); 3070 } 3071 3072 /* Alloc memory for sg translation */ 3073 host->sg_cpu = dmam_alloc_coherent(host->dev, 3074 DESC_RING_BUF_SZ, 3075 &host->sg_dma, GFP_KERNEL); 3076 if (!host->sg_cpu) { 3077 dev_err(host->dev, 3078 "%s: could not alloc DMA memory\n", 3079 __func__); 3080 goto no_dma; 3081 } 3082 3083 host->dma_ops = &dw_mci_idmac_ops; 3084 dev_info(host->dev, "Using internal DMA controller.\n"); 3085 } else { 3086 /* TRANS_MODE_EDMAC: check dma bindings again */ 3087 if ((device_property_string_array_count(dev, "dma-names") < 0) || 3088 !device_property_present(dev, "dmas")) { 3089 goto no_dma; 3090 } 3091 host->dma_ops = &dw_mci_edmac_ops; 3092 dev_info(host->dev, "Using external DMA controller.\n"); 3093 } 3094 3095 if (host->dma_ops->init && host->dma_ops->start && 3096 host->dma_ops->stop && host->dma_ops->cleanup) { 3097 if (host->dma_ops->init(host)) { 3098 dev_err(host->dev, "%s: Unable to initialize DMA Controller.\n", 3099 __func__); 3100 goto no_dma; 3101 } 3102 } else { 3103 dev_err(host->dev, "DMA initialization not found.\n"); 3104 goto no_dma; 3105 } 3106 3107 return; 3108 3109 no_dma: 3110 dev_info(host->dev, "Using PIO mode.\n"); 3111 host->use_dma = TRANS_MODE_PIO; 3112 } 3113 3114 static int dw_mci_parse_dt(struct dw_mci *host) 3115 { 3116 struct device *dev = host->dev; 3117 const struct dw_mci_drv_data *drv_data = host->drv_data; 3118 int ret; 3119 u32 clock_frequency; 3120 3121 /* find reset controller when exist */ 3122 host->rstc = devm_reset_control_get_optional_exclusive(dev, "reset"); 3123 if (IS_ERR(host->rstc)) 3124 return PTR_ERR(host->rstc); 3125 3126 if (!host->fifo_depth && device_property_read_u32(dev, "fifo-depth", &host->fifo_depth)) 3127 dev_info(dev, 3128 "fifo-depth property not found, using value of FIFOTH register as default\n"); 3129 3130 if (!host->detect_delay_ms) 3131 device_property_read_u32(dev, "card-detect-delay", 3132 &host->detect_delay_ms); 3133 3134 if (!host->data_addr_override) 3135 device_property_read_u32(dev, "data-addr", &host->data_addr_override); 3136 3137 if (device_property_present(dev, "fifo-watermark-aligned")) 3138 host->wm_aligned = true; 3139 3140 if (!host->bus_hz && !device_property_read_u32(dev, "clock-frequency", &clock_frequency)) 3141 host->bus_hz = clock_frequency; 3142 3143 if (drv_data && drv_data->parse_dt) { 3144 ret = drv_data->parse_dt(host); 3145 if (ret) 3146 return ret; 3147 } 3148 3149 return 0; 3150 } 3151 3152 static void dw_mci_enable_cd(struct dw_mci *host) 3153 { 3154 unsigned long irqflags; 3155 u32 temp; 3156 3157 /* 3158 * No need for CD if host has a non-error GPIO 3159 * as well as broken card detection is found. 3160 */ 3161 if (host->mmc->caps & MMC_CAP_NEEDS_POLL) 3162 return; 3163 3164 if (mmc_gpio_get_cd(host->mmc) < 0) { 3165 spin_lock_irqsave(&host->irq_lock, irqflags); 3166 temp = mci_readl(host, INTMASK); 3167 temp |= SDMMC_INT_CD; 3168 mci_writel(host, INTMASK, temp); 3169 spin_unlock_irqrestore(&host->irq_lock, irqflags); 3170 } 3171 } 3172 3173 struct dw_mci *dw_mci_alloc_host(struct device *dev) 3174 { 3175 struct mmc_host *mmc; 3176 struct dw_mci *host; 3177 3178 mmc = devm_mmc_alloc_host(dev, sizeof(struct dw_mci)); 3179 if (!mmc) 3180 return ERR_PTR(-ENOMEM); 3181 3182 host = mmc_priv(mmc); 3183 host->mmc = mmc; 3184 host->dev = dev; 3185 host->dma_threshold = 16; 3186 3187 return host; 3188 } 3189 EXPORT_SYMBOL(dw_mci_alloc_host); 3190 3191 int dw_mci_probe(struct dw_mci *host) 3192 { 3193 const struct dw_mci_drv_data *drv_data = host->drv_data; 3194 int width, i, ret = 0; 3195 u32 fifo_size; 3196 3197 ret = dw_mci_parse_dt(host); 3198 if (ret) 3199 return dev_err_probe(host->dev, ret, "parse dt failed\n"); 3200 3201 host->biu_clk = devm_clk_get(host->dev, "biu"); 3202 if (IS_ERR(host->biu_clk)) { 3203 dev_dbg(host->dev, "biu clock not available\n"); 3204 ret = PTR_ERR(host->biu_clk); 3205 if (ret == -EPROBE_DEFER) 3206 return ret; 3207 3208 } else { 3209 ret = clk_prepare_enable(host->biu_clk); 3210 if (ret) { 3211 dev_err(host->dev, "failed to enable biu clock\n"); 3212 return ret; 3213 } 3214 } 3215 3216 host->ciu_clk = devm_clk_get(host->dev, "ciu"); 3217 if (IS_ERR(host->ciu_clk)) { 3218 dev_dbg(host->dev, "ciu clock not available\n"); 3219 ret = PTR_ERR(host->ciu_clk); 3220 if (ret == -EPROBE_DEFER) 3221 goto err_clk_biu; 3222 } else { 3223 ret = clk_prepare_enable(host->ciu_clk); 3224 if (ret) { 3225 dev_err(host->dev, "failed to enable ciu clock\n"); 3226 goto err_clk_biu; 3227 } 3228 3229 if (host->bus_hz) { 3230 ret = clk_set_rate(host->ciu_clk, host->bus_hz); 3231 if (ret) 3232 dev_warn(host->dev, 3233 "Unable to set bus rate to %uHz\n", 3234 host->bus_hz); 3235 } 3236 host->bus_hz = clk_get_rate(host->ciu_clk); 3237 } 3238 3239 if (!host->bus_hz) { 3240 dev_err(host->dev, 3241 "Platform data must supply bus speed\n"); 3242 ret = -ENODEV; 3243 goto err_clk_ciu; 3244 } 3245 3246 if (host->rstc) { 3247 reset_control_assert(host->rstc); 3248 usleep_range(10, 50); 3249 reset_control_deassert(host->rstc); 3250 } 3251 3252 if (drv_data && drv_data->init) { 3253 ret = drv_data->init(host); 3254 if (ret) { 3255 dev_err(host->dev, 3256 "implementation specific init failed\n"); 3257 goto err_clk_ciu; 3258 } 3259 } 3260 3261 hrtimer_setup(&host->wd_timer, dw_mci_watchdog_fn, CLOCK_MONOTONIC, 3262 HRTIMER_MODE_REL); 3263 3264 spin_lock_init(&host->lock); 3265 spin_lock_init(&host->irq_lock); 3266 3267 dw_mci_init_fault(host); 3268 3269 /* 3270 * Get the host data width - this assumes that HCON has been set with 3271 * the correct values. 3272 */ 3273 i = SDMMC_GET_HDATA_WIDTH(mci_readl(host, HCON)); 3274 if (!i) { 3275 host->push_data = dw_mci_push_data16; 3276 host->pull_data = dw_mci_pull_data16; 3277 width = 16; 3278 host->data_shift = 1; 3279 } else if (i == 2) { 3280 if ((host->quirks & DW_MMC_QUIRK_FIFO64_32)) { 3281 host->push_data = dw_mci_push_data64_32; 3282 host->pull_data = dw_mci_pull_data64_32; 3283 } else { 3284 host->push_data = dw_mci_push_data64; 3285 host->pull_data = dw_mci_pull_data64; 3286 } 3287 width = 64; 3288 host->data_shift = 3; 3289 } else { 3290 /* Check for a reserved value, and warn if it is */ 3291 WARN((i != 1), 3292 "HCON reports a reserved host data width!\n" 3293 "Defaulting to 32-bit access.\n"); 3294 host->push_data = dw_mci_push_data32; 3295 host->pull_data = dw_mci_pull_data32; 3296 width = 32; 3297 host->data_shift = 2; 3298 } 3299 3300 /* Reset all blocks */ 3301 if (!dw_mci_ctrl_reset(host, SDMMC_CTRL_ALL_RESET_FLAGS)) { 3302 ret = -ENODEV; 3303 goto err_clk_ciu; 3304 } 3305 3306 dw_mci_init_dma(host); 3307 3308 /* Clear the interrupts for the host controller */ 3309 mci_writel(host, RINTSTS, 0xFFFFFFFF); 3310 mci_writel(host, INTMASK, 0); /* disable all mmc interrupt first */ 3311 3312 /* Put in max timeout */ 3313 mci_writel(host, TMOUT, 0xFFFFFFFF); 3314 3315 /* 3316 * FIFO threshold settings RxMark = fifo_size / 2 - 1, 3317 * Tx Mark = fifo_size / 2 DMA Size = 8 3318 */ 3319 if (!host->fifo_depth) { 3320 /* 3321 * Power-on value of RX_WMark is FIFO_DEPTH-1, but this may 3322 * have been overwritten by the bootloader, just like we're 3323 * about to do, so if you know the value for your hardware, you 3324 * should put it in the platform data. 3325 */ 3326 fifo_size = mci_readl(host, FIFOTH); 3327 fifo_size = 1 + ((fifo_size >> 16) & 0xfff); 3328 } else { 3329 fifo_size = host->fifo_depth; 3330 } 3331 host->fifo_depth = fifo_size; 3332 host->fifoth_val = 3333 SDMMC_SET_FIFOTH(0x2, fifo_size / 2 - 1, fifo_size / 2); 3334 mci_writel(host, FIFOTH, host->fifoth_val); 3335 3336 /* disable clock to CIU */ 3337 mci_writel(host, CLKENA, 0); 3338 mci_writel(host, CLKSRC, 0); 3339 3340 /* 3341 * In 2.40a spec, Data offset is changed. 3342 * Need to check the version-id and set data-offset for DATA register. 3343 */ 3344 host->verid = SDMMC_GET_VERID(mci_readl(host, VERID)); 3345 dev_info(host->dev, "Version ID is %04x\n", host->verid); 3346 3347 if (host->data_addr_override) 3348 host->fifo_reg = host->regs + host->data_addr_override; 3349 else if (host->verid < DW_MMC_240A) 3350 host->fifo_reg = host->regs + DATA_OFFSET; 3351 else 3352 host->fifo_reg = host->regs + DATA_240A_OFFSET; 3353 3354 INIT_WORK(&host->bh_work, dw_mci_work_func); 3355 ret = devm_request_irq(host->dev, host->irq, dw_mci_interrupt, 3356 host->irq_flags, "dw-mci", host); 3357 if (ret) 3358 goto err_dmaunmap; 3359 3360 /* 3361 * Enable interrupts for command done, data over, data empty, 3362 * receive ready and error such as transmit, receive timeout, crc error 3363 */ 3364 mci_writel(host, INTMASK, SDMMC_INT_CMD_DONE | SDMMC_INT_DATA_OVER | 3365 SDMMC_INT_TXDR | SDMMC_INT_RXDR | 3366 DW_MCI_ERROR_FLAGS); 3367 /* Enable mci interrupt */ 3368 mci_writel(host, CTRL, SDMMC_CTRL_INT_ENABLE); 3369 3370 dev_info(host->dev, 3371 "DW MMC controller at irq %d,%d bit host data width,%u deep fifo\n", 3372 host->irq, width, fifo_size); 3373 3374 ret = dw_mci_init_host(host); 3375 if (ret) { 3376 dev_dbg(host->dev, "host init failed\n"); 3377 goto err_dmaunmap; 3378 } 3379 3380 /* Now that host is setup, we can enable card detect */ 3381 dw_mci_enable_cd(host); 3382 3383 return 0; 3384 3385 err_dmaunmap: 3386 if (host->use_dma && host->dma_ops->exit) 3387 host->dma_ops->exit(host); 3388 3389 reset_control_assert(host->rstc); 3390 3391 err_clk_ciu: 3392 clk_disable_unprepare(host->ciu_clk); 3393 3394 err_clk_biu: 3395 clk_disable_unprepare(host->biu_clk); 3396 3397 return ret; 3398 } 3399 EXPORT_SYMBOL(dw_mci_probe); 3400 3401 void dw_mci_remove(struct dw_mci *host) 3402 { 3403 dev_dbg(host->dev, "remove host\n"); 3404 dw_mci_cleanup_host(host); 3405 3406 mci_writel(host, RINTSTS, 0xFFFFFFFF); 3407 mci_writel(host, INTMASK, 0); /* disable all mmc interrupt first */ 3408 3409 /* disable clock to CIU */ 3410 mci_writel(host, CLKENA, 0); 3411 mci_writel(host, CLKSRC, 0); 3412 3413 if (host->use_dma && host->dma_ops->exit) 3414 host->dma_ops->exit(host); 3415 3416 reset_control_assert(host->rstc); 3417 3418 clk_disable_unprepare(host->ciu_clk); 3419 clk_disable_unprepare(host->biu_clk); 3420 } 3421 EXPORT_SYMBOL(dw_mci_remove); 3422 3423 int dw_mci_runtime_suspend(struct device *dev) 3424 { 3425 struct dw_mci *host = dev_get_drvdata(dev); 3426 3427 if (host->use_dma && host->dma_ops->exit) 3428 host->dma_ops->exit(host); 3429 3430 clk_disable_unprepare(host->ciu_clk); 3431 3432 if (mmc_host_can_gpio_cd(host->mmc) || 3433 !mmc_card_is_removable(host->mmc)) 3434 clk_disable_unprepare(host->biu_clk); 3435 3436 return 0; 3437 } 3438 EXPORT_SYMBOL(dw_mci_runtime_suspend); 3439 3440 int dw_mci_runtime_resume(struct device *dev) 3441 { 3442 int ret = 0; 3443 struct dw_mci *host = dev_get_drvdata(dev); 3444 3445 if (mmc_host_can_gpio_cd(host->mmc) || 3446 !mmc_card_is_removable(host->mmc)) { 3447 ret = clk_prepare_enable(host->biu_clk); 3448 if (ret) 3449 return ret; 3450 } 3451 3452 ret = clk_prepare_enable(host->ciu_clk); 3453 if (ret) 3454 goto err; 3455 3456 if (!dw_mci_ctrl_reset(host, SDMMC_CTRL_ALL_RESET_FLAGS)) { 3457 clk_disable_unprepare(host->ciu_clk); 3458 ret = -ENODEV; 3459 goto err; 3460 } 3461 3462 if (host->use_dma && host->dma_ops->init) { 3463 ret = host->dma_ops->init(host); 3464 if (ret) 3465 return ret; 3466 } 3467 3468 /* 3469 * Restore the initial value at FIFOTH register 3470 * And Invalidate the prev_blksz with zero 3471 */ 3472 mci_writel(host, FIFOTH, host->fifoth_val); 3473 host->prev_blksz = 0; 3474 3475 /* Put in max timeout */ 3476 mci_writel(host, TMOUT, 0xFFFFFFFF); 3477 3478 mci_writel(host, RINTSTS, 0xFFFFFFFF); 3479 mci_writel(host, INTMASK, SDMMC_INT_CMD_DONE | SDMMC_INT_DATA_OVER | 3480 SDMMC_INT_TXDR | SDMMC_INT_RXDR | 3481 DW_MCI_ERROR_FLAGS); 3482 mci_writel(host, CTRL, SDMMC_CTRL_INT_ENABLE); 3483 3484 3485 if (host->mmc->pm_flags & MMC_PM_KEEP_POWER) 3486 dw_mci_set_ios(host->mmc, &host->mmc->ios); 3487 3488 /* Force setup bus to guarantee available clock output */ 3489 dw_mci_setup_bus(host, true); 3490 3491 /* Re-enable SDIO interrupts. */ 3492 if (sdio_irq_claimed(host->mmc)) 3493 __dw_mci_enable_sdio_irq(host, 1); 3494 3495 /* Now that host is setup, we can enable card detect */ 3496 dw_mci_enable_cd(host); 3497 3498 return 0; 3499 3500 err: 3501 if (mmc_host_can_gpio_cd(host->mmc) || 3502 !mmc_card_is_removable(host->mmc)) 3503 clk_disable_unprepare(host->biu_clk); 3504 3505 return ret; 3506 } 3507 EXPORT_SYMBOL(dw_mci_runtime_resume); 3508 3509 const struct dev_pm_ops dw_mci_pmops = { 3510 SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, pm_runtime_force_resume) 3511 RUNTIME_PM_OPS(dw_mci_runtime_suspend, dw_mci_runtime_resume, NULL) 3512 }; 3513 EXPORT_SYMBOL_GPL(dw_mci_pmops); 3514 3515 MODULE_DESCRIPTION("DW Multimedia Card Interface driver"); 3516 MODULE_AUTHOR("NXP Semiconductor VietNam"); 3517 MODULE_AUTHOR("Imagination Technologies Ltd"); 3518 MODULE_LICENSE("GPL v2"); 3519