1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2012 Oleksandr Tymoshenko <gonzo@freebsd.org> 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 * 28 */ 29 #include <sys/cdefs.h> 30 __FBSDID("$FreeBSD$"); 31 32 #include <sys/param.h> 33 #include <sys/systm.h> 34 #include <sys/bus.h> 35 #include <sys/kernel.h> 36 #include <sys/lock.h> 37 #include <sys/malloc.h> 38 #include <sys/module.h> 39 #include <sys/mutex.h> 40 #include <sys/rman.h> 41 #include <sys/sysctl.h> 42 #include <sys/taskqueue.h> 43 44 #include <machine/bus.h> 45 46 #include <dev/ofw/ofw_bus.h> 47 #include <dev/ofw/ofw_bus_subr.h> 48 49 #include <dev/mmc/bridge.h> 50 #include <dev/mmc/mmcreg.h> 51 #include <dev/mmc/mmc_fdt_helpers.h> 52 53 #include <dev/sdhci/sdhci.h> 54 55 #include "mmcbr_if.h" 56 #include "sdhci_if.h" 57 58 #include "opt_mmccam.h" 59 60 #include "bcm2835_dma.h" 61 #include <arm/broadcom/bcm2835/bcm2835_mbox_prop.h> 62 #ifdef NOTYET 63 #include <arm/broadcom/bcm2835/bcm2835_clkman.h> 64 #endif 65 #include <arm/broadcom/bcm2835/bcm2835_vcbus.h> 66 67 #define BCM2835_DEFAULT_SDHCI_FREQ 50 68 #define BCM2838_DEFAULT_SDHCI_FREQ 100 69 70 #define BCM_SDHCI_BUFFER_SIZE 512 71 /* 72 * NUM_DMA_SEGS is the number of DMA segments we want to accommodate on average. 73 * We add in a number of segments based on how much we may need to spill into 74 * another segment due to crossing page boundaries. e.g. up to PAGE_SIZE, an 75 * extra page is needed as we can cross a page boundary exactly once. 76 */ 77 #define NUM_DMA_SEGS 1 78 #define NUM_DMA_SPILL_SEGS \ 79 ((((NUM_DMA_SEGS * BCM_SDHCI_BUFFER_SIZE) - 1) / PAGE_SIZE) + 1) 80 #define ALLOCATED_DMA_SEGS (NUM_DMA_SEGS + NUM_DMA_SPILL_SEGS) 81 #define BCM_DMA_MAXSIZE (NUM_DMA_SEGS * BCM_SDHCI_BUFFER_SIZE) 82 83 #define BCM_SDHCI_SLOT_LEFT(slot) \ 84 ((slot)->curcmd->data->len - (slot)->offset) 85 86 #define BCM_SDHCI_SEGSZ_LEFT(slot) \ 87 min(BCM_DMA_MAXSIZE, \ 88 rounddown(BCM_SDHCI_SLOT_LEFT(slot), BCM_SDHCI_BUFFER_SIZE)) 89 90 #define DATA_PENDING_MASK (SDHCI_INT_DATA_AVAIL | SDHCI_INT_SPACE_AVAIL) 91 #define DATA_XFER_MASK (DATA_PENDING_MASK | SDHCI_INT_DATA_END) 92 93 #ifdef DEBUG 94 static int bcm2835_sdhci_debug = 0; 95 96 TUNABLE_INT("hw.bcm2835.sdhci.debug", &bcm2835_sdhci_debug); 97 SYSCTL_INT(_hw_sdhci, OID_AUTO, bcm2835_sdhci_debug, CTLFLAG_RWTUN, 98 &bcm2835_sdhci_debug, 0, "bcm2835 SDHCI debug level"); 99 100 #define dprintf(fmt, args...) \ 101 do { \ 102 if (bcm2835_sdhci_debug) \ 103 printf("%s: " fmt, __func__, ##args); \ 104 } while (0) 105 #else 106 #define dprintf(fmt, args...) 107 #endif 108 109 static int bcm2835_sdhci_hs = 1; 110 static int bcm2835_sdhci_pio_mode = 0; 111 112 struct bcm_mmc_conf { 113 int clock_id; 114 int clock_src; 115 int default_freq; 116 int quirks; 117 int emmc_dreq; 118 }; 119 120 struct bcm_mmc_conf bcm2835_sdhci_conf = { 121 .clock_id = BCM2835_MBOX_CLOCK_ID_EMMC, 122 .clock_src = -1, 123 .default_freq = BCM2835_DEFAULT_SDHCI_FREQ, 124 .quirks = SDHCI_QUIRK_DATA_TIMEOUT_USES_SDCLK | 125 SDHCI_QUIRK_BROKEN_TIMEOUT_VAL | SDHCI_QUIRK_DONT_SET_HISPD_BIT | 126 SDHCI_QUIRK_MISSING_CAPS, 127 .emmc_dreq = BCM_DMA_DREQ_EMMC, 128 }; 129 130 struct bcm_mmc_conf bcm2838_emmc2_conf = { 131 .clock_id = BCM2838_MBOX_CLOCK_ID_EMMC2, 132 .clock_src = -1, 133 .default_freq = BCM2838_DEFAULT_SDHCI_FREQ, 134 .quirks = 0, 135 .emmc_dreq = BCM_DMA_DREQ_NONE, 136 }; 137 138 static struct ofw_compat_data compat_data[] = { 139 {"broadcom,bcm2835-sdhci", (uintptr_t)&bcm2835_sdhci_conf}, 140 {"brcm,bcm2835-sdhci", (uintptr_t)&bcm2835_sdhci_conf}, 141 {"brcm,bcm2835-mmc", (uintptr_t)&bcm2835_sdhci_conf}, 142 {"brcm,bcm2711-emmc2", (uintptr_t)&bcm2838_emmc2_conf}, 143 {"brcm,bcm2838-emmc2", (uintptr_t)&bcm2838_emmc2_conf}, 144 {NULL, 0} 145 }; 146 147 TUNABLE_INT("hw.bcm2835.sdhci.hs", &bcm2835_sdhci_hs); 148 TUNABLE_INT("hw.bcm2835.sdhci.pio_mode", &bcm2835_sdhci_pio_mode); 149 150 struct bcm_sdhci_softc { 151 device_t sc_dev; 152 struct resource * sc_mem_res; 153 struct resource * sc_irq_res; 154 bus_space_tag_t sc_bst; 155 bus_space_handle_t sc_bsh; 156 void * sc_intrhand; 157 struct mmc_request * sc_req; 158 struct sdhci_slot sc_slot; 159 struct mmc_fdt_helper sc_mmc_helper; 160 int sc_dma_ch; 161 bus_dma_tag_t sc_dma_tag; 162 bus_dmamap_t sc_dma_map; 163 vm_paddr_t sc_sdhci_buffer_phys; 164 bus_addr_t dmamap_seg_addrs[ALLOCATED_DMA_SEGS]; 165 bus_size_t dmamap_seg_sizes[ALLOCATED_DMA_SEGS]; 166 int dmamap_seg_count; 167 int dmamap_seg_index; 168 int dmamap_status; 169 uint32_t blksz_and_count; 170 uint32_t cmd_and_mode; 171 bool need_update_blk; 172 #ifdef NOTYET 173 device_t clkman; 174 #endif 175 struct bcm_mmc_conf * conf; 176 }; 177 178 static int bcm_sdhci_probe(device_t); 179 static int bcm_sdhci_attach(device_t); 180 static int bcm_sdhci_detach(device_t); 181 static void bcm_sdhci_intr(void *); 182 183 static int bcm_sdhci_get_ro(device_t, device_t); 184 static void bcm_sdhci_dma_intr(int ch, void *arg); 185 static void bcm_sdhci_start_dma(struct sdhci_slot *slot); 186 187 static void 188 bcm_sdhci_dmacb(void *arg, bus_dma_segment_t *segs, int nseg, int err) 189 { 190 struct bcm_sdhci_softc *sc = arg; 191 int i; 192 193 /* Sanity check: we can only ever have one mapping at a time. */ 194 KASSERT(sc->dmamap_seg_count == 0, ("leaked DMA segment")); 195 sc->dmamap_status = err; 196 sc->dmamap_seg_count = nseg; 197 198 /* Note nseg is guaranteed to be zero if err is non-zero. */ 199 for (i = 0; i < nseg; i++) { 200 sc->dmamap_seg_addrs[i] = segs[i].ds_addr; 201 sc->dmamap_seg_sizes[i] = segs[i].ds_len; 202 } 203 } 204 205 static int 206 bcm_sdhci_probe(device_t dev) 207 { 208 209 if (!ofw_bus_status_okay(dev)) 210 return (ENXIO); 211 212 if (ofw_bus_search_compatible(dev, compat_data)->ocd_data == 0) 213 return (ENXIO); 214 215 device_set_desc(dev, "Broadcom 2708 SDHCI controller"); 216 217 return (BUS_PROBE_DEFAULT); 218 } 219 220 static int 221 bcm_sdhci_attach(device_t dev) 222 { 223 struct bcm_sdhci_softc *sc = device_get_softc(dev); 224 int rid, err; 225 phandle_t node; 226 pcell_t cell; 227 u_int default_freq; 228 229 sc->sc_dev = dev; 230 sc->sc_req = NULL; 231 232 sc->conf = (struct bcm_mmc_conf *)ofw_bus_search_compatible(dev, 233 compat_data)->ocd_data; 234 if (sc->conf == 0) 235 return (ENXIO); 236 237 err = bcm2835_mbox_set_power_state(BCM2835_MBOX_POWER_ID_EMMC, TRUE); 238 if (err != 0) { 239 if (bootverbose) 240 device_printf(dev, "Unable to enable the power\n"); 241 return (err); 242 } 243 244 default_freq = 0; 245 err = bcm2835_mbox_get_clock_rate(sc->conf->clock_id, &default_freq); 246 if (err == 0) { 247 /* Convert to MHz */ 248 default_freq /= 1000000; 249 } 250 if (default_freq == 0) { 251 node = ofw_bus_get_node(sc->sc_dev); 252 if ((OF_getencprop(node, "clock-frequency", &cell, 253 sizeof(cell))) > 0) 254 default_freq = cell / 1000000; 255 } 256 if (default_freq == 0) 257 default_freq = sc->conf->default_freq; 258 259 if (bootverbose) 260 device_printf(dev, "SDHCI frequency: %dMHz\n", default_freq); 261 #ifdef NOTYET 262 if (sc->conf->clock_src > 0) { 263 uint32_t f; 264 sc->clkman = devclass_get_device( 265 devclass_find("bcm2835_clkman"), 0); 266 if (sc->clkman == NULL) { 267 device_printf(dev, "cannot find Clock Manager\n"); 268 return (ENXIO); 269 } 270 271 f = bcm2835_clkman_set_frequency(sc->clkman, 272 sc->conf->clock_src, default_freq); 273 if (f == 0) 274 return (EINVAL); 275 276 if (bootverbose) 277 device_printf(dev, "Clock source frequency: %dMHz\n", 278 f); 279 } 280 #endif 281 282 rid = 0; 283 sc->sc_mem_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, 284 RF_ACTIVE); 285 if (!sc->sc_mem_res) { 286 device_printf(dev, "cannot allocate memory window\n"); 287 err = ENXIO; 288 goto fail; 289 } 290 291 sc->sc_bst = rman_get_bustag(sc->sc_mem_res); 292 sc->sc_bsh = rman_get_bushandle(sc->sc_mem_res); 293 294 rid = 0; 295 sc->sc_irq_res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, 296 RF_ACTIVE | RF_SHAREABLE); 297 if (!sc->sc_irq_res) { 298 device_printf(dev, "cannot allocate interrupt\n"); 299 err = ENXIO; 300 goto fail; 301 } 302 303 if (bus_setup_intr(dev, sc->sc_irq_res, INTR_TYPE_BIO | INTR_MPSAFE, 304 NULL, bcm_sdhci_intr, sc, &sc->sc_intrhand)) { 305 device_printf(dev, "cannot setup interrupt handler\n"); 306 err = ENXIO; 307 goto fail; 308 } 309 310 if (!bcm2835_sdhci_pio_mode) 311 sc->sc_slot.opt = SDHCI_PLATFORM_TRANSFER; 312 313 sc->sc_slot.caps = SDHCI_CAN_VDD_330 | SDHCI_CAN_VDD_180; 314 if (bcm2835_sdhci_hs) 315 sc->sc_slot.caps |= SDHCI_CAN_DO_HISPD; 316 sc->sc_slot.caps |= (default_freq << SDHCI_CLOCK_BASE_SHIFT); 317 sc->sc_slot.quirks = sc->conf->quirks; 318 319 sdhci_init_slot(dev, &sc->sc_slot, 0); 320 mmc_fdt_parse(dev, 0, &sc->sc_mmc_helper, &sc->sc_slot.host); 321 322 sc->sc_dma_ch = bcm_dma_allocate(BCM_DMA_CH_ANY); 323 if (sc->sc_dma_ch == BCM_DMA_CH_INVALID) 324 goto fail; 325 326 err = bcm_dma_setup_intr(sc->sc_dma_ch, bcm_sdhci_dma_intr, sc); 327 if (err != 0) { 328 device_printf(dev, 329 "cannot setup dma interrupt handler\n"); 330 err = ENXIO; 331 goto fail; 332 } 333 334 /* Allocate bus_dma resources. */ 335 err = bus_dma_tag_create(bus_get_dma_tag(dev), 336 1, 0, bcm283x_dmabus_peripheral_lowaddr(), 337 BUS_SPACE_MAXADDR, NULL, NULL, 338 BCM_DMA_MAXSIZE, ALLOCATED_DMA_SEGS, BCM_SDHCI_BUFFER_SIZE, 339 BUS_DMA_ALLOCNOW, NULL, NULL, 340 &sc->sc_dma_tag); 341 342 if (err) { 343 device_printf(dev, "failed allocate DMA tag"); 344 goto fail; 345 } 346 347 err = bus_dmamap_create(sc->sc_dma_tag, 0, &sc->sc_dma_map); 348 if (err) { 349 device_printf(dev, "bus_dmamap_create failed\n"); 350 goto fail; 351 } 352 353 /* FIXME: Fix along with other BUS_SPACE_PHYSADDR instances */ 354 sc->sc_sdhci_buffer_phys = rman_get_start(sc->sc_mem_res) + 355 SDHCI_BUFFER; 356 357 bus_generic_probe(dev); 358 bus_generic_attach(dev); 359 360 sdhci_start_slot(&sc->sc_slot); 361 362 /* Seed our copies. */ 363 sc->blksz_and_count = SDHCI_READ_4(dev, &sc->sc_slot, SDHCI_BLOCK_SIZE); 364 sc->cmd_and_mode = SDHCI_READ_4(dev, &sc->sc_slot, SDHCI_TRANSFER_MODE); 365 366 return (0); 367 368 fail: 369 if (sc->sc_intrhand) 370 bus_teardown_intr(dev, sc->sc_irq_res, sc->sc_intrhand); 371 if (sc->sc_irq_res) 372 bus_release_resource(dev, SYS_RES_IRQ, 0, sc->sc_irq_res); 373 if (sc->sc_mem_res) 374 bus_release_resource(dev, SYS_RES_MEMORY, 0, sc->sc_mem_res); 375 376 return (err); 377 } 378 379 static int 380 bcm_sdhci_detach(device_t dev) 381 { 382 383 return (EBUSY); 384 } 385 386 static void 387 bcm_sdhci_intr(void *arg) 388 { 389 struct bcm_sdhci_softc *sc = arg; 390 391 sdhci_generic_intr(&sc->sc_slot); 392 } 393 394 static int 395 bcm_sdhci_update_ios(device_t bus, device_t child) 396 { 397 #ifdef EXT_RESOURCES 398 struct bcm_sdhci_softc *sc; 399 struct mmc_ios *ios; 400 #endif 401 int rv; 402 403 #ifdef EXT_RESOURCES 404 sc = device_get_softc(bus); 405 ios = &sc->sc_slot.host.ios; 406 407 if (ios->power_mode == power_up) { 408 if (sc->sc_mmc_helper.vmmc_supply) 409 regulator_enable(sc->sc_mmc_helper.vmmc_supply); 410 if (sc->sc_mmc_helper.vqmmc_supply) 411 regulator_enable(sc->sc_mmc_helper.vqmmc_supply); 412 } 413 #endif 414 415 rv = sdhci_generic_update_ios(bus, child); 416 if (rv != 0) 417 return (rv); 418 419 #ifdef EXT_RESOURCES 420 if (ios->power_mode == power_off) { 421 if (sc->sc_mmc_helper.vmmc_supply) 422 regulator_disable(sc->sc_mmc_helper.vmmc_supply); 423 if (sc->sc_mmc_helper.vqmmc_supply) 424 regulator_disable(sc->sc_mmc_helper.vqmmc_supply); 425 } 426 #endif 427 428 return (0); 429 } 430 431 static int 432 bcm_sdhci_get_ro(device_t bus, device_t child) 433 { 434 435 return (0); 436 } 437 438 static inline uint32_t 439 RD4(struct bcm_sdhci_softc *sc, bus_size_t off) 440 { 441 uint32_t val = bus_space_read_4(sc->sc_bst, sc->sc_bsh, off); 442 return val; 443 } 444 445 static inline void 446 WR4(struct bcm_sdhci_softc *sc, bus_size_t off, uint32_t val) 447 { 448 449 bus_space_write_4(sc->sc_bst, sc->sc_bsh, off, val); 450 /* 451 * The Arasan HC has a bug where it may lose the content of 452 * consecutive writes to registers that are within two SD-card 453 * clock cycles of each other (a clock domain crossing problem). 454 */ 455 if (sc->sc_slot.clock > 0) 456 DELAY(((2 * 1000000) / sc->sc_slot.clock) + 1); 457 } 458 459 static uint8_t 460 bcm_sdhci_read_1(device_t dev, struct sdhci_slot *slot, bus_size_t off) 461 { 462 struct bcm_sdhci_softc *sc = device_get_softc(dev); 463 uint32_t val = RD4(sc, off & ~3); 464 465 return ((val >> (off & 3)*8) & 0xff); 466 } 467 468 static uint16_t 469 bcm_sdhci_read_2(device_t dev, struct sdhci_slot *slot, bus_size_t off) 470 { 471 struct bcm_sdhci_softc *sc = device_get_softc(dev); 472 uint32_t val32; 473 474 /* 475 * Standard 32-bit handling of command and transfer mode, as 476 * well as block size and count. 477 */ 478 if ((off == SDHCI_BLOCK_SIZE || off == SDHCI_BLOCK_COUNT) && 479 sc->need_update_blk) 480 val32 = sc->blksz_and_count; 481 else if (off == SDHCI_TRANSFER_MODE || off == SDHCI_COMMAND_FLAGS) 482 val32 = sc->cmd_and_mode; 483 else 484 val32 = RD4(sc, off & ~3); 485 486 return ((val32 >> (off & 3)*8) & 0xffff); 487 } 488 489 static uint32_t 490 bcm_sdhci_read_4(device_t dev, struct sdhci_slot *slot, bus_size_t off) 491 { 492 struct bcm_sdhci_softc *sc = device_get_softc(dev); 493 494 return RD4(sc, off); 495 } 496 497 static void 498 bcm_sdhci_read_multi_4(device_t dev, struct sdhci_slot *slot, bus_size_t off, 499 uint32_t *data, bus_size_t count) 500 { 501 struct bcm_sdhci_softc *sc = device_get_softc(dev); 502 503 bus_space_read_multi_4(sc->sc_bst, sc->sc_bsh, off, data, count); 504 } 505 506 static void 507 bcm_sdhci_write_1(device_t dev, struct sdhci_slot *slot, bus_size_t off, 508 uint8_t val) 509 { 510 struct bcm_sdhci_softc *sc = device_get_softc(dev); 511 uint32_t val32 = RD4(sc, off & ~3); 512 val32 &= ~(0xff << (off & 3)*8); 513 val32 |= (val << (off & 3)*8); 514 WR4(sc, off & ~3, val32); 515 } 516 517 static void 518 bcm_sdhci_write_2(device_t dev, struct sdhci_slot *slot, bus_size_t off, 519 uint16_t val) 520 { 521 struct bcm_sdhci_softc *sc = device_get_softc(dev); 522 uint32_t val32; 523 524 /* 525 * If we have a queued up 16bit value for blk size or count, use and 526 * update the saved value rather than doing any real register access. 527 * If we did not touch either since the last write, then read from 528 * register as at least block count can change. 529 * Similarly, if we are about to issue a command, always use the saved 530 * value for transfer mode as we can never write that without issuing 531 * a command. 532 */ 533 if ((off == SDHCI_BLOCK_SIZE || off == SDHCI_BLOCK_COUNT) && 534 sc->need_update_blk) 535 val32 = sc->blksz_and_count; 536 else if (off == SDHCI_COMMAND_FLAGS) 537 val32 = sc->cmd_and_mode; 538 else 539 val32 = RD4(sc, off & ~3); 540 541 val32 &= ~(0xffff << (off & 3)*8); 542 val32 |= (val << (off & 3)*8); 543 544 if (off == SDHCI_TRANSFER_MODE) 545 sc->cmd_and_mode = val32; 546 else if (off == SDHCI_BLOCK_SIZE || off == SDHCI_BLOCK_COUNT) { 547 sc->blksz_and_count = val32; 548 sc->need_update_blk = true; 549 } else { 550 if (off == SDHCI_COMMAND_FLAGS) { 551 /* If we saved blk writes, do them now before cmd. */ 552 if (sc->need_update_blk) { 553 WR4(sc, SDHCI_BLOCK_SIZE, sc->blksz_and_count); 554 sc->need_update_blk = false; 555 } 556 /* Always save cmd and mode registers. */ 557 sc->cmd_and_mode = val32; 558 } 559 WR4(sc, off & ~3, val32); 560 } 561 } 562 563 static void 564 bcm_sdhci_write_4(device_t dev, struct sdhci_slot *slot, bus_size_t off, 565 uint32_t val) 566 { 567 struct bcm_sdhci_softc *sc = device_get_softc(dev); 568 WR4(sc, off, val); 569 } 570 571 static void 572 bcm_sdhci_write_multi_4(device_t dev, struct sdhci_slot *slot, bus_size_t off, 573 uint32_t *data, bus_size_t count) 574 { 575 struct bcm_sdhci_softc *sc = device_get_softc(dev); 576 577 bus_space_write_multi_4(sc->sc_bst, sc->sc_bsh, off, data, count); 578 } 579 580 static void 581 bcm_sdhci_start_dma_seg(struct bcm_sdhci_softc *sc) 582 { 583 struct sdhci_slot *slot; 584 vm_paddr_t pdst, psrc; 585 int err, idx, len, sync_op, width; 586 587 slot = &sc->sc_slot; 588 mtx_assert(&slot->mtx, MA_OWNED); 589 idx = sc->dmamap_seg_index++; 590 len = sc->dmamap_seg_sizes[idx]; 591 slot->offset += len; 592 width = (len & 0xf ? BCM_DMA_32BIT : BCM_DMA_128BIT); 593 594 if (slot->curcmd->data->flags & MMC_DATA_READ) { 595 /* 596 * Peripherals on the AXI bus do not need DREQ pacing for reads 597 * from the ARM core, so we can safely set this to NONE. 598 */ 599 bcm_dma_setup_src(sc->sc_dma_ch, BCM_DMA_DREQ_NONE, 600 BCM_DMA_SAME_ADDR, BCM_DMA_32BIT); 601 bcm_dma_setup_dst(sc->sc_dma_ch, BCM_DMA_DREQ_NONE, 602 BCM_DMA_INC_ADDR, width); 603 psrc = sc->sc_sdhci_buffer_phys; 604 pdst = sc->dmamap_seg_addrs[idx]; 605 sync_op = BUS_DMASYNC_PREREAD; 606 } else { 607 /* 608 * The ordering here is important, because the last write to 609 * dst/src in the dma control block writes the real dreq value. 610 */ 611 bcm_dma_setup_src(sc->sc_dma_ch, BCM_DMA_DREQ_NONE, 612 BCM_DMA_INC_ADDR, width); 613 bcm_dma_setup_dst(sc->sc_dma_ch, sc->conf->emmc_dreq, 614 BCM_DMA_SAME_ADDR, BCM_DMA_32BIT); 615 psrc = sc->dmamap_seg_addrs[idx]; 616 pdst = sc->sc_sdhci_buffer_phys; 617 sync_op = BUS_DMASYNC_PREWRITE; 618 } 619 620 /* 621 * When starting a new DMA operation do the busdma sync operation, and 622 * disable SDCHI data interrrupts because we'll be driven by DMA 623 * interrupts (or SDHCI error interrupts) until the IO is done. 624 */ 625 if (idx == 0) { 626 bus_dmamap_sync(sc->sc_dma_tag, sc->sc_dma_map, sync_op); 627 628 slot->intmask &= ~DATA_XFER_MASK; 629 bcm_sdhci_write_4(sc->sc_dev, slot, SDHCI_SIGNAL_ENABLE, 630 slot->intmask); 631 } 632 633 /* 634 * Start the DMA transfer. Only programming errors (like failing to 635 * allocate a channel) cause a non-zero return from bcm_dma_start(). 636 */ 637 err = bcm_dma_start(sc->sc_dma_ch, psrc, pdst, len); 638 KASSERT((err == 0), ("bcm2835_sdhci: failed DMA start")); 639 } 640 641 static void 642 bcm_sdhci_dma_exit(struct bcm_sdhci_softc *sc) 643 { 644 struct sdhci_slot *slot = &sc->sc_slot; 645 646 mtx_assert(&slot->mtx, MA_OWNED); 647 648 /* Re-enable interrupts */ 649 slot->intmask |= DATA_XFER_MASK; 650 bcm_sdhci_write_4(slot->bus, slot, SDHCI_SIGNAL_ENABLE, 651 slot->intmask); 652 } 653 654 static void 655 bcm_sdhci_dma_unload(struct bcm_sdhci_softc *sc) 656 { 657 struct sdhci_slot *slot = &sc->sc_slot; 658 659 if (sc->dmamap_seg_count == 0) 660 return; 661 if ((slot->curcmd->data->flags & MMC_DATA_READ) != 0) 662 bus_dmamap_sync(sc->sc_dma_tag, sc->sc_dma_map, 663 BUS_DMASYNC_POSTREAD); 664 else 665 bus_dmamap_sync(sc->sc_dma_tag, sc->sc_dma_map, 666 BUS_DMASYNC_POSTWRITE); 667 bus_dmamap_unload(sc->sc_dma_tag, sc->sc_dma_map); 668 669 sc->dmamap_seg_count = 0; 670 sc->dmamap_seg_index = 0; 671 } 672 673 static void 674 bcm_sdhci_dma_intr(int ch, void *arg) 675 { 676 struct bcm_sdhci_softc *sc = (struct bcm_sdhci_softc *)arg; 677 struct sdhci_slot *slot = &sc->sc_slot; 678 uint32_t reg; 679 680 mtx_lock(&slot->mtx); 681 if (slot->curcmd == NULL) 682 goto out; 683 /* 684 * If there are more segments for the current dma, start the next one. 685 * Otherwise unload the dma map and decide what to do next based on the 686 * status of the sdhci controller and whether there's more data left. 687 */ 688 if (sc->dmamap_seg_index < sc->dmamap_seg_count) { 689 bcm_sdhci_start_dma_seg(sc); 690 goto out; 691 } 692 693 bcm_sdhci_dma_unload(sc); 694 695 /* 696 * If we had no further segments pending, we need to determine how to 697 * proceed next. If the 'data/space pending' bit is already set and we 698 * can continue via DMA, do so. Otherwise, re-enable interrupts and 699 * return. 700 */ 701 reg = bcm_sdhci_read_4(slot->bus, slot, SDHCI_INT_STATUS) & 702 DATA_XFER_MASK; 703 if ((reg & DATA_PENDING_MASK) != 0 && 704 BCM_SDHCI_SEGSZ_LEFT(slot) >= BCM_SDHCI_BUFFER_SIZE) { 705 /* ACK any pending interrupts */ 706 bcm_sdhci_write_4(slot->bus, slot, SDHCI_INT_STATUS, 707 DATA_PENDING_MASK); 708 709 bcm_sdhci_start_dma(slot); 710 if (slot->curcmd->error != 0) { 711 /* We won't recover from this error for this command. */ 712 bcm_sdhci_dma_unload(sc); 713 bcm_sdhci_dma_exit(sc); 714 sdhci_finish_data(slot); 715 } 716 } else if ((reg & SDHCI_INT_DATA_END) != 0) { 717 bcm_sdhci_dma_exit(sc); 718 bcm_sdhci_write_4(slot->bus, slot, SDHCI_INT_STATUS, 719 reg); 720 slot->flags &= ~PLATFORM_DATA_STARTED; 721 sdhci_finish_data(slot); 722 } else { 723 bcm_sdhci_dma_exit(sc); 724 } 725 out: 726 mtx_unlock(&slot->mtx); 727 } 728 729 static void 730 bcm_sdhci_start_dma(struct sdhci_slot *slot) 731 { 732 struct bcm_sdhci_softc *sc = device_get_softc(slot->bus); 733 uint8_t *buf; 734 size_t left; 735 736 mtx_assert(&slot->mtx, MA_OWNED); 737 738 left = BCM_SDHCI_SEGSZ_LEFT(slot); 739 buf = (uint8_t *)slot->curcmd->data->data + slot->offset; 740 KASSERT(left != 0, 741 ("%s: DMA handling incorrectly indicated", __func__)); 742 743 /* 744 * No need to check segment count here; if we've not yet unloaded 745 * previous segments, we'll catch that in bcm_sdhci_dmacb. 746 */ 747 if (bus_dmamap_load(sc->sc_dma_tag, sc->sc_dma_map, buf, left, 748 bcm_sdhci_dmacb, sc, BUS_DMA_NOWAIT) != 0 || 749 sc->dmamap_status != 0) { 750 slot->curcmd->error = MMC_ERR_NO_MEMORY; 751 return; 752 } 753 754 /* DMA start */ 755 bcm_sdhci_start_dma_seg(sc); 756 } 757 758 static int 759 bcm_sdhci_will_handle_transfer(device_t dev, struct sdhci_slot *slot) 760 { 761 #ifdef INVARIANTS 762 struct bcm_sdhci_softc *sc = device_get_softc(slot->bus); 763 #endif 764 765 /* 766 * This indicates that we somehow let a data interrupt slip by into the 767 * SDHCI framework, when it should not have. This really needs to be 768 * caught and fixed ASAP, as it really shouldn't happen. 769 */ 770 KASSERT(sc->dmamap_seg_count == 0, 771 ("data pending interrupt pushed through SDHCI framework")); 772 773 /* 774 * Do not use DMA for transfers less than our block size. Checking 775 * alignment serves little benefit, as we round transfer sizes down to 776 * a multiple of the block size and push the transfer back to 777 * SDHCI-driven PIO once we're below the block size. 778 */ 779 if (BCM_SDHCI_SEGSZ_LEFT(slot) < BCM_DMA_BLOCK_SIZE) 780 return (0); 781 782 return (1); 783 } 784 785 static void 786 bcm_sdhci_start_transfer(device_t dev, struct sdhci_slot *slot, 787 uint32_t *intmask) 788 { 789 790 /* DMA transfer FIFO 1KB */ 791 bcm_sdhci_start_dma(slot); 792 } 793 794 static void 795 bcm_sdhci_finish_transfer(device_t dev, struct sdhci_slot *slot) 796 { 797 struct bcm_sdhci_softc *sc = device_get_softc(slot->bus); 798 799 /* 800 * Clean up. Interrupts are clearly enabled, because we received an 801 * SDHCI_INT_DATA_END to get this far -- just make sure we don't leave 802 * anything laying around. 803 */ 804 if (sc->dmamap_seg_count != 0) { 805 /* 806 * Our segment math should have worked out such that we would 807 * never finish the transfer without having used up all of the 808 * segments. If we haven't, that means we must have erroneously 809 * regressed to SDHCI-driven PIO to finish the operation and 810 * this is certainly caused by developer-error. 811 */ 812 bcm_sdhci_dma_unload(sc); 813 } 814 815 sdhci_finish_data(slot); 816 } 817 818 static device_method_t bcm_sdhci_methods[] = { 819 /* Device interface */ 820 DEVMETHOD(device_probe, bcm_sdhci_probe), 821 DEVMETHOD(device_attach, bcm_sdhci_attach), 822 DEVMETHOD(device_detach, bcm_sdhci_detach), 823 824 /* Bus interface */ 825 DEVMETHOD(bus_read_ivar, sdhci_generic_read_ivar), 826 DEVMETHOD(bus_write_ivar, sdhci_generic_write_ivar), 827 DEVMETHOD(bus_add_child, bus_generic_add_child), 828 829 /* MMC bridge interface */ 830 DEVMETHOD(mmcbr_update_ios, bcm_sdhci_update_ios), 831 DEVMETHOD(mmcbr_request, sdhci_generic_request), 832 DEVMETHOD(mmcbr_get_ro, bcm_sdhci_get_ro), 833 DEVMETHOD(mmcbr_acquire_host, sdhci_generic_acquire_host), 834 DEVMETHOD(mmcbr_release_host, sdhci_generic_release_host), 835 836 /* Platform transfer methods */ 837 DEVMETHOD(sdhci_platform_will_handle, bcm_sdhci_will_handle_transfer), 838 DEVMETHOD(sdhci_platform_start_transfer, bcm_sdhci_start_transfer), 839 DEVMETHOD(sdhci_platform_finish_transfer, bcm_sdhci_finish_transfer), 840 /* SDHCI registers accessors */ 841 DEVMETHOD(sdhci_read_1, bcm_sdhci_read_1), 842 DEVMETHOD(sdhci_read_2, bcm_sdhci_read_2), 843 DEVMETHOD(sdhci_read_4, bcm_sdhci_read_4), 844 DEVMETHOD(sdhci_read_multi_4, bcm_sdhci_read_multi_4), 845 DEVMETHOD(sdhci_write_1, bcm_sdhci_write_1), 846 DEVMETHOD(sdhci_write_2, bcm_sdhci_write_2), 847 DEVMETHOD(sdhci_write_4, bcm_sdhci_write_4), 848 DEVMETHOD(sdhci_write_multi_4, bcm_sdhci_write_multi_4), 849 850 DEVMETHOD_END 851 }; 852 853 static devclass_t bcm_sdhci_devclass; 854 855 static driver_t bcm_sdhci_driver = { 856 "sdhci_bcm", 857 bcm_sdhci_methods, 858 sizeof(struct bcm_sdhci_softc), 859 }; 860 861 DRIVER_MODULE(sdhci_bcm, simplebus, bcm_sdhci_driver, bcm_sdhci_devclass, 862 NULL, NULL); 863 #ifdef NOTYET 864 MODULE_DEPEND(sdhci_bcm, bcm2835_clkman, 1, 1, 1); 865 #endif 866 SDHCI_DEPEND(sdhci_bcm); 867 #ifndef MMCCAM 868 MMC_DECLARE_BRIDGE(sdhci_bcm); 869 #endif 870