1 /*- 2 * Copyright (c) 2017-2018 Ruslan Bukin <br@bsdpad.com> 3 * All rights reserved. 4 * 5 * This software was developed by SRI International and the University of 6 * Cambridge Computer Laboratory under DARPA/AFRL contract FA8750-10-C-0237 7 * ("CTSRD"), as part of the DARPA CRASH research programme. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * SUCH DAMAGE. 29 */ 30 31 /* ARM PrimeCell DMA Controller (PL330) driver. */ 32 33 #include <sys/cdefs.h> 34 __FBSDID("$FreeBSD$"); 35 36 #include "opt_platform.h" 37 #include <sys/param.h> 38 #include <sys/endian.h> 39 #include <sys/systm.h> 40 #include <sys/conf.h> 41 #include <sys/bus.h> 42 #include <sys/kernel.h> 43 #include <sys/kthread.h> 44 #include <sys/sglist.h> 45 #include <sys/module.h> 46 #include <sys/lock.h> 47 #include <sys/resource.h> 48 #include <sys/rman.h> 49 50 #include <vm/vm.h> 51 #include <vm/vm_extern.h> 52 #include <vm/vm_kern.h> 53 #include <vm/pmap.h> 54 55 #include <machine/bus.h> 56 57 #ifdef FDT 58 #include <dev/fdt/fdt_common.h> 59 #include <dev/ofw/ofw_bus.h> 60 #include <dev/ofw/ofw_bus_subr.h> 61 #endif 62 63 #include <dev/xdma/xdma.h> 64 #include <dev/xdma/controller/pl330.h> 65 66 #include "xdma_if.h" 67 68 #define PL330_DEBUG 69 #undef PL330_DEBUG 70 71 #ifdef PL330_DEBUG 72 #define dprintf(fmt, ...) printf(fmt, ##__VA_ARGS__) 73 #else 74 #define dprintf(fmt, ...) 75 #endif 76 77 #define READ4(_sc, _reg) \ 78 bus_read_4(_sc->res[0], _reg) 79 #define WRITE4(_sc, _reg, _val) \ 80 bus_write_4(_sc->res[0], _reg, _val) 81 82 #define PL330_NCHANNELS 32 83 #define PL330_MAXLOAD 2048 84 85 struct pl330_channel { 86 struct pl330_softc *sc; 87 xdma_channel_t *xchan; 88 int used; 89 int index; 90 uint8_t *ibuf; 91 bus_addr_t ibuf_phys; 92 uint32_t enqueued; 93 uint32_t capacity; 94 }; 95 96 struct pl330_fdt_data { 97 uint32_t periph_id; 98 }; 99 100 struct pl330_softc { 101 device_t dev; 102 struct resource *res[PL330_NCHANNELS + 1]; 103 void *ih[PL330_NCHANNELS]; 104 struct pl330_channel channels[PL330_NCHANNELS]; 105 }; 106 107 static struct resource_spec pl330_spec[] = { 108 { SYS_RES_MEMORY, 0, RF_ACTIVE }, 109 { SYS_RES_IRQ, 0, RF_ACTIVE }, 110 { SYS_RES_IRQ, 1, RF_ACTIVE | RF_OPTIONAL }, 111 { SYS_RES_IRQ, 2, RF_ACTIVE | RF_OPTIONAL }, 112 { SYS_RES_IRQ, 3, RF_ACTIVE | RF_OPTIONAL }, 113 { SYS_RES_IRQ, 4, RF_ACTIVE | RF_OPTIONAL }, 114 { SYS_RES_IRQ, 5, RF_ACTIVE | RF_OPTIONAL }, 115 { SYS_RES_IRQ, 6, RF_ACTIVE | RF_OPTIONAL }, 116 { SYS_RES_IRQ, 7, RF_ACTIVE | RF_OPTIONAL }, 117 { SYS_RES_IRQ, 8, RF_ACTIVE | RF_OPTIONAL }, 118 { SYS_RES_IRQ, 9, RF_ACTIVE | RF_OPTIONAL }, 119 { SYS_RES_IRQ, 10, RF_ACTIVE | RF_OPTIONAL }, 120 { SYS_RES_IRQ, 11, RF_ACTIVE | RF_OPTIONAL }, 121 { SYS_RES_IRQ, 12, RF_ACTIVE | RF_OPTIONAL }, 122 { SYS_RES_IRQ, 13, RF_ACTIVE | RF_OPTIONAL }, 123 { SYS_RES_IRQ, 14, RF_ACTIVE | RF_OPTIONAL }, 124 { SYS_RES_IRQ, 15, RF_ACTIVE | RF_OPTIONAL }, 125 { SYS_RES_IRQ, 16, RF_ACTIVE | RF_OPTIONAL }, 126 { SYS_RES_IRQ, 17, RF_ACTIVE | RF_OPTIONAL }, 127 { SYS_RES_IRQ, 18, RF_ACTIVE | RF_OPTIONAL }, 128 { SYS_RES_IRQ, 19, RF_ACTIVE | RF_OPTIONAL }, 129 { SYS_RES_IRQ, 20, RF_ACTIVE | RF_OPTIONAL }, 130 { SYS_RES_IRQ, 21, RF_ACTIVE | RF_OPTIONAL }, 131 { SYS_RES_IRQ, 22, RF_ACTIVE | RF_OPTIONAL }, 132 { SYS_RES_IRQ, 23, RF_ACTIVE | RF_OPTIONAL }, 133 { SYS_RES_IRQ, 24, RF_ACTIVE | RF_OPTIONAL }, 134 { SYS_RES_IRQ, 25, RF_ACTIVE | RF_OPTIONAL }, 135 { SYS_RES_IRQ, 26, RF_ACTIVE | RF_OPTIONAL }, 136 { SYS_RES_IRQ, 27, RF_ACTIVE | RF_OPTIONAL }, 137 { SYS_RES_IRQ, 28, RF_ACTIVE | RF_OPTIONAL }, 138 { SYS_RES_IRQ, 29, RF_ACTIVE | RF_OPTIONAL }, 139 { SYS_RES_IRQ, 30, RF_ACTIVE | RF_OPTIONAL }, 140 { SYS_RES_IRQ, 31, RF_ACTIVE | RF_OPTIONAL }, 141 { -1, 0 } 142 }; 143 144 #define HWTYPE_NONE 0 145 #define HWTYPE_STD 1 146 147 static struct ofw_compat_data compat_data[] = { 148 { "arm,pl330", HWTYPE_STD }, 149 { NULL, HWTYPE_NONE }, 150 }; 151 152 static void 153 pl330_intr(void *arg) 154 { 155 xdma_transfer_status_t status; 156 struct xdma_transfer_status st; 157 struct pl330_channel *chan; 158 struct xdma_channel *xchan; 159 struct pl330_softc *sc; 160 uint32_t pending; 161 int i; 162 int c; 163 164 sc = arg; 165 166 pending = READ4(sc, INTMIS); 167 168 dprintf("%s: 0x%x, LC0 %x, SAR %x DAR %x\n", 169 __func__, pending, READ4(sc, LC0(0)), 170 READ4(sc, SAR(0)), READ4(sc, DAR(0))); 171 172 WRITE4(sc, INTCLR, pending); 173 174 for (c = 0; c < PL330_NCHANNELS; c++) { 175 if ((pending & (1 << c)) == 0) { 176 continue; 177 } 178 chan = &sc->channels[c]; 179 xchan = chan->xchan; 180 st.error = 0; 181 st.transferred = 0; 182 for (i = 0; i < chan->enqueued; i++) { 183 xchan_seg_done(xchan, &st); 184 } 185 186 /* Accept new requests. */ 187 chan->capacity = PL330_MAXLOAD; 188 189 /* Finish operation */ 190 status.error = 0; 191 status.transferred = 0; 192 xdma_callback(chan->xchan, &status); 193 } 194 } 195 196 static uint32_t 197 emit_mov(uint8_t *buf, uint32_t reg, uint32_t val) 198 { 199 200 buf[0] = DMAMOV; 201 buf[1] = reg; 202 buf[2] = val; 203 buf[3] = val >> 8; 204 buf[4] = val >> 16; 205 buf[5] = val >> 24; 206 207 return (6); 208 } 209 210 static uint32_t 211 emit_lp(uint8_t *buf, uint8_t idx, uint32_t iter) 212 { 213 214 if (idx > 1) 215 return (0); /* We have two loops only. */ 216 217 buf[0] = DMALP; 218 buf[0] |= (idx << 1); 219 buf[1] = (iter - 1) & 0xff; 220 221 return (2); 222 } 223 224 static uint32_t 225 emit_lpend(uint8_t *buf, uint8_t idx, 226 uint8_t burst, uint8_t jump_addr_relative) 227 { 228 229 buf[0] = DMALPEND; 230 buf[0] |= DMALPEND_NF; 231 buf[0] |= (idx << 2); 232 if (burst) 233 buf[0] |= (1 << 1) | (1 << 0); 234 else 235 buf[0] |= (0 << 1) | (1 << 0); 236 buf[1] = jump_addr_relative; 237 238 return (2); 239 } 240 241 static uint32_t 242 emit_ld(uint8_t *buf, uint8_t burst) 243 { 244 245 buf[0] = DMALD; 246 if (burst) 247 buf[0] |= (1 << 1) | (1 << 0); 248 else 249 buf[0] |= (0 << 1) | (1 << 0); 250 251 return (1); 252 } 253 254 static uint32_t 255 emit_st(uint8_t *buf, uint8_t burst) 256 { 257 258 buf[0] = DMAST; 259 if (burst) 260 buf[0] |= (1 << 1) | (1 << 0); 261 else 262 buf[0] |= (0 << 1) | (1 << 0); 263 264 return (1); 265 } 266 267 static uint32_t 268 emit_end(uint8_t *buf) 269 { 270 271 buf[0] = DMAEND; 272 273 return (1); 274 } 275 276 static uint32_t 277 emit_sev(uint8_t *buf, uint32_t ev) 278 { 279 280 buf[0] = DMASEV; 281 buf[1] = (ev << 3); 282 283 return (2); 284 } 285 286 static uint32_t 287 emit_wfp(uint8_t *buf, uint32_t p_id) 288 { 289 290 buf[0] = DMAWFP; 291 buf[0] |= (1 << 0); 292 buf[1] = (p_id << 3); 293 294 return (2); 295 } 296 297 static uint32_t 298 emit_go(uint8_t *buf, uint32_t chan_id, 299 uint32_t addr, uint8_t non_secure) 300 { 301 302 buf[0] = DMAGO; 303 buf[0] |= (non_secure << 1); 304 305 buf[1] = chan_id; 306 buf[2] = addr; 307 buf[3] = addr >> 8; 308 buf[4] = addr >> 16; 309 buf[5] = addr >> 24; 310 311 return (6); 312 } 313 314 static int 315 pl330_probe(device_t dev) 316 { 317 int hwtype; 318 319 if (!ofw_bus_status_okay(dev)) 320 return (ENXIO); 321 322 hwtype = ofw_bus_search_compatible(dev, compat_data)->ocd_data; 323 if (hwtype == HWTYPE_NONE) 324 return (ENXIO); 325 326 device_set_desc(dev, "ARM PrimeCell DMA Controller (PL330)"); 327 328 return (BUS_PROBE_DEFAULT); 329 } 330 331 static int 332 pl330_attach(device_t dev) 333 { 334 struct pl330_softc *sc; 335 phandle_t xref, node; 336 int err; 337 int i; 338 339 sc = device_get_softc(dev); 340 sc->dev = dev; 341 342 if (bus_alloc_resources(dev, pl330_spec, sc->res)) { 343 device_printf(dev, "could not allocate resources for device\n"); 344 return (ENXIO); 345 } 346 347 /* Setup interrupt handler */ 348 for (i = 0; i < PL330_NCHANNELS; i++) { 349 if (sc->res[i + 1] == NULL) 350 break; 351 err = bus_setup_intr(dev, sc->res[i + 1], INTR_TYPE_MISC | INTR_MPSAFE, 352 NULL, pl330_intr, sc, sc->ih[i]); 353 if (err) { 354 device_printf(dev, "Unable to alloc interrupt resource.\n"); 355 return (ENXIO); 356 } 357 } 358 359 node = ofw_bus_get_node(dev); 360 xref = OF_xref_from_node(node); 361 OF_device_register_xref(xref, dev); 362 363 return (0); 364 } 365 366 static int 367 pl330_detach(device_t dev) 368 { 369 struct pl330_softc *sc; 370 371 sc = device_get_softc(dev); 372 373 return (0); 374 } 375 376 static int 377 pl330_channel_alloc(device_t dev, struct xdma_channel *xchan) 378 { 379 struct pl330_channel *chan; 380 struct pl330_softc *sc; 381 int i; 382 383 sc = device_get_softc(dev); 384 385 for (i = 0; i < PL330_NCHANNELS; i++) { 386 chan = &sc->channels[i]; 387 if (chan->used == 0) { 388 chan->xchan = xchan; 389 xchan->chan = (void *)chan; 390 xchan->caps |= XCHAN_CAP_BUSDMA; 391 chan->index = i; 392 chan->sc = sc; 393 chan->used = 1; 394 395 chan->ibuf = (void *)kmem_alloc_contig(kernel_arena, 396 PAGE_SIZE*8, M_ZERO, 0, ~0, PAGE_SIZE, 0, 397 VM_MEMATTR_UNCACHEABLE); 398 chan->ibuf_phys = vtophys(chan->ibuf); 399 400 return (0); 401 } 402 } 403 404 return (-1); 405 } 406 407 static int 408 pl330_channel_free(device_t dev, struct xdma_channel *xchan) 409 { 410 struct pl330_channel *chan; 411 struct pl330_softc *sc; 412 413 sc = device_get_softc(dev); 414 415 chan = (struct pl330_channel *)xchan->chan; 416 chan->used = 0; 417 418 return (0); 419 } 420 421 static int 422 pl330_channel_capacity(device_t dev, xdma_channel_t *xchan, 423 uint32_t *capacity) 424 { 425 struct pl330_channel *chan; 426 427 chan = (struct pl330_channel *)xchan->chan; 428 429 *capacity = chan->capacity; 430 431 return (0); 432 } 433 434 static int 435 pl330_ccr_port_width(struct xdma_sglist *sg, uint32_t *addr) 436 { 437 uint32_t reg; 438 439 reg = 0; 440 441 switch (sg->src_width) { 442 case 1: 443 reg |= CCR_SRC_BURST_SIZE_1; 444 break; 445 case 2: 446 reg |= CCR_SRC_BURST_SIZE_2; 447 break; 448 case 4: 449 reg |= CCR_SRC_BURST_SIZE_4; 450 break; 451 default: 452 return (-1); 453 } 454 455 switch (sg->dst_width) { 456 case 1: 457 reg |= CCR_DST_BURST_SIZE_1; 458 break; 459 case 2: 460 reg |= CCR_DST_BURST_SIZE_2; 461 break; 462 case 4: 463 reg |= CCR_DST_BURST_SIZE_4; 464 break; 465 default: 466 return (-1); 467 } 468 469 *addr |= reg; 470 471 return (0); 472 } 473 474 static int 475 pl330_channel_submit_sg(device_t dev, struct xdma_channel *xchan, 476 struct xdma_sglist *sg, uint32_t sg_n) 477 { 478 struct pl330_fdt_data *data; 479 xdma_controller_t *xdma; 480 struct pl330_channel *chan; 481 struct pl330_softc *sc; 482 uint32_t src_addr_lo; 483 uint32_t dst_addr_lo; 484 uint32_t len; 485 uint32_t reg; 486 uint32_t offs; 487 uint32_t cnt; 488 uint8_t *ibuf; 489 uint8_t dbuf[6]; 490 uint8_t offs0, offs1; 491 int err; 492 int i; 493 494 sc = device_get_softc(dev); 495 496 xdma = xchan->xdma; 497 data = (struct pl330_fdt_data *)xdma->data; 498 499 chan = (struct pl330_channel *)xchan->chan; 500 ibuf = chan->ibuf; 501 502 dprintf("%s: chan->index %d\n", __func__, chan->index); 503 504 offs = 0; 505 506 for (i = 0; i < sg_n; i++) { 507 if (sg[i].direction == XDMA_DEV_TO_MEM) 508 reg = CCR_DST_INC; 509 else { 510 reg = CCR_SRC_INC; 511 reg |= (CCR_DST_PROT_PRIV); 512 } 513 514 err = pl330_ccr_port_width(&sg[i], ®); 515 if (err != 0) 516 return (err); 517 518 offs += emit_mov(&chan->ibuf[offs], R_CCR, reg); 519 520 src_addr_lo = (uint32_t)sg[i].src_addr; 521 dst_addr_lo = (uint32_t)sg[i].dst_addr; 522 len = (uint32_t)sg[i].len; 523 524 dprintf("%s: src %x dst %x len %d periph_id %d\n", __func__, 525 src_addr_lo, dst_addr_lo, len, data->periph_id); 526 527 offs += emit_mov(&ibuf[offs], R_SAR, src_addr_lo); 528 offs += emit_mov(&ibuf[offs], R_DAR, dst_addr_lo); 529 530 if (sg[i].src_width != sg[i].dst_width) 531 return (-1); /* Not supported. */ 532 533 cnt = (len / sg[i].src_width); 534 if (cnt > 128) { 535 offs += emit_lp(&ibuf[offs], 0, cnt / 128); 536 offs0 = offs; 537 offs += emit_lp(&ibuf[offs], 1, 128); 538 offs1 = offs; 539 } else { 540 offs += emit_lp(&ibuf[offs], 0, cnt); 541 offs0 = offs; 542 } 543 offs += emit_wfp(&ibuf[offs], data->periph_id); 544 offs += emit_ld(&ibuf[offs], 1); 545 offs += emit_st(&ibuf[offs], 1); 546 547 if (cnt > 128) 548 offs += emit_lpend(&ibuf[offs], 1, 1, (offs - offs1)); 549 550 offs += emit_lpend(&ibuf[offs], 0, 1, (offs - offs0)); 551 } 552 553 offs += emit_sev(&ibuf[offs], chan->index); 554 offs += emit_end(&ibuf[offs]); 555 556 emit_go(dbuf, chan->index, chan->ibuf_phys, 0); 557 558 reg = (dbuf[1] << 24) | (dbuf[0] << 16); 559 WRITE4(sc, DBGINST0, reg); 560 reg = (dbuf[5] << 24) | (dbuf[4] << 16) | (dbuf[3] << 8) | dbuf[2]; 561 WRITE4(sc, DBGINST1, reg); 562 563 WRITE4(sc, INTCLR, 0xffffffff); 564 WRITE4(sc, INTEN, (1 << chan->index)); 565 566 chan->enqueued = sg_n; 567 chan->capacity = 0; 568 569 /* Start operation */ 570 WRITE4(sc, DBGCMD, 0); 571 572 return (0); 573 } 574 575 static int 576 pl330_channel_prep_sg(device_t dev, struct xdma_channel *xchan) 577 { 578 struct pl330_channel *chan; 579 struct pl330_softc *sc; 580 581 sc = device_get_softc(dev); 582 583 dprintf("%s(%d)\n", __func__, device_get_unit(dev)); 584 585 chan = (struct pl330_channel *)xchan->chan; 586 chan->capacity = PL330_MAXLOAD; 587 588 return (0); 589 } 590 591 static int 592 pl330_channel_control(device_t dev, xdma_channel_t *xchan, int cmd) 593 { 594 struct pl330_channel *chan; 595 struct pl330_softc *sc; 596 597 sc = device_get_softc(dev); 598 599 chan = (struct pl330_channel *)xchan->chan; 600 601 switch (cmd) { 602 case XDMA_CMD_BEGIN: 603 case XDMA_CMD_TERMINATE: 604 case XDMA_CMD_PAUSE: 605 /* TODO: implement me */ 606 return (-1); 607 } 608 609 return (0); 610 } 611 612 #ifdef FDT 613 static int 614 pl330_ofw_md_data(device_t dev, pcell_t *cells, int ncells, void **ptr) 615 { 616 struct pl330_fdt_data *data; 617 618 if (ncells != 1) 619 return (-1); 620 621 data = malloc(sizeof(struct pl330_fdt_data), 622 M_DEVBUF, (M_WAITOK | M_ZERO)); 623 data->periph_id = cells[0]; 624 625 *ptr = data; 626 627 return (0); 628 } 629 #endif 630 631 static device_method_t pl330_methods[] = { 632 /* Device interface */ 633 DEVMETHOD(device_probe, pl330_probe), 634 DEVMETHOD(device_attach, pl330_attach), 635 DEVMETHOD(device_detach, pl330_detach), 636 637 /* xDMA Interface */ 638 DEVMETHOD(xdma_channel_alloc, pl330_channel_alloc), 639 DEVMETHOD(xdma_channel_free, pl330_channel_free), 640 DEVMETHOD(xdma_channel_control, pl330_channel_control), 641 642 /* xDMA SG Interface */ 643 DEVMETHOD(xdma_channel_capacity, pl330_channel_capacity), 644 DEVMETHOD(xdma_channel_prep_sg, pl330_channel_prep_sg), 645 DEVMETHOD(xdma_channel_submit_sg, pl330_channel_submit_sg), 646 647 #ifdef FDT 648 DEVMETHOD(xdma_ofw_md_data, pl330_ofw_md_data), 649 #endif 650 651 DEVMETHOD_END 652 }; 653 654 static driver_t pl330_driver = { 655 "pl330", 656 pl330_methods, 657 sizeof(struct pl330_softc), 658 }; 659 660 static devclass_t pl330_devclass; 661 662 EARLY_DRIVER_MODULE(pl330, simplebus, pl330_driver, pl330_devclass, 0, 0, 663 BUS_PASS_INTERRUPT + BUS_PASS_ORDER_LATE); 664