1 /*- 2 * Copyright (c) 2017-2018 Ruslan Bukin <br@bsdpad.com> 3 * All rights reserved. 4 * 5 * This software was developed by SRI International and the University of 6 * Cambridge Computer Laboratory under DARPA/AFRL contract FA8750-10-C-0237 7 * ("CTSRD"), as part of the DARPA CRASH research programme. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * SUCH DAMAGE. 29 */ 30 31 /* ARM PrimeCell DMA Controller (PL330) driver. */ 32 33 #include <sys/cdefs.h> 34 #include "opt_platform.h" 35 #include <sys/param.h> 36 #include <sys/endian.h> 37 #include <sys/systm.h> 38 #include <sys/conf.h> 39 #include <sys/bus.h> 40 #include <sys/kernel.h> 41 #include <sys/kthread.h> 42 #include <sys/sglist.h> 43 #include <sys/module.h> 44 #include <sys/lock.h> 45 #include <sys/resource.h> 46 #include <sys/rman.h> 47 48 #include <vm/vm.h> 49 #include <vm/vm_extern.h> 50 #include <vm/vm_kern.h> 51 #include <vm/pmap.h> 52 53 #include <machine/bus.h> 54 55 #ifdef FDT 56 #include <dev/fdt/fdt_common.h> 57 #include <dev/ofw/ofw_bus.h> 58 #include <dev/ofw/ofw_bus_subr.h> 59 #endif 60 61 #include <dev/xdma/xdma.h> 62 #include <dev/xdma/controller/pl330.h> 63 64 #include "xdma_if.h" 65 66 #define PL330_DEBUG 67 #undef PL330_DEBUG 68 69 #ifdef PL330_DEBUG 70 #define dprintf(fmt, ...) printf(fmt, ##__VA_ARGS__) 71 #else 72 #define dprintf(fmt, ...) 73 #endif 74 75 #define READ4(_sc, _reg) \ 76 bus_read_4(_sc->res[0], _reg) 77 #define WRITE4(_sc, _reg, _val) \ 78 bus_write_4(_sc->res[0], _reg, _val) 79 80 #define PL330_NCHANNELS 32 81 #define PL330_MAXLOAD 2048 82 83 struct pl330_channel { 84 struct pl330_softc *sc; 85 xdma_channel_t *xchan; 86 int used; 87 int index; 88 uint8_t *ibuf; 89 bus_addr_t ibuf_phys; 90 uint32_t enqueued; 91 uint32_t capacity; 92 }; 93 94 struct pl330_fdt_data { 95 uint32_t periph_id; 96 }; 97 98 struct pl330_softc { 99 device_t dev; 100 struct resource *res[PL330_NCHANNELS + 1]; 101 void *ih[PL330_NCHANNELS]; 102 struct pl330_channel channels[PL330_NCHANNELS]; 103 }; 104 105 static struct resource_spec pl330_spec[] = { 106 { SYS_RES_MEMORY, 0, RF_ACTIVE }, 107 { SYS_RES_IRQ, 0, RF_ACTIVE }, 108 { SYS_RES_IRQ, 1, RF_ACTIVE | RF_OPTIONAL }, 109 { SYS_RES_IRQ, 2, RF_ACTIVE | RF_OPTIONAL }, 110 { SYS_RES_IRQ, 3, RF_ACTIVE | RF_OPTIONAL }, 111 { SYS_RES_IRQ, 4, RF_ACTIVE | RF_OPTIONAL }, 112 { SYS_RES_IRQ, 5, RF_ACTIVE | RF_OPTIONAL }, 113 { SYS_RES_IRQ, 6, RF_ACTIVE | RF_OPTIONAL }, 114 { SYS_RES_IRQ, 7, RF_ACTIVE | RF_OPTIONAL }, 115 { SYS_RES_IRQ, 8, RF_ACTIVE | RF_OPTIONAL }, 116 { SYS_RES_IRQ, 9, RF_ACTIVE | RF_OPTIONAL }, 117 { SYS_RES_IRQ, 10, RF_ACTIVE | RF_OPTIONAL }, 118 { SYS_RES_IRQ, 11, RF_ACTIVE | RF_OPTIONAL }, 119 { SYS_RES_IRQ, 12, RF_ACTIVE | RF_OPTIONAL }, 120 { SYS_RES_IRQ, 13, RF_ACTIVE | RF_OPTIONAL }, 121 { SYS_RES_IRQ, 14, RF_ACTIVE | RF_OPTIONAL }, 122 { SYS_RES_IRQ, 15, RF_ACTIVE | RF_OPTIONAL }, 123 { SYS_RES_IRQ, 16, RF_ACTIVE | RF_OPTIONAL }, 124 { SYS_RES_IRQ, 17, RF_ACTIVE | RF_OPTIONAL }, 125 { SYS_RES_IRQ, 18, RF_ACTIVE | RF_OPTIONAL }, 126 { SYS_RES_IRQ, 19, RF_ACTIVE | RF_OPTIONAL }, 127 { SYS_RES_IRQ, 20, RF_ACTIVE | RF_OPTIONAL }, 128 { SYS_RES_IRQ, 21, RF_ACTIVE | RF_OPTIONAL }, 129 { SYS_RES_IRQ, 22, RF_ACTIVE | RF_OPTIONAL }, 130 { SYS_RES_IRQ, 23, RF_ACTIVE | RF_OPTIONAL }, 131 { SYS_RES_IRQ, 24, RF_ACTIVE | RF_OPTIONAL }, 132 { SYS_RES_IRQ, 25, RF_ACTIVE | RF_OPTIONAL }, 133 { SYS_RES_IRQ, 26, RF_ACTIVE | RF_OPTIONAL }, 134 { SYS_RES_IRQ, 27, RF_ACTIVE | RF_OPTIONAL }, 135 { SYS_RES_IRQ, 28, RF_ACTIVE | RF_OPTIONAL }, 136 { SYS_RES_IRQ, 29, RF_ACTIVE | RF_OPTIONAL }, 137 { SYS_RES_IRQ, 30, RF_ACTIVE | RF_OPTIONAL }, 138 { SYS_RES_IRQ, 31, RF_ACTIVE | RF_OPTIONAL }, 139 { -1, 0 } 140 }; 141 142 #define HWTYPE_NONE 0 143 #define HWTYPE_STD 1 144 145 static struct ofw_compat_data compat_data[] = { 146 { "arm,pl330", HWTYPE_STD }, 147 { NULL, HWTYPE_NONE }, 148 }; 149 150 static void 151 pl330_intr(void *arg) 152 { 153 xdma_transfer_status_t status; 154 struct xdma_transfer_status st; 155 struct pl330_channel *chan; 156 struct xdma_channel *xchan; 157 struct pl330_softc *sc; 158 uint32_t pending; 159 int i; 160 int c; 161 162 sc = arg; 163 164 pending = READ4(sc, INTMIS); 165 166 dprintf("%s: 0x%x, LC0 %x, SAR %x DAR %x\n", 167 __func__, pending, READ4(sc, LC0(0)), 168 READ4(sc, SAR(0)), READ4(sc, DAR(0))); 169 170 WRITE4(sc, INTCLR, pending); 171 172 for (c = 0; c < PL330_NCHANNELS; c++) { 173 if ((pending & (1 << c)) == 0) { 174 continue; 175 } 176 chan = &sc->channels[c]; 177 xchan = chan->xchan; 178 st.error = 0; 179 st.transferred = 0; 180 for (i = 0; i < chan->enqueued; i++) { 181 xchan_seg_done(xchan, &st); 182 } 183 184 /* Accept new requests. */ 185 chan->capacity = PL330_MAXLOAD; 186 187 /* Finish operation */ 188 status.error = 0; 189 status.transferred = 0; 190 xdma_callback(chan->xchan, &status); 191 } 192 } 193 194 static uint32_t 195 emit_mov(uint8_t *buf, uint32_t reg, uint32_t val) 196 { 197 198 buf[0] = DMAMOV; 199 buf[1] = reg; 200 buf[2] = val; 201 buf[3] = val >> 8; 202 buf[4] = val >> 16; 203 buf[5] = val >> 24; 204 205 return (6); 206 } 207 208 static uint32_t 209 emit_lp(uint8_t *buf, uint8_t idx, uint32_t iter) 210 { 211 212 if (idx > 1) 213 return (0); /* We have two loops only. */ 214 215 buf[0] = DMALP; 216 buf[0] |= (idx << 1); 217 buf[1] = (iter - 1) & 0xff; 218 219 return (2); 220 } 221 222 static uint32_t 223 emit_lpend(uint8_t *buf, uint8_t idx, 224 uint8_t burst, uint8_t jump_addr_relative) 225 { 226 227 buf[0] = DMALPEND; 228 buf[0] |= DMALPEND_NF; 229 buf[0] |= (idx << 2); 230 if (burst) 231 buf[0] |= (1 << 1) | (1 << 0); 232 else 233 buf[0] |= (0 << 1) | (1 << 0); 234 buf[1] = jump_addr_relative; 235 236 return (2); 237 } 238 239 static uint32_t 240 emit_ld(uint8_t *buf, uint8_t burst) 241 { 242 243 buf[0] = DMALD; 244 if (burst) 245 buf[0] |= (1 << 1) | (1 << 0); 246 else 247 buf[0] |= (0 << 1) | (1 << 0); 248 249 return (1); 250 } 251 252 static uint32_t 253 emit_st(uint8_t *buf, uint8_t burst) 254 { 255 256 buf[0] = DMAST; 257 if (burst) 258 buf[0] |= (1 << 1) | (1 << 0); 259 else 260 buf[0] |= (0 << 1) | (1 << 0); 261 262 return (1); 263 } 264 265 static uint32_t 266 emit_end(uint8_t *buf) 267 { 268 269 buf[0] = DMAEND; 270 271 return (1); 272 } 273 274 static uint32_t 275 emit_sev(uint8_t *buf, uint32_t ev) 276 { 277 278 buf[0] = DMASEV; 279 buf[1] = (ev << 3); 280 281 return (2); 282 } 283 284 static uint32_t 285 emit_wfp(uint8_t *buf, uint32_t p_id) 286 { 287 288 buf[0] = DMAWFP; 289 buf[0] |= (1 << 0); 290 buf[1] = (p_id << 3); 291 292 return (2); 293 } 294 295 static uint32_t 296 emit_go(uint8_t *buf, uint32_t chan_id, 297 uint32_t addr, uint8_t non_secure) 298 { 299 300 buf[0] = DMAGO; 301 buf[0] |= (non_secure << 1); 302 303 buf[1] = chan_id; 304 buf[2] = addr; 305 buf[3] = addr >> 8; 306 buf[4] = addr >> 16; 307 buf[5] = addr >> 24; 308 309 return (6); 310 } 311 312 static int 313 pl330_probe(device_t dev) 314 { 315 int hwtype; 316 317 if (!ofw_bus_status_okay(dev)) 318 return (ENXIO); 319 320 hwtype = ofw_bus_search_compatible(dev, compat_data)->ocd_data; 321 if (hwtype == HWTYPE_NONE) 322 return (ENXIO); 323 324 device_set_desc(dev, "ARM PrimeCell DMA Controller (PL330)"); 325 326 return (BUS_PROBE_DEFAULT); 327 } 328 329 static int 330 pl330_attach(device_t dev) 331 { 332 struct pl330_softc *sc; 333 phandle_t xref, node; 334 int err; 335 int i; 336 337 sc = device_get_softc(dev); 338 sc->dev = dev; 339 340 if (bus_alloc_resources(dev, pl330_spec, sc->res)) { 341 device_printf(dev, "could not allocate resources for device\n"); 342 return (ENXIO); 343 } 344 345 /* Setup interrupt handler */ 346 for (i = 0; i < PL330_NCHANNELS; i++) { 347 if (sc->res[i + 1] == NULL) 348 break; 349 err = bus_setup_intr(dev, sc->res[i + 1], INTR_TYPE_MISC | INTR_MPSAFE, 350 NULL, pl330_intr, sc, sc->ih[i]); 351 if (err) { 352 device_printf(dev, "Unable to alloc interrupt resource.\n"); 353 return (ENXIO); 354 } 355 } 356 357 node = ofw_bus_get_node(dev); 358 xref = OF_xref_from_node(node); 359 OF_device_register_xref(xref, dev); 360 361 return (0); 362 } 363 364 static int 365 pl330_detach(device_t dev) 366 { 367 return (0); 368 } 369 370 static int 371 pl330_channel_alloc(device_t dev, struct xdma_channel *xchan) 372 { 373 struct pl330_channel *chan; 374 struct pl330_softc *sc; 375 int i; 376 377 sc = device_get_softc(dev); 378 379 for (i = 0; i < PL330_NCHANNELS; i++) { 380 chan = &sc->channels[i]; 381 if (chan->used == 0) { 382 chan->xchan = xchan; 383 xchan->chan = (void *)chan; 384 xchan->caps |= XCHAN_CAP_BUSDMA; 385 chan->index = i; 386 chan->sc = sc; 387 chan->used = 1; 388 389 chan->ibuf = (void *)kmem_alloc_contig(PAGE_SIZE * 8, 390 M_ZERO, 0, ~0, PAGE_SIZE, 0, 391 VM_MEMATTR_UNCACHEABLE); 392 chan->ibuf_phys = vtophys(chan->ibuf); 393 394 return (0); 395 } 396 } 397 398 return (-1); 399 } 400 401 static int 402 pl330_channel_free(device_t dev, struct xdma_channel *xchan) 403 { 404 struct pl330_channel *chan; 405 406 chan = (struct pl330_channel *)xchan->chan; 407 chan->used = 0; 408 409 return (0); 410 } 411 412 static int 413 pl330_channel_capacity(device_t dev, xdma_channel_t *xchan, 414 uint32_t *capacity) 415 { 416 struct pl330_channel *chan; 417 418 chan = (struct pl330_channel *)xchan->chan; 419 420 *capacity = chan->capacity; 421 422 return (0); 423 } 424 425 static int 426 pl330_ccr_port_width(struct xdma_sglist *sg, uint32_t *addr) 427 { 428 uint32_t reg; 429 430 reg = 0; 431 432 switch (sg->src_width) { 433 case 1: 434 reg |= CCR_SRC_BURST_SIZE_1; 435 break; 436 case 2: 437 reg |= CCR_SRC_BURST_SIZE_2; 438 break; 439 case 4: 440 reg |= CCR_SRC_BURST_SIZE_4; 441 break; 442 default: 443 return (-1); 444 } 445 446 switch (sg->dst_width) { 447 case 1: 448 reg |= CCR_DST_BURST_SIZE_1; 449 break; 450 case 2: 451 reg |= CCR_DST_BURST_SIZE_2; 452 break; 453 case 4: 454 reg |= CCR_DST_BURST_SIZE_4; 455 break; 456 default: 457 return (-1); 458 } 459 460 *addr |= reg; 461 462 return (0); 463 } 464 465 static int 466 pl330_channel_submit_sg(device_t dev, struct xdma_channel *xchan, 467 struct xdma_sglist *sg, uint32_t sg_n) 468 { 469 struct pl330_fdt_data *data; 470 xdma_controller_t *xdma; 471 struct pl330_channel *chan; 472 struct pl330_softc *sc; 473 uint32_t src_addr_lo; 474 uint32_t dst_addr_lo; 475 uint32_t len; 476 uint32_t reg; 477 uint32_t offs; 478 uint32_t cnt; 479 uint8_t *ibuf; 480 uint8_t dbuf[6]; 481 uint8_t offs0, offs1; 482 int err; 483 int i; 484 485 sc = device_get_softc(dev); 486 487 xdma = xchan->xdma; 488 data = (struct pl330_fdt_data *)xdma->data; 489 490 chan = (struct pl330_channel *)xchan->chan; 491 ibuf = chan->ibuf; 492 493 dprintf("%s: chan->index %d\n", __func__, chan->index); 494 495 offs = 0; 496 497 for (i = 0; i < sg_n; i++) { 498 if (sg[i].direction == XDMA_DEV_TO_MEM) 499 reg = CCR_DST_INC; 500 else { 501 reg = CCR_SRC_INC; 502 reg |= (CCR_DST_PROT_PRIV); 503 } 504 505 err = pl330_ccr_port_width(&sg[i], ®); 506 if (err != 0) 507 return (err); 508 509 offs += emit_mov(&chan->ibuf[offs], R_CCR, reg); 510 511 src_addr_lo = (uint32_t)sg[i].src_addr; 512 dst_addr_lo = (uint32_t)sg[i].dst_addr; 513 len = (uint32_t)sg[i].len; 514 515 dprintf("%s: src %x dst %x len %d periph_id %d\n", __func__, 516 src_addr_lo, dst_addr_lo, len, data->periph_id); 517 518 offs += emit_mov(&ibuf[offs], R_SAR, src_addr_lo); 519 offs += emit_mov(&ibuf[offs], R_DAR, dst_addr_lo); 520 521 if (sg[i].src_width != sg[i].dst_width) 522 return (-1); /* Not supported. */ 523 524 cnt = (len / sg[i].src_width); 525 if (cnt > 128) { 526 offs += emit_lp(&ibuf[offs], 0, cnt / 128); 527 offs0 = offs; 528 offs += emit_lp(&ibuf[offs], 1, 128); 529 offs1 = offs; 530 } else { 531 offs += emit_lp(&ibuf[offs], 0, cnt); 532 offs0 = offs; 533 } 534 offs += emit_wfp(&ibuf[offs], data->periph_id); 535 offs += emit_ld(&ibuf[offs], 1); 536 offs += emit_st(&ibuf[offs], 1); 537 538 if (cnt > 128) 539 offs += emit_lpend(&ibuf[offs], 1, 1, (offs - offs1)); 540 541 offs += emit_lpend(&ibuf[offs], 0, 1, (offs - offs0)); 542 } 543 544 offs += emit_sev(&ibuf[offs], chan->index); 545 offs += emit_end(&ibuf[offs]); 546 547 emit_go(dbuf, chan->index, chan->ibuf_phys, 0); 548 549 reg = (dbuf[1] << 24) | (dbuf[0] << 16); 550 WRITE4(sc, DBGINST0, reg); 551 reg = (dbuf[5] << 24) | (dbuf[4] << 16) | (dbuf[3] << 8) | dbuf[2]; 552 WRITE4(sc, DBGINST1, reg); 553 554 WRITE4(sc, INTCLR, 0xffffffff); 555 WRITE4(sc, INTEN, (1 << chan->index)); 556 557 chan->enqueued = sg_n; 558 chan->capacity = 0; 559 560 /* Start operation */ 561 WRITE4(sc, DBGCMD, 0); 562 563 return (0); 564 } 565 566 static int 567 pl330_channel_prep_sg(device_t dev, struct xdma_channel *xchan) 568 { 569 struct pl330_channel *chan; 570 571 dprintf("%s(%d)\n", __func__, device_get_unit(dev)); 572 573 chan = (struct pl330_channel *)xchan->chan; 574 chan->capacity = PL330_MAXLOAD; 575 576 return (0); 577 } 578 579 static int 580 pl330_channel_control(device_t dev, xdma_channel_t *xchan, int cmd) 581 { 582 switch (cmd) { 583 case XDMA_CMD_BEGIN: 584 case XDMA_CMD_TERMINATE: 585 case XDMA_CMD_PAUSE: 586 /* TODO: implement me */ 587 return (-1); 588 } 589 590 return (0); 591 } 592 593 #ifdef FDT 594 static int 595 pl330_ofw_md_data(device_t dev, pcell_t *cells, int ncells, void **ptr) 596 { 597 struct pl330_fdt_data *data; 598 599 if (ncells != 1) 600 return (-1); 601 602 data = malloc(sizeof(struct pl330_fdt_data), 603 M_DEVBUF, (M_WAITOK | M_ZERO)); 604 data->periph_id = cells[0]; 605 606 *ptr = data; 607 608 return (0); 609 } 610 #endif 611 612 static device_method_t pl330_methods[] = { 613 /* Device interface */ 614 DEVMETHOD(device_probe, pl330_probe), 615 DEVMETHOD(device_attach, pl330_attach), 616 DEVMETHOD(device_detach, pl330_detach), 617 618 /* xDMA Interface */ 619 DEVMETHOD(xdma_channel_alloc, pl330_channel_alloc), 620 DEVMETHOD(xdma_channel_free, pl330_channel_free), 621 DEVMETHOD(xdma_channel_control, pl330_channel_control), 622 623 /* xDMA SG Interface */ 624 DEVMETHOD(xdma_channel_capacity, pl330_channel_capacity), 625 DEVMETHOD(xdma_channel_prep_sg, pl330_channel_prep_sg), 626 DEVMETHOD(xdma_channel_submit_sg, pl330_channel_submit_sg), 627 628 #ifdef FDT 629 DEVMETHOD(xdma_ofw_md_data, pl330_ofw_md_data), 630 #endif 631 632 DEVMETHOD_END 633 }; 634 635 static driver_t pl330_driver = { 636 "pl330", 637 pl330_methods, 638 sizeof(struct pl330_softc), 639 }; 640 641 EARLY_DRIVER_MODULE(pl330, simplebus, pl330_driver, 0, 0, 642 BUS_PASS_INTERRUPT + BUS_PASS_ORDER_LATE); 643