1 /*- 2 * Copyright (c) 2017-2018 Ruslan Bukin <br@bsdpad.com> 3 * All rights reserved. 4 * 5 * This software was developed by SRI International and the University of 6 * Cambridge Computer Laboratory under DARPA/AFRL contract FA8750-10-C-0237 7 * ("CTSRD"), as part of the DARPA CRASH research programme. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * SUCH DAMAGE. 29 */ 30 31 /* ARM PrimeCell DMA Controller (PL330) driver. */ 32 33 #include <sys/cdefs.h> 34 __FBSDID("$FreeBSD$"); 35 36 #include "opt_platform.h" 37 #include <sys/param.h> 38 #include <sys/endian.h> 39 #include <sys/systm.h> 40 #include <sys/conf.h> 41 #include <sys/bus.h> 42 #include <sys/kernel.h> 43 #include <sys/kthread.h> 44 #include <sys/sglist.h> 45 #include <sys/module.h> 46 #include <sys/lock.h> 47 #include <sys/resource.h> 48 #include <sys/rman.h> 49 50 #include <vm/vm.h> 51 #include <vm/vm_extern.h> 52 #include <vm/vm_kern.h> 53 #include <vm/pmap.h> 54 55 #include <machine/bus.h> 56 57 #ifdef FDT 58 #include <dev/fdt/fdt_common.h> 59 #include <dev/ofw/ofw_bus.h> 60 #include <dev/ofw/ofw_bus_subr.h> 61 #endif 62 63 #include <dev/xdma/xdma.h> 64 #include <dev/xdma/controller/pl330.h> 65 66 #include "xdma_if.h" 67 68 #define PL330_DEBUG 69 #undef PL330_DEBUG 70 71 #ifdef PL330_DEBUG 72 #define dprintf(fmt, ...) printf(fmt, ##__VA_ARGS__) 73 #else 74 #define dprintf(fmt, ...) 75 #endif 76 77 #define READ4(_sc, _reg) \ 78 bus_read_4(_sc->res[0], _reg) 79 #define WRITE4(_sc, _reg, _val) \ 80 bus_write_4(_sc->res[0], _reg, _val) 81 82 #define PL330_NCHANNELS 32 83 #define PL330_MAXLOAD 2048 84 85 struct pl330_channel { 86 struct pl330_softc *sc; 87 xdma_channel_t *xchan; 88 int used; 89 int index; 90 uint8_t *ibuf; 91 bus_addr_t ibuf_phys; 92 uint32_t enqueued; 93 uint32_t capacity; 94 }; 95 96 struct pl330_fdt_data { 97 uint32_t periph_id; 98 }; 99 100 struct pl330_softc { 101 device_t dev; 102 struct resource *res[PL330_NCHANNELS + 1]; 103 void *ih[PL330_NCHANNELS]; 104 struct pl330_channel channels[PL330_NCHANNELS]; 105 }; 106 107 static struct resource_spec pl330_spec[] = { 108 { SYS_RES_MEMORY, 0, RF_ACTIVE }, 109 { SYS_RES_IRQ, 0, RF_ACTIVE }, 110 { SYS_RES_IRQ, 1, RF_ACTIVE | RF_OPTIONAL }, 111 { SYS_RES_IRQ, 2, RF_ACTIVE | RF_OPTIONAL }, 112 { SYS_RES_IRQ, 3, RF_ACTIVE | RF_OPTIONAL }, 113 { SYS_RES_IRQ, 4, RF_ACTIVE | RF_OPTIONAL }, 114 { SYS_RES_IRQ, 5, RF_ACTIVE | RF_OPTIONAL }, 115 { SYS_RES_IRQ, 6, RF_ACTIVE | RF_OPTIONAL }, 116 { SYS_RES_IRQ, 7, RF_ACTIVE | RF_OPTIONAL }, 117 { SYS_RES_IRQ, 8, RF_ACTIVE | RF_OPTIONAL }, 118 { SYS_RES_IRQ, 9, RF_ACTIVE | RF_OPTIONAL }, 119 { SYS_RES_IRQ, 10, RF_ACTIVE | RF_OPTIONAL }, 120 { SYS_RES_IRQ, 11, RF_ACTIVE | RF_OPTIONAL }, 121 { SYS_RES_IRQ, 12, RF_ACTIVE | RF_OPTIONAL }, 122 { SYS_RES_IRQ, 13, RF_ACTIVE | RF_OPTIONAL }, 123 { SYS_RES_IRQ, 14, RF_ACTIVE | RF_OPTIONAL }, 124 { SYS_RES_IRQ, 15, RF_ACTIVE | RF_OPTIONAL }, 125 { SYS_RES_IRQ, 16, RF_ACTIVE | RF_OPTIONAL }, 126 { SYS_RES_IRQ, 17, RF_ACTIVE | RF_OPTIONAL }, 127 { SYS_RES_IRQ, 18, RF_ACTIVE | RF_OPTIONAL }, 128 { SYS_RES_IRQ, 19, RF_ACTIVE | RF_OPTIONAL }, 129 { SYS_RES_IRQ, 20, RF_ACTIVE | RF_OPTIONAL }, 130 { SYS_RES_IRQ, 21, RF_ACTIVE | RF_OPTIONAL }, 131 { SYS_RES_IRQ, 22, RF_ACTIVE | RF_OPTIONAL }, 132 { SYS_RES_IRQ, 23, RF_ACTIVE | RF_OPTIONAL }, 133 { SYS_RES_IRQ, 24, RF_ACTIVE | RF_OPTIONAL }, 134 { SYS_RES_IRQ, 25, RF_ACTIVE | RF_OPTIONAL }, 135 { SYS_RES_IRQ, 26, RF_ACTIVE | RF_OPTIONAL }, 136 { SYS_RES_IRQ, 27, RF_ACTIVE | RF_OPTIONAL }, 137 { SYS_RES_IRQ, 28, RF_ACTIVE | RF_OPTIONAL }, 138 { SYS_RES_IRQ, 29, RF_ACTIVE | RF_OPTIONAL }, 139 { SYS_RES_IRQ, 30, RF_ACTIVE | RF_OPTIONAL }, 140 { SYS_RES_IRQ, 31, RF_ACTIVE | RF_OPTIONAL }, 141 { -1, 0 } 142 }; 143 144 #define HWTYPE_NONE 0 145 #define HWTYPE_STD 1 146 147 static struct ofw_compat_data compat_data[] = { 148 { "arm,pl330", HWTYPE_STD }, 149 { NULL, HWTYPE_NONE }, 150 }; 151 152 static void 153 pl330_intr(void *arg) 154 { 155 xdma_transfer_status_t status; 156 struct xdma_transfer_status st; 157 struct pl330_channel *chan; 158 struct xdma_channel *xchan; 159 struct pl330_softc *sc; 160 uint32_t pending; 161 int i; 162 int c; 163 164 sc = arg; 165 166 pending = READ4(sc, INTMIS); 167 168 dprintf("%s: 0x%x, LC0 %x, SAR %x DAR %x\n", 169 __func__, pending, READ4(sc, LC0(0)), 170 READ4(sc, SAR(0)), READ4(sc, DAR(0))); 171 172 WRITE4(sc, INTCLR, pending); 173 174 for (c = 0; c < PL330_NCHANNELS; c++) { 175 if ((pending & (1 << c)) == 0) { 176 continue; 177 } 178 chan = &sc->channels[c]; 179 xchan = chan->xchan; 180 st.error = 0; 181 st.transferred = 0; 182 for (i = 0; i < chan->enqueued; i++) { 183 xchan_seg_done(xchan, &st); 184 } 185 186 /* Accept new requests. */ 187 chan->capacity = PL330_MAXLOAD; 188 189 /* Finish operation */ 190 status.error = 0; 191 status.transferred = 0; 192 xdma_callback(chan->xchan, &status); 193 } 194 } 195 196 static uint32_t 197 emit_mov(uint8_t *buf, uint32_t reg, uint32_t val) 198 { 199 200 buf[0] = DMAMOV; 201 buf[1] = reg; 202 buf[2] = val; 203 buf[3] = val >> 8; 204 buf[4] = val >> 16; 205 buf[5] = val >> 24; 206 207 return (6); 208 } 209 210 static uint32_t 211 emit_lp(uint8_t *buf, uint8_t idx, uint32_t iter) 212 { 213 214 if (idx > 1) 215 return (0); /* We have two loops only. */ 216 217 buf[0] = DMALP; 218 buf[0] |= (idx << 1); 219 buf[1] = (iter - 1) & 0xff; 220 221 return (2); 222 } 223 224 static uint32_t 225 emit_lpend(uint8_t *buf, uint8_t idx, 226 uint8_t burst, uint8_t jump_addr_relative) 227 { 228 229 buf[0] = DMALPEND; 230 buf[0] |= DMALPEND_NF; 231 buf[0] |= (idx << 2); 232 if (burst) 233 buf[0] |= (1 << 1) | (1 << 0); 234 else 235 buf[0] |= (0 << 1) | (1 << 0); 236 buf[1] = jump_addr_relative; 237 238 return (2); 239 } 240 241 static uint32_t 242 emit_ld(uint8_t *buf, uint8_t burst) 243 { 244 245 buf[0] = DMALD; 246 if (burst) 247 buf[0] |= (1 << 1) | (1 << 0); 248 else 249 buf[0] |= (0 << 1) | (1 << 0); 250 251 return (1); 252 } 253 254 static uint32_t 255 emit_st(uint8_t *buf, uint8_t burst) 256 { 257 258 buf[0] = DMAST; 259 if (burst) 260 buf[0] |= (1 << 1) | (1 << 0); 261 else 262 buf[0] |= (0 << 1) | (1 << 0); 263 264 return (1); 265 } 266 267 static uint32_t 268 emit_end(uint8_t *buf) 269 { 270 271 buf[0] = DMAEND; 272 273 return (1); 274 } 275 276 static uint32_t 277 emit_sev(uint8_t *buf, uint32_t ev) 278 { 279 280 buf[0] = DMASEV; 281 buf[1] = (ev << 3); 282 283 return (2); 284 } 285 286 static uint32_t 287 emit_wfp(uint8_t *buf, uint32_t p_id) 288 { 289 290 buf[0] = DMAWFP; 291 buf[0] |= (1 << 0); 292 buf[1] = (p_id << 3); 293 294 return (2); 295 } 296 297 static uint32_t 298 emit_go(uint8_t *buf, uint32_t chan_id, 299 uint32_t addr, uint8_t non_secure) 300 { 301 302 buf[0] = DMAGO; 303 buf[0] |= (non_secure << 1); 304 305 buf[1] = chan_id; 306 buf[2] = addr; 307 buf[3] = addr >> 8; 308 buf[4] = addr >> 16; 309 buf[5] = addr >> 24; 310 311 return (6); 312 } 313 314 static int 315 pl330_probe(device_t dev) 316 { 317 int hwtype; 318 319 if (!ofw_bus_status_okay(dev)) 320 return (ENXIO); 321 322 hwtype = ofw_bus_search_compatible(dev, compat_data)->ocd_data; 323 if (hwtype == HWTYPE_NONE) 324 return (ENXIO); 325 326 device_set_desc(dev, "ARM PrimeCell DMA Controller (PL330)"); 327 328 return (BUS_PROBE_DEFAULT); 329 } 330 331 static int 332 pl330_attach(device_t dev) 333 { 334 struct pl330_softc *sc; 335 phandle_t xref, node; 336 int err; 337 int i; 338 339 sc = device_get_softc(dev); 340 sc->dev = dev; 341 342 if (bus_alloc_resources(dev, pl330_spec, sc->res)) { 343 device_printf(dev, "could not allocate resources for device\n"); 344 return (ENXIO); 345 } 346 347 /* Setup interrupt handler */ 348 for (i = 0; i < PL330_NCHANNELS; i++) { 349 if (sc->res[i + 1] == NULL) 350 break; 351 err = bus_setup_intr(dev, sc->res[i + 1], INTR_TYPE_MISC | INTR_MPSAFE, 352 NULL, pl330_intr, sc, sc->ih[i]); 353 if (err) { 354 device_printf(dev, "Unable to alloc interrupt resource.\n"); 355 return (ENXIO); 356 } 357 } 358 359 node = ofw_bus_get_node(dev); 360 xref = OF_xref_from_node(node); 361 OF_device_register_xref(xref, dev); 362 363 return (0); 364 } 365 366 static int 367 pl330_detach(device_t dev) 368 { 369 return (0); 370 } 371 372 static int 373 pl330_channel_alloc(device_t dev, struct xdma_channel *xchan) 374 { 375 struct pl330_channel *chan; 376 struct pl330_softc *sc; 377 int i; 378 379 sc = device_get_softc(dev); 380 381 for (i = 0; i < PL330_NCHANNELS; i++) { 382 chan = &sc->channels[i]; 383 if (chan->used == 0) { 384 chan->xchan = xchan; 385 xchan->chan = (void *)chan; 386 xchan->caps |= XCHAN_CAP_BUSDMA; 387 chan->index = i; 388 chan->sc = sc; 389 chan->used = 1; 390 391 chan->ibuf = (void *)kmem_alloc_contig(PAGE_SIZE * 8, 392 M_ZERO, 0, ~0, PAGE_SIZE, 0, 393 VM_MEMATTR_UNCACHEABLE); 394 chan->ibuf_phys = vtophys(chan->ibuf); 395 396 return (0); 397 } 398 } 399 400 return (-1); 401 } 402 403 static int 404 pl330_channel_free(device_t dev, struct xdma_channel *xchan) 405 { 406 struct pl330_channel *chan; 407 408 chan = (struct pl330_channel *)xchan->chan; 409 chan->used = 0; 410 411 return (0); 412 } 413 414 static int 415 pl330_channel_capacity(device_t dev, xdma_channel_t *xchan, 416 uint32_t *capacity) 417 { 418 struct pl330_channel *chan; 419 420 chan = (struct pl330_channel *)xchan->chan; 421 422 *capacity = chan->capacity; 423 424 return (0); 425 } 426 427 static int 428 pl330_ccr_port_width(struct xdma_sglist *sg, uint32_t *addr) 429 { 430 uint32_t reg; 431 432 reg = 0; 433 434 switch (sg->src_width) { 435 case 1: 436 reg |= CCR_SRC_BURST_SIZE_1; 437 break; 438 case 2: 439 reg |= CCR_SRC_BURST_SIZE_2; 440 break; 441 case 4: 442 reg |= CCR_SRC_BURST_SIZE_4; 443 break; 444 default: 445 return (-1); 446 } 447 448 switch (sg->dst_width) { 449 case 1: 450 reg |= CCR_DST_BURST_SIZE_1; 451 break; 452 case 2: 453 reg |= CCR_DST_BURST_SIZE_2; 454 break; 455 case 4: 456 reg |= CCR_DST_BURST_SIZE_4; 457 break; 458 default: 459 return (-1); 460 } 461 462 *addr |= reg; 463 464 return (0); 465 } 466 467 static int 468 pl330_channel_submit_sg(device_t dev, struct xdma_channel *xchan, 469 struct xdma_sglist *sg, uint32_t sg_n) 470 { 471 struct pl330_fdt_data *data; 472 xdma_controller_t *xdma; 473 struct pl330_channel *chan; 474 struct pl330_softc *sc; 475 uint32_t src_addr_lo; 476 uint32_t dst_addr_lo; 477 uint32_t len; 478 uint32_t reg; 479 uint32_t offs; 480 uint32_t cnt; 481 uint8_t *ibuf; 482 uint8_t dbuf[6]; 483 uint8_t offs0, offs1; 484 int err; 485 int i; 486 487 sc = device_get_softc(dev); 488 489 xdma = xchan->xdma; 490 data = (struct pl330_fdt_data *)xdma->data; 491 492 chan = (struct pl330_channel *)xchan->chan; 493 ibuf = chan->ibuf; 494 495 dprintf("%s: chan->index %d\n", __func__, chan->index); 496 497 offs = 0; 498 499 for (i = 0; i < sg_n; i++) { 500 if (sg[i].direction == XDMA_DEV_TO_MEM) 501 reg = CCR_DST_INC; 502 else { 503 reg = CCR_SRC_INC; 504 reg |= (CCR_DST_PROT_PRIV); 505 } 506 507 err = pl330_ccr_port_width(&sg[i], ®); 508 if (err != 0) 509 return (err); 510 511 offs += emit_mov(&chan->ibuf[offs], R_CCR, reg); 512 513 src_addr_lo = (uint32_t)sg[i].src_addr; 514 dst_addr_lo = (uint32_t)sg[i].dst_addr; 515 len = (uint32_t)sg[i].len; 516 517 dprintf("%s: src %x dst %x len %d periph_id %d\n", __func__, 518 src_addr_lo, dst_addr_lo, len, data->periph_id); 519 520 offs += emit_mov(&ibuf[offs], R_SAR, src_addr_lo); 521 offs += emit_mov(&ibuf[offs], R_DAR, dst_addr_lo); 522 523 if (sg[i].src_width != sg[i].dst_width) 524 return (-1); /* Not supported. */ 525 526 cnt = (len / sg[i].src_width); 527 if (cnt > 128) { 528 offs += emit_lp(&ibuf[offs], 0, cnt / 128); 529 offs0 = offs; 530 offs += emit_lp(&ibuf[offs], 1, 128); 531 offs1 = offs; 532 } else { 533 offs += emit_lp(&ibuf[offs], 0, cnt); 534 offs0 = offs; 535 } 536 offs += emit_wfp(&ibuf[offs], data->periph_id); 537 offs += emit_ld(&ibuf[offs], 1); 538 offs += emit_st(&ibuf[offs], 1); 539 540 if (cnt > 128) 541 offs += emit_lpend(&ibuf[offs], 1, 1, (offs - offs1)); 542 543 offs += emit_lpend(&ibuf[offs], 0, 1, (offs - offs0)); 544 } 545 546 offs += emit_sev(&ibuf[offs], chan->index); 547 offs += emit_end(&ibuf[offs]); 548 549 emit_go(dbuf, chan->index, chan->ibuf_phys, 0); 550 551 reg = (dbuf[1] << 24) | (dbuf[0] << 16); 552 WRITE4(sc, DBGINST0, reg); 553 reg = (dbuf[5] << 24) | (dbuf[4] << 16) | (dbuf[3] << 8) | dbuf[2]; 554 WRITE4(sc, DBGINST1, reg); 555 556 WRITE4(sc, INTCLR, 0xffffffff); 557 WRITE4(sc, INTEN, (1 << chan->index)); 558 559 chan->enqueued = sg_n; 560 chan->capacity = 0; 561 562 /* Start operation */ 563 WRITE4(sc, DBGCMD, 0); 564 565 return (0); 566 } 567 568 static int 569 pl330_channel_prep_sg(device_t dev, struct xdma_channel *xchan) 570 { 571 struct pl330_channel *chan; 572 573 dprintf("%s(%d)\n", __func__, device_get_unit(dev)); 574 575 chan = (struct pl330_channel *)xchan->chan; 576 chan->capacity = PL330_MAXLOAD; 577 578 return (0); 579 } 580 581 static int 582 pl330_channel_control(device_t dev, xdma_channel_t *xchan, int cmd) 583 { 584 switch (cmd) { 585 case XDMA_CMD_BEGIN: 586 case XDMA_CMD_TERMINATE: 587 case XDMA_CMD_PAUSE: 588 /* TODO: implement me */ 589 return (-1); 590 } 591 592 return (0); 593 } 594 595 #ifdef FDT 596 static int 597 pl330_ofw_md_data(device_t dev, pcell_t *cells, int ncells, void **ptr) 598 { 599 struct pl330_fdt_data *data; 600 601 if (ncells != 1) 602 return (-1); 603 604 data = malloc(sizeof(struct pl330_fdt_data), 605 M_DEVBUF, (M_WAITOK | M_ZERO)); 606 data->periph_id = cells[0]; 607 608 *ptr = data; 609 610 return (0); 611 } 612 #endif 613 614 static device_method_t pl330_methods[] = { 615 /* Device interface */ 616 DEVMETHOD(device_probe, pl330_probe), 617 DEVMETHOD(device_attach, pl330_attach), 618 DEVMETHOD(device_detach, pl330_detach), 619 620 /* xDMA Interface */ 621 DEVMETHOD(xdma_channel_alloc, pl330_channel_alloc), 622 DEVMETHOD(xdma_channel_free, pl330_channel_free), 623 DEVMETHOD(xdma_channel_control, pl330_channel_control), 624 625 /* xDMA SG Interface */ 626 DEVMETHOD(xdma_channel_capacity, pl330_channel_capacity), 627 DEVMETHOD(xdma_channel_prep_sg, pl330_channel_prep_sg), 628 DEVMETHOD(xdma_channel_submit_sg, pl330_channel_submit_sg), 629 630 #ifdef FDT 631 DEVMETHOD(xdma_ofw_md_data, pl330_ofw_md_data), 632 #endif 633 634 DEVMETHOD_END 635 }; 636 637 static driver_t pl330_driver = { 638 "pl330", 639 pl330_methods, 640 sizeof(struct pl330_softc), 641 }; 642 643 EARLY_DRIVER_MODULE(pl330, simplebus, pl330_driver, 0, 0, 644 BUS_PASS_INTERRUPT + BUS_PASS_ORDER_LATE); 645