1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2019 Ruslan Bukin <br@bsdpad.com> 5 * 6 * This software was developed by SRI International and the University of 7 * Cambridge Computer Laboratory (Department of Computer Science and 8 * Technology) under DARPA contract HR0011-18-C-0016 ("ECATS"), as part of the 9 * DARPA SSITH research programme. 10 * 11 * Redistribution and use in source and binary forms, with or without 12 * modification, are permitted provided that the following conditions 13 * are met: 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 21 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 24 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 26 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 27 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 28 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 29 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 30 * SUCH DAMAGE. 31 */ 32 33 /* Xilinx AXI DMA controller driver. */ 34 35 #include <sys/cdefs.h> 36 __FBSDID("$FreeBSD$"); 37 38 #include "opt_platform.h" 39 #include <sys/param.h> 40 #include <sys/systm.h> 41 #include <sys/conf.h> 42 #include <sys/bus.h> 43 #include <sys/kernel.h> 44 #include <sys/module.h> 45 #include <sys/rman.h> 46 47 #include <machine/bus.h> 48 49 #include <vm/vm.h> 50 #include <vm/vm_extern.h> 51 #include <vm/vm_page.h> 52 53 #ifdef FDT 54 #include <dev/fdt/fdt_common.h> 55 #include <dev/ofw/ofw_bus.h> 56 #include <dev/ofw/ofw_bus_subr.h> 57 #endif 58 59 #include <dev/xdma/xdma.h> 60 #include <dev/xilinx/axidma.h> 61 62 #include "xdma_if.h" 63 64 #define READ4(_sc, _reg) \ 65 bus_space_read_4(_sc->bst, _sc->bsh, _reg) 66 #define WRITE4(_sc, _reg, _val) \ 67 bus_space_write_4(_sc->bst, _sc->bsh, _reg, _val) 68 #define READ8(_sc, _reg) \ 69 bus_space_read_8(_sc->bst, _sc->bsh, _reg) 70 #define WRITE8(_sc, _reg, _val) \ 71 bus_space_write_8(_sc->bst, _sc->bsh, _reg, _val) 72 73 #define AXIDMA_DEBUG 74 #undef AXIDMA_DEBUG 75 76 #ifdef AXIDMA_DEBUG 77 #define dprintf(fmt, ...) printf(fmt, ##__VA_ARGS__) 78 #else 79 #define dprintf(fmt, ...) 80 #endif 81 82 extern struct bus_space memmap_bus; 83 84 struct axidma_channel { 85 struct axidma_softc *sc; 86 xdma_channel_t *xchan; 87 bool used; 88 int idx_head; 89 int idx_tail; 90 91 struct axidma_desc **descs; 92 vm_paddr_t *descs_phys; 93 uint32_t descs_num; 94 95 vm_size_t mem_size; 96 vm_offset_t mem_paddr; 97 vm_offset_t mem_vaddr; 98 99 uint32_t descs_used_count; 100 }; 101 102 struct axidma_softc { 103 device_t dev; 104 struct resource *res[3]; 105 bus_space_tag_t bst; 106 bus_space_handle_t bsh; 107 void *ih[2]; 108 struct axidma_desc desc; 109 struct axidma_channel channels[AXIDMA_NCHANNELS]; 110 }; 111 112 static struct resource_spec axidma_spec[] = { 113 { SYS_RES_MEMORY, 0, RF_ACTIVE }, 114 { SYS_RES_IRQ, 0, RF_ACTIVE }, 115 { SYS_RES_IRQ, 1, RF_ACTIVE }, 116 { -1, 0 } 117 }; 118 119 #define HWTYPE_NONE 0 120 #define HWTYPE_STD 1 121 122 static struct ofw_compat_data compat_data[] = { 123 { "xlnx,eth-dma", HWTYPE_STD }, 124 { NULL, HWTYPE_NONE }, 125 }; 126 127 static int axidma_probe(device_t dev); 128 static int axidma_attach(device_t dev); 129 static int axidma_detach(device_t dev); 130 131 static inline uint32_t 132 axidma_next_desc(struct axidma_channel *chan, uint32_t curidx) 133 { 134 135 return ((curidx + 1) % chan->descs_num); 136 } 137 138 static void 139 axidma_intr(struct axidma_softc *sc, 140 struct axidma_channel *chan) 141 { 142 xdma_transfer_status_t status; 143 xdma_transfer_status_t st; 144 struct axidma_fdt_data *data; 145 xdma_controller_t *xdma; 146 struct axidma_desc *desc; 147 struct xdma_channel *xchan; 148 uint32_t tot_copied; 149 int pending; 150 int errors; 151 152 xchan = chan->xchan; 153 xdma = xchan->xdma; 154 data = xdma->data; 155 156 pending = READ4(sc, AXI_DMASR(data->id)); 157 WRITE4(sc, AXI_DMASR(data->id), pending); 158 159 errors = (pending & (DMASR_DMAINTERR | DMASR_DMASLVERR 160 | DMASR_DMADECOREERR | DMASR_SGINTERR 161 | DMASR_SGSLVERR | DMASR_SGDECERR)); 162 163 dprintf("%s: AXI_DMASR %x\n", __func__, 164 READ4(sc, AXI_DMASR(data->id))); 165 dprintf("%s: AXI_CURDESC %x\n", __func__, 166 READ4(sc, AXI_CURDESC(data->id))); 167 dprintf("%s: AXI_TAILDESC %x\n", __func__, 168 READ4(sc, AXI_TAILDESC(data->id))); 169 170 tot_copied = 0; 171 172 while (chan->idx_tail != chan->idx_head) { 173 desc = chan->descs[chan->idx_tail]; 174 if ((desc->status & BD_STATUS_CMPLT) == 0) 175 break; 176 177 st.error = errors; 178 st.transferred = desc->status & BD_CONTROL_LEN_M; 179 tot_copied += st.transferred; 180 xchan_seg_done(xchan, &st); 181 182 chan->idx_tail = axidma_next_desc(chan, chan->idx_tail); 183 atomic_subtract_int(&chan->descs_used_count, 1); 184 } 185 186 /* Finish operation */ 187 status.error = errors; 188 status.transferred = tot_copied; 189 xdma_callback(chan->xchan, &status); 190 } 191 192 static void 193 axidma_intr_rx(void *arg) 194 { 195 struct axidma_softc *sc; 196 struct axidma_channel *chan; 197 198 dprintf("%s\n", __func__); 199 200 sc = arg; 201 chan = &sc->channels[AXIDMA_RX_CHAN]; 202 203 axidma_intr(sc, chan); 204 } 205 206 static void 207 axidma_intr_tx(void *arg) 208 { 209 struct axidma_softc *sc; 210 struct axidma_channel *chan; 211 212 dprintf("%s\n", __func__); 213 214 sc = arg; 215 chan = &sc->channels[AXIDMA_TX_CHAN]; 216 217 axidma_intr(sc, chan); 218 } 219 220 static int 221 axidma_reset(struct axidma_softc *sc, int chan_id) 222 { 223 int timeout; 224 225 WRITE4(sc, AXI_DMACR(chan_id), DMACR_RESET); 226 227 timeout = 100; 228 do { 229 if ((READ4(sc, AXI_DMACR(chan_id)) & DMACR_RESET) == 0) 230 break; 231 } while (timeout--); 232 233 dprintf("timeout %d\n", timeout); 234 235 if (timeout == 0) 236 return (-1); 237 238 dprintf("%s: read control after reset: %x\n", 239 __func__, READ4(sc, AXI_DMACR(chan_id))); 240 241 return (0); 242 } 243 244 static int 245 axidma_probe(device_t dev) 246 { 247 int hwtype; 248 249 if (!ofw_bus_status_okay(dev)) 250 return (ENXIO); 251 252 hwtype = ofw_bus_search_compatible(dev, compat_data)->ocd_data; 253 if (hwtype == HWTYPE_NONE) 254 return (ENXIO); 255 256 device_set_desc(dev, "Xilinx AXI DMA"); 257 258 return (BUS_PROBE_DEFAULT); 259 } 260 261 static int 262 axidma_attach(device_t dev) 263 { 264 struct axidma_softc *sc; 265 phandle_t xref, node; 266 int err; 267 268 sc = device_get_softc(dev); 269 sc->dev = dev; 270 271 if (bus_alloc_resources(dev, axidma_spec, sc->res)) { 272 device_printf(dev, "could not allocate resources.\n"); 273 return (ENXIO); 274 } 275 276 /* CSR memory interface */ 277 sc->bst = rman_get_bustag(sc->res[0]); 278 sc->bsh = rman_get_bushandle(sc->res[0]); 279 280 /* Setup interrupt handler */ 281 err = bus_setup_intr(dev, sc->res[1], INTR_TYPE_MISC | INTR_MPSAFE, 282 NULL, axidma_intr_tx, sc, &sc->ih[0]); 283 if (err) { 284 device_printf(dev, "Unable to alloc interrupt resource.\n"); 285 return (ENXIO); 286 } 287 288 /* Setup interrupt handler */ 289 err = bus_setup_intr(dev, sc->res[2], INTR_TYPE_MISC | INTR_MPSAFE, 290 NULL, axidma_intr_rx, sc, &sc->ih[1]); 291 if (err) { 292 device_printf(dev, "Unable to alloc interrupt resource.\n"); 293 return (ENXIO); 294 } 295 296 node = ofw_bus_get_node(dev); 297 xref = OF_xref_from_node(node); 298 OF_device_register_xref(xref, dev); 299 300 return (0); 301 } 302 303 static int 304 axidma_detach(device_t dev) 305 { 306 struct axidma_softc *sc; 307 308 sc = device_get_softc(dev); 309 310 bus_teardown_intr(dev, sc->res[1], sc->ih[0]); 311 bus_teardown_intr(dev, sc->res[2], sc->ih[1]); 312 bus_release_resources(dev, axidma_spec, sc->res); 313 314 return (0); 315 } 316 317 static int 318 axidma_desc_free(struct axidma_softc *sc, struct axidma_channel *chan) 319 { 320 struct xdma_channel *xchan; 321 322 xchan = chan->xchan; 323 324 free(chan->descs, M_DEVBUF); 325 free(chan->descs_phys, M_DEVBUF); 326 327 pmap_kremove_device(chan->mem_vaddr, chan->mem_size); 328 kva_free(chan->mem_vaddr, chan->mem_size); 329 vmem_free(xchan->vmem, chan->mem_paddr, chan->mem_size); 330 331 return (0); 332 } 333 334 static int 335 axidma_desc_alloc(struct axidma_softc *sc, struct xdma_channel *xchan, 336 uint32_t desc_size) 337 { 338 struct axidma_channel *chan; 339 int nsegments; 340 int i; 341 342 chan = (struct axidma_channel *)xchan->chan; 343 nsegments = chan->descs_num; 344 345 chan->descs = malloc(nsegments * sizeof(struct axidma_desc *), 346 M_DEVBUF, M_NOWAIT | M_ZERO); 347 if (chan->descs == NULL) { 348 device_printf(sc->dev, 349 "%s: Can't allocate memory.\n", __func__); 350 return (-1); 351 } 352 353 chan->descs_phys = malloc(nsegments * sizeof(bus_dma_segment_t), 354 M_DEVBUF, M_NOWAIT | M_ZERO); 355 chan->mem_size = desc_size * nsegments; 356 if (vmem_alloc(xchan->vmem, chan->mem_size, M_FIRSTFIT | M_NOWAIT, 357 &chan->mem_paddr)) { 358 device_printf(sc->dev, "Failed to allocate memory.\n"); 359 return (-1); 360 } 361 chan->mem_vaddr = kva_alloc(chan->mem_size); 362 pmap_kenter_device(chan->mem_vaddr, chan->mem_size, chan->mem_paddr); 363 364 device_printf(sc->dev, "Allocated chunk %lx %lu\n", 365 chan->mem_paddr, chan->mem_size); 366 367 for (i = 0; i < nsegments; i++) { 368 chan->descs[i] = (struct axidma_desc *) 369 ((uint64_t)chan->mem_vaddr + desc_size * i); 370 chan->descs_phys[i] = chan->mem_paddr + desc_size * i; 371 } 372 373 return (0); 374 } 375 376 static int 377 axidma_channel_alloc(device_t dev, struct xdma_channel *xchan) 378 { 379 xdma_controller_t *xdma; 380 struct axidma_fdt_data *data; 381 struct axidma_channel *chan; 382 struct axidma_softc *sc; 383 384 sc = device_get_softc(dev); 385 386 if (xchan->caps & XCHAN_CAP_BUSDMA) { 387 device_printf(sc->dev, 388 "Error: busdma operation is not implemented."); 389 return (-1); 390 } 391 392 xdma = xchan->xdma; 393 data = xdma->data; 394 395 chan = &sc->channels[data->id]; 396 if (chan->used == false) { 397 if (axidma_reset(sc, data->id) != 0) 398 return (-1); 399 chan->xchan = xchan; 400 xchan->caps |= XCHAN_CAP_BOUNCE; 401 xchan->chan = (void *)chan; 402 chan->sc = sc; 403 chan->used = true; 404 chan->idx_head = 0; 405 chan->idx_tail = 0; 406 chan->descs_used_count = 0; 407 chan->descs_num = AXIDMA_DESCS_NUM; 408 409 return (0); 410 } 411 412 return (-1); 413 } 414 415 static int 416 axidma_channel_free(device_t dev, struct xdma_channel *xchan) 417 { 418 struct axidma_channel *chan; 419 struct axidma_softc *sc; 420 421 sc = device_get_softc(dev); 422 423 chan = (struct axidma_channel *)xchan->chan; 424 425 axidma_desc_free(sc, chan); 426 427 chan->used = false; 428 429 return (0); 430 } 431 432 static int 433 axidma_channel_capacity(device_t dev, xdma_channel_t *xchan, 434 uint32_t *capacity) 435 { 436 struct axidma_channel *chan; 437 uint32_t c; 438 439 chan = (struct axidma_channel *)xchan->chan; 440 441 /* At least one descriptor must be left empty. */ 442 c = (chan->descs_num - chan->descs_used_count - 1); 443 444 *capacity = c; 445 446 return (0); 447 } 448 449 static int 450 axidma_channel_submit_sg(device_t dev, struct xdma_channel *xchan, 451 struct xdma_sglist *sg, uint32_t sg_n) 452 { 453 xdma_controller_t *xdma; 454 struct axidma_fdt_data *data; 455 struct axidma_channel *chan; 456 struct axidma_desc *desc; 457 struct axidma_softc *sc; 458 uint32_t src_addr; 459 uint32_t dst_addr; 460 uint32_t addr; 461 uint32_t len; 462 uint32_t tmp; 463 int i; 464 465 dprintf("%s: sg_n %d\n", __func__, sg_n); 466 467 sc = device_get_softc(dev); 468 469 chan = (struct axidma_channel *)xchan->chan; 470 xdma = xchan->xdma; 471 data = xdma->data; 472 473 if (sg_n == 0) 474 return (0); 475 476 tmp = 0; 477 478 for (i = 0; i < sg_n; i++) { 479 src_addr = (uint32_t)sg[i].src_addr; 480 dst_addr = (uint32_t)sg[i].dst_addr; 481 len = (uint32_t)sg[i].len; 482 483 dprintf("%s(%d): src %x dst %x len %d\n", __func__, 484 data->id, src_addr, dst_addr, len); 485 486 desc = chan->descs[chan->idx_head]; 487 if (sg[i].direction == XDMA_MEM_TO_DEV) 488 desc->phys = src_addr; 489 else 490 desc->phys = dst_addr; 491 desc->status = 0; 492 desc->control = len; 493 if (sg[i].first == 1) 494 desc->control |= BD_CONTROL_TXSOF; 495 if (sg[i].last == 1) 496 desc->control |= BD_CONTROL_TXEOF; 497 498 tmp = chan->idx_head; 499 500 atomic_add_int(&chan->descs_used_count, 1); 501 chan->idx_head = axidma_next_desc(chan, chan->idx_head); 502 } 503 504 dprintf("%s(%d): _curdesc %x\n", __func__, data->id, 505 READ8(sc, AXI_CURDESC(data->id))); 506 dprintf("%s(%d): _curdesc %x\n", __func__, data->id, 507 READ8(sc, AXI_CURDESC(data->id))); 508 dprintf("%s(%d): status %x\n", __func__, data->id, 509 READ4(sc, AXI_DMASR(data->id))); 510 511 addr = chan->descs_phys[tmp]; 512 WRITE8(sc, AXI_TAILDESC(data->id), addr); 513 514 return (0); 515 } 516 517 static int 518 axidma_channel_prep_sg(device_t dev, struct xdma_channel *xchan) 519 { 520 xdma_controller_t *xdma; 521 struct axidma_fdt_data *data; 522 struct axidma_channel *chan; 523 struct axidma_desc *desc; 524 struct axidma_softc *sc; 525 uint32_t addr; 526 uint32_t reg; 527 int ret; 528 int i; 529 530 sc = device_get_softc(dev); 531 532 chan = (struct axidma_channel *)xchan->chan; 533 xdma = xchan->xdma; 534 data = xdma->data; 535 536 dprintf("%s(%d)\n", __func__, data->id); 537 538 ret = axidma_desc_alloc(sc, xchan, sizeof(struct axidma_desc)); 539 if (ret != 0) { 540 device_printf(sc->dev, 541 "%s: Can't allocate descriptors.\n", __func__); 542 return (-1); 543 } 544 545 for (i = 0; i < chan->descs_num; i++) { 546 desc = chan->descs[i]; 547 bzero(desc, sizeof(struct axidma_desc)); 548 549 if (i == (chan->descs_num - 1)) 550 desc->next = chan->descs_phys[0]; 551 else 552 desc->next = chan->descs_phys[i + 1]; 553 desc->status = 0; 554 desc->control = 0; 555 556 dprintf("%s(%d): desc %d vaddr %lx next paddr %x\n", __func__, 557 data->id, i, (uint64_t)desc, le32toh(desc->next)); 558 } 559 560 addr = chan->descs_phys[0]; 561 WRITE8(sc, AXI_CURDESC(data->id), addr); 562 563 reg = READ4(sc, AXI_DMACR(data->id)); 564 reg |= DMACR_IOC_IRQEN | DMACR_DLY_IRQEN | DMACR_ERR_IRQEN; 565 WRITE4(sc, AXI_DMACR(data->id), reg); 566 reg |= DMACR_RS; 567 WRITE4(sc, AXI_DMACR(data->id), reg); 568 569 return (0); 570 } 571 572 static int 573 axidma_channel_control(device_t dev, xdma_channel_t *xchan, int cmd) 574 { 575 576 switch (cmd) { 577 case XDMA_CMD_BEGIN: 578 case XDMA_CMD_TERMINATE: 579 case XDMA_CMD_PAUSE: 580 /* TODO: implement me */ 581 return (-1); 582 } 583 584 return (0); 585 } 586 587 #ifdef FDT 588 static int 589 axidma_ofw_md_data(device_t dev, pcell_t *cells, int ncells, void **ptr) 590 { 591 struct axidma_fdt_data *data; 592 593 if (ncells != 1) 594 return (-1); 595 596 data = malloc(sizeof(struct axidma_fdt_data), 597 M_DEVBUF, (M_WAITOK | M_ZERO)); 598 data->id = cells[0]; 599 600 *ptr = data; 601 602 return (0); 603 } 604 #endif 605 606 static device_method_t axidma_methods[] = { 607 /* Device interface */ 608 DEVMETHOD(device_probe, axidma_probe), 609 DEVMETHOD(device_attach, axidma_attach), 610 DEVMETHOD(device_detach, axidma_detach), 611 612 /* xDMA Interface */ 613 DEVMETHOD(xdma_channel_alloc, axidma_channel_alloc), 614 DEVMETHOD(xdma_channel_free, axidma_channel_free), 615 DEVMETHOD(xdma_channel_control, axidma_channel_control), 616 617 /* xDMA SG Interface */ 618 DEVMETHOD(xdma_channel_capacity, axidma_channel_capacity), 619 DEVMETHOD(xdma_channel_prep_sg, axidma_channel_prep_sg), 620 DEVMETHOD(xdma_channel_submit_sg, axidma_channel_submit_sg), 621 622 #ifdef FDT 623 DEVMETHOD(xdma_ofw_md_data, axidma_ofw_md_data), 624 #endif 625 626 DEVMETHOD_END 627 }; 628 629 static driver_t axidma_driver = { 630 "axidma", 631 axidma_methods, 632 sizeof(struct axidma_softc), 633 }; 634 635 EARLY_DRIVER_MODULE(axidma, simplebus, axidma_driver, 0, 0, 636 BUS_PASS_INTERRUPT + BUS_PASS_ORDER_LATE); 637