1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2019 Ruslan Bukin <br@bsdpad.com> 5 * 6 * This software was developed by SRI International and the University of 7 * Cambridge Computer Laboratory (Department of Computer Science and 8 * Technology) under DARPA contract HR0011-18-C-0016 ("ECATS"), as part of the 9 * DARPA SSITH research programme. 10 * 11 * Redistribution and use in source and binary forms, with or without 12 * modification, are permitted provided that the following conditions 13 * are met: 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 21 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 24 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 26 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 27 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 28 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 29 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 30 * SUCH DAMAGE. 31 */ 32 33 /* Xilinx AXI DMA controller driver. */ 34 35 #include <sys/cdefs.h> 36 #include "opt_platform.h" 37 #include <sys/param.h> 38 #include <sys/systm.h> 39 #include <sys/conf.h> 40 #include <sys/bus.h> 41 #include <sys/kernel.h> 42 #include <sys/module.h> 43 #include <sys/rman.h> 44 45 #include <machine/bus.h> 46 47 #include <vm/vm.h> 48 #include <vm/vm_extern.h> 49 #include <vm/vm_page.h> 50 51 #ifdef FDT 52 #include <dev/fdt/fdt_common.h> 53 #include <dev/ofw/ofw_bus.h> 54 #include <dev/ofw/ofw_bus_subr.h> 55 #endif 56 57 #include <dev/xdma/xdma.h> 58 #include <dev/xilinx/axidma.h> 59 60 #include "xdma_if.h" 61 62 #define READ4(_sc, _reg) \ 63 bus_space_read_4(_sc->bst, _sc->bsh, _reg) 64 #define WRITE4(_sc, _reg, _val) \ 65 bus_space_write_4(_sc->bst, _sc->bsh, _reg, _val) 66 #define READ8(_sc, _reg) \ 67 bus_space_read_8(_sc->bst, _sc->bsh, _reg) 68 #define WRITE8(_sc, _reg, _val) \ 69 bus_space_write_8(_sc->bst, _sc->bsh, _reg, _val) 70 71 #define AXIDMA_DEBUG 72 #undef AXIDMA_DEBUG 73 74 #ifdef AXIDMA_DEBUG 75 #define dprintf(fmt, ...) printf(fmt, ##__VA_ARGS__) 76 #else 77 #define dprintf(fmt, ...) 78 #endif 79 80 extern struct bus_space memmap_bus; 81 82 struct axidma_channel { 83 struct axidma_softc *sc; 84 xdma_channel_t *xchan; 85 bool used; 86 int idx_head; 87 int idx_tail; 88 89 struct axidma_desc **descs; 90 vm_paddr_t *descs_phys; 91 uint32_t descs_num; 92 93 vm_size_t mem_size; 94 vm_offset_t mem_paddr; 95 vm_offset_t mem_vaddr; 96 97 uint32_t descs_used_count; 98 }; 99 100 struct axidma_softc { 101 device_t dev; 102 struct resource *res[3]; 103 bus_space_tag_t bst; 104 bus_space_handle_t bsh; 105 void *ih[2]; 106 struct axidma_desc desc; 107 struct axidma_channel channels[AXIDMA_NCHANNELS]; 108 }; 109 110 static struct resource_spec axidma_spec[] = { 111 { SYS_RES_MEMORY, 0, RF_ACTIVE }, 112 { SYS_RES_IRQ, 0, RF_ACTIVE }, 113 { SYS_RES_IRQ, 1, RF_ACTIVE }, 114 { -1, 0 } 115 }; 116 117 #define HWTYPE_NONE 0 118 #define HWTYPE_STD 1 119 120 static struct ofw_compat_data compat_data[] = { 121 { "xlnx,eth-dma", HWTYPE_STD }, 122 { NULL, HWTYPE_NONE }, 123 }; 124 125 static int axidma_probe(device_t dev); 126 static int axidma_attach(device_t dev); 127 static int axidma_detach(device_t dev); 128 129 static inline uint32_t 130 axidma_next_desc(struct axidma_channel *chan, uint32_t curidx) 131 { 132 133 return ((curidx + 1) % chan->descs_num); 134 } 135 136 static void 137 axidma_intr(struct axidma_softc *sc, 138 struct axidma_channel *chan) 139 { 140 xdma_transfer_status_t status; 141 xdma_transfer_status_t st; 142 struct axidma_fdt_data *data; 143 xdma_controller_t *xdma; 144 struct axidma_desc *desc; 145 struct xdma_channel *xchan; 146 uint32_t tot_copied; 147 int pending; 148 int errors; 149 150 xchan = chan->xchan; 151 xdma = xchan->xdma; 152 data = xdma->data; 153 154 pending = READ4(sc, AXI_DMASR(data->id)); 155 WRITE4(sc, AXI_DMASR(data->id), pending); 156 157 errors = (pending & (DMASR_DMAINTERR | DMASR_DMASLVERR 158 | DMASR_DMADECOREERR | DMASR_SGINTERR 159 | DMASR_SGSLVERR | DMASR_SGDECERR)); 160 161 dprintf("%s: AXI_DMASR %x\n", __func__, 162 READ4(sc, AXI_DMASR(data->id))); 163 dprintf("%s: AXI_CURDESC %x\n", __func__, 164 READ4(sc, AXI_CURDESC(data->id))); 165 dprintf("%s: AXI_TAILDESC %x\n", __func__, 166 READ4(sc, AXI_TAILDESC(data->id))); 167 168 tot_copied = 0; 169 170 while (chan->idx_tail != chan->idx_head) { 171 desc = chan->descs[chan->idx_tail]; 172 if ((desc->status & BD_STATUS_CMPLT) == 0) 173 break; 174 175 st.error = errors; 176 st.transferred = desc->status & BD_CONTROL_LEN_M; 177 tot_copied += st.transferred; 178 xchan_seg_done(xchan, &st); 179 180 chan->idx_tail = axidma_next_desc(chan, chan->idx_tail); 181 atomic_subtract_int(&chan->descs_used_count, 1); 182 } 183 184 /* Finish operation */ 185 status.error = errors; 186 status.transferred = tot_copied; 187 xdma_callback(chan->xchan, &status); 188 } 189 190 static void 191 axidma_intr_rx(void *arg) 192 { 193 struct axidma_softc *sc; 194 struct axidma_channel *chan; 195 196 dprintf("%s\n", __func__); 197 198 sc = arg; 199 chan = &sc->channels[AXIDMA_RX_CHAN]; 200 201 axidma_intr(sc, chan); 202 } 203 204 static void 205 axidma_intr_tx(void *arg) 206 { 207 struct axidma_softc *sc; 208 struct axidma_channel *chan; 209 210 dprintf("%s\n", __func__); 211 212 sc = arg; 213 chan = &sc->channels[AXIDMA_TX_CHAN]; 214 215 axidma_intr(sc, chan); 216 } 217 218 static int 219 axidma_reset(struct axidma_softc *sc, int chan_id) 220 { 221 int timeout; 222 223 WRITE4(sc, AXI_DMACR(chan_id), DMACR_RESET); 224 225 timeout = 100; 226 do { 227 if ((READ4(sc, AXI_DMACR(chan_id)) & DMACR_RESET) == 0) 228 break; 229 } while (timeout--); 230 231 dprintf("timeout %d\n", timeout); 232 233 if (timeout == 0) 234 return (-1); 235 236 dprintf("%s: read control after reset: %x\n", 237 __func__, READ4(sc, AXI_DMACR(chan_id))); 238 239 return (0); 240 } 241 242 static int 243 axidma_probe(device_t dev) 244 { 245 int hwtype; 246 247 if (!ofw_bus_status_okay(dev)) 248 return (ENXIO); 249 250 hwtype = ofw_bus_search_compatible(dev, compat_data)->ocd_data; 251 if (hwtype == HWTYPE_NONE) 252 return (ENXIO); 253 254 device_set_desc(dev, "Xilinx AXI DMA"); 255 256 return (BUS_PROBE_DEFAULT); 257 } 258 259 static int 260 axidma_attach(device_t dev) 261 { 262 struct axidma_softc *sc; 263 phandle_t xref, node; 264 int err; 265 266 sc = device_get_softc(dev); 267 sc->dev = dev; 268 269 if (bus_alloc_resources(dev, axidma_spec, sc->res)) { 270 device_printf(dev, "could not allocate resources.\n"); 271 return (ENXIO); 272 } 273 274 /* CSR memory interface */ 275 sc->bst = rman_get_bustag(sc->res[0]); 276 sc->bsh = rman_get_bushandle(sc->res[0]); 277 278 /* Setup interrupt handler */ 279 err = bus_setup_intr(dev, sc->res[1], INTR_TYPE_MISC | INTR_MPSAFE, 280 NULL, axidma_intr_tx, sc, &sc->ih[0]); 281 if (err) { 282 device_printf(dev, "Unable to alloc interrupt resource.\n"); 283 return (ENXIO); 284 } 285 286 /* Setup interrupt handler */ 287 err = bus_setup_intr(dev, sc->res[2], INTR_TYPE_MISC | INTR_MPSAFE, 288 NULL, axidma_intr_rx, sc, &sc->ih[1]); 289 if (err) { 290 device_printf(dev, "Unable to alloc interrupt resource.\n"); 291 return (ENXIO); 292 } 293 294 node = ofw_bus_get_node(dev); 295 xref = OF_xref_from_node(node); 296 OF_device_register_xref(xref, dev); 297 298 return (0); 299 } 300 301 static int 302 axidma_detach(device_t dev) 303 { 304 struct axidma_softc *sc; 305 306 sc = device_get_softc(dev); 307 308 bus_teardown_intr(dev, sc->res[1], sc->ih[0]); 309 bus_teardown_intr(dev, sc->res[2], sc->ih[1]); 310 bus_release_resources(dev, axidma_spec, sc->res); 311 312 return (0); 313 } 314 315 static int 316 axidma_desc_free(struct axidma_softc *sc, struct axidma_channel *chan) 317 { 318 struct xdma_channel *xchan; 319 320 xchan = chan->xchan; 321 322 free(chan->descs, M_DEVBUF); 323 free(chan->descs_phys, M_DEVBUF); 324 325 pmap_kremove_device(chan->mem_vaddr, chan->mem_size); 326 kva_free(chan->mem_vaddr, chan->mem_size); 327 vmem_free(xchan->vmem, chan->mem_paddr, chan->mem_size); 328 329 return (0); 330 } 331 332 static int 333 axidma_desc_alloc(struct axidma_softc *sc, struct xdma_channel *xchan, 334 uint32_t desc_size) 335 { 336 struct axidma_channel *chan; 337 int nsegments; 338 int i; 339 340 chan = (struct axidma_channel *)xchan->chan; 341 nsegments = chan->descs_num; 342 343 chan->descs = malloc(nsegments * sizeof(struct axidma_desc *), 344 M_DEVBUF, M_NOWAIT | M_ZERO); 345 if (chan->descs == NULL) { 346 device_printf(sc->dev, 347 "%s: Can't allocate memory.\n", __func__); 348 return (-1); 349 } 350 351 chan->descs_phys = malloc(nsegments * sizeof(bus_dma_segment_t), 352 M_DEVBUF, M_NOWAIT | M_ZERO); 353 chan->mem_size = desc_size * nsegments; 354 if (vmem_alloc(xchan->vmem, chan->mem_size, M_FIRSTFIT | M_NOWAIT, 355 &chan->mem_paddr)) { 356 device_printf(sc->dev, "Failed to allocate memory.\n"); 357 return (-1); 358 } 359 chan->mem_vaddr = kva_alloc(chan->mem_size); 360 pmap_kenter_device(chan->mem_vaddr, chan->mem_size, chan->mem_paddr); 361 362 device_printf(sc->dev, "Allocated chunk %lx %lu\n", 363 chan->mem_paddr, chan->mem_size); 364 365 for (i = 0; i < nsegments; i++) { 366 chan->descs[i] = (struct axidma_desc *) 367 ((uint64_t)chan->mem_vaddr + desc_size * i); 368 chan->descs_phys[i] = chan->mem_paddr + desc_size * i; 369 } 370 371 return (0); 372 } 373 374 static int 375 axidma_channel_alloc(device_t dev, struct xdma_channel *xchan) 376 { 377 xdma_controller_t *xdma; 378 struct axidma_fdt_data *data; 379 struct axidma_channel *chan; 380 struct axidma_softc *sc; 381 382 sc = device_get_softc(dev); 383 384 if (xchan->caps & XCHAN_CAP_BUSDMA) { 385 device_printf(sc->dev, 386 "Error: busdma operation is not implemented."); 387 return (-1); 388 } 389 390 xdma = xchan->xdma; 391 data = xdma->data; 392 393 chan = &sc->channels[data->id]; 394 if (chan->used == false) { 395 if (axidma_reset(sc, data->id) != 0) 396 return (-1); 397 chan->xchan = xchan; 398 xchan->caps |= XCHAN_CAP_BOUNCE; 399 xchan->chan = (void *)chan; 400 chan->sc = sc; 401 chan->used = true; 402 chan->idx_head = 0; 403 chan->idx_tail = 0; 404 chan->descs_used_count = 0; 405 chan->descs_num = AXIDMA_DESCS_NUM; 406 407 return (0); 408 } 409 410 return (-1); 411 } 412 413 static int 414 axidma_channel_free(device_t dev, struct xdma_channel *xchan) 415 { 416 struct axidma_channel *chan; 417 struct axidma_softc *sc; 418 419 sc = device_get_softc(dev); 420 421 chan = (struct axidma_channel *)xchan->chan; 422 423 axidma_desc_free(sc, chan); 424 425 chan->used = false; 426 427 return (0); 428 } 429 430 static int 431 axidma_channel_capacity(device_t dev, xdma_channel_t *xchan, 432 uint32_t *capacity) 433 { 434 struct axidma_channel *chan; 435 uint32_t c; 436 437 chan = (struct axidma_channel *)xchan->chan; 438 439 /* At least one descriptor must be left empty. */ 440 c = (chan->descs_num - chan->descs_used_count - 1); 441 442 *capacity = c; 443 444 return (0); 445 } 446 447 static int 448 axidma_channel_submit_sg(device_t dev, struct xdma_channel *xchan, 449 struct xdma_sglist *sg, uint32_t sg_n) 450 { 451 xdma_controller_t *xdma; 452 struct axidma_fdt_data *data; 453 struct axidma_channel *chan; 454 struct axidma_desc *desc; 455 struct axidma_softc *sc; 456 uint32_t src_addr; 457 uint32_t dst_addr; 458 uint32_t addr; 459 uint32_t len; 460 uint32_t tmp; 461 int i; 462 463 dprintf("%s: sg_n %d\n", __func__, sg_n); 464 465 sc = device_get_softc(dev); 466 467 chan = (struct axidma_channel *)xchan->chan; 468 xdma = xchan->xdma; 469 data = xdma->data; 470 471 if (sg_n == 0) 472 return (0); 473 474 tmp = 0; 475 476 for (i = 0; i < sg_n; i++) { 477 src_addr = (uint32_t)sg[i].src_addr; 478 dst_addr = (uint32_t)sg[i].dst_addr; 479 len = (uint32_t)sg[i].len; 480 481 dprintf("%s(%d): src %x dst %x len %d\n", __func__, 482 data->id, src_addr, dst_addr, len); 483 484 desc = chan->descs[chan->idx_head]; 485 if (sg[i].direction == XDMA_MEM_TO_DEV) 486 desc->phys = src_addr; 487 else 488 desc->phys = dst_addr; 489 desc->status = 0; 490 desc->control = len; 491 if (sg[i].first == 1) 492 desc->control |= BD_CONTROL_TXSOF; 493 if (sg[i].last == 1) 494 desc->control |= BD_CONTROL_TXEOF; 495 496 tmp = chan->idx_head; 497 498 atomic_add_int(&chan->descs_used_count, 1); 499 chan->idx_head = axidma_next_desc(chan, chan->idx_head); 500 } 501 502 dprintf("%s(%d): _curdesc %x\n", __func__, data->id, 503 READ8(sc, AXI_CURDESC(data->id))); 504 dprintf("%s(%d): _curdesc %x\n", __func__, data->id, 505 READ8(sc, AXI_CURDESC(data->id))); 506 dprintf("%s(%d): status %x\n", __func__, data->id, 507 READ4(sc, AXI_DMASR(data->id))); 508 509 addr = chan->descs_phys[tmp]; 510 WRITE8(sc, AXI_TAILDESC(data->id), addr); 511 512 return (0); 513 } 514 515 static int 516 axidma_channel_prep_sg(device_t dev, struct xdma_channel *xchan) 517 { 518 xdma_controller_t *xdma; 519 struct axidma_fdt_data *data; 520 struct axidma_channel *chan; 521 struct axidma_desc *desc; 522 struct axidma_softc *sc; 523 uint32_t addr; 524 uint32_t reg; 525 int ret; 526 int i; 527 528 sc = device_get_softc(dev); 529 530 chan = (struct axidma_channel *)xchan->chan; 531 xdma = xchan->xdma; 532 data = xdma->data; 533 534 dprintf("%s(%d)\n", __func__, data->id); 535 536 ret = axidma_desc_alloc(sc, xchan, sizeof(struct axidma_desc)); 537 if (ret != 0) { 538 device_printf(sc->dev, 539 "%s: Can't allocate descriptors.\n", __func__); 540 return (-1); 541 } 542 543 for (i = 0; i < chan->descs_num; i++) { 544 desc = chan->descs[i]; 545 bzero(desc, sizeof(struct axidma_desc)); 546 547 if (i == (chan->descs_num - 1)) 548 desc->next = chan->descs_phys[0]; 549 else 550 desc->next = chan->descs_phys[i + 1]; 551 desc->status = 0; 552 desc->control = 0; 553 554 dprintf("%s(%d): desc %d vaddr %lx next paddr %x\n", __func__, 555 data->id, i, (uint64_t)desc, le32toh(desc->next)); 556 } 557 558 addr = chan->descs_phys[0]; 559 WRITE8(sc, AXI_CURDESC(data->id), addr); 560 561 reg = READ4(sc, AXI_DMACR(data->id)); 562 reg |= DMACR_IOC_IRQEN | DMACR_DLY_IRQEN | DMACR_ERR_IRQEN; 563 WRITE4(sc, AXI_DMACR(data->id), reg); 564 reg |= DMACR_RS; 565 WRITE4(sc, AXI_DMACR(data->id), reg); 566 567 return (0); 568 } 569 570 static int 571 axidma_channel_control(device_t dev, xdma_channel_t *xchan, int cmd) 572 { 573 574 switch (cmd) { 575 case XDMA_CMD_BEGIN: 576 case XDMA_CMD_TERMINATE: 577 case XDMA_CMD_PAUSE: 578 /* TODO: implement me */ 579 return (-1); 580 } 581 582 return (0); 583 } 584 585 #ifdef FDT 586 static int 587 axidma_ofw_md_data(device_t dev, pcell_t *cells, int ncells, void **ptr) 588 { 589 struct axidma_fdt_data *data; 590 591 if (ncells != 1) 592 return (-1); 593 594 data = malloc(sizeof(struct axidma_fdt_data), 595 M_DEVBUF, (M_WAITOK | M_ZERO)); 596 data->id = cells[0]; 597 598 *ptr = data; 599 600 return (0); 601 } 602 #endif 603 604 static device_method_t axidma_methods[] = { 605 /* Device interface */ 606 DEVMETHOD(device_probe, axidma_probe), 607 DEVMETHOD(device_attach, axidma_attach), 608 DEVMETHOD(device_detach, axidma_detach), 609 610 /* xDMA Interface */ 611 DEVMETHOD(xdma_channel_alloc, axidma_channel_alloc), 612 DEVMETHOD(xdma_channel_free, axidma_channel_free), 613 DEVMETHOD(xdma_channel_control, axidma_channel_control), 614 615 /* xDMA SG Interface */ 616 DEVMETHOD(xdma_channel_capacity, axidma_channel_capacity), 617 DEVMETHOD(xdma_channel_prep_sg, axidma_channel_prep_sg), 618 DEVMETHOD(xdma_channel_submit_sg, axidma_channel_submit_sg), 619 620 #ifdef FDT 621 DEVMETHOD(xdma_ofw_md_data, axidma_ofw_md_data), 622 #endif 623 624 DEVMETHOD_END 625 }; 626 627 static driver_t axidma_driver = { 628 "axidma", 629 axidma_methods, 630 sizeof(struct axidma_softc), 631 }; 632 633 EARLY_DRIVER_MODULE(axidma, simplebus, axidma_driver, 0, 0, 634 BUS_PASS_INTERRUPT + BUS_PASS_ORDER_LATE); 635