1 /*- 2 * Copyright (c) 2016 Ruslan Bukin <br@bsdpad.com> 3 * All rights reserved. 4 * 5 * This software was developed by SRI International and the University of 6 * Cambridge Computer Laboratory under DARPA/AFRL contract FA8750-10-C-0237 7 * ("CTSRD"), as part of the DARPA CRASH research programme. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * SUCH DAMAGE. 29 */ 30 31 #include <sys/cdefs.h> 32 __FBSDID("$FreeBSD$"); 33 34 #include "opt_platform.h" 35 #include <sys/param.h> 36 #include <sys/conf.h> 37 #include <sys/bus.h> 38 #include <sys/kernel.h> 39 #include <sys/queue.h> 40 #include <sys/kobj.h> 41 #include <sys/malloc.h> 42 #include <sys/mutex.h> 43 #include <sys/limits.h> 44 #include <sys/lock.h> 45 #include <sys/sysctl.h> 46 #include <sys/systm.h> 47 #include <sys/sx.h> 48 #include <sys/bus_dma.h> 49 50 #include <machine/bus.h> 51 52 #ifdef FDT 53 #include <dev/fdt/fdt_common.h> 54 #include <dev/ofw/ofw_bus.h> 55 #include <dev/ofw/ofw_bus_subr.h> 56 #endif 57 58 #include <dev/xdma/xdma.h> 59 60 #include <xdma_if.h> 61 62 MALLOC_DEFINE(M_XDMA, "xdma", "xDMA framework"); 63 64 /* 65 * Multiple xDMA controllers may work with single DMA device, 66 * so we have global lock for physical channel management. 67 */ 68 static struct mtx xdma_mtx; 69 #define XDMA_LOCK() mtx_lock(&xdma_mtx) 70 #define XDMA_UNLOCK() mtx_unlock(&xdma_mtx) 71 #define XDMA_ASSERT_LOCKED() mtx_assert(&xdma_mtx, MA_OWNED) 72 73 /* 74 * Per channel locks. 75 */ 76 #define XCHAN_LOCK(xchan) mtx_lock(&(xchan)->mtx_lock) 77 #define XCHAN_UNLOCK(xchan) mtx_unlock(&(xchan)->mtx_lock) 78 #define XCHAN_ASSERT_LOCKED(xchan) mtx_assert(&(xchan)->mtx_lock, MA_OWNED) 79 80 /* 81 * Allocate virtual xDMA channel. 82 */ 83 xdma_channel_t * 84 xdma_channel_alloc(xdma_controller_t *xdma) 85 { 86 xdma_channel_t *xchan; 87 int ret; 88 89 xchan = malloc(sizeof(xdma_channel_t), M_XDMA, M_WAITOK | M_ZERO); 90 if (xchan == NULL) { 91 device_printf(xdma->dev, 92 "%s: Can't allocate memory for channel.\n", __func__); 93 return (NULL); 94 } 95 xchan->xdma = xdma; 96 97 XDMA_LOCK(); 98 99 /* Request a real channel from hardware driver. */ 100 ret = XDMA_CHANNEL_ALLOC(xdma->dma_dev, xchan); 101 if (ret != 0) { 102 device_printf(xdma->dev, 103 "%s: Can't request hardware channel.\n", __func__); 104 XDMA_UNLOCK(); 105 free(xchan, M_XDMA); 106 107 return (NULL); 108 } 109 110 TAILQ_INIT(&xchan->ie_handlers); 111 mtx_init(&xchan->mtx_lock, "xDMA", NULL, MTX_DEF); 112 113 TAILQ_INSERT_TAIL(&xdma->channels, xchan, xchan_next); 114 115 XDMA_UNLOCK(); 116 117 return (xchan); 118 } 119 120 int 121 xdma_channel_free(xdma_channel_t *xchan) 122 { 123 xdma_controller_t *xdma; 124 int err; 125 126 xdma = xchan->xdma; 127 128 XDMA_LOCK(); 129 130 /* Free the real DMA channel. */ 131 err = XDMA_CHANNEL_FREE(xdma->dma_dev, xchan); 132 if (err != 0) { 133 device_printf(xdma->dev, 134 "%s: Can't free real hw channel.\n", __func__); 135 XDMA_UNLOCK(); 136 return (-1); 137 } 138 139 xdma_teardown_all_intr(xchan); 140 141 /* Deallocate descriptors, if any. */ 142 xdma_desc_free(xchan); 143 144 mtx_destroy(&xchan->mtx_lock); 145 146 TAILQ_REMOVE(&xdma->channels, xchan, xchan_next); 147 148 free(xchan, M_XDMA); 149 150 XDMA_UNLOCK(); 151 152 return (0); 153 } 154 155 int 156 xdma_setup_intr(xdma_channel_t *xchan, int (*cb)(void *), void *arg, 157 void **ihandler) 158 { 159 struct xdma_intr_handler *ih; 160 xdma_controller_t *xdma; 161 162 xdma = xchan->xdma; 163 KASSERT(xdma != NULL, ("xdma is NULL")); 164 165 /* Sanity check. */ 166 if (cb == NULL) { 167 device_printf(xdma->dev, 168 "%s: Can't setup interrupt handler.\n", 169 __func__); 170 171 return (-1); 172 } 173 174 ih = malloc(sizeof(struct xdma_intr_handler), 175 M_XDMA, M_WAITOK | M_ZERO); 176 if (ih == NULL) { 177 device_printf(xdma->dev, 178 "%s: Can't allocate memory for interrupt handler.\n", 179 __func__); 180 181 return (-1); 182 } 183 184 ih->cb = cb; 185 ih->cb_user = arg; 186 187 TAILQ_INSERT_TAIL(&xchan->ie_handlers, ih, ih_next); 188 189 if (ihandler != NULL) { 190 *ihandler = ih; 191 } 192 193 return (0); 194 } 195 196 int 197 xdma_teardown_intr(xdma_channel_t *xchan, struct xdma_intr_handler *ih) 198 { 199 xdma_controller_t *xdma; 200 201 xdma = xchan->xdma; 202 KASSERT(xdma != NULL, ("xdma is NULL")); 203 204 /* Sanity check. */ 205 if (ih == NULL) { 206 device_printf(xdma->dev, 207 "%s: Can't teardown interrupt.\n", __func__); 208 return (-1); 209 } 210 211 TAILQ_REMOVE(&xchan->ie_handlers, ih, ih_next); 212 free(ih, M_XDMA); 213 214 return (0); 215 } 216 217 int 218 xdma_teardown_all_intr(xdma_channel_t *xchan) 219 { 220 struct xdma_intr_handler *ih_tmp; 221 struct xdma_intr_handler *ih; 222 xdma_controller_t *xdma; 223 224 xdma = xchan->xdma; 225 KASSERT(xdma != NULL, ("xdma is NULL")); 226 227 TAILQ_FOREACH_SAFE(ih, &xchan->ie_handlers, ih_next, ih_tmp) { 228 TAILQ_REMOVE(&xchan->ie_handlers, ih, ih_next); 229 free(ih, M_XDMA); 230 } 231 232 return (0); 233 } 234 235 static void 236 xdma_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int err) 237 { 238 xdma_channel_t *xchan; 239 int i; 240 241 xchan = (xdma_channel_t *)arg; 242 KASSERT(xchan != NULL, ("xchan is NULL")); 243 244 if (err) { 245 xchan->map_err = 1; 246 return; 247 } 248 249 for (i = 0; i < nseg; i++) { 250 xchan->descs_phys[i].ds_addr = segs[i].ds_addr; 251 xchan->descs_phys[i].ds_len = segs[i].ds_len; 252 } 253 } 254 255 static int 256 xdma_desc_alloc_bus_dma(xdma_channel_t *xchan, uint32_t desc_size, 257 uint32_t align) 258 { 259 xdma_controller_t *xdma; 260 bus_size_t all_desc_sz; 261 xdma_config_t *conf; 262 int nsegments; 263 int err; 264 265 xdma = xchan->xdma; 266 conf = &xchan->conf; 267 268 nsegments = conf->block_num; 269 all_desc_sz = (nsegments * desc_size); 270 271 err = bus_dma_tag_create( 272 bus_get_dma_tag(xdma->dev), 273 align, desc_size, /* alignment, boundary */ 274 BUS_SPACE_MAXADDR_32BIT, /* lowaddr */ 275 BUS_SPACE_MAXADDR, /* highaddr */ 276 NULL, NULL, /* filter, filterarg */ 277 all_desc_sz, nsegments, /* maxsize, nsegments*/ 278 desc_size, 0, /* maxsegsize, flags */ 279 NULL, NULL, /* lockfunc, lockarg */ 280 &xchan->dma_tag); 281 if (err) { 282 device_printf(xdma->dev, 283 "%s: Can't create bus_dma tag.\n", __func__); 284 return (-1); 285 } 286 287 err = bus_dmamem_alloc(xchan->dma_tag, (void **)&xchan->descs, 288 BUS_DMA_WAITOK | BUS_DMA_COHERENT, &xchan->dma_map); 289 if (err) { 290 device_printf(xdma->dev, 291 "%s: Can't allocate memory for descriptors.\n", __func__); 292 return (-1); 293 } 294 295 xchan->descs_phys = malloc(nsegments * sizeof(xdma_descriptor_t), M_XDMA, 296 (M_WAITOK | M_ZERO)); 297 298 xchan->map_err = 0; 299 err = bus_dmamap_load(xchan->dma_tag, xchan->dma_map, xchan->descs, 300 all_desc_sz, xdma_dmamap_cb, xchan, BUS_DMA_WAITOK); 301 if (err) { 302 device_printf(xdma->dev, 303 "%s: Can't load DMA map.\n", __func__); 304 return (-1); 305 } 306 307 if (xchan->map_err != 0) { 308 device_printf(xdma->dev, 309 "%s: Can't load DMA map.\n", __func__); 310 return (-1); 311 } 312 313 return (0); 314 } 315 316 /* 317 * This function called by DMA controller driver. 318 */ 319 int 320 xdma_desc_alloc(xdma_channel_t *xchan, uint32_t desc_size, uint32_t align) 321 { 322 xdma_controller_t *xdma; 323 xdma_config_t *conf; 324 int ret; 325 326 XCHAN_ASSERT_LOCKED(xchan); 327 328 xdma = xchan->xdma; 329 if (xdma == NULL) { 330 device_printf(xdma->dev, 331 "%s: Channel was not allocated properly.\n", __func__); 332 return (-1); 333 } 334 335 if (xchan->flags & XCHAN_DESC_ALLOCATED) { 336 device_printf(xdma->dev, 337 "%s: Descriptors already allocated.\n", __func__); 338 return (-1); 339 } 340 341 if ((xchan->flags & XCHAN_CONFIGURED) == 0) { 342 device_printf(xdma->dev, 343 "%s: Channel has no configuration.\n", __func__); 344 return (-1); 345 } 346 347 conf = &xchan->conf; 348 349 XCHAN_UNLOCK(xchan); 350 ret = xdma_desc_alloc_bus_dma(xchan, desc_size, align); 351 XCHAN_LOCK(xchan); 352 if (ret != 0) { 353 device_printf(xdma->dev, 354 "%s: Can't allocate memory for descriptors.\n", 355 __func__); 356 return (-1); 357 } 358 359 xchan->flags |= XCHAN_DESC_ALLOCATED; 360 361 /* We are going to write to descriptors. */ 362 bus_dmamap_sync(xchan->dma_tag, xchan->dma_map, BUS_DMASYNC_PREWRITE); 363 364 return (0); 365 } 366 367 int 368 xdma_desc_free(xdma_channel_t *xchan) 369 { 370 371 if ((xchan->flags & XCHAN_DESC_ALLOCATED) == 0) { 372 /* No descriptors allocated. */ 373 return (-1); 374 } 375 376 bus_dmamap_unload(xchan->dma_tag, xchan->dma_map); 377 bus_dmamem_free(xchan->dma_tag, xchan->descs, xchan->dma_map); 378 bus_dma_tag_destroy(xchan->dma_tag); 379 free(xchan->descs_phys, M_XDMA); 380 381 xchan->flags &= ~(XCHAN_DESC_ALLOCATED); 382 383 return (0); 384 } 385 386 int 387 xdma_prep_memcpy(xdma_channel_t *xchan, uintptr_t src_addr, 388 uintptr_t dst_addr, size_t len) 389 { 390 xdma_controller_t *xdma; 391 xdma_config_t *conf; 392 int ret; 393 394 xdma = xchan->xdma; 395 KASSERT(xdma != NULL, ("xdma is NULL")); 396 397 conf = &xchan->conf; 398 conf->direction = XDMA_MEM_TO_MEM; 399 conf->src_addr = src_addr; 400 conf->dst_addr = dst_addr; 401 conf->block_len = len; 402 conf->block_num = 1; 403 404 xchan->flags |= (XCHAN_CONFIGURED | XCHAN_TYPE_MEMCPY); 405 406 XCHAN_LOCK(xchan); 407 408 /* Deallocate old descriptors, if any. */ 409 xdma_desc_free(xchan); 410 411 ret = XDMA_CHANNEL_PREP_MEMCPY(xdma->dma_dev, xchan); 412 if (ret != 0) { 413 device_printf(xdma->dev, 414 "%s: Can't prepare memcpy transfer.\n", __func__); 415 XCHAN_UNLOCK(xchan); 416 417 return (-1); 418 } 419 420 if (xchan->flags & XCHAN_DESC_ALLOCATED) { 421 /* Driver created xDMA descriptors. */ 422 bus_dmamap_sync(xchan->dma_tag, xchan->dma_map, 423 BUS_DMASYNC_POSTWRITE); 424 } 425 426 XCHAN_UNLOCK(xchan); 427 428 return (0); 429 } 430 431 int 432 xdma_prep_cyclic(xdma_channel_t *xchan, enum xdma_direction dir, 433 uintptr_t src_addr, uintptr_t dst_addr, int block_len, 434 int block_num, int src_width, int dst_width) 435 { 436 xdma_controller_t *xdma; 437 xdma_config_t *conf; 438 int ret; 439 440 xdma = xchan->xdma; 441 KASSERT(xdma != NULL, ("xdma is NULL")); 442 443 conf = &xchan->conf; 444 conf->direction = dir; 445 conf->src_addr = src_addr; 446 conf->dst_addr = dst_addr; 447 conf->block_len = block_len; 448 conf->block_num = block_num; 449 conf->src_width = src_width; 450 conf->dst_width = dst_width; 451 452 xchan->flags |= (XCHAN_CONFIGURED | XCHAN_TYPE_CYCLIC); 453 454 XCHAN_LOCK(xchan); 455 456 /* Deallocate old descriptors, if any. */ 457 xdma_desc_free(xchan); 458 459 ret = XDMA_CHANNEL_PREP_CYCLIC(xdma->dma_dev, xchan); 460 if (ret != 0) { 461 device_printf(xdma->dev, 462 "%s: Can't prepare cyclic transfer.\n", __func__); 463 XCHAN_UNLOCK(xchan); 464 465 return (-1); 466 } 467 468 if (xchan->flags & XCHAN_DESC_ALLOCATED) { 469 /* Driver has created xDMA descriptors. */ 470 bus_dmamap_sync(xchan->dma_tag, xchan->dma_map, 471 BUS_DMASYNC_POSTWRITE); 472 } 473 474 XCHAN_UNLOCK(xchan); 475 476 return (0); 477 } 478 479 int 480 xdma_begin(xdma_channel_t *xchan) 481 { 482 xdma_controller_t *xdma; 483 int ret; 484 485 xdma = xchan->xdma; 486 487 ret = XDMA_CHANNEL_CONTROL(xdma->dma_dev, xchan, XDMA_CMD_BEGIN); 488 if (ret != 0) { 489 device_printf(xdma->dev, 490 "%s: Can't begin the channel operation.\n", __func__); 491 return (-1); 492 } 493 494 return (0); 495 } 496 497 int 498 xdma_terminate(xdma_channel_t *xchan) 499 { 500 xdma_controller_t *xdma; 501 int ret; 502 503 xdma = xchan->xdma; 504 505 ret = XDMA_CHANNEL_CONTROL(xdma->dma_dev, xchan, XDMA_CMD_TERMINATE); 506 if (ret != 0) { 507 device_printf(xdma->dev, 508 "%s: Can't terminate the channel operation.\n", __func__); 509 return (-1); 510 } 511 512 return (0); 513 } 514 515 int 516 xdma_pause(xdma_channel_t *xchan) 517 { 518 xdma_controller_t *xdma; 519 int ret; 520 521 xdma = xchan->xdma; 522 523 ret = XDMA_CHANNEL_CONTROL(xdma->dma_dev, xchan, XDMA_CMD_PAUSE); 524 if (ret != 0) { 525 device_printf(xdma->dev, 526 "%s: Can't pause the channel operation.\n", __func__); 527 return (-1); 528 } 529 530 return (ret); 531 } 532 533 int 534 xdma_callback(xdma_channel_t *xchan) 535 { 536 struct xdma_intr_handler *ih_tmp; 537 struct xdma_intr_handler *ih; 538 539 TAILQ_FOREACH_SAFE(ih, &xchan->ie_handlers, ih_next, ih_tmp) { 540 if (ih->cb != NULL) { 541 ih->cb(ih->cb_user); 542 } 543 } 544 545 return (0); 546 } 547 548 void 549 xdma_assert_locked(void) 550 { 551 552 XDMA_ASSERT_LOCKED(); 553 } 554 555 #ifdef FDT 556 /* 557 * Notify the DMA driver we have machine-dependent data in FDT. 558 */ 559 static int 560 xdma_ofw_md_data(xdma_controller_t *xdma, pcell_t *cells, int ncells) 561 { 562 uint32_t ret; 563 564 ret = XDMA_OFW_MD_DATA(xdma->dma_dev, cells, ncells, (void **)&xdma->data); 565 566 return (ret); 567 } 568 569 /* 570 * Allocate xdma controller. 571 */ 572 xdma_controller_t * 573 xdma_ofw_get(device_t dev, const char *prop) 574 { 575 phandle_t node, parent; 576 xdma_controller_t *xdma; 577 device_t dma_dev; 578 pcell_t *cells; 579 int ncells; 580 int error; 581 int ndmas; 582 int idx; 583 584 node = ofw_bus_get_node(dev); 585 if (node <= 0) { 586 device_printf(dev, 587 "%s called on not ofw based device.\n", __func__); 588 } 589 590 error = ofw_bus_parse_xref_list_get_length(node, 591 "dmas", "#dma-cells", &ndmas); 592 if (error) { 593 device_printf(dev, 594 "%s can't get dmas list.\n", __func__); 595 return (NULL); 596 } 597 598 if (ndmas == 0) { 599 device_printf(dev, 600 "%s dmas list is empty.\n", __func__); 601 return (NULL); 602 } 603 604 error = ofw_bus_find_string_index(node, "dma-names", prop, &idx); 605 if (error != 0) { 606 device_printf(dev, 607 "%s can't find string index.\n", __func__); 608 return (NULL); 609 } 610 611 error = ofw_bus_parse_xref_list_alloc(node, "dmas", "#dma-cells", 612 idx, &parent, &ncells, &cells); 613 if (error != 0) { 614 device_printf(dev, 615 "%s can't get dma device xref.\n", __func__); 616 return (NULL); 617 } 618 619 dma_dev = OF_device_from_xref(parent); 620 if (dma_dev == NULL) { 621 device_printf(dev, 622 "%s can't get dma device.\n", __func__); 623 return (NULL); 624 } 625 626 xdma = malloc(sizeof(struct xdma_controller), M_XDMA, M_WAITOK | M_ZERO); 627 if (xdma == NULL) { 628 device_printf(dev, 629 "%s can't allocate memory for xdma.\n", __func__); 630 return (NULL); 631 } 632 xdma->dev = dev; 633 xdma->dma_dev = dma_dev; 634 635 TAILQ_INIT(&xdma->channels); 636 637 xdma_ofw_md_data(xdma, cells, ncells); 638 free(cells, M_OFWPROP); 639 640 return (xdma); 641 } 642 #endif 643 644 /* 645 * Free xDMA controller object. 646 */ 647 int 648 xdma_put(xdma_controller_t *xdma) 649 { 650 651 XDMA_LOCK(); 652 653 /* Ensure no channels allocated. */ 654 if (!TAILQ_EMPTY(&xdma->channels)) { 655 device_printf(xdma->dev, "%s: Can't free xDMA\n", __func__); 656 return (-1); 657 } 658 659 free(xdma->data, M_DEVBUF); 660 free(xdma, M_XDMA); 661 662 XDMA_UNLOCK(); 663 664 return (0); 665 } 666 667 static void 668 xdma_init(void) 669 { 670 671 mtx_init(&xdma_mtx, "xDMA", NULL, MTX_DEF); 672 } 673 674 SYSINIT(xdma, SI_SUB_DRIVERS, SI_ORDER_FIRST, xdma_init, NULL); 675