1 /*- 2 * Copyright (c) 2016 Ruslan Bukin <br@bsdpad.com> 3 * All rights reserved. 4 * 5 * This software was developed by SRI International and the University of 6 * Cambridge Computer Laboratory under DARPA/AFRL contract FA8750-10-C-0237 7 * ("CTSRD"), as part of the DARPA CRASH research programme. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * SUCH DAMAGE. 29 */ 30 31 #include <sys/cdefs.h> 32 __FBSDID("$FreeBSD$"); 33 34 #include "opt_platform.h" 35 #include <sys/param.h> 36 #include <sys/conf.h> 37 #include <sys/bus.h> 38 #include <sys/kernel.h> 39 #include <sys/queue.h> 40 #include <sys/kobj.h> 41 #include <sys/malloc.h> 42 #include <sys/mutex.h> 43 #include <sys/limits.h> 44 #include <sys/lock.h> 45 #include <sys/sysctl.h> 46 #include <sys/systm.h> 47 #include <sys/sx.h> 48 49 #include <machine/bus.h> 50 51 #ifdef FDT 52 #include <dev/fdt/fdt_common.h> 53 #include <dev/ofw/ofw_bus.h> 54 #include <dev/ofw/ofw_bus_subr.h> 55 #endif 56 57 #include <dev/xdma/xdma.h> 58 59 #include <xdma_if.h> 60 61 MALLOC_DEFINE(M_XDMA, "xdma", "xDMA framework"); 62 63 /* 64 * Multiple xDMA controllers may work with single DMA device, 65 * so we have global lock for physical channel management. 66 */ 67 static struct mtx xdma_mtx; 68 #define XDMA_LOCK() mtx_lock(&xdma_mtx) 69 #define XDMA_UNLOCK() mtx_unlock(&xdma_mtx) 70 #define XDMA_ASSERT_LOCKED() mtx_assert(&xdma_mtx, MA_OWNED) 71 72 /* 73 * Per channel locks. 74 */ 75 #define XCHAN_LOCK(xchan) mtx_lock(&(xchan)->mtx_lock) 76 #define XCHAN_UNLOCK(xchan) mtx_unlock(&(xchan)->mtx_lock) 77 #define XCHAN_ASSERT_LOCKED(xchan) mtx_assert(&(xchan)->mtx_lock, MA_OWNED) 78 79 /* 80 * Allocate virtual xDMA channel. 81 */ 82 xdma_channel_t * 83 xdma_channel_alloc(xdma_controller_t *xdma) 84 { 85 xdma_channel_t *xchan; 86 int ret; 87 88 xchan = malloc(sizeof(xdma_channel_t), M_XDMA, M_WAITOK | M_ZERO); 89 if (xchan == NULL) { 90 device_printf(xdma->dev, 91 "%s: Can't allocate memory for channel.\n", __func__); 92 return (NULL); 93 } 94 xchan->xdma = xdma; 95 96 XDMA_LOCK(); 97 98 /* Request a real channel from hardware driver. */ 99 ret = XDMA_CHANNEL_ALLOC(xdma->dma_dev, xchan); 100 if (ret != 0) { 101 device_printf(xdma->dev, 102 "%s: Can't request hardware channel.\n", __func__); 103 XDMA_UNLOCK(); 104 free(xchan, M_XDMA); 105 106 return (NULL); 107 } 108 109 TAILQ_INIT(&xchan->ie_handlers); 110 mtx_init(&xchan->mtx_lock, "xDMA", NULL, MTX_DEF); 111 112 TAILQ_INSERT_TAIL(&xdma->channels, xchan, xchan_next); 113 114 XDMA_UNLOCK(); 115 116 return (xchan); 117 } 118 119 int 120 xdma_channel_free(xdma_channel_t *xchan) 121 { 122 xdma_controller_t *xdma; 123 int err; 124 125 xdma = xchan->xdma; 126 127 XDMA_LOCK(); 128 129 /* Free the real DMA channel. */ 130 err = XDMA_CHANNEL_FREE(xdma->dma_dev, xchan); 131 if (err != 0) { 132 device_printf(xdma->dev, 133 "%s: Can't free real hw channel.\n", __func__); 134 XDMA_UNLOCK(); 135 return (-1); 136 } 137 138 xdma_teardown_all_intr(xchan); 139 140 /* Deallocate descriptors, if any. */ 141 xdma_desc_free(xchan); 142 143 mtx_destroy(&xchan->mtx_lock); 144 145 TAILQ_REMOVE(&xdma->channels, xchan, xchan_next); 146 147 free(xchan, M_XDMA); 148 149 XDMA_UNLOCK(); 150 151 return (0); 152 } 153 154 int 155 xdma_setup_intr(xdma_channel_t *xchan, int (*cb)(void *), void *arg, 156 void **ihandler) 157 { 158 struct xdma_intr_handler *ih; 159 xdma_controller_t *xdma; 160 161 xdma = xchan->xdma; 162 KASSERT(xdma != NULL, ("xdma is NULL")); 163 164 /* Sanity check. */ 165 if (cb == NULL) { 166 device_printf(xdma->dev, 167 "%s: Can't setup interrupt handler.\n", 168 __func__); 169 170 return (-1); 171 } 172 173 ih = malloc(sizeof(struct xdma_intr_handler), 174 M_XDMA, M_WAITOK | M_ZERO); 175 if (ih == NULL) { 176 device_printf(xdma->dev, 177 "%s: Can't allocate memory for interrupt handler.\n", 178 __func__); 179 180 return (-1); 181 } 182 183 ih->cb = cb; 184 ih->cb_user = arg; 185 186 TAILQ_INSERT_TAIL(&xchan->ie_handlers, ih, ih_next); 187 188 if (ihandler != NULL) { 189 *ihandler = ih; 190 } 191 192 return (0); 193 } 194 195 int 196 xdma_teardown_intr(xdma_channel_t *xchan, struct xdma_intr_handler *ih) 197 { 198 xdma_controller_t *xdma; 199 200 xdma = xchan->xdma; 201 KASSERT(xdma != NULL, ("xdma is NULL")); 202 203 /* Sanity check. */ 204 if (ih == NULL) { 205 device_printf(xdma->dev, 206 "%s: Can't teardown interrupt.\n", __func__); 207 return (-1); 208 } 209 210 TAILQ_REMOVE(&xchan->ie_handlers, ih, ih_next); 211 free(ih, M_XDMA); 212 213 return (0); 214 } 215 216 int 217 xdma_teardown_all_intr(xdma_channel_t *xchan) 218 { 219 struct xdma_intr_handler *ih_tmp; 220 struct xdma_intr_handler *ih; 221 xdma_controller_t *xdma; 222 223 xdma = xchan->xdma; 224 KASSERT(xdma != NULL, ("xdma is NULL")); 225 226 TAILQ_FOREACH_SAFE(ih, &xchan->ie_handlers, ih_next, ih_tmp) { 227 TAILQ_REMOVE(&xchan->ie_handlers, ih, ih_next); 228 free(ih, M_XDMA); 229 } 230 231 return (0); 232 } 233 234 static void 235 xdma_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int err) 236 { 237 xdma_channel_t *xchan; 238 int i; 239 240 xchan = (xdma_channel_t *)arg; 241 KASSERT(xchan != NULL, ("xchan is NULL")); 242 243 if (err) { 244 xchan->map_err = 1; 245 return; 246 } 247 248 for (i = 0; i < nseg; i++) { 249 xchan->descs_phys[i].ds_addr = segs[i].ds_addr; 250 xchan->descs_phys[i].ds_len = segs[i].ds_len; 251 } 252 } 253 254 static int 255 xdma_desc_alloc_bus_dma(xdma_channel_t *xchan, uint32_t desc_size, 256 uint32_t align) 257 { 258 xdma_controller_t *xdma; 259 bus_size_t all_desc_sz; 260 xdma_config_t *conf; 261 int nsegments; 262 int err; 263 264 xdma = xchan->xdma; 265 conf = &xchan->conf; 266 267 nsegments = conf->block_num; 268 all_desc_sz = (nsegments * desc_size); 269 270 err = bus_dma_tag_create( 271 bus_get_dma_tag(xdma->dev), 272 align, desc_size, /* alignment, boundary */ 273 BUS_SPACE_MAXADDR_32BIT, /* lowaddr */ 274 BUS_SPACE_MAXADDR, /* highaddr */ 275 NULL, NULL, /* filter, filterarg */ 276 all_desc_sz, nsegments, /* maxsize, nsegments*/ 277 desc_size, 0, /* maxsegsize, flags */ 278 NULL, NULL, /* lockfunc, lockarg */ 279 &xchan->dma_tag); 280 if (err) { 281 device_printf(xdma->dev, 282 "%s: Can't create bus_dma tag.\n", __func__); 283 return (-1); 284 } 285 286 err = bus_dmamem_alloc(xchan->dma_tag, (void **)&xchan->descs, 287 BUS_DMA_WAITOK | BUS_DMA_COHERENT, &xchan->dma_map); 288 if (err) { 289 device_printf(xdma->dev, 290 "%s: Can't allocate memory for descriptors.\n", __func__); 291 return (-1); 292 } 293 294 xchan->descs_phys = malloc(nsegments * sizeof(xdma_descriptor_t), M_XDMA, 295 (M_WAITOK | M_ZERO)); 296 297 xchan->map_err = 0; 298 err = bus_dmamap_load(xchan->dma_tag, xchan->dma_map, xchan->descs, 299 all_desc_sz, xdma_dmamap_cb, xchan, BUS_DMA_WAITOK); 300 if (err) { 301 device_printf(xdma->dev, 302 "%s: Can't load DMA map.\n", __func__); 303 return (-1); 304 } 305 306 if (xchan->map_err != 0) { 307 device_printf(xdma->dev, 308 "%s: Can't load DMA map.\n", __func__); 309 return (-1); 310 } 311 312 return (0); 313 } 314 315 /* 316 * This function called by DMA controller driver. 317 */ 318 int 319 xdma_desc_alloc(xdma_channel_t *xchan, uint32_t desc_size, uint32_t align) 320 { 321 xdma_controller_t *xdma; 322 xdma_config_t *conf; 323 int ret; 324 325 XCHAN_ASSERT_LOCKED(xchan); 326 327 xdma = xchan->xdma; 328 if (xdma == NULL) { 329 device_printf(xdma->dev, 330 "%s: Channel was not allocated properly.\n", __func__); 331 return (-1); 332 } 333 334 if (xchan->flags & XCHAN_DESC_ALLOCATED) { 335 device_printf(xdma->dev, 336 "%s: Descriptors already allocated.\n", __func__); 337 return (-1); 338 } 339 340 if ((xchan->flags & XCHAN_CONFIGURED) == 0) { 341 device_printf(xdma->dev, 342 "%s: Channel has no configuration.\n", __func__); 343 return (-1); 344 } 345 346 conf = &xchan->conf; 347 348 XCHAN_UNLOCK(xchan); 349 ret = xdma_desc_alloc_bus_dma(xchan, desc_size, align); 350 XCHAN_LOCK(xchan); 351 if (ret != 0) { 352 device_printf(xdma->dev, 353 "%s: Can't allocate memory for descriptors.\n", 354 __func__); 355 return (-1); 356 } 357 358 xchan->flags |= XCHAN_DESC_ALLOCATED; 359 360 /* We are going to write to descriptors. */ 361 bus_dmamap_sync(xchan->dma_tag, xchan->dma_map, BUS_DMASYNC_PREWRITE); 362 363 return (0); 364 } 365 366 int 367 xdma_desc_free(xdma_channel_t *xchan) 368 { 369 370 if ((xchan->flags & XCHAN_DESC_ALLOCATED) == 0) { 371 /* No descriptors allocated. */ 372 return (-1); 373 } 374 375 bus_dmamap_unload(xchan->dma_tag, xchan->dma_map); 376 bus_dmamem_free(xchan->dma_tag, xchan->descs, xchan->dma_map); 377 bus_dma_tag_destroy(xchan->dma_tag); 378 free(xchan->descs_phys, M_XDMA); 379 380 xchan->flags &= ~(XCHAN_DESC_ALLOCATED); 381 382 return (0); 383 } 384 385 int 386 xdma_prep_memcpy(xdma_channel_t *xchan, uintptr_t src_addr, 387 uintptr_t dst_addr, size_t len) 388 { 389 xdma_controller_t *xdma; 390 xdma_config_t *conf; 391 int ret; 392 393 xdma = xchan->xdma; 394 KASSERT(xdma != NULL, ("xdma is NULL")); 395 396 conf = &xchan->conf; 397 conf->direction = XDMA_MEM_TO_MEM; 398 conf->src_addr = src_addr; 399 conf->dst_addr = dst_addr; 400 conf->block_len = len; 401 conf->block_num = 1; 402 403 xchan->flags |= (XCHAN_CONFIGURED | XCHAN_TYPE_MEMCPY); 404 405 XCHAN_LOCK(xchan); 406 407 /* Deallocate old descriptors, if any. */ 408 xdma_desc_free(xchan); 409 410 ret = XDMA_CHANNEL_PREP_MEMCPY(xdma->dma_dev, xchan); 411 if (ret != 0) { 412 device_printf(xdma->dev, 413 "%s: Can't prepare memcpy transfer.\n", __func__); 414 XCHAN_UNLOCK(xchan); 415 416 return (-1); 417 } 418 419 if (xchan->flags & XCHAN_DESC_ALLOCATED) { 420 /* Driver created xDMA descriptors. */ 421 bus_dmamap_sync(xchan->dma_tag, xchan->dma_map, 422 BUS_DMASYNC_POSTWRITE); 423 } 424 425 XCHAN_UNLOCK(xchan); 426 427 return (0); 428 } 429 430 int 431 xdma_prep_cyclic(xdma_channel_t *xchan, enum xdma_direction dir, 432 uintptr_t src_addr, uintptr_t dst_addr, int block_len, 433 int block_num, int src_width, int dst_width) 434 { 435 xdma_controller_t *xdma; 436 xdma_config_t *conf; 437 int ret; 438 439 xdma = xchan->xdma; 440 KASSERT(xdma != NULL, ("xdma is NULL")); 441 442 conf = &xchan->conf; 443 conf->direction = dir; 444 conf->src_addr = src_addr; 445 conf->dst_addr = dst_addr; 446 conf->block_len = block_len; 447 conf->block_num = block_num; 448 conf->src_width = src_width; 449 conf->dst_width = dst_width; 450 451 xchan->flags |= (XCHAN_CONFIGURED | XCHAN_TYPE_CYCLIC); 452 453 XCHAN_LOCK(xchan); 454 455 /* Deallocate old descriptors, if any. */ 456 xdma_desc_free(xchan); 457 458 ret = XDMA_CHANNEL_PREP_CYCLIC(xdma->dma_dev, xchan); 459 if (ret != 0) { 460 device_printf(xdma->dev, 461 "%s: Can't prepare cyclic transfer.\n", __func__); 462 XCHAN_UNLOCK(xchan); 463 464 return (-1); 465 } 466 467 if (xchan->flags & XCHAN_DESC_ALLOCATED) { 468 /* Driver has created xDMA descriptors. */ 469 bus_dmamap_sync(xchan->dma_tag, xchan->dma_map, 470 BUS_DMASYNC_POSTWRITE); 471 } 472 473 XCHAN_UNLOCK(xchan); 474 475 return (0); 476 } 477 478 int 479 xdma_begin(xdma_channel_t *xchan) 480 { 481 xdma_controller_t *xdma; 482 int ret; 483 484 xdma = xchan->xdma; 485 486 ret = XDMA_CHANNEL_CONTROL(xdma->dma_dev, xchan, XDMA_CMD_BEGIN); 487 if (ret != 0) { 488 device_printf(xdma->dev, 489 "%s: Can't begin the channel operation.\n", __func__); 490 return (-1); 491 } 492 493 return (0); 494 } 495 496 int 497 xdma_terminate(xdma_channel_t *xchan) 498 { 499 xdma_controller_t *xdma; 500 int ret; 501 502 xdma = xchan->xdma; 503 504 ret = XDMA_CHANNEL_CONTROL(xdma->dma_dev, xchan, XDMA_CMD_TERMINATE); 505 if (ret != 0) { 506 device_printf(xdma->dev, 507 "%s: Can't terminate the channel operation.\n", __func__); 508 return (-1); 509 } 510 511 return (0); 512 } 513 514 int 515 xdma_pause(xdma_channel_t *xchan) 516 { 517 xdma_controller_t *xdma; 518 int ret; 519 520 xdma = xchan->xdma; 521 522 ret = XDMA_CHANNEL_CONTROL(xdma->dma_dev, xchan, XDMA_CMD_PAUSE); 523 if (ret != 0) { 524 device_printf(xdma->dev, 525 "%s: Can't pause the channel operation.\n", __func__); 526 return (-1); 527 } 528 529 return (ret); 530 } 531 532 int 533 xdma_callback(xdma_channel_t *xchan) 534 { 535 struct xdma_intr_handler *ih_tmp; 536 struct xdma_intr_handler *ih; 537 538 TAILQ_FOREACH_SAFE(ih, &xchan->ie_handlers, ih_next, ih_tmp) { 539 if (ih->cb != NULL) { 540 ih->cb(ih->cb_user); 541 } 542 } 543 544 return (0); 545 } 546 547 void 548 xdma_assert_locked(void) 549 { 550 551 XDMA_ASSERT_LOCKED(); 552 } 553 554 #ifdef FDT 555 /* 556 * Notify the DMA driver we have machine-dependent data in FDT. 557 */ 558 static int 559 xdma_ofw_md_data(xdma_controller_t *xdma, pcell_t *cells, int ncells) 560 { 561 uint32_t ret; 562 563 ret = XDMA_OFW_MD_DATA(xdma->dma_dev, cells, ncells, (void **)&xdma->data); 564 565 return (ret); 566 } 567 568 /* 569 * Allocate xdma controller. 570 */ 571 xdma_controller_t * 572 xdma_ofw_get(device_t dev, const char *prop) 573 { 574 phandle_t node, parent; 575 xdma_controller_t *xdma; 576 device_t dma_dev; 577 pcell_t *cells; 578 int ncells; 579 int error; 580 int ndmas; 581 int idx; 582 583 node = ofw_bus_get_node(dev); 584 if (node <= 0) { 585 device_printf(dev, 586 "%s called on not ofw based device.\n", __func__); 587 } 588 589 error = ofw_bus_parse_xref_list_get_length(node, 590 "dmas", "#dma-cells", &ndmas); 591 if (error) { 592 device_printf(dev, 593 "%s can't get dmas list.\n", __func__); 594 return (NULL); 595 } 596 597 if (ndmas == 0) { 598 device_printf(dev, 599 "%s dmas list is empty.\n", __func__); 600 return (NULL); 601 } 602 603 error = ofw_bus_find_string_index(node, "dma-names", prop, &idx); 604 if (error != 0) { 605 device_printf(dev, 606 "%s can't find string index.\n", __func__); 607 return (NULL); 608 } 609 610 error = ofw_bus_parse_xref_list_alloc(node, "dmas", "#dma-cells", 611 idx, &parent, &ncells, &cells); 612 if (error != 0) { 613 device_printf(dev, 614 "%s can't get dma device xref.\n", __func__); 615 return (NULL); 616 } 617 618 dma_dev = OF_device_from_xref(parent); 619 if (dma_dev == NULL) { 620 device_printf(dev, 621 "%s can't get dma device.\n", __func__); 622 return (NULL); 623 } 624 625 xdma = malloc(sizeof(struct xdma_controller), M_XDMA, M_WAITOK | M_ZERO); 626 if (xdma == NULL) { 627 device_printf(dev, 628 "%s can't allocate memory for xdma.\n", __func__); 629 return (NULL); 630 } 631 xdma->dev = dev; 632 xdma->dma_dev = dma_dev; 633 634 TAILQ_INIT(&xdma->channels); 635 636 xdma_ofw_md_data(xdma, cells, ncells); 637 free(cells, M_OFWPROP); 638 639 return (xdma); 640 } 641 #endif 642 643 /* 644 * Free xDMA controller object. 645 */ 646 int 647 xdma_put(xdma_controller_t *xdma) 648 { 649 650 XDMA_LOCK(); 651 652 /* Ensure no channels allocated. */ 653 if (!TAILQ_EMPTY(&xdma->channels)) { 654 device_printf(xdma->dev, "%s: Can't free xDMA\n", __func__); 655 return (-1); 656 } 657 658 free(xdma->data, M_DEVBUF); 659 free(xdma, M_XDMA); 660 661 XDMA_UNLOCK(); 662 663 return (0); 664 } 665 666 static void 667 xdma_init(void) 668 { 669 670 mtx_init(&xdma_mtx, "xDMA", NULL, MTX_DEF); 671 } 672 673 SYSINIT(xdma, SI_SUB_DRIVERS, SI_ORDER_FIRST, xdma_init, NULL); 674