1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2016-2019 Ruslan Bukin <br@bsdpad.com> 5 * 6 * This software was developed by SRI International and the University of 7 * Cambridge Computer Laboratory under DARPA/AFRL contract FA8750-10-C-0237 8 * ("CTSRD"), as part of the DARPA CRASH research programme. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 29 * SUCH DAMAGE. 30 */ 31 32 #include <sys/cdefs.h> 33 __FBSDID("$FreeBSD$"); 34 35 #include "opt_platform.h" 36 #include <sys/param.h> 37 #include <sys/conf.h> 38 #include <sys/bus.h> 39 #include <sys/epoch.h> 40 #include <sys/kernel.h> 41 #include <sys/queue.h> 42 #include <sys/kobj.h> 43 #include <sys/malloc.h> 44 #include <sys/limits.h> 45 #include <sys/lock.h> 46 #include <sys/mutex.h> 47 #include <sys/sysctl.h> 48 #include <sys/systm.h> 49 50 #include <machine/bus.h> 51 52 #ifdef FDT 53 #include <dev/fdt/fdt_common.h> 54 #include <dev/ofw/ofw_bus.h> 55 #include <dev/ofw/ofw_bus_subr.h> 56 #endif 57 58 #include <dev/xdma/xdma.h> 59 60 #include <xdma_if.h> 61 62 /* 63 * Multiple xDMA controllers may work with single DMA device, 64 * so we have global lock for physical channel management. 65 */ 66 static struct mtx xdma_mtx; 67 68 #define XDMA_LOCK() mtx_lock(&xdma_mtx) 69 #define XDMA_UNLOCK() mtx_unlock(&xdma_mtx) 70 #define XDMA_ASSERT_LOCKED() mtx_assert(&xdma_mtx, MA_OWNED) 71 72 #define FDT_REG_CELLS 4 73 74 #ifdef FDT 75 static int 76 xdma_get_iommu_fdt(xdma_controller_t *xdma, xdma_channel_t *xchan) 77 { 78 struct xdma_iommu *xio; 79 phandle_t node; 80 pcell_t prop; 81 size_t len; 82 83 node = ofw_bus_get_node(xdma->dma_dev); 84 if (OF_getproplen(node, "xdma,iommu") <= 0) 85 return (0); 86 87 len = OF_getencprop(node, "xdma,iommu", &prop, sizeof(prop)); 88 if (len != sizeof(prop)) { 89 device_printf(xdma->dev, 90 "%s: Can't get iommu device node\n", __func__); 91 return (0); 92 } 93 94 xio = &xchan->xio; 95 xio->dev = OF_device_from_xref(prop); 96 if (xio->dev == NULL) { 97 device_printf(xdma->dev, 98 "%s: Can't get iommu device\n", __func__); 99 return (0); 100 } 101 102 /* Found */ 103 return (1); 104 } 105 #endif 106 107 /* 108 * Allocate virtual xDMA channel. 109 */ 110 xdma_channel_t * 111 xdma_channel_alloc(xdma_controller_t *xdma, uint32_t caps) 112 { 113 xdma_channel_t *xchan; 114 int ret; 115 116 xchan = malloc(sizeof(xdma_channel_t), M_XDMA, M_WAITOK | M_ZERO); 117 xchan->xdma = xdma; 118 119 #ifdef FDT 120 /* Check if this DMA controller supports IOMMU. */ 121 if (xdma_get_iommu_fdt(xdma, xchan)) 122 caps |= XCHAN_CAP_IOMMU | XCHAN_CAP_NOSEG; 123 #endif 124 125 xchan->caps = caps; 126 127 XDMA_LOCK(); 128 129 /* Request a real channel from hardware driver. */ 130 ret = XDMA_CHANNEL_ALLOC(xdma->dma_dev, xchan); 131 if (ret != 0) { 132 device_printf(xdma->dev, 133 "%s: Can't request hardware channel.\n", __func__); 134 XDMA_UNLOCK(); 135 free(xchan, M_XDMA); 136 137 return (NULL); 138 } 139 140 TAILQ_INIT(&xchan->ie_handlers); 141 142 mtx_init(&xchan->mtx_lock, "xDMA chan", NULL, MTX_DEF); 143 mtx_init(&xchan->mtx_qin_lock, "xDMA qin", NULL, MTX_DEF); 144 mtx_init(&xchan->mtx_qout_lock, "xDMA qout", NULL, MTX_DEF); 145 mtx_init(&xchan->mtx_bank_lock, "xDMA bank", NULL, MTX_DEF); 146 mtx_init(&xchan->mtx_proc_lock, "xDMA proc", NULL, MTX_DEF); 147 148 TAILQ_INIT(&xchan->bank); 149 TAILQ_INIT(&xchan->queue_in); 150 TAILQ_INIT(&xchan->queue_out); 151 TAILQ_INIT(&xchan->processing); 152 153 if (xchan->caps & XCHAN_CAP_IOMMU) 154 xdma_iommu_init(&xchan->xio); 155 156 TAILQ_INSERT_TAIL(&xdma->channels, xchan, xchan_next); 157 158 XDMA_UNLOCK(); 159 160 return (xchan); 161 } 162 163 int 164 xdma_channel_free(xdma_channel_t *xchan) 165 { 166 xdma_controller_t *xdma; 167 int err; 168 169 xdma = xchan->xdma; 170 KASSERT(xdma != NULL, ("xdma is NULL")); 171 172 XDMA_LOCK(); 173 174 /* Free the real DMA channel. */ 175 err = XDMA_CHANNEL_FREE(xdma->dma_dev, xchan); 176 if (err != 0) { 177 device_printf(xdma->dev, 178 "%s: Can't free real hw channel.\n", __func__); 179 XDMA_UNLOCK(); 180 return (-1); 181 } 182 183 if (xchan->flags & XCHAN_TYPE_SG) 184 xdma_channel_free_sg(xchan); 185 186 if (xchan->caps & XCHAN_CAP_IOMMU) 187 xdma_iommu_release(&xchan->xio); 188 189 xdma_teardown_all_intr(xchan); 190 191 mtx_destroy(&xchan->mtx_lock); 192 mtx_destroy(&xchan->mtx_qin_lock); 193 mtx_destroy(&xchan->mtx_qout_lock); 194 mtx_destroy(&xchan->mtx_bank_lock); 195 mtx_destroy(&xchan->mtx_proc_lock); 196 197 TAILQ_REMOVE(&xdma->channels, xchan, xchan_next); 198 199 free(xchan, M_XDMA); 200 201 XDMA_UNLOCK(); 202 203 return (0); 204 } 205 206 int 207 xdma_setup_intr(xdma_channel_t *xchan, int flags, 208 int (*cb)(void *, xdma_transfer_status_t *), 209 void *arg, void **ihandler) 210 { 211 struct xdma_intr_handler *ih; 212 xdma_controller_t *xdma; 213 214 xdma = xchan->xdma; 215 KASSERT(xdma != NULL, ("xdma is NULL")); 216 217 /* Sanity check. */ 218 if (cb == NULL) { 219 device_printf(xdma->dev, 220 "%s: Can't setup interrupt handler.\n", 221 __func__); 222 223 return (-1); 224 } 225 226 ih = malloc(sizeof(struct xdma_intr_handler), 227 M_XDMA, M_WAITOK | M_ZERO); 228 ih->flags = flags; 229 ih->cb = cb; 230 ih->cb_user = arg; 231 232 XCHAN_LOCK(xchan); 233 TAILQ_INSERT_TAIL(&xchan->ie_handlers, ih, ih_next); 234 XCHAN_UNLOCK(xchan); 235 236 if (ihandler != NULL) 237 *ihandler = ih; 238 239 return (0); 240 } 241 242 int 243 xdma_teardown_intr(xdma_channel_t *xchan, struct xdma_intr_handler *ih) 244 { 245 xdma_controller_t *xdma; 246 247 xdma = xchan->xdma; 248 KASSERT(xdma != NULL, ("xdma is NULL")); 249 250 /* Sanity check. */ 251 if (ih == NULL) { 252 device_printf(xdma->dev, 253 "%s: Can't teardown interrupt.\n", __func__); 254 return (-1); 255 } 256 257 TAILQ_REMOVE(&xchan->ie_handlers, ih, ih_next); 258 free(ih, M_XDMA); 259 260 return (0); 261 } 262 263 int 264 xdma_teardown_all_intr(xdma_channel_t *xchan) 265 { 266 struct xdma_intr_handler *ih_tmp; 267 struct xdma_intr_handler *ih; 268 269 KASSERT(xchan->xdma != NULL, ("xdma is NULL")); 270 271 TAILQ_FOREACH_SAFE(ih, &xchan->ie_handlers, ih_next, ih_tmp) { 272 TAILQ_REMOVE(&xchan->ie_handlers, ih, ih_next); 273 free(ih, M_XDMA); 274 } 275 276 return (0); 277 } 278 279 int 280 xdma_request(xdma_channel_t *xchan, struct xdma_request *req) 281 { 282 xdma_controller_t *xdma; 283 int ret; 284 285 xdma = xchan->xdma; 286 287 KASSERT(xdma != NULL, ("xdma is NULL")); 288 289 XCHAN_LOCK(xchan); 290 ret = XDMA_CHANNEL_REQUEST(xdma->dma_dev, xchan, req); 291 if (ret != 0) { 292 device_printf(xdma->dev, 293 "%s: Can't request a transfer.\n", __func__); 294 XCHAN_UNLOCK(xchan); 295 296 return (-1); 297 } 298 XCHAN_UNLOCK(xchan); 299 300 return (0); 301 } 302 303 int 304 xdma_control(xdma_channel_t *xchan, enum xdma_command cmd) 305 { 306 xdma_controller_t *xdma; 307 int ret; 308 309 xdma = xchan->xdma; 310 KASSERT(xdma != NULL, ("xdma is NULL")); 311 312 ret = XDMA_CHANNEL_CONTROL(xdma->dma_dev, xchan, cmd); 313 if (ret != 0) { 314 device_printf(xdma->dev, 315 "%s: Can't process command.\n", __func__); 316 return (-1); 317 } 318 319 return (0); 320 } 321 322 void 323 xdma_callback(xdma_channel_t *xchan, xdma_transfer_status_t *status) 324 { 325 struct xdma_intr_handler *ih_tmp; 326 struct xdma_intr_handler *ih; 327 struct epoch_tracker et; 328 329 KASSERT(xchan->xdma != NULL, ("xdma is NULL")); 330 331 TAILQ_FOREACH_SAFE(ih, &xchan->ie_handlers, ih_next, ih_tmp) { 332 if (ih->cb != NULL) { 333 if (ih->flags & XDMA_INTR_NET) 334 NET_EPOCH_ENTER(et); 335 ih->cb(ih->cb_user, status); 336 if (ih->flags & XDMA_INTR_NET) 337 NET_EPOCH_EXIT(et); 338 } 339 } 340 341 if (xchan->flags & XCHAN_TYPE_SG) 342 xdma_queue_submit(xchan); 343 } 344 345 #ifdef FDT 346 /* 347 * Notify the DMA driver we have machine-dependent data in FDT. 348 */ 349 static int 350 xdma_ofw_md_data(xdma_controller_t *xdma, pcell_t *cells, int ncells) 351 { 352 uint32_t ret; 353 354 ret = XDMA_OFW_MD_DATA(xdma->dma_dev, 355 cells, ncells, (void **)&xdma->data); 356 357 return (ret); 358 } 359 360 int 361 xdma_handle_mem_node(vmem_t *vmem, phandle_t memory) 362 { 363 pcell_t reg[FDT_REG_CELLS * FDT_MEM_REGIONS]; 364 pcell_t *regp; 365 int addr_cells, size_cells; 366 int i, reg_len, ret, tuple_size, tuples; 367 u_long mem_start, mem_size; 368 369 if ((ret = fdt_addrsize_cells(OF_parent(memory), &addr_cells, 370 &size_cells)) != 0) 371 return (ret); 372 373 if (addr_cells > 2) 374 return (ERANGE); 375 376 tuple_size = sizeof(pcell_t) * (addr_cells + size_cells); 377 reg_len = OF_getproplen(memory, "reg"); 378 if (reg_len <= 0 || reg_len > sizeof(reg)) 379 return (ERANGE); 380 381 if (OF_getprop(memory, "reg", reg, reg_len) <= 0) 382 return (ENXIO); 383 384 tuples = reg_len / tuple_size; 385 regp = (pcell_t *)® 386 for (i = 0; i < tuples; i++) { 387 ret = fdt_data_to_res(regp, addr_cells, size_cells, 388 &mem_start, &mem_size); 389 if (ret != 0) 390 return (ret); 391 392 vmem_add(vmem, mem_start, mem_size, 0); 393 regp += addr_cells + size_cells; 394 } 395 396 return (0); 397 } 398 399 vmem_t * 400 xdma_get_memory(device_t dev) 401 { 402 phandle_t mem_node, node; 403 pcell_t mem_handle; 404 vmem_t *vmem; 405 406 node = ofw_bus_get_node(dev); 407 if (node <= 0) { 408 device_printf(dev, 409 "%s called on not ofw based device.\n", __func__); 410 return (NULL); 411 } 412 413 if (!OF_hasprop(node, "memory-region")) 414 return (NULL); 415 416 if (OF_getencprop(node, "memory-region", (void *)&mem_handle, 417 sizeof(mem_handle)) <= 0) 418 return (NULL); 419 420 vmem = vmem_create("xDMA vmem", 0, 0, PAGE_SIZE, 421 PAGE_SIZE, M_BESTFIT | M_WAITOK); 422 if (vmem == NULL) 423 return (NULL); 424 425 mem_node = OF_node_from_xref(mem_handle); 426 if (xdma_handle_mem_node(vmem, mem_node) != 0) { 427 vmem_destroy(vmem); 428 return (NULL); 429 } 430 431 return (vmem); 432 } 433 434 void 435 xdma_put_memory(vmem_t *vmem) 436 { 437 438 vmem_destroy(vmem); 439 } 440 441 void 442 xchan_set_memory(xdma_channel_t *xchan, vmem_t *vmem) 443 { 444 445 xchan->vmem = vmem; 446 } 447 448 /* 449 * Allocate xdma controller. 450 */ 451 xdma_controller_t * 452 xdma_ofw_get(device_t dev, const char *prop) 453 { 454 phandle_t node, parent; 455 xdma_controller_t *xdma; 456 device_t dma_dev; 457 pcell_t *cells; 458 int ncells; 459 int error; 460 int ndmas; 461 int idx; 462 463 node = ofw_bus_get_node(dev); 464 if (node <= 0) 465 device_printf(dev, 466 "%s called on not ofw based device.\n", __func__); 467 468 error = ofw_bus_parse_xref_list_get_length(node, 469 "dmas", "#dma-cells", &ndmas); 470 if (error) { 471 device_printf(dev, 472 "%s can't get dmas list.\n", __func__); 473 return (NULL); 474 } 475 476 if (ndmas == 0) { 477 device_printf(dev, 478 "%s dmas list is empty.\n", __func__); 479 return (NULL); 480 } 481 482 error = ofw_bus_find_string_index(node, "dma-names", prop, &idx); 483 if (error != 0) { 484 device_printf(dev, 485 "%s can't find string index.\n", __func__); 486 return (NULL); 487 } 488 489 error = ofw_bus_parse_xref_list_alloc(node, "dmas", "#dma-cells", 490 idx, &parent, &ncells, &cells); 491 if (error != 0) { 492 device_printf(dev, 493 "%s can't get dma device xref.\n", __func__); 494 return (NULL); 495 } 496 497 dma_dev = OF_device_from_xref(parent); 498 if (dma_dev == NULL) { 499 device_printf(dev, 500 "%s can't get dma device.\n", __func__); 501 return (NULL); 502 } 503 504 xdma = malloc(sizeof(struct xdma_controller), 505 M_XDMA, M_WAITOK | M_ZERO); 506 xdma->dev = dev; 507 xdma->dma_dev = dma_dev; 508 509 TAILQ_INIT(&xdma->channels); 510 511 xdma_ofw_md_data(xdma, cells, ncells); 512 free(cells, M_OFWPROP); 513 514 return (xdma); 515 } 516 #endif 517 518 /* 519 * Allocate xdma controller. 520 */ 521 xdma_controller_t * 522 xdma_get(device_t dev, device_t dma_dev) 523 { 524 xdma_controller_t *xdma; 525 526 xdma = malloc(sizeof(struct xdma_controller), 527 M_XDMA, M_WAITOK | M_ZERO); 528 xdma->dev = dev; 529 xdma->dma_dev = dma_dev; 530 531 TAILQ_INIT(&xdma->channels); 532 533 return (xdma); 534 } 535 536 /* 537 * Free xDMA controller object. 538 */ 539 int 540 xdma_put(xdma_controller_t *xdma) 541 { 542 543 XDMA_LOCK(); 544 545 /* Ensure no channels allocated. */ 546 if (!TAILQ_EMPTY(&xdma->channels)) { 547 device_printf(xdma->dev, "%s: Can't free xDMA\n", __func__); 548 return (-1); 549 } 550 551 free(xdma->data, M_DEVBUF); 552 free(xdma, M_XDMA); 553 554 XDMA_UNLOCK(); 555 556 return (0); 557 } 558 559 static void 560 xdma_init(void) 561 { 562 563 mtx_init(&xdma_mtx, "xDMA", NULL, MTX_DEF); 564 } 565 566 SYSINIT(xdma, SI_SUB_DRIVERS, SI_ORDER_FIRST, xdma_init, NULL); 567