1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2018-2019 Ruslan Bukin <br@bsdpad.com> 5 * 6 * This software was developed by SRI International and the University of 7 * Cambridge Computer Laboratory under DARPA/AFRL contract FA8750-10-C-0237 8 * ("CTSRD"), as part of the DARPA CRASH research programme. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 29 * SUCH DAMAGE. 30 */ 31 32 #include <sys/cdefs.h> 33 #include "opt_platform.h" 34 #include <sys/param.h> 35 #include <sys/conf.h> 36 #include <sys/bus.h> 37 #include <sys/kernel.h> 38 #include <sys/lock.h> 39 #include <sys/malloc.h> 40 #include <sys/mbuf.h> 41 #include <sys/mutex.h> 42 #include <sys/rwlock.h> 43 44 #include <machine/bus.h> 45 46 #include <vm/vm.h> 47 #include <vm/pmap.h> 48 #include <vm/vm_extern.h> 49 #include <vm/vm_page.h> 50 51 #ifdef FDT 52 #include <dev/fdt/fdt_common.h> 53 #include <dev/ofw/ofw_bus.h> 54 #include <dev/ofw/ofw_bus_subr.h> 55 #endif 56 57 #include <dev/xdma/xdma.h> 58 59 #include <xdma_if.h> 60 61 struct seg_load_request { 62 struct bus_dma_segment *seg; 63 uint32_t nsegs; 64 uint32_t error; 65 }; 66 67 static void 68 xchan_bufs_free_reserved(xdma_channel_t *xchan) 69 { 70 struct xdma_request *xr; 71 vm_size_t size; 72 int i; 73 74 for (i = 0; i < xchan->xr_num; i++) { 75 xr = &xchan->xr_mem[i]; 76 size = xr->buf.size; 77 if (xr->buf.vaddr) { 78 pmap_kremove_device(xr->buf.vaddr, size); 79 kva_free(xr->buf.vaddr, size); 80 xr->buf.vaddr = 0; 81 } 82 if (xr->buf.paddr) { 83 vmem_free(xchan->vmem, xr->buf.paddr, size); 84 xr->buf.paddr = 0; 85 } 86 xr->buf.size = 0; 87 } 88 } 89 90 static int 91 xchan_bufs_alloc_reserved(xdma_channel_t *xchan) 92 { 93 xdma_controller_t *xdma; 94 struct xdma_request *xr; 95 vmem_addr_t addr; 96 vm_size_t size; 97 int i; 98 99 xdma = xchan->xdma; 100 101 if (xchan->vmem == NULL) 102 return (ENOBUFS); 103 104 for (i = 0; i < xchan->xr_num; i++) { 105 xr = &xchan->xr_mem[i]; 106 size = round_page(xchan->maxsegsize); 107 if (vmem_alloc(xchan->vmem, size, 108 M_BESTFIT | M_NOWAIT, &addr)) { 109 device_printf(xdma->dev, 110 "%s: Can't allocate memory\n", __func__); 111 xchan_bufs_free_reserved(xchan); 112 return (ENOMEM); 113 } 114 115 xr->buf.size = size; 116 xr->buf.paddr = addr; 117 xr->buf.vaddr = kva_alloc(size); 118 if (xr->buf.vaddr == 0) { 119 device_printf(xdma->dev, 120 "%s: Can't allocate KVA\n", __func__); 121 xchan_bufs_free_reserved(xchan); 122 return (ENOMEM); 123 } 124 pmap_kenter_device(xr->buf.vaddr, size, addr); 125 } 126 127 return (0); 128 } 129 130 static int 131 xchan_bufs_alloc_busdma(xdma_channel_t *xchan) 132 { 133 xdma_controller_t *xdma; 134 struct xdma_request *xr; 135 int err; 136 int i; 137 138 xdma = xchan->xdma; 139 140 /* Create bus_dma tag */ 141 err = bus_dma_tag_create( 142 bus_get_dma_tag(xdma->dev), /* Parent tag. */ 143 xchan->alignment, /* alignment */ 144 xchan->boundary, /* boundary */ 145 xchan->lowaddr, /* lowaddr */ 146 xchan->highaddr, /* highaddr */ 147 NULL, NULL, /* filter, filterarg */ 148 xchan->maxsegsize * xchan->maxnsegs, /* maxsize */ 149 xchan->maxnsegs, /* nsegments */ 150 xchan->maxsegsize, /* maxsegsize */ 151 0, /* flags */ 152 NULL, NULL, /* lockfunc, lockarg */ 153 &xchan->dma_tag_bufs); 154 if (err != 0) { 155 device_printf(xdma->dev, 156 "%s: Can't create bus_dma tag.\n", __func__); 157 return (-1); 158 } 159 160 for (i = 0; i < xchan->xr_num; i++) { 161 xr = &xchan->xr_mem[i]; 162 err = bus_dmamap_create(xchan->dma_tag_bufs, 0, 163 &xr->buf.map); 164 if (err != 0) { 165 device_printf(xdma->dev, 166 "%s: Can't create buf DMA map.\n", __func__); 167 168 /* Cleanup. */ 169 bus_dma_tag_destroy(xchan->dma_tag_bufs); 170 171 return (-1); 172 } 173 } 174 175 return (0); 176 } 177 178 static int 179 xchan_bufs_alloc(xdma_channel_t *xchan) 180 { 181 xdma_controller_t *xdma; 182 int ret; 183 184 xdma = xchan->xdma; 185 186 if (xdma == NULL) { 187 printf("%s: Channel was not allocated properly.\n", __func__); 188 return (-1); 189 } 190 191 if (xchan->caps & XCHAN_CAP_BUSDMA) 192 ret = xchan_bufs_alloc_busdma(xchan); 193 else { 194 ret = xchan_bufs_alloc_reserved(xchan); 195 } 196 if (ret != 0) { 197 device_printf(xdma->dev, 198 "%s: Can't allocate bufs.\n", __func__); 199 return (-1); 200 } 201 202 xchan->flags |= XCHAN_BUFS_ALLOCATED; 203 204 return (0); 205 } 206 207 static int 208 xchan_bufs_free(xdma_channel_t *xchan) 209 { 210 struct xdma_request *xr; 211 struct xchan_buf *b; 212 int i; 213 214 if ((xchan->flags & XCHAN_BUFS_ALLOCATED) == 0) 215 return (-1); 216 217 if (xchan->caps & XCHAN_CAP_BUSDMA) { 218 for (i = 0; i < xchan->xr_num; i++) { 219 xr = &xchan->xr_mem[i]; 220 b = &xr->buf; 221 bus_dmamap_destroy(xchan->dma_tag_bufs, b->map); 222 } 223 bus_dma_tag_destroy(xchan->dma_tag_bufs); 224 } else 225 xchan_bufs_free_reserved(xchan); 226 227 xchan->flags &= ~XCHAN_BUFS_ALLOCATED; 228 229 return (0); 230 } 231 232 void 233 xdma_channel_free_sg(xdma_channel_t *xchan) 234 { 235 236 xchan_bufs_free(xchan); 237 xchan_sglist_free(xchan); 238 xchan_bank_free(xchan); 239 } 240 241 /* 242 * Prepare xchan for a scatter-gather transfer. 243 * xr_num - xdma requests queue size, 244 * maxsegsize - maximum allowed scatter-gather list element size in bytes 245 */ 246 int 247 xdma_prep_sg(xdma_channel_t *xchan, uint32_t xr_num, 248 bus_size_t maxsegsize, bus_size_t maxnsegs, 249 bus_size_t alignment, bus_addr_t boundary, 250 bus_addr_t lowaddr, bus_addr_t highaddr) 251 { 252 xdma_controller_t *xdma; 253 int ret; 254 255 xdma = xchan->xdma; 256 257 KASSERT(xdma != NULL, ("xdma is NULL")); 258 259 if (xchan->flags & XCHAN_CONFIGURED) { 260 device_printf(xdma->dev, 261 "%s: Channel is already configured.\n", __func__); 262 return (-1); 263 } 264 265 xchan->xr_num = xr_num; 266 xchan->maxsegsize = maxsegsize; 267 xchan->maxnsegs = maxnsegs; 268 xchan->alignment = alignment; 269 xchan->boundary = boundary; 270 xchan->lowaddr = lowaddr; 271 xchan->highaddr = highaddr; 272 273 if (xchan->maxnsegs > XDMA_MAX_SEG) { 274 device_printf(xdma->dev, "%s: maxnsegs is too big\n", 275 __func__); 276 return (-1); 277 } 278 279 xchan_bank_init(xchan); 280 281 /* Allocate sglist. */ 282 ret = xchan_sglist_alloc(xchan); 283 if (ret != 0) { 284 device_printf(xdma->dev, 285 "%s: Can't allocate sglist.\n", __func__); 286 return (-1); 287 } 288 289 /* Allocate buffers if required. */ 290 if (xchan->caps & (XCHAN_CAP_BUSDMA | XCHAN_CAP_BOUNCE)) { 291 ret = xchan_bufs_alloc(xchan); 292 if (ret != 0) { 293 device_printf(xdma->dev, 294 "%s: Can't allocate bufs.\n", __func__); 295 296 /* Cleanup */ 297 xchan_sglist_free(xchan); 298 xchan_bank_free(xchan); 299 300 return (-1); 301 } 302 } 303 304 xchan->flags |= (XCHAN_CONFIGURED | XCHAN_TYPE_SG); 305 306 XCHAN_LOCK(xchan); 307 ret = XDMA_CHANNEL_PREP_SG(xdma->dma_dev, xchan); 308 if (ret != 0) { 309 device_printf(xdma->dev, 310 "%s: Can't prepare SG transfer.\n", __func__); 311 XCHAN_UNLOCK(xchan); 312 313 return (-1); 314 } 315 XCHAN_UNLOCK(xchan); 316 317 return (0); 318 } 319 320 void 321 xchan_seg_done(xdma_channel_t *xchan, 322 struct xdma_transfer_status *st) 323 { 324 struct xdma_request *xr; 325 struct xchan_buf *b; 326 bus_addr_t addr; 327 328 xr = TAILQ_FIRST(&xchan->processing); 329 if (xr == NULL) 330 panic("request not found\n"); 331 332 b = &xr->buf; 333 334 atomic_subtract_int(&b->nsegs_left, 1); 335 336 if (b->nsegs_left == 0) { 337 if (xchan->caps & XCHAN_CAP_BUSDMA) { 338 if (xr->direction == XDMA_MEM_TO_DEV) 339 bus_dmamap_sync(xchan->dma_tag_bufs, b->map, 340 BUS_DMASYNC_POSTWRITE); 341 else 342 bus_dmamap_sync(xchan->dma_tag_bufs, b->map, 343 BUS_DMASYNC_POSTREAD); 344 bus_dmamap_unload(xchan->dma_tag_bufs, b->map); 345 } else if (xchan->caps & XCHAN_CAP_BOUNCE) { 346 if (xr->req_type == XR_TYPE_MBUF && 347 xr->direction == XDMA_DEV_TO_MEM) 348 m_copyback(xr->m, 0, st->transferred, 349 (void *)xr->buf.vaddr); 350 } else if (xchan->caps & XCHAN_CAP_IOMMU) { 351 if (xr->direction == XDMA_MEM_TO_DEV) 352 addr = xr->src_addr; 353 else 354 addr = xr->dst_addr; 355 xdma_iommu_remove_entry(xchan, addr); 356 } 357 xr->status.error = st->error; 358 xr->status.transferred = st->transferred; 359 360 QUEUE_PROC_LOCK(xchan); 361 TAILQ_REMOVE(&xchan->processing, xr, xr_next); 362 QUEUE_PROC_UNLOCK(xchan); 363 364 QUEUE_OUT_LOCK(xchan); 365 TAILQ_INSERT_TAIL(&xchan->queue_out, xr, xr_next); 366 QUEUE_OUT_UNLOCK(xchan); 367 } 368 } 369 370 static void 371 xdma_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nsegs, int error) 372 { 373 struct seg_load_request *slr; 374 struct bus_dma_segment *seg; 375 int i; 376 377 slr = arg; 378 seg = slr->seg; 379 380 if (error != 0) { 381 slr->error = error; 382 return; 383 } 384 385 slr->nsegs = nsegs; 386 387 for (i = 0; i < nsegs; i++) { 388 seg[i].ds_addr = segs[i].ds_addr; 389 seg[i].ds_len = segs[i].ds_len; 390 } 391 } 392 393 static int 394 _xdma_load_data_busdma(xdma_channel_t *xchan, struct xdma_request *xr, 395 struct bus_dma_segment *seg) 396 { 397 xdma_controller_t *xdma; 398 struct seg_load_request slr; 399 uint32_t nsegs; 400 void *addr; 401 int error; 402 403 xdma = xchan->xdma; 404 405 error = 0; 406 nsegs = 0; 407 408 switch (xr->req_type) { 409 case XR_TYPE_MBUF: 410 error = bus_dmamap_load_mbuf_sg(xchan->dma_tag_bufs, 411 xr->buf.map, xr->m, seg, &nsegs, BUS_DMA_NOWAIT); 412 break; 413 case XR_TYPE_BIO: 414 slr.nsegs = 0; 415 slr.error = 0; 416 slr.seg = seg; 417 error = bus_dmamap_load_bio(xchan->dma_tag_bufs, 418 xr->buf.map, xr->bp, xdma_dmamap_cb, &slr, BUS_DMA_NOWAIT); 419 if (slr.error != 0) { 420 device_printf(xdma->dma_dev, 421 "%s: bus_dmamap_load failed, err %d\n", 422 __func__, slr.error); 423 return (0); 424 } 425 nsegs = slr.nsegs; 426 break; 427 case XR_TYPE_VIRT: 428 switch (xr->direction) { 429 case XDMA_MEM_TO_DEV: 430 addr = (void *)xr->src_addr; 431 break; 432 case XDMA_DEV_TO_MEM: 433 addr = (void *)xr->dst_addr; 434 break; 435 default: 436 device_printf(xdma->dma_dev, 437 "%s: Direction is not supported\n", __func__); 438 return (0); 439 } 440 slr.nsegs = 0; 441 slr.error = 0; 442 slr.seg = seg; 443 error = bus_dmamap_load(xchan->dma_tag_bufs, xr->buf.map, 444 addr, (xr->block_len * xr->block_num), 445 xdma_dmamap_cb, &slr, BUS_DMA_NOWAIT); 446 if (slr.error != 0) { 447 device_printf(xdma->dma_dev, 448 "%s: bus_dmamap_load failed, err %d\n", 449 __func__, slr.error); 450 return (0); 451 } 452 nsegs = slr.nsegs; 453 break; 454 default: 455 break; 456 } 457 458 if (error != 0) { 459 if (error == ENOMEM) { 460 /* 461 * Out of memory. Try again later. 462 * TODO: count errors. 463 */ 464 } else 465 device_printf(xdma->dma_dev, 466 "%s: bus_dmamap_load failed with err %d\n", 467 __func__, error); 468 return (0); 469 } 470 471 if (xr->direction == XDMA_MEM_TO_DEV) 472 bus_dmamap_sync(xchan->dma_tag_bufs, xr->buf.map, 473 BUS_DMASYNC_PREWRITE); 474 else 475 bus_dmamap_sync(xchan->dma_tag_bufs, xr->buf.map, 476 BUS_DMASYNC_PREREAD); 477 478 return (nsegs); 479 } 480 481 static int 482 _xdma_load_data(xdma_channel_t *xchan, struct xdma_request *xr, 483 struct bus_dma_segment *seg) 484 { 485 struct mbuf *m; 486 uint32_t nsegs; 487 vm_offset_t va, addr; 488 bus_addr_t pa; 489 vm_prot_t prot; 490 491 m = xr->m; 492 493 KASSERT(xchan->caps & (XCHAN_CAP_NOSEG | XCHAN_CAP_BOUNCE), 494 ("Handling segmented data is not implemented here.")); 495 496 nsegs = 1; 497 498 switch (xr->req_type) { 499 case XR_TYPE_MBUF: 500 if (xchan->caps & XCHAN_CAP_BOUNCE) { 501 if (xr->direction == XDMA_MEM_TO_DEV) 502 m_copydata(m, 0, m->m_pkthdr.len, 503 (void *)xr->buf.vaddr); 504 seg[0].ds_addr = (bus_addr_t)xr->buf.paddr; 505 } else if (xchan->caps & XCHAN_CAP_IOMMU) { 506 addr = mtod(m, bus_addr_t); 507 pa = vtophys(addr); 508 509 if (xr->direction == XDMA_MEM_TO_DEV) 510 prot = VM_PROT_READ; 511 else 512 prot = VM_PROT_WRITE; 513 514 xdma_iommu_add_entry(xchan, &va, 515 pa, m->m_pkthdr.len, prot); 516 517 /* 518 * Save VA so we can unload data later 519 * after completion of this transfer. 520 */ 521 if (xr->direction == XDMA_MEM_TO_DEV) 522 xr->src_addr = va; 523 else 524 xr->dst_addr = va; 525 seg[0].ds_addr = va; 526 } else 527 seg[0].ds_addr = mtod(m, bus_addr_t); 528 seg[0].ds_len = m->m_pkthdr.len; 529 break; 530 case XR_TYPE_BIO: 531 case XR_TYPE_VIRT: 532 default: 533 panic("implement me\n"); 534 } 535 536 return (nsegs); 537 } 538 539 static int 540 xdma_load_data(xdma_channel_t *xchan, 541 struct xdma_request *xr, struct bus_dma_segment *seg) 542 { 543 int nsegs; 544 545 nsegs = 0; 546 547 if (xchan->caps & XCHAN_CAP_BUSDMA) 548 nsegs = _xdma_load_data_busdma(xchan, xr, seg); 549 else 550 nsegs = _xdma_load_data(xchan, xr, seg); 551 if (nsegs == 0) 552 return (0); /* Try again later. */ 553 554 xr->buf.nsegs = nsegs; 555 xr->buf.nsegs_left = nsegs; 556 557 return (nsegs); 558 } 559 560 static int 561 xdma_process(xdma_channel_t *xchan, 562 struct xdma_sglist *sg) 563 { 564 struct bus_dma_segment seg[XDMA_MAX_SEG]; 565 struct xdma_request *xr; 566 struct xdma_request *xr_tmp; 567 xdma_controller_t *xdma; 568 uint32_t capacity; 569 uint32_t n; 570 uint32_t c; 571 int nsegs; 572 int ret; 573 574 XCHAN_ASSERT_LOCKED(xchan); 575 576 xdma = xchan->xdma; 577 578 n = 0; 579 c = 0; 580 581 ret = XDMA_CHANNEL_CAPACITY(xdma->dma_dev, xchan, &capacity); 582 if (ret != 0) { 583 device_printf(xdma->dev, 584 "%s: Can't get DMA controller capacity.\n", __func__); 585 return (-1); 586 } 587 588 TAILQ_FOREACH_SAFE(xr, &xchan->queue_in, xr_next, xr_tmp) { 589 switch (xr->req_type) { 590 case XR_TYPE_MBUF: 591 if ((xchan->caps & XCHAN_CAP_NOSEG) || 592 (c > xchan->maxnsegs)) 593 c = xdma_mbuf_defrag(xchan, xr); 594 break; 595 case XR_TYPE_BIO: 596 case XR_TYPE_VIRT: 597 default: 598 c = 1; 599 } 600 601 if (capacity <= (c + n)) { 602 /* 603 * No space yet available for the entire 604 * request in the DMA engine. 605 */ 606 break; 607 } 608 609 if ((c + n + xchan->maxnsegs) >= XDMA_SGLIST_MAXLEN) { 610 /* Sglist is full. */ 611 break; 612 } 613 614 nsegs = xdma_load_data(xchan, xr, seg); 615 if (nsegs == 0) 616 break; 617 618 xdma_sglist_add(&sg[n], seg, nsegs, xr); 619 n += nsegs; 620 621 QUEUE_IN_LOCK(xchan); 622 TAILQ_REMOVE(&xchan->queue_in, xr, xr_next); 623 QUEUE_IN_UNLOCK(xchan); 624 625 QUEUE_PROC_LOCK(xchan); 626 TAILQ_INSERT_TAIL(&xchan->processing, xr, xr_next); 627 QUEUE_PROC_UNLOCK(xchan); 628 } 629 630 return (n); 631 } 632 633 int 634 xdma_queue_submit_sg(xdma_channel_t *xchan) 635 { 636 struct xdma_sglist *sg; 637 xdma_controller_t *xdma; 638 uint32_t sg_n; 639 int ret; 640 641 xdma = xchan->xdma; 642 KASSERT(xdma != NULL, ("xdma is NULL")); 643 644 XCHAN_ASSERT_LOCKED(xchan); 645 646 sg = xchan->sg; 647 648 if ((xchan->caps & (XCHAN_CAP_BOUNCE | XCHAN_CAP_BUSDMA)) && 649 (xchan->flags & XCHAN_BUFS_ALLOCATED) == 0) { 650 device_printf(xdma->dev, 651 "%s: Can't submit a transfer: no bufs\n", 652 __func__); 653 return (-1); 654 } 655 656 sg_n = xdma_process(xchan, sg); 657 if (sg_n == 0) 658 return (0); /* Nothing to submit */ 659 660 /* Now submit sglist to DMA engine driver. */ 661 ret = XDMA_CHANNEL_SUBMIT_SG(xdma->dma_dev, xchan, sg, sg_n); 662 if (ret != 0) { 663 device_printf(xdma->dev, 664 "%s: Can't submit an sglist.\n", __func__); 665 return (-1); 666 } 667 668 return (0); 669 } 670