1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2008 Hans Petter Selasky. All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28 #ifdef USB_GLOBAL_INCLUDE_FILE 29 #include USB_GLOBAL_INCLUDE_FILE 30 #else 31 #include <sys/stdint.h> 32 #include <sys/stddef.h> 33 #include <sys/param.h> 34 #include <sys/queue.h> 35 #include <sys/types.h> 36 #include <sys/systm.h> 37 #include <sys/kernel.h> 38 #include <sys/bus.h> 39 #include <sys/module.h> 40 #include <sys/lock.h> 41 #include <sys/mutex.h> 42 #include <sys/condvar.h> 43 #include <sys/sysctl.h> 44 #include <sys/sx.h> 45 #include <sys/unistd.h> 46 #include <sys/callout.h> 47 #include <sys/malloc.h> 48 #include <sys/priv.h> 49 50 #include <dev/usb/usb.h> 51 #include <dev/usb/usbdi.h> 52 #include <dev/usb/usbdi_util.h> 53 54 #define USB_DEBUG_VAR usb_debug 55 56 #include <dev/usb/usb_core.h> 57 #include <dev/usb/usb_busdma.h> 58 #include <dev/usb/usb_process.h> 59 #include <dev/usb/usb_transfer.h> 60 #include <dev/usb/usb_device.h> 61 #include <dev/usb/usb_util.h> 62 #include <dev/usb/usb_debug.h> 63 64 #include <dev/usb/usb_controller.h> 65 #include <dev/usb/usb_bus.h> 66 #endif /* USB_GLOBAL_INCLUDE_FILE */ 67 68 #if USB_HAVE_BUSDMA 69 static void usb_dma_tag_create(struct usb_dma_tag *, usb_size_t, usb_size_t); 70 static void usb_dma_tag_destroy(struct usb_dma_tag *); 71 static void usb_dma_lock_cb(void *, bus_dma_lock_op_t); 72 static void usb_pc_alloc_mem_cb(void *, bus_dma_segment_t *, int, int); 73 static void usb_pc_load_mem_cb(void *, bus_dma_segment_t *, int, int); 74 static void usb_pc_common_mem_cb(void *, bus_dma_segment_t *, int, int, 75 uint8_t); 76 #endif 77 78 /*------------------------------------------------------------------------* 79 * usbd_get_page - lookup DMA-able memory for the given offset 80 * 81 * NOTE: Only call this function when the "page_cache" structure has 82 * been properly initialized ! 83 *------------------------------------------------------------------------*/ 84 void 85 usbd_get_page(struct usb_page_cache *pc, usb_frlength_t offset, 86 struct usb_page_search *res) 87 { 88 #if USB_HAVE_BUSDMA 89 struct usb_page *page; 90 91 if (pc->page_start) { 92 /* Case 1 - something has been loaded into DMA */ 93 94 if (pc->buffer) { 95 /* Case 1a - Kernel Virtual Address */ 96 97 res->buffer = USB_ADD_BYTES(pc->buffer, offset); 98 } 99 offset += pc->page_offset_buf; 100 101 /* compute destination page */ 102 103 page = pc->page_start; 104 105 if (pc->ismultiseg) { 106 page += (offset / USB_PAGE_SIZE); 107 108 offset %= USB_PAGE_SIZE; 109 110 res->length = USB_PAGE_SIZE - offset; 111 res->physaddr = page->physaddr + offset; 112 } else { 113 res->length = (usb_size_t)-1; 114 res->physaddr = page->physaddr + offset; 115 } 116 if (!pc->buffer) { 117 /* Case 1b - Non Kernel Virtual Address */ 118 119 res->buffer = USB_ADD_BYTES(page->buffer, offset); 120 } 121 return; 122 } 123 #endif 124 /* Case 2 - Plain PIO */ 125 126 res->buffer = USB_ADD_BYTES(pc->buffer, offset); 127 res->length = (usb_size_t)-1; 128 #if USB_HAVE_BUSDMA 129 res->physaddr = 0; 130 #endif 131 } 132 133 /*------------------------------------------------------------------------* 134 * usb_pc_buffer_is_aligned - verify alignment 135 * 136 * This function is used to check if a page cache buffer is properly 137 * aligned to reduce the use of bounce buffers in PIO mode. 138 *------------------------------------------------------------------------*/ 139 uint8_t 140 usb_pc_buffer_is_aligned(struct usb_page_cache *pc, usb_frlength_t offset, 141 usb_frlength_t len, usb_frlength_t mask) 142 { 143 struct usb_page_search buf_res; 144 145 while (len != 0) { 146 usbd_get_page(pc, offset, &buf_res); 147 148 if (buf_res.length > len) 149 buf_res.length = len; 150 if (USB_P2U(buf_res.buffer) & mask) 151 return (0); 152 if (buf_res.length & mask) 153 return (0); 154 155 offset += buf_res.length; 156 len -= buf_res.length; 157 } 158 return (1); 159 } 160 161 /*------------------------------------------------------------------------* 162 * usbd_copy_in - copy directly to DMA-able memory 163 *------------------------------------------------------------------------*/ 164 void 165 usbd_copy_in(struct usb_page_cache *cache, usb_frlength_t offset, 166 const void *ptr, usb_frlength_t len) 167 { 168 struct usb_page_search buf_res; 169 170 while (len != 0) { 171 usbd_get_page(cache, offset, &buf_res); 172 173 if (buf_res.length > len) { 174 buf_res.length = len; 175 } 176 memcpy(buf_res.buffer, ptr, buf_res.length); 177 178 offset += buf_res.length; 179 len -= buf_res.length; 180 ptr = USB_ADD_BYTES(ptr, buf_res.length); 181 } 182 } 183 184 /*------------------------------------------------------------------------* 185 * usbd_copy_in_user - copy directly to DMA-able memory from userland 186 * 187 * Return values: 188 * 0: Success 189 * Else: Failure 190 *------------------------------------------------------------------------*/ 191 #if USB_HAVE_USER_IO 192 int 193 usbd_copy_in_user(struct usb_page_cache *cache, usb_frlength_t offset, 194 const void *ptr, usb_frlength_t len) 195 { 196 struct usb_page_search buf_res; 197 int error; 198 199 while (len != 0) { 200 usbd_get_page(cache, offset, &buf_res); 201 202 if (buf_res.length > len) { 203 buf_res.length = len; 204 } 205 error = copyin(ptr, buf_res.buffer, buf_res.length); 206 if (error) 207 return (error); 208 209 offset += buf_res.length; 210 len -= buf_res.length; 211 ptr = USB_ADD_BYTES(ptr, buf_res.length); 212 } 213 return (0); /* success */ 214 } 215 #endif 216 217 /*------------------------------------------------------------------------* 218 * usbd_m_copy_in - copy a mbuf chain directly into DMA-able memory 219 *------------------------------------------------------------------------*/ 220 #if USB_HAVE_MBUF 221 struct usb_m_copy_in_arg { 222 struct usb_page_cache *cache; 223 usb_frlength_t dst_offset; 224 }; 225 226 static int 227 usbd_m_copy_in_cb(void *arg, void *src, uint32_t count) 228 { 229 struct usb_m_copy_in_arg *ua = arg; 230 231 usbd_copy_in(ua->cache, ua->dst_offset, src, count); 232 ua->dst_offset += count; 233 return (0); 234 } 235 236 void 237 usbd_m_copy_in(struct usb_page_cache *cache, usb_frlength_t dst_offset, 238 struct mbuf *m, usb_size_t src_offset, usb_frlength_t src_len) 239 { 240 struct usb_m_copy_in_arg arg = {cache, dst_offset}; 241 (void) m_apply(m, src_offset, src_len, &usbd_m_copy_in_cb, &arg); 242 } 243 #endif 244 245 /*------------------------------------------------------------------------* 246 * usb_uiomove - factored out code 247 *------------------------------------------------------------------------*/ 248 #if USB_HAVE_USER_IO 249 int 250 usb_uiomove(struct usb_page_cache *pc, struct uio *uio, 251 usb_frlength_t pc_offset, usb_frlength_t len) 252 { 253 struct usb_page_search res; 254 int error = 0; 255 256 while (len != 0) { 257 usbd_get_page(pc, pc_offset, &res); 258 259 if (res.length > len) { 260 res.length = len; 261 } 262 /* 263 * "uiomove()" can sleep so one needs to make a wrapper, 264 * exiting the mutex and checking things 265 */ 266 error = uiomove(res.buffer, res.length, uio); 267 268 if (error) { 269 break; 270 } 271 pc_offset += res.length; 272 len -= res.length; 273 } 274 return (error); 275 } 276 #endif 277 278 /*------------------------------------------------------------------------* 279 * usbd_copy_out - copy directly from DMA-able memory 280 *------------------------------------------------------------------------*/ 281 void 282 usbd_copy_out(struct usb_page_cache *cache, usb_frlength_t offset, 283 void *ptr, usb_frlength_t len) 284 { 285 struct usb_page_search res; 286 287 while (len != 0) { 288 usbd_get_page(cache, offset, &res); 289 290 if (res.length > len) { 291 res.length = len; 292 } 293 memcpy(ptr, res.buffer, res.length); 294 295 offset += res.length; 296 len -= res.length; 297 ptr = USB_ADD_BYTES(ptr, res.length); 298 } 299 } 300 301 /*------------------------------------------------------------------------* 302 * usbd_copy_out_user - copy directly from DMA-able memory to userland 303 * 304 * Return values: 305 * 0: Success 306 * Else: Failure 307 *------------------------------------------------------------------------*/ 308 #if USB_HAVE_USER_IO 309 int 310 usbd_copy_out_user(struct usb_page_cache *cache, usb_frlength_t offset, 311 void *ptr, usb_frlength_t len) 312 { 313 struct usb_page_search res; 314 int error; 315 316 while (len != 0) { 317 usbd_get_page(cache, offset, &res); 318 319 if (res.length > len) { 320 res.length = len; 321 } 322 error = copyout(res.buffer, ptr, res.length); 323 if (error) 324 return (error); 325 326 offset += res.length; 327 len -= res.length; 328 ptr = USB_ADD_BYTES(ptr, res.length); 329 } 330 return (0); /* success */ 331 } 332 #endif 333 334 /*------------------------------------------------------------------------* 335 * usbd_frame_zero - zero DMA-able memory 336 *------------------------------------------------------------------------*/ 337 void 338 usbd_frame_zero(struct usb_page_cache *cache, usb_frlength_t offset, 339 usb_frlength_t len) 340 { 341 struct usb_page_search res; 342 343 while (len != 0) { 344 usbd_get_page(cache, offset, &res); 345 346 if (res.length > len) { 347 res.length = len; 348 } 349 memset(res.buffer, 0, res.length); 350 351 offset += res.length; 352 len -= res.length; 353 } 354 } 355 356 #if USB_HAVE_BUSDMA 357 358 /*------------------------------------------------------------------------* 359 * usb_dma_lock_cb - dummy callback 360 *------------------------------------------------------------------------*/ 361 static void 362 usb_dma_lock_cb(void *arg, bus_dma_lock_op_t op) 363 { 364 /* we use "mtx_owned()" instead of this function */ 365 } 366 367 /*------------------------------------------------------------------------* 368 * usb_dma_tag_create - allocate a DMA tag 369 * 370 * NOTE: If the "align" parameter has a value of 1 the DMA-tag will 371 * allow multi-segment mappings. Else all mappings are single-segment. 372 *------------------------------------------------------------------------*/ 373 static void 374 usb_dma_tag_create(struct usb_dma_tag *udt, 375 usb_size_t size, usb_size_t align) 376 { 377 bus_dma_tag_t tag; 378 379 if (bus_dma_tag_create 380 ( /* parent */ udt->tag_parent->tag, 381 /* alignment */ align, 382 /* boundary */ 0, 383 /* lowaddr */ (2ULL << (udt->tag_parent->dma_bits - 1)) - 1, 384 /* highaddr */ BUS_SPACE_MAXADDR, 385 /* filter */ NULL, 386 /* filterarg */ NULL, 387 /* maxsize */ size, 388 /* nsegments */ (align == 1 && size > 1) ? 389 (2 + (size / USB_PAGE_SIZE)) : 1, 390 /* maxsegsz */ (align == 1 && size > USB_PAGE_SIZE) ? 391 USB_PAGE_SIZE : size, 392 /* flags */ BUS_DMA_KEEP_PG_OFFSET, 393 /* lockfn */ &usb_dma_lock_cb, 394 /* lockarg */ NULL, 395 &tag)) { 396 tag = NULL; 397 } 398 udt->tag = tag; 399 } 400 401 /*------------------------------------------------------------------------* 402 * usb_dma_tag_free - free a DMA tag 403 *------------------------------------------------------------------------*/ 404 static void 405 usb_dma_tag_destroy(struct usb_dma_tag *udt) 406 { 407 bus_dma_tag_destroy(udt->tag); 408 } 409 410 /*------------------------------------------------------------------------* 411 * usb_pc_alloc_mem_cb - BUS-DMA callback function 412 *------------------------------------------------------------------------*/ 413 static void 414 usb_pc_alloc_mem_cb(void *arg, bus_dma_segment_t *segs, 415 int nseg, int error) 416 { 417 usb_pc_common_mem_cb(arg, segs, nseg, error, 0); 418 } 419 420 /*------------------------------------------------------------------------* 421 * usb_pc_load_mem_cb - BUS-DMA callback function 422 *------------------------------------------------------------------------*/ 423 static void 424 usb_pc_load_mem_cb(void *arg, bus_dma_segment_t *segs, 425 int nseg, int error) 426 { 427 usb_pc_common_mem_cb(arg, segs, nseg, error, 1); 428 } 429 430 /*------------------------------------------------------------------------* 431 * usb_pc_common_mem_cb - BUS-DMA callback function 432 *------------------------------------------------------------------------*/ 433 static void 434 usb_pc_common_mem_cb(void *arg, bus_dma_segment_t *segs, 435 int nseg, int error, uint8_t isload) 436 { 437 struct usb_dma_parent_tag *uptag; 438 struct usb_page_cache *pc; 439 struct usb_page *pg; 440 usb_size_t rem; 441 bus_size_t off; 442 uint8_t owned; 443 444 pc = arg; 445 uptag = pc->tag_parent; 446 447 /* 448 * XXX There is sometimes recursive locking here. 449 * XXX We should try to find a better solution. 450 * XXX Until further the "owned" variable does 451 * XXX the trick. 452 */ 453 454 if (error) { 455 goto done; 456 } 457 458 off = 0; 459 pg = pc->page_start; 460 pg->physaddr = rounddown2(segs->ds_addr, USB_PAGE_SIZE); 461 rem = segs->ds_addr & (USB_PAGE_SIZE - 1); 462 pc->page_offset_buf = rem; 463 pc->page_offset_end += rem; 464 #ifdef USB_DEBUG 465 if (nseg > 1) { 466 int x; 467 468 for (x = 0; x != nseg - 1; x++) { 469 if (((segs[x].ds_addr + segs[x].ds_len) & (USB_PAGE_SIZE - 1)) == 470 ((segs[x + 1].ds_addr & (USB_PAGE_SIZE - 1)))) 471 continue; 472 /* 473 * This check verifies there is no page offset 474 * hole between any of the segments. See the 475 * BUS_DMA_KEEP_PG_OFFSET flag. 476 */ 477 DPRINTFN(0, "Page offset was not preserved\n"); 478 error = 1; 479 goto done; 480 } 481 } 482 #endif 483 while (pc->ismultiseg) { 484 off += USB_PAGE_SIZE; 485 if (off >= (segs->ds_len + rem)) { 486 /* page crossing */ 487 nseg--; 488 segs++; 489 off = 0; 490 rem = 0; 491 if (nseg == 0) 492 break; 493 } 494 pg++; 495 pg->physaddr = rounddown2(segs->ds_addr + off, USB_PAGE_SIZE); 496 } 497 498 done: 499 owned = mtx_owned(uptag->mtx); 500 if (!owned) 501 USB_MTX_LOCK(uptag->mtx); 502 503 uptag->dma_error = (error ? 1 : 0); 504 if (isload) { 505 (uptag->func) (uptag); 506 } else { 507 cv_broadcast(uptag->cv); 508 } 509 if (!owned) 510 USB_MTX_UNLOCK(uptag->mtx); 511 } 512 513 /*------------------------------------------------------------------------* 514 * usb_pc_alloc_mem - allocate DMA'able memory 515 * 516 * Returns: 517 * 0: Success 518 * Else: Failure 519 *------------------------------------------------------------------------*/ 520 uint8_t 521 usb_pc_alloc_mem(struct usb_page_cache *pc, struct usb_page *pg, 522 usb_size_t size, usb_size_t align) 523 { 524 struct usb_dma_parent_tag *uptag; 525 struct usb_dma_tag *utag; 526 bus_dmamap_t map; 527 void *ptr; 528 int err; 529 530 uptag = pc->tag_parent; 531 532 if (align != 1) { 533 /* 534 * The alignment must be greater or equal to the 535 * "size" else the object can be split between two 536 * memory pages and we get a problem! 537 */ 538 while (align < size) { 539 align *= 2; 540 if (align == 0) { 541 goto error; 542 } 543 } 544 #if 1 545 /* 546 * XXX BUS-DMA workaround - FIXME later: 547 * 548 * We assume that the alignment at this point of 549 * the code is greater than or equal to the size and 550 * less than two times the size, so that if we double 551 * the size, the size will be greater than the 552 * alignment. 553 * 554 * The bus-dma system has a check for "alignment" 555 * being less than "size". If that check fails we end 556 * up using contigmalloc which is page based even for 557 * small allocations. Try to avoid that to save 558 * memory, hence we sometimes to a large number of 559 * small allocations! 560 */ 561 if (size <= (USB_PAGE_SIZE / 2)) { 562 size *= 2; 563 } 564 #endif 565 } 566 /* get the correct DMA tag */ 567 utag = usb_dma_tag_find(uptag, size, align); 568 if (utag == NULL) { 569 goto error; 570 } 571 /* allocate memory */ 572 if (bus_dmamem_alloc( 573 utag->tag, &ptr, (BUS_DMA_WAITOK | BUS_DMA_COHERENT), &map)) { 574 goto error; 575 } 576 /* setup page cache */ 577 pc->buffer = ptr; 578 pc->page_start = pg; 579 pc->page_offset_buf = 0; 580 pc->page_offset_end = size; 581 pc->map = map; 582 pc->tag = utag->tag; 583 pc->ismultiseg = (align == 1); 584 585 USB_MTX_LOCK(uptag->mtx); 586 587 /* load memory into DMA */ 588 err = bus_dmamap_load( 589 utag->tag, map, ptr, size, &usb_pc_alloc_mem_cb, 590 pc, (BUS_DMA_WAITOK | BUS_DMA_COHERENT)); 591 592 if (err == EINPROGRESS) { 593 cv_wait(uptag->cv, uptag->mtx); 594 err = 0; 595 } 596 USB_MTX_UNLOCK(uptag->mtx); 597 598 if (err || uptag->dma_error) { 599 bus_dmamem_free(utag->tag, ptr, map); 600 goto error; 601 } 602 pc->isloaded = 1; 603 memset(ptr, 0, size); 604 605 usb_pc_cpu_flush(pc); 606 607 return (0); 608 609 error: 610 /* reset most of the page cache */ 611 pc->buffer = NULL; 612 pc->page_start = NULL; 613 pc->page_offset_buf = 0; 614 pc->page_offset_end = 0; 615 pc->isloaded = 0; 616 pc->map = NULL; 617 pc->tag = NULL; 618 return (1); 619 } 620 621 /*------------------------------------------------------------------------* 622 * usb_pc_free_mem - free DMA memory 623 * 624 * This function is NULL safe. 625 *------------------------------------------------------------------------*/ 626 void 627 usb_pc_free_mem(struct usb_page_cache *pc) 628 { 629 if (pc && pc->buffer) { 630 if (pc->isloaded) 631 bus_dmamap_unload(pc->tag, pc->map); 632 633 bus_dmamem_free(pc->tag, pc->buffer, pc->map); 634 635 pc->buffer = NULL; 636 pc->isloaded = 0; 637 } 638 } 639 640 /*------------------------------------------------------------------------* 641 * usb_pc_load_mem - load virtual memory into DMA 642 * 643 * Return values: 644 * 0: Success 645 * Else: Error 646 *------------------------------------------------------------------------*/ 647 uint8_t 648 usb_pc_load_mem(struct usb_page_cache *pc, usb_size_t size, uint8_t sync) 649 { 650 /* setup page cache */ 651 pc->page_offset_buf = 0; 652 pc->page_offset_end = size; 653 pc->ismultiseg = 1; 654 655 USB_MTX_ASSERT(pc->tag_parent->mtx, MA_OWNED); 656 657 if (size > 0) { 658 if (sync) { 659 struct usb_dma_parent_tag *uptag; 660 int err; 661 662 uptag = pc->tag_parent; 663 664 /* 665 * We have to unload the previous loaded DMA 666 * pages before trying to load a new one! 667 */ 668 if (pc->isloaded) 669 bus_dmamap_unload(pc->tag, pc->map); 670 671 /* 672 * Try to load memory into DMA. 673 */ 674 err = bus_dmamap_load( 675 pc->tag, pc->map, pc->buffer, size, 676 &usb_pc_alloc_mem_cb, pc, BUS_DMA_WAITOK); 677 if (err == EINPROGRESS) { 678 cv_wait(uptag->cv, uptag->mtx); 679 err = 0; 680 } 681 if (err || uptag->dma_error) { 682 pc->isloaded = 0; 683 return (1); 684 } 685 } else { 686 /* 687 * We have to unload the previous loaded DMA 688 * pages before trying to load a new one! 689 */ 690 if (pc->isloaded) 691 bus_dmamap_unload(pc->tag, pc->map); 692 693 /* 694 * Try to load memory into DMA. The callback 695 * will be called in all cases: 696 */ 697 if (bus_dmamap_load( 698 pc->tag, pc->map, pc->buffer, size, 699 &usb_pc_load_mem_cb, pc, BUS_DMA_WAITOK)) { 700 } 701 } 702 pc->isloaded = 1; 703 } else { 704 if (!sync) { 705 /* 706 * Call callback so that refcount is decremented 707 * properly: 708 */ 709 pc->tag_parent->dma_error = 0; 710 (pc->tag_parent->func) (pc->tag_parent); 711 } 712 } 713 return (0); 714 } 715 716 /*------------------------------------------------------------------------* 717 * usb_pc_cpu_invalidate - invalidate CPU cache 718 *------------------------------------------------------------------------*/ 719 void 720 usb_pc_cpu_invalidate(struct usb_page_cache *pc) 721 { 722 if (pc->page_offset_end == pc->page_offset_buf) { 723 /* nothing has been loaded into this page cache! */ 724 return; 725 } 726 727 /* 728 * TODO: We currently do XXX_POSTREAD and XXX_PREREAD at the 729 * same time, but in the future we should try to isolate the 730 * different cases to optimise the code. --HPS 731 */ 732 bus_dmamap_sync(pc->tag, pc->map, BUS_DMASYNC_POSTREAD); 733 bus_dmamap_sync(pc->tag, pc->map, BUS_DMASYNC_PREREAD); 734 } 735 736 /*------------------------------------------------------------------------* 737 * usb_pc_cpu_flush - flush CPU cache 738 *------------------------------------------------------------------------*/ 739 void 740 usb_pc_cpu_flush(struct usb_page_cache *pc) 741 { 742 if (pc->page_offset_end == pc->page_offset_buf) { 743 /* nothing has been loaded into this page cache! */ 744 return; 745 } 746 bus_dmamap_sync(pc->tag, pc->map, BUS_DMASYNC_PREWRITE); 747 } 748 749 /*------------------------------------------------------------------------* 750 * usb_pc_dmamap_create - create a DMA map 751 * 752 * Returns: 753 * 0: Success 754 * Else: Failure 755 *------------------------------------------------------------------------*/ 756 uint8_t 757 usb_pc_dmamap_create(struct usb_page_cache *pc, usb_size_t size) 758 { 759 struct usb_xfer_root *info; 760 struct usb_dma_tag *utag; 761 762 /* get info */ 763 info = USB_DMATAG_TO_XROOT(pc->tag_parent); 764 765 /* sanity check */ 766 if (info == NULL) { 767 goto error; 768 } 769 utag = usb_dma_tag_find(pc->tag_parent, size, 1); 770 if (utag == NULL) { 771 goto error; 772 } 773 /* create DMA map */ 774 if (bus_dmamap_create(utag->tag, 0, &pc->map)) { 775 goto error; 776 } 777 pc->tag = utag->tag; 778 return 0; /* success */ 779 780 error: 781 pc->map = NULL; 782 pc->tag = NULL; 783 return 1; /* failure */ 784 } 785 786 /*------------------------------------------------------------------------* 787 * usb_pc_dmamap_destroy 788 * 789 * This function is NULL safe. 790 *------------------------------------------------------------------------*/ 791 void 792 usb_pc_dmamap_destroy(struct usb_page_cache *pc) 793 { 794 if (pc && pc->tag) { 795 if (pc->isloaded) 796 bus_dmamap_unload(pc->tag, pc->map); 797 bus_dmamap_destroy(pc->tag, pc->map); 798 pc->tag = NULL; 799 pc->map = NULL; 800 } 801 } 802 803 /*------------------------------------------------------------------------* 804 * usb_dma_tag_find - factored out code 805 *------------------------------------------------------------------------*/ 806 struct usb_dma_tag * 807 usb_dma_tag_find(struct usb_dma_parent_tag *udpt, 808 usb_size_t size, usb_size_t align) 809 { 810 struct usb_dma_tag *udt; 811 uint8_t nudt; 812 813 USB_ASSERT(align > 0, ("Invalid parameter align = 0\n")); 814 USB_ASSERT(size > 0, ("Invalid parameter size = 0\n")); 815 816 udt = udpt->utag_first; 817 nudt = udpt->utag_max; 818 819 while (nudt--) { 820 if (udt->align == 0) { 821 usb_dma_tag_create(udt, size, align); 822 if (udt->tag == NULL) { 823 return (NULL); 824 } 825 udt->align = align; 826 udt->size = size; 827 return (udt); 828 } 829 if ((udt->align == align) && (udt->size == size)) { 830 return (udt); 831 } 832 udt++; 833 } 834 return (NULL); 835 } 836 837 /*------------------------------------------------------------------------* 838 * usb_dma_tag_setup - initialise USB DMA tags 839 *------------------------------------------------------------------------*/ 840 void 841 usb_dma_tag_setup(struct usb_dma_parent_tag *udpt, 842 struct usb_dma_tag *udt, bus_dma_tag_t dmat, 843 struct mtx *mtx, usb_dma_callback_t *func, 844 uint8_t ndmabits, uint8_t nudt) 845 { 846 memset(udpt, 0, sizeof(*udpt)); 847 848 /* sanity checking */ 849 if ((nudt == 0) || 850 (ndmabits == 0) || 851 (mtx == NULL)) { 852 /* something is corrupt */ 853 return; 854 } 855 /* initialise condition variable */ 856 cv_init(udpt->cv, "USB DMA CV"); 857 858 /* store some information */ 859 udpt->mtx = mtx; 860 udpt->func = func; 861 udpt->tag = dmat; 862 udpt->utag_first = udt; 863 udpt->utag_max = nudt; 864 udpt->dma_bits = ndmabits; 865 866 while (nudt--) { 867 memset(udt, 0, sizeof(*udt)); 868 udt->tag_parent = udpt; 869 udt++; 870 } 871 } 872 873 /*------------------------------------------------------------------------* 874 * usb_bus_tag_unsetup - factored out code 875 *------------------------------------------------------------------------*/ 876 void 877 usb_dma_tag_unsetup(struct usb_dma_parent_tag *udpt) 878 { 879 struct usb_dma_tag *udt; 880 uint8_t nudt; 881 882 udt = udpt->utag_first; 883 nudt = udpt->utag_max; 884 885 while (nudt--) { 886 if (udt->align) { 887 /* destroy the USB DMA tag */ 888 usb_dma_tag_destroy(udt); 889 udt->align = 0; 890 } 891 udt++; 892 } 893 894 if (udpt->utag_max) { 895 /* destroy the condition variable */ 896 cv_destroy(udpt->cv); 897 } 898 } 899 900 /*------------------------------------------------------------------------* 901 * usb_bdma_work_loop 902 * 903 * This function handles loading of virtual buffers into DMA and is 904 * only called when "dma_refcount" is zero. 905 *------------------------------------------------------------------------*/ 906 void 907 usb_bdma_work_loop(struct usb_xfer_queue *pq) 908 { 909 struct usb_xfer_root *info; 910 struct usb_xfer *xfer; 911 usb_frcount_t nframes; 912 913 xfer = pq->curr; 914 info = xfer->xroot; 915 916 USB_MTX_ASSERT(info->xfer_mtx, MA_OWNED); 917 918 if (xfer->error) { 919 /* some error happened */ 920 USB_BUS_LOCK(info->bus); 921 usbd_transfer_done(xfer, 0); 922 USB_BUS_UNLOCK(info->bus); 923 return; 924 } 925 if (!xfer->flags_int.bdma_setup) { 926 struct usb_page *pg; 927 usb_frlength_t frlength_0; 928 uint8_t isread; 929 930 xfer->flags_int.bdma_setup = 1; 931 932 /* reset BUS-DMA load state */ 933 934 info->dma_error = 0; 935 936 if (xfer->flags_int.isochronous_xfr) { 937 /* only one frame buffer */ 938 nframes = 1; 939 frlength_0 = xfer->sumlen; 940 } else { 941 /* can be multiple frame buffers */ 942 nframes = xfer->nframes; 943 frlength_0 = xfer->frlengths[0]; 944 } 945 946 /* 947 * Set DMA direction first. This is needed to 948 * select the correct cache invalidate and cache 949 * flush operations. 950 */ 951 isread = USB_GET_DATA_ISREAD(xfer); 952 pg = xfer->dma_page_ptr; 953 954 if (xfer->flags_int.control_xfr && 955 xfer->flags_int.control_hdr) { 956 /* special case */ 957 if (xfer->flags_int.usb_mode == USB_MODE_DEVICE) { 958 /* The device controller writes to memory */ 959 xfer->frbuffers[0].isread = 1; 960 } else { 961 /* The host controller reads from memory */ 962 xfer->frbuffers[0].isread = 0; 963 } 964 } else { 965 /* default case */ 966 xfer->frbuffers[0].isread = isread; 967 } 968 969 /* 970 * Setup the "page_start" pointer which points to an array of 971 * USB pages where information about the physical address of a 972 * page will be stored. Also initialise the "isread" field of 973 * the USB page caches. 974 */ 975 xfer->frbuffers[0].page_start = pg; 976 977 info->dma_nframes = nframes; 978 info->dma_currframe = 0; 979 info->dma_frlength_0 = frlength_0; 980 981 pg += (frlength_0 / USB_PAGE_SIZE); 982 pg += 2; 983 984 while (--nframes > 0) { 985 xfer->frbuffers[nframes].isread = isread; 986 xfer->frbuffers[nframes].page_start = pg; 987 988 pg += (xfer->frlengths[nframes] / USB_PAGE_SIZE); 989 pg += 2; 990 } 991 } 992 if (info->dma_error) { 993 USB_BUS_LOCK(info->bus); 994 usbd_transfer_done(xfer, USB_ERR_DMA_LOAD_FAILED); 995 USB_BUS_UNLOCK(info->bus); 996 return; 997 } 998 if (info->dma_currframe != info->dma_nframes) { 999 if (info->dma_currframe == 0) { 1000 /* special case */ 1001 usb_pc_load_mem(xfer->frbuffers, 1002 info->dma_frlength_0, 0); 1003 } else { 1004 /* default case */ 1005 nframes = info->dma_currframe; 1006 usb_pc_load_mem(xfer->frbuffers + nframes, 1007 xfer->frlengths[nframes], 0); 1008 } 1009 1010 /* advance frame index */ 1011 info->dma_currframe++; 1012 1013 return; 1014 } 1015 /* go ahead */ 1016 usb_bdma_pre_sync(xfer); 1017 1018 /* start loading next USB transfer, if any */ 1019 usb_command_wrapper(pq, NULL); 1020 1021 /* finally start the hardware */ 1022 usbd_pipe_enter(xfer); 1023 } 1024 1025 /*------------------------------------------------------------------------* 1026 * usb_bdma_done_event 1027 * 1028 * This function is called when the BUS-DMA has loaded virtual memory 1029 * into DMA, if any. 1030 *------------------------------------------------------------------------*/ 1031 void 1032 usb_bdma_done_event(struct usb_dma_parent_tag *udpt) 1033 { 1034 struct usb_xfer_root *info; 1035 1036 info = USB_DMATAG_TO_XROOT(udpt); 1037 1038 USB_MTX_ASSERT(info->xfer_mtx, MA_OWNED); 1039 1040 /* copy error */ 1041 info->dma_error = udpt->dma_error; 1042 1043 /* enter workloop again */ 1044 usb_command_wrapper(&info->dma_q, 1045 info->dma_q.curr); 1046 } 1047 1048 /*------------------------------------------------------------------------* 1049 * usb_bdma_pre_sync 1050 * 1051 * This function handles DMA synchronisation that must be done before 1052 * an USB transfer is started. 1053 *------------------------------------------------------------------------*/ 1054 void 1055 usb_bdma_pre_sync(struct usb_xfer *xfer) 1056 { 1057 struct usb_page_cache *pc; 1058 usb_frcount_t nframes; 1059 1060 if (xfer->flags_int.isochronous_xfr) { 1061 /* only one frame buffer */ 1062 nframes = 1; 1063 } else { 1064 /* can be multiple frame buffers */ 1065 nframes = xfer->nframes; 1066 } 1067 1068 pc = xfer->frbuffers; 1069 1070 while (nframes--) { 1071 if (pc->isread) { 1072 usb_pc_cpu_invalidate(pc); 1073 } else { 1074 usb_pc_cpu_flush(pc); 1075 } 1076 pc++; 1077 } 1078 } 1079 1080 /*------------------------------------------------------------------------* 1081 * usb_bdma_post_sync 1082 * 1083 * This function handles DMA synchronisation that must be done after 1084 * an USB transfer is complete. 1085 *------------------------------------------------------------------------*/ 1086 void 1087 usb_bdma_post_sync(struct usb_xfer *xfer) 1088 { 1089 struct usb_page_cache *pc; 1090 usb_frcount_t nframes; 1091 1092 if (xfer->flags_int.isochronous_xfr) { 1093 /* only one frame buffer */ 1094 nframes = 1; 1095 } else { 1096 /* can be multiple frame buffers */ 1097 nframes = xfer->nframes; 1098 } 1099 1100 pc = xfer->frbuffers; 1101 1102 while (nframes--) { 1103 if (pc->isread) { 1104 usb_pc_cpu_invalidate(pc); 1105 } 1106 pc++; 1107 } 1108 } 1109 1110 #endif 1111