1 /* $FreeBSD$ */ 2 /*- 3 * Copyright (c) 2008 Hans Petter Selasky. All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 */ 26 27 #include <sys/stdint.h> 28 #include <sys/stddef.h> 29 #include <sys/param.h> 30 #include <sys/queue.h> 31 #include <sys/types.h> 32 #include <sys/systm.h> 33 #include <sys/kernel.h> 34 #include <sys/bus.h> 35 #include <sys/linker_set.h> 36 #include <sys/module.h> 37 #include <sys/lock.h> 38 #include <sys/mutex.h> 39 #include <sys/condvar.h> 40 #include <sys/sysctl.h> 41 #include <sys/sx.h> 42 #include <sys/unistd.h> 43 #include <sys/callout.h> 44 #include <sys/malloc.h> 45 #include <sys/priv.h> 46 47 #include <dev/usb/usb.h> 48 #include <dev/usb/usbdi.h> 49 #include <dev/usb/usbdi_util.h> 50 51 #define USB_DEBUG_VAR usb_debug 52 53 #include <dev/usb/usb_core.h> 54 #include <dev/usb/usb_busdma.h> 55 #include <dev/usb/usb_process.h> 56 #include <dev/usb/usb_transfer.h> 57 #include <dev/usb/usb_device.h> 58 #include <dev/usb/usb_util.h> 59 #include <dev/usb/usb_debug.h> 60 61 #include <dev/usb/usb_controller.h> 62 #include <dev/usb/usb_bus.h> 63 64 #if USB_HAVE_BUSDMA 65 static void usb_dma_tag_create(struct usb_dma_tag *, usb_size_t, usb_size_t); 66 static void usb_dma_tag_destroy(struct usb_dma_tag *); 67 static void usb_dma_lock_cb(void *, bus_dma_lock_op_t); 68 static void usb_pc_alloc_mem_cb(void *, bus_dma_segment_t *, int, int); 69 static void usb_pc_load_mem_cb(void *, bus_dma_segment_t *, int, int); 70 static void usb_pc_common_mem_cb(void *, bus_dma_segment_t *, int, int, 71 uint8_t); 72 #endif 73 74 /*------------------------------------------------------------------------* 75 * usbd_get_page - lookup DMA-able memory for the given offset 76 * 77 * NOTE: Only call this function when the "page_cache" structure has 78 * been properly initialized ! 79 *------------------------------------------------------------------------*/ 80 void 81 usbd_get_page(struct usb_page_cache *pc, usb_frlength_t offset, 82 struct usb_page_search *res) 83 { 84 struct usb_page *page; 85 86 #if USB_HAVE_BUSDMA 87 if (pc->page_start) { 88 89 /* Case 1 - something has been loaded into DMA */ 90 91 if (pc->buffer) { 92 93 /* Case 1a - Kernel Virtual Address */ 94 95 res->buffer = USB_ADD_BYTES(pc->buffer, offset); 96 } 97 offset += pc->page_offset_buf; 98 99 /* compute destination page */ 100 101 page = pc->page_start; 102 103 if (pc->ismultiseg) { 104 105 page += (offset / USB_PAGE_SIZE); 106 107 offset %= USB_PAGE_SIZE; 108 109 res->length = USB_PAGE_SIZE - offset; 110 res->physaddr = page->physaddr + offset; 111 } else { 112 res->length = 0 - 1; 113 res->physaddr = page->physaddr + offset; 114 } 115 if (!pc->buffer) { 116 117 /* Case 1b - Non Kernel Virtual Address */ 118 119 res->buffer = USB_ADD_BYTES(page->buffer, offset); 120 } 121 return; 122 } 123 #endif 124 /* Case 2 - Plain PIO */ 125 126 res->buffer = USB_ADD_BYTES(pc->buffer, offset); 127 res->length = 0 - 1; 128 #if USB_HAVE_BUSDMA 129 res->physaddr = 0; 130 #endif 131 } 132 133 /*------------------------------------------------------------------------* 134 * usbd_copy_in - copy directly to DMA-able memory 135 *------------------------------------------------------------------------*/ 136 void 137 usbd_copy_in(struct usb_page_cache *cache, usb_frlength_t offset, 138 const void *ptr, usb_frlength_t len) 139 { 140 struct usb_page_search buf_res; 141 142 while (len != 0) { 143 144 usbd_get_page(cache, offset, &buf_res); 145 146 if (buf_res.length > len) { 147 buf_res.length = len; 148 } 149 bcopy(ptr, buf_res.buffer, buf_res.length); 150 151 offset += buf_res.length; 152 len -= buf_res.length; 153 ptr = USB_ADD_BYTES(ptr, buf_res.length); 154 } 155 } 156 157 /*------------------------------------------------------------------------* 158 * usbd_copy_in_user - copy directly to DMA-able memory from userland 159 * 160 * Return values: 161 * 0: Success 162 * Else: Failure 163 *------------------------------------------------------------------------*/ 164 #if USB_HAVE_USER_IO 165 int 166 usbd_copy_in_user(struct usb_page_cache *cache, usb_frlength_t offset, 167 const void *ptr, usb_frlength_t len) 168 { 169 struct usb_page_search buf_res; 170 int error; 171 172 while (len != 0) { 173 174 usbd_get_page(cache, offset, &buf_res); 175 176 if (buf_res.length > len) { 177 buf_res.length = len; 178 } 179 error = copyin(ptr, buf_res.buffer, buf_res.length); 180 if (error) 181 return (error); 182 183 offset += buf_res.length; 184 len -= buf_res.length; 185 ptr = USB_ADD_BYTES(ptr, buf_res.length); 186 } 187 return (0); /* success */ 188 } 189 #endif 190 191 /*------------------------------------------------------------------------* 192 * usbd_m_copy_in - copy a mbuf chain directly into DMA-able memory 193 *------------------------------------------------------------------------*/ 194 #if USB_HAVE_MBUF 195 struct usb_m_copy_in_arg { 196 struct usb_page_cache *cache; 197 usb_frlength_t dst_offset; 198 }; 199 200 static int 201 usbd_m_copy_in_cb(void *arg, void *src, uint32_t count) 202 { 203 register struct usb_m_copy_in_arg *ua = arg; 204 205 usbd_copy_in(ua->cache, ua->dst_offset, src, count); 206 ua->dst_offset += count; 207 return (0); 208 } 209 210 void 211 usbd_m_copy_in(struct usb_page_cache *cache, usb_frlength_t dst_offset, 212 struct mbuf *m, usb_size_t src_offset, usb_frlength_t src_len) 213 { 214 struct usb_m_copy_in_arg arg = {cache, dst_offset}; 215 int error; 216 217 error = m_apply(m, src_offset, src_len, &usbd_m_copy_in_cb, &arg); 218 } 219 #endif 220 221 /*------------------------------------------------------------------------* 222 * usb_uiomove - factored out code 223 *------------------------------------------------------------------------*/ 224 #if USB_HAVE_USER_IO 225 int 226 usb_uiomove(struct usb_page_cache *pc, struct uio *uio, 227 usb_frlength_t pc_offset, usb_frlength_t len) 228 { 229 struct usb_page_search res; 230 int error = 0; 231 232 while (len != 0) { 233 234 usbd_get_page(pc, pc_offset, &res); 235 236 if (res.length > len) { 237 res.length = len; 238 } 239 /* 240 * "uiomove()" can sleep so one needs to make a wrapper, 241 * exiting the mutex and checking things 242 */ 243 error = uiomove(res.buffer, res.length, uio); 244 245 if (error) { 246 break; 247 } 248 pc_offset += res.length; 249 len -= res.length; 250 } 251 return (error); 252 } 253 #endif 254 255 /*------------------------------------------------------------------------* 256 * usbd_copy_out - copy directly from DMA-able memory 257 *------------------------------------------------------------------------*/ 258 void 259 usbd_copy_out(struct usb_page_cache *cache, usb_frlength_t offset, 260 void *ptr, usb_frlength_t len) 261 { 262 struct usb_page_search res; 263 264 while (len != 0) { 265 266 usbd_get_page(cache, offset, &res); 267 268 if (res.length > len) { 269 res.length = len; 270 } 271 bcopy(res.buffer, ptr, res.length); 272 273 offset += res.length; 274 len -= res.length; 275 ptr = USB_ADD_BYTES(ptr, res.length); 276 } 277 } 278 279 /*------------------------------------------------------------------------* 280 * usbd_copy_out_user - copy directly from DMA-able memory to userland 281 * 282 * Return values: 283 * 0: Success 284 * Else: Failure 285 *------------------------------------------------------------------------*/ 286 #if USB_HAVE_USER_IO 287 int 288 usbd_copy_out_user(struct usb_page_cache *cache, usb_frlength_t offset, 289 void *ptr, usb_frlength_t len) 290 { 291 struct usb_page_search res; 292 int error; 293 294 while (len != 0) { 295 296 usbd_get_page(cache, offset, &res); 297 298 if (res.length > len) { 299 res.length = len; 300 } 301 error = copyout(res.buffer, ptr, res.length); 302 if (error) 303 return (error); 304 305 offset += res.length; 306 len -= res.length; 307 ptr = USB_ADD_BYTES(ptr, res.length); 308 } 309 return (0); /* success */ 310 } 311 #endif 312 313 /*------------------------------------------------------------------------* 314 * usbd_frame_zero - zero DMA-able memory 315 *------------------------------------------------------------------------*/ 316 void 317 usbd_frame_zero(struct usb_page_cache *cache, usb_frlength_t offset, 318 usb_frlength_t len) 319 { 320 struct usb_page_search res; 321 322 while (len != 0) { 323 324 usbd_get_page(cache, offset, &res); 325 326 if (res.length > len) { 327 res.length = len; 328 } 329 bzero(res.buffer, res.length); 330 331 offset += res.length; 332 len -= res.length; 333 } 334 } 335 336 #if USB_HAVE_BUSDMA 337 338 /*------------------------------------------------------------------------* 339 * usb_dma_lock_cb - dummy callback 340 *------------------------------------------------------------------------*/ 341 static void 342 usb_dma_lock_cb(void *arg, bus_dma_lock_op_t op) 343 { 344 /* we use "mtx_owned()" instead of this function */ 345 } 346 347 /*------------------------------------------------------------------------* 348 * usb_dma_tag_create - allocate a DMA tag 349 * 350 * NOTE: If the "align" parameter has a value of 1 the DMA-tag will 351 * allow multi-segment mappings. Else all mappings are single-segment. 352 *------------------------------------------------------------------------*/ 353 static void 354 usb_dma_tag_create(struct usb_dma_tag *udt, 355 usb_size_t size, usb_size_t align) 356 { 357 bus_dma_tag_t tag; 358 359 if (bus_dma_tag_create 360 ( /* parent */ udt->tag_parent->tag, 361 /* alignment */ align, 362 /* boundary */ USB_PAGE_SIZE, 363 /* lowaddr */ (2ULL << (udt->tag_parent->dma_bits - 1)) - 1, 364 /* highaddr */ BUS_SPACE_MAXADDR, 365 /* filter */ NULL, 366 /* filterarg */ NULL, 367 /* maxsize */ size, 368 /* nsegments */ (align == 1) ? 369 (2 + (size / USB_PAGE_SIZE)) : 1, 370 /* maxsegsz */ (align == 1) ? 371 USB_PAGE_SIZE : size, 372 /* flags */ BUS_DMA_KEEP_PG_OFFSET, 373 /* lockfn */ &usb_dma_lock_cb, 374 /* lockarg */ NULL, 375 &tag)) { 376 tag = NULL; 377 } 378 udt->tag = tag; 379 } 380 381 /*------------------------------------------------------------------------* 382 * usb_dma_tag_free - free a DMA tag 383 *------------------------------------------------------------------------*/ 384 static void 385 usb_dma_tag_destroy(struct usb_dma_tag *udt) 386 { 387 bus_dma_tag_destroy(udt->tag); 388 } 389 390 /*------------------------------------------------------------------------* 391 * usb_pc_alloc_mem_cb - BUS-DMA callback function 392 *------------------------------------------------------------------------*/ 393 static void 394 usb_pc_alloc_mem_cb(void *arg, bus_dma_segment_t *segs, 395 int nseg, int error) 396 { 397 usb_pc_common_mem_cb(arg, segs, nseg, error, 0); 398 } 399 400 /*------------------------------------------------------------------------* 401 * usb_pc_load_mem_cb - BUS-DMA callback function 402 *------------------------------------------------------------------------*/ 403 static void 404 usb_pc_load_mem_cb(void *arg, bus_dma_segment_t *segs, 405 int nseg, int error) 406 { 407 usb_pc_common_mem_cb(arg, segs, nseg, error, 1); 408 } 409 410 /*------------------------------------------------------------------------* 411 * usb_pc_common_mem_cb - BUS-DMA callback function 412 *------------------------------------------------------------------------*/ 413 static void 414 usb_pc_common_mem_cb(void *arg, bus_dma_segment_t *segs, 415 int nseg, int error, uint8_t isload) 416 { 417 struct usb_dma_parent_tag *uptag; 418 struct usb_page_cache *pc; 419 struct usb_page *pg; 420 usb_size_t rem; 421 uint8_t owned; 422 423 pc = arg; 424 uptag = pc->tag_parent; 425 426 /* 427 * XXX There is sometimes recursive locking here. 428 * XXX We should try to find a better solution. 429 * XXX Until further the "owned" variable does 430 * XXX the trick. 431 */ 432 433 if (error) { 434 goto done; 435 } 436 pg = pc->page_start; 437 pg->physaddr = segs->ds_addr & ~(USB_PAGE_SIZE - 1); 438 rem = segs->ds_addr & (USB_PAGE_SIZE - 1); 439 pc->page_offset_buf = rem; 440 pc->page_offset_end += rem; 441 nseg--; 442 #ifdef USB_DEBUG 443 if (rem != (USB_P2U(pc->buffer) & (USB_PAGE_SIZE - 1))) { 444 /* 445 * This check verifies that the physical address is correct: 446 */ 447 DPRINTFN(0, "Page offset was not preserved!\n"); 448 error = 1; 449 goto done; 450 } 451 #endif 452 while (nseg > 0) { 453 nseg--; 454 segs++; 455 pg++; 456 pg->physaddr = segs->ds_addr & ~(USB_PAGE_SIZE - 1); 457 } 458 459 done: 460 owned = mtx_owned(uptag->mtx); 461 if (!owned) 462 mtx_lock(uptag->mtx); 463 464 uptag->dma_error = (error ? 1 : 0); 465 if (isload) { 466 (uptag->func) (uptag); 467 } else { 468 cv_broadcast(uptag->cv); 469 } 470 if (!owned) 471 mtx_unlock(uptag->mtx); 472 } 473 474 /*------------------------------------------------------------------------* 475 * usb_pc_alloc_mem - allocate DMA'able memory 476 * 477 * Returns: 478 * 0: Success 479 * Else: Failure 480 *------------------------------------------------------------------------*/ 481 uint8_t 482 usb_pc_alloc_mem(struct usb_page_cache *pc, struct usb_page *pg, 483 usb_size_t size, usb_size_t align) 484 { 485 struct usb_dma_parent_tag *uptag; 486 struct usb_dma_tag *utag; 487 bus_dmamap_t map; 488 void *ptr; 489 int err; 490 491 uptag = pc->tag_parent; 492 493 if (align != 1) { 494 /* 495 * The alignment must be greater or equal to the 496 * "size" else the object can be split between two 497 * memory pages and we get a problem! 498 */ 499 while (align < size) { 500 align *= 2; 501 if (align == 0) { 502 goto error; 503 } 504 } 505 #if 1 506 /* 507 * XXX BUS-DMA workaround - FIXME later: 508 * 509 * We assume that that the aligment at this point of 510 * the code is greater than or equal to the size and 511 * less than two times the size, so that if we double 512 * the size, the size will be greater than the 513 * alignment. 514 * 515 * The bus-dma system has a check for "alignment" 516 * being less than "size". If that check fails we end 517 * up using contigmalloc which is page based even for 518 * small allocations. Try to avoid that to save 519 * memory, hence we sometimes to a large number of 520 * small allocations! 521 */ 522 if (size <= (USB_PAGE_SIZE / 2)) { 523 size *= 2; 524 } 525 #endif 526 } 527 /* get the correct DMA tag */ 528 utag = usb_dma_tag_find(uptag, size, align); 529 if (utag == NULL) { 530 goto error; 531 } 532 /* allocate memory */ 533 if (bus_dmamem_alloc( 534 utag->tag, &ptr, (BUS_DMA_WAITOK | BUS_DMA_COHERENT), &map)) { 535 goto error; 536 } 537 /* setup page cache */ 538 pc->buffer = ptr; 539 pc->page_start = pg; 540 pc->page_offset_buf = 0; 541 pc->page_offset_end = size; 542 pc->map = map; 543 pc->tag = utag->tag; 544 pc->ismultiseg = (align == 1); 545 546 mtx_lock(uptag->mtx); 547 548 /* load memory into DMA */ 549 err = bus_dmamap_load( 550 utag->tag, map, ptr, size, &usb_pc_alloc_mem_cb, 551 pc, (BUS_DMA_WAITOK | BUS_DMA_COHERENT)); 552 553 if (err == EINPROGRESS) { 554 cv_wait(uptag->cv, uptag->mtx); 555 err = 0; 556 } 557 mtx_unlock(uptag->mtx); 558 559 if (err || uptag->dma_error) { 560 bus_dmamem_free(utag->tag, ptr, map); 561 goto error; 562 } 563 bzero(ptr, size); 564 565 usb_pc_cpu_flush(pc); 566 567 return (0); 568 569 error: 570 /* reset most of the page cache */ 571 pc->buffer = NULL; 572 pc->page_start = NULL; 573 pc->page_offset_buf = 0; 574 pc->page_offset_end = 0; 575 pc->map = NULL; 576 pc->tag = NULL; 577 return (1); 578 } 579 580 /*------------------------------------------------------------------------* 581 * usb_pc_free_mem - free DMA memory 582 * 583 * This function is NULL safe. 584 *------------------------------------------------------------------------*/ 585 void 586 usb_pc_free_mem(struct usb_page_cache *pc) 587 { 588 if (pc && pc->buffer) { 589 590 bus_dmamap_unload(pc->tag, pc->map); 591 592 bus_dmamem_free(pc->tag, pc->buffer, pc->map); 593 594 pc->buffer = NULL; 595 } 596 } 597 598 /*------------------------------------------------------------------------* 599 * usb_pc_load_mem - load virtual memory into DMA 600 * 601 * Return values: 602 * 0: Success 603 * Else: Error 604 *------------------------------------------------------------------------*/ 605 uint8_t 606 usb_pc_load_mem(struct usb_page_cache *pc, usb_size_t size, uint8_t sync) 607 { 608 /* setup page cache */ 609 pc->page_offset_buf = 0; 610 pc->page_offset_end = size; 611 pc->ismultiseg = 1; 612 613 mtx_assert(pc->tag_parent->mtx, MA_OWNED); 614 615 if (size > 0) { 616 if (sync) { 617 struct usb_dma_parent_tag *uptag; 618 int err; 619 620 uptag = pc->tag_parent; 621 622 /* 623 * We have to unload the previous loaded DMA 624 * pages before trying to load a new one! 625 */ 626 bus_dmamap_unload(pc->tag, pc->map); 627 628 /* 629 * Try to load memory into DMA. 630 */ 631 err = bus_dmamap_load( 632 pc->tag, pc->map, pc->buffer, size, 633 &usb_pc_alloc_mem_cb, pc, BUS_DMA_WAITOK); 634 if (err == EINPROGRESS) { 635 cv_wait(uptag->cv, uptag->mtx); 636 err = 0; 637 } 638 if (err || uptag->dma_error) { 639 return (1); 640 } 641 } else { 642 643 /* 644 * We have to unload the previous loaded DMA 645 * pages before trying to load a new one! 646 */ 647 bus_dmamap_unload(pc->tag, pc->map); 648 649 /* 650 * Try to load memory into DMA. The callback 651 * will be called in all cases: 652 */ 653 if (bus_dmamap_load( 654 pc->tag, pc->map, pc->buffer, size, 655 &usb_pc_load_mem_cb, pc, BUS_DMA_WAITOK)) { 656 } 657 } 658 } else { 659 if (!sync) { 660 /* 661 * Call callback so that refcount is decremented 662 * properly: 663 */ 664 pc->tag_parent->dma_error = 0; 665 (pc->tag_parent->func) (pc->tag_parent); 666 } 667 } 668 return (0); 669 } 670 671 /*------------------------------------------------------------------------* 672 * usb_pc_cpu_invalidate - invalidate CPU cache 673 *------------------------------------------------------------------------*/ 674 void 675 usb_pc_cpu_invalidate(struct usb_page_cache *pc) 676 { 677 if (pc->page_offset_end == pc->page_offset_buf) { 678 /* nothing has been loaded into this page cache! */ 679 return; 680 } 681 bus_dmamap_sync(pc->tag, pc->map, 682 BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD); 683 } 684 685 /*------------------------------------------------------------------------* 686 * usb_pc_cpu_flush - flush CPU cache 687 *------------------------------------------------------------------------*/ 688 void 689 usb_pc_cpu_flush(struct usb_page_cache *pc) 690 { 691 if (pc->page_offset_end == pc->page_offset_buf) { 692 /* nothing has been loaded into this page cache! */ 693 return; 694 } 695 bus_dmamap_sync(pc->tag, pc->map, 696 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD); 697 } 698 699 /*------------------------------------------------------------------------* 700 * usb_pc_dmamap_create - create a DMA map 701 * 702 * Returns: 703 * 0: Success 704 * Else: Failure 705 *------------------------------------------------------------------------*/ 706 uint8_t 707 usb_pc_dmamap_create(struct usb_page_cache *pc, usb_size_t size) 708 { 709 struct usb_xfer_root *info; 710 struct usb_dma_tag *utag; 711 712 /* get info */ 713 info = USB_DMATAG_TO_XROOT(pc->tag_parent); 714 715 /* sanity check */ 716 if (info == NULL) { 717 goto error; 718 } 719 utag = usb_dma_tag_find(pc->tag_parent, size, 1); 720 if (utag == NULL) { 721 goto error; 722 } 723 /* create DMA map */ 724 if (bus_dmamap_create(utag->tag, 0, &pc->map)) { 725 goto error; 726 } 727 pc->tag = utag->tag; 728 return 0; /* success */ 729 730 error: 731 pc->map = NULL; 732 pc->tag = NULL; 733 return 1; /* failure */ 734 } 735 736 /*------------------------------------------------------------------------* 737 * usb_pc_dmamap_destroy 738 * 739 * This function is NULL safe. 740 *------------------------------------------------------------------------*/ 741 void 742 usb_pc_dmamap_destroy(struct usb_page_cache *pc) 743 { 744 if (pc && pc->tag) { 745 bus_dmamap_destroy(pc->tag, pc->map); 746 pc->tag = NULL; 747 pc->map = NULL; 748 } 749 } 750 751 /*------------------------------------------------------------------------* 752 * usb_dma_tag_find - factored out code 753 *------------------------------------------------------------------------*/ 754 struct usb_dma_tag * 755 usb_dma_tag_find(struct usb_dma_parent_tag *udpt, 756 usb_size_t size, usb_size_t align) 757 { 758 struct usb_dma_tag *udt; 759 uint8_t nudt; 760 761 USB_ASSERT(align > 0, ("Invalid parameter align = 0!\n")); 762 USB_ASSERT(size > 0, ("Invalid parameter size = 0!\n")); 763 764 udt = udpt->utag_first; 765 nudt = udpt->utag_max; 766 767 while (nudt--) { 768 769 if (udt->align == 0) { 770 usb_dma_tag_create(udt, size, align); 771 if (udt->tag == NULL) { 772 return (NULL); 773 } 774 udt->align = align; 775 udt->size = size; 776 return (udt); 777 } 778 if ((udt->align == align) && (udt->size == size)) { 779 return (udt); 780 } 781 udt++; 782 } 783 return (NULL); 784 } 785 786 /*------------------------------------------------------------------------* 787 * usb_dma_tag_setup - initialise USB DMA tags 788 *------------------------------------------------------------------------*/ 789 void 790 usb_dma_tag_setup(struct usb_dma_parent_tag *udpt, 791 struct usb_dma_tag *udt, bus_dma_tag_t dmat, 792 struct mtx *mtx, usb_dma_callback_t *func, 793 uint8_t ndmabits, uint8_t nudt) 794 { 795 bzero(udpt, sizeof(*udpt)); 796 797 /* sanity checking */ 798 if ((nudt == 0) || 799 (ndmabits == 0) || 800 (mtx == NULL)) { 801 /* something is corrupt */ 802 return; 803 } 804 /* initialise condition variable */ 805 cv_init(udpt->cv, "USB DMA CV"); 806 807 /* store some information */ 808 udpt->mtx = mtx; 809 udpt->func = func; 810 udpt->tag = dmat; 811 udpt->utag_first = udt; 812 udpt->utag_max = nudt; 813 udpt->dma_bits = ndmabits; 814 815 while (nudt--) { 816 bzero(udt, sizeof(*udt)); 817 udt->tag_parent = udpt; 818 udt++; 819 } 820 } 821 822 /*------------------------------------------------------------------------* 823 * usb_bus_tag_unsetup - factored out code 824 *------------------------------------------------------------------------*/ 825 void 826 usb_dma_tag_unsetup(struct usb_dma_parent_tag *udpt) 827 { 828 struct usb_dma_tag *udt; 829 uint8_t nudt; 830 831 udt = udpt->utag_first; 832 nudt = udpt->utag_max; 833 834 while (nudt--) { 835 836 if (udt->align) { 837 /* destroy the USB DMA tag */ 838 usb_dma_tag_destroy(udt); 839 udt->align = 0; 840 } 841 udt++; 842 } 843 844 if (udpt->utag_max) { 845 /* destroy the condition variable */ 846 cv_destroy(udpt->cv); 847 } 848 } 849 850 /*------------------------------------------------------------------------* 851 * usb_bdma_work_loop 852 * 853 * This function handles loading of virtual buffers into DMA and is 854 * only called when "dma_refcount" is zero. 855 *------------------------------------------------------------------------*/ 856 void 857 usb_bdma_work_loop(struct usb_xfer_queue *pq) 858 { 859 struct usb_xfer_root *info; 860 struct usb_xfer *xfer; 861 usb_frcount_t nframes; 862 863 xfer = pq->curr; 864 info = xfer->xroot; 865 866 mtx_assert(info->xfer_mtx, MA_OWNED); 867 868 if (xfer->error) { 869 /* some error happened */ 870 USB_BUS_LOCK(info->bus); 871 usbd_transfer_done(xfer, 0); 872 USB_BUS_UNLOCK(info->bus); 873 return; 874 } 875 if (!xfer->flags_int.bdma_setup) { 876 struct usb_page *pg; 877 usb_frlength_t frlength_0; 878 uint8_t isread; 879 880 xfer->flags_int.bdma_setup = 1; 881 882 /* reset BUS-DMA load state */ 883 884 info->dma_error = 0; 885 886 if (xfer->flags_int.isochronous_xfr) { 887 /* only one frame buffer */ 888 nframes = 1; 889 frlength_0 = xfer->sumlen; 890 } else { 891 /* can be multiple frame buffers */ 892 nframes = xfer->nframes; 893 frlength_0 = xfer->frlengths[0]; 894 } 895 896 /* 897 * Set DMA direction first. This is needed to 898 * select the correct cache invalidate and cache 899 * flush operations. 900 */ 901 isread = USB_GET_DATA_ISREAD(xfer); 902 pg = xfer->dma_page_ptr; 903 904 if (xfer->flags_int.control_xfr && 905 xfer->flags_int.control_hdr) { 906 /* special case */ 907 if (xfer->flags_int.usb_mode == USB_MODE_DEVICE) { 908 /* The device controller writes to memory */ 909 xfer->frbuffers[0].isread = 1; 910 } else { 911 /* The host controller reads from memory */ 912 xfer->frbuffers[0].isread = 0; 913 } 914 } else { 915 /* default case */ 916 xfer->frbuffers[0].isread = isread; 917 } 918 919 /* 920 * Setup the "page_start" pointer which points to an array of 921 * USB pages where information about the physical address of a 922 * page will be stored. Also initialise the "isread" field of 923 * the USB page caches. 924 */ 925 xfer->frbuffers[0].page_start = pg; 926 927 info->dma_nframes = nframes; 928 info->dma_currframe = 0; 929 info->dma_frlength_0 = frlength_0; 930 931 pg += (frlength_0 / USB_PAGE_SIZE); 932 pg += 2; 933 934 while (--nframes > 0) { 935 xfer->frbuffers[nframes].isread = isread; 936 xfer->frbuffers[nframes].page_start = pg; 937 938 pg += (xfer->frlengths[nframes] / USB_PAGE_SIZE); 939 pg += 2; 940 } 941 942 } 943 if (info->dma_error) { 944 USB_BUS_LOCK(info->bus); 945 usbd_transfer_done(xfer, USB_ERR_DMA_LOAD_FAILED); 946 USB_BUS_UNLOCK(info->bus); 947 return; 948 } 949 if (info->dma_currframe != info->dma_nframes) { 950 951 if (info->dma_currframe == 0) { 952 /* special case */ 953 usb_pc_load_mem(xfer->frbuffers, 954 info->dma_frlength_0, 0); 955 } else { 956 /* default case */ 957 nframes = info->dma_currframe; 958 usb_pc_load_mem(xfer->frbuffers + nframes, 959 xfer->frlengths[nframes], 0); 960 } 961 962 /* advance frame index */ 963 info->dma_currframe++; 964 965 return; 966 } 967 /* go ahead */ 968 usb_bdma_pre_sync(xfer); 969 970 /* start loading next USB transfer, if any */ 971 usb_command_wrapper(pq, NULL); 972 973 /* finally start the hardware */ 974 usbd_pipe_enter(xfer); 975 } 976 977 /*------------------------------------------------------------------------* 978 * usb_bdma_done_event 979 * 980 * This function is called when the BUS-DMA has loaded virtual memory 981 * into DMA, if any. 982 *------------------------------------------------------------------------*/ 983 void 984 usb_bdma_done_event(struct usb_dma_parent_tag *udpt) 985 { 986 struct usb_xfer_root *info; 987 988 info = USB_DMATAG_TO_XROOT(udpt); 989 990 mtx_assert(info->xfer_mtx, MA_OWNED); 991 992 /* copy error */ 993 info->dma_error = udpt->dma_error; 994 995 /* enter workloop again */ 996 usb_command_wrapper(&info->dma_q, 997 info->dma_q.curr); 998 } 999 1000 /*------------------------------------------------------------------------* 1001 * usb_bdma_pre_sync 1002 * 1003 * This function handles DMA synchronisation that must be done before 1004 * an USB transfer is started. 1005 *------------------------------------------------------------------------*/ 1006 void 1007 usb_bdma_pre_sync(struct usb_xfer *xfer) 1008 { 1009 struct usb_page_cache *pc; 1010 usb_frcount_t nframes; 1011 1012 if (xfer->flags_int.isochronous_xfr) { 1013 /* only one frame buffer */ 1014 nframes = 1; 1015 } else { 1016 /* can be multiple frame buffers */ 1017 nframes = xfer->nframes; 1018 } 1019 1020 pc = xfer->frbuffers; 1021 1022 while (nframes--) { 1023 1024 if (pc->isread) { 1025 usb_pc_cpu_invalidate(pc); 1026 } else { 1027 usb_pc_cpu_flush(pc); 1028 } 1029 pc++; 1030 } 1031 } 1032 1033 /*------------------------------------------------------------------------* 1034 * usb_bdma_post_sync 1035 * 1036 * This function handles DMA synchronisation that must be done after 1037 * an USB transfer is complete. 1038 *------------------------------------------------------------------------*/ 1039 void 1040 usb_bdma_post_sync(struct usb_xfer *xfer) 1041 { 1042 struct usb_page_cache *pc; 1043 usb_frcount_t nframes; 1044 1045 if (xfer->flags_int.isochronous_xfr) { 1046 /* only one frame buffer */ 1047 nframes = 1; 1048 } else { 1049 /* can be multiple frame buffers */ 1050 nframes = xfer->nframes; 1051 } 1052 1053 pc = xfer->frbuffers; 1054 1055 while (nframes--) { 1056 if (pc->isread) { 1057 usb_pc_cpu_invalidate(pc); 1058 } 1059 pc++; 1060 } 1061 } 1062 1063 #endif 1064