1 /* $FreeBSD$ */ 2 /*- 3 * Copyright (c) 2008 Hans Petter Selasky. All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 */ 26 27 #ifdef USB_GLOBAL_INCLUDE_FILE 28 #include USB_GLOBAL_INCLUDE_FILE 29 #else 30 #include <sys/stdint.h> 31 #include <sys/stddef.h> 32 #include <sys/param.h> 33 #include <sys/queue.h> 34 #include <sys/types.h> 35 #include <sys/systm.h> 36 #include <sys/kernel.h> 37 #include <sys/bus.h> 38 #include <sys/module.h> 39 #include <sys/lock.h> 40 #include <sys/mutex.h> 41 #include <sys/condvar.h> 42 #include <sys/sysctl.h> 43 #include <sys/sx.h> 44 #include <sys/unistd.h> 45 #include <sys/callout.h> 46 #include <sys/malloc.h> 47 #include <sys/priv.h> 48 49 #include <dev/usb/usb.h> 50 #include <dev/usb/usbdi.h> 51 #include <dev/usb/usbdi_util.h> 52 53 #define USB_DEBUG_VAR usb_debug 54 55 #include <dev/usb/usb_core.h> 56 #include <dev/usb/usb_busdma.h> 57 #include <dev/usb/usb_process.h> 58 #include <dev/usb/usb_transfer.h> 59 #include <dev/usb/usb_device.h> 60 #include <dev/usb/usb_util.h> 61 #include <dev/usb/usb_debug.h> 62 63 #include <dev/usb/usb_controller.h> 64 #include <dev/usb/usb_bus.h> 65 #endif /* USB_GLOBAL_INCLUDE_FILE */ 66 67 #if USB_HAVE_BUSDMA 68 static void usb_dma_tag_create(struct usb_dma_tag *, usb_size_t, usb_size_t); 69 static void usb_dma_tag_destroy(struct usb_dma_tag *); 70 static void usb_dma_lock_cb(void *, bus_dma_lock_op_t); 71 static void usb_pc_alloc_mem_cb(void *, bus_dma_segment_t *, int, int); 72 static void usb_pc_load_mem_cb(void *, bus_dma_segment_t *, int, int); 73 static void usb_pc_common_mem_cb(void *, bus_dma_segment_t *, int, int, 74 uint8_t); 75 #endif 76 77 /*------------------------------------------------------------------------* 78 * usbd_get_page - lookup DMA-able memory for the given offset 79 * 80 * NOTE: Only call this function when the "page_cache" structure has 81 * been properly initialized ! 82 *------------------------------------------------------------------------*/ 83 void 84 usbd_get_page(struct usb_page_cache *pc, usb_frlength_t offset, 85 struct usb_page_search *res) 86 { 87 #if USB_HAVE_BUSDMA 88 struct usb_page *page; 89 90 if (pc->page_start) { 91 92 /* Case 1 - something has been loaded into DMA */ 93 94 if (pc->buffer) { 95 96 /* Case 1a - Kernel Virtual Address */ 97 98 res->buffer = USB_ADD_BYTES(pc->buffer, offset); 99 } 100 offset += pc->page_offset_buf; 101 102 /* compute destination page */ 103 104 page = pc->page_start; 105 106 if (pc->ismultiseg) { 107 108 page += (offset / USB_PAGE_SIZE); 109 110 offset %= USB_PAGE_SIZE; 111 112 res->length = USB_PAGE_SIZE - offset; 113 res->physaddr = page->physaddr + offset; 114 } else { 115 res->length = (usb_size_t)-1; 116 res->physaddr = page->physaddr + offset; 117 } 118 if (!pc->buffer) { 119 120 /* Case 1b - Non Kernel Virtual Address */ 121 122 res->buffer = USB_ADD_BYTES(page->buffer, offset); 123 } 124 return; 125 } 126 #endif 127 /* Case 2 - Plain PIO */ 128 129 res->buffer = USB_ADD_BYTES(pc->buffer, offset); 130 res->length = (usb_size_t)-1; 131 #if USB_HAVE_BUSDMA 132 res->physaddr = 0; 133 #endif 134 } 135 136 /*------------------------------------------------------------------------* 137 * usb_pc_buffer_is_aligned - verify alignment 138 * 139 * This function is used to check if a page cache buffer is properly 140 * aligned to reduce the use of bounce buffers in PIO mode. 141 *------------------------------------------------------------------------*/ 142 uint8_t 143 usb_pc_buffer_is_aligned(struct usb_page_cache *pc, usb_frlength_t offset, 144 usb_frlength_t len, usb_frlength_t mask) 145 { 146 struct usb_page_search buf_res; 147 148 while (len != 0) { 149 150 usbd_get_page(pc, offset, &buf_res); 151 152 if (buf_res.length > len) 153 buf_res.length = len; 154 if (USB_P2U(buf_res.buffer) & mask) 155 return (0); 156 if (buf_res.length & mask) 157 return (0); 158 159 offset += buf_res.length; 160 len -= buf_res.length; 161 } 162 return (1); 163 } 164 165 /*------------------------------------------------------------------------* 166 * usbd_copy_in - copy directly to DMA-able memory 167 *------------------------------------------------------------------------*/ 168 void 169 usbd_copy_in(struct usb_page_cache *cache, usb_frlength_t offset, 170 const void *ptr, usb_frlength_t len) 171 { 172 struct usb_page_search buf_res; 173 174 while (len != 0) { 175 176 usbd_get_page(cache, offset, &buf_res); 177 178 if (buf_res.length > len) { 179 buf_res.length = len; 180 } 181 memcpy(buf_res.buffer, ptr, buf_res.length); 182 183 offset += buf_res.length; 184 len -= buf_res.length; 185 ptr = USB_ADD_BYTES(ptr, buf_res.length); 186 } 187 } 188 189 /*------------------------------------------------------------------------* 190 * usbd_copy_in_user - copy directly to DMA-able memory from userland 191 * 192 * Return values: 193 * 0: Success 194 * Else: Failure 195 *------------------------------------------------------------------------*/ 196 #if USB_HAVE_USER_IO 197 int 198 usbd_copy_in_user(struct usb_page_cache *cache, usb_frlength_t offset, 199 const void *ptr, usb_frlength_t len) 200 { 201 struct usb_page_search buf_res; 202 int error; 203 204 while (len != 0) { 205 206 usbd_get_page(cache, offset, &buf_res); 207 208 if (buf_res.length > len) { 209 buf_res.length = len; 210 } 211 error = copyin(ptr, buf_res.buffer, buf_res.length); 212 if (error) 213 return (error); 214 215 offset += buf_res.length; 216 len -= buf_res.length; 217 ptr = USB_ADD_BYTES(ptr, buf_res.length); 218 } 219 return (0); /* success */ 220 } 221 #endif 222 223 /*------------------------------------------------------------------------* 224 * usbd_m_copy_in - copy a mbuf chain directly into DMA-able memory 225 *------------------------------------------------------------------------*/ 226 #if USB_HAVE_MBUF 227 struct usb_m_copy_in_arg { 228 struct usb_page_cache *cache; 229 usb_frlength_t dst_offset; 230 }; 231 232 static int 233 usbd_m_copy_in_cb(void *arg, void *src, uint32_t count) 234 { 235 register struct usb_m_copy_in_arg *ua = arg; 236 237 usbd_copy_in(ua->cache, ua->dst_offset, src, count); 238 ua->dst_offset += count; 239 return (0); 240 } 241 242 void 243 usbd_m_copy_in(struct usb_page_cache *cache, usb_frlength_t dst_offset, 244 struct mbuf *m, usb_size_t src_offset, usb_frlength_t src_len) 245 { 246 struct usb_m_copy_in_arg arg = {cache, dst_offset}; 247 (void) m_apply(m, src_offset, src_len, &usbd_m_copy_in_cb, &arg); 248 } 249 #endif 250 251 /*------------------------------------------------------------------------* 252 * usb_uiomove - factored out code 253 *------------------------------------------------------------------------*/ 254 #if USB_HAVE_USER_IO 255 int 256 usb_uiomove(struct usb_page_cache *pc, struct uio *uio, 257 usb_frlength_t pc_offset, usb_frlength_t len) 258 { 259 struct usb_page_search res; 260 int error = 0; 261 262 while (len != 0) { 263 264 usbd_get_page(pc, pc_offset, &res); 265 266 if (res.length > len) { 267 res.length = len; 268 } 269 /* 270 * "uiomove()" can sleep so one needs to make a wrapper, 271 * exiting the mutex and checking things 272 */ 273 error = uiomove(res.buffer, res.length, uio); 274 275 if (error) { 276 break; 277 } 278 pc_offset += res.length; 279 len -= res.length; 280 } 281 return (error); 282 } 283 #endif 284 285 /*------------------------------------------------------------------------* 286 * usbd_copy_out - copy directly from DMA-able memory 287 *------------------------------------------------------------------------*/ 288 void 289 usbd_copy_out(struct usb_page_cache *cache, usb_frlength_t offset, 290 void *ptr, usb_frlength_t len) 291 { 292 struct usb_page_search res; 293 294 while (len != 0) { 295 296 usbd_get_page(cache, offset, &res); 297 298 if (res.length > len) { 299 res.length = len; 300 } 301 memcpy(ptr, res.buffer, res.length); 302 303 offset += res.length; 304 len -= res.length; 305 ptr = USB_ADD_BYTES(ptr, res.length); 306 } 307 } 308 309 /*------------------------------------------------------------------------* 310 * usbd_copy_out_user - copy directly from DMA-able memory to userland 311 * 312 * Return values: 313 * 0: Success 314 * Else: Failure 315 *------------------------------------------------------------------------*/ 316 #if USB_HAVE_USER_IO 317 int 318 usbd_copy_out_user(struct usb_page_cache *cache, usb_frlength_t offset, 319 void *ptr, usb_frlength_t len) 320 { 321 struct usb_page_search res; 322 int error; 323 324 while (len != 0) { 325 326 usbd_get_page(cache, offset, &res); 327 328 if (res.length > len) { 329 res.length = len; 330 } 331 error = copyout(res.buffer, ptr, res.length); 332 if (error) 333 return (error); 334 335 offset += res.length; 336 len -= res.length; 337 ptr = USB_ADD_BYTES(ptr, res.length); 338 } 339 return (0); /* success */ 340 } 341 #endif 342 343 /*------------------------------------------------------------------------* 344 * usbd_frame_zero - zero DMA-able memory 345 *------------------------------------------------------------------------*/ 346 void 347 usbd_frame_zero(struct usb_page_cache *cache, usb_frlength_t offset, 348 usb_frlength_t len) 349 { 350 struct usb_page_search res; 351 352 while (len != 0) { 353 354 usbd_get_page(cache, offset, &res); 355 356 if (res.length > len) { 357 res.length = len; 358 } 359 memset(res.buffer, 0, res.length); 360 361 offset += res.length; 362 len -= res.length; 363 } 364 } 365 366 #if USB_HAVE_BUSDMA 367 368 /*------------------------------------------------------------------------* 369 * usb_dma_lock_cb - dummy callback 370 *------------------------------------------------------------------------*/ 371 static void 372 usb_dma_lock_cb(void *arg, bus_dma_lock_op_t op) 373 { 374 /* we use "mtx_owned()" instead of this function */ 375 } 376 377 /*------------------------------------------------------------------------* 378 * usb_dma_tag_create - allocate a DMA tag 379 * 380 * NOTE: If the "align" parameter has a value of 1 the DMA-tag will 381 * allow multi-segment mappings. Else all mappings are single-segment. 382 *------------------------------------------------------------------------*/ 383 static void 384 usb_dma_tag_create(struct usb_dma_tag *udt, 385 usb_size_t size, usb_size_t align) 386 { 387 bus_dma_tag_t tag; 388 389 if (bus_dma_tag_create 390 ( /* parent */ udt->tag_parent->tag, 391 /* alignment */ align, 392 /* boundary */ 0, 393 /* lowaddr */ (2ULL << (udt->tag_parent->dma_bits - 1)) - 1, 394 /* highaddr */ BUS_SPACE_MAXADDR, 395 /* filter */ NULL, 396 /* filterarg */ NULL, 397 /* maxsize */ size, 398 /* nsegments */ (align == 1 && size > 1) ? 399 (2 + (size / USB_PAGE_SIZE)) : 1, 400 /* maxsegsz */ (align == 1 && size > USB_PAGE_SIZE) ? 401 USB_PAGE_SIZE : size, 402 /* flags */ BUS_DMA_KEEP_PG_OFFSET, 403 /* lockfn */ &usb_dma_lock_cb, 404 /* lockarg */ NULL, 405 &tag)) { 406 tag = NULL; 407 } 408 udt->tag = tag; 409 } 410 411 /*------------------------------------------------------------------------* 412 * usb_dma_tag_free - free a DMA tag 413 *------------------------------------------------------------------------*/ 414 static void 415 usb_dma_tag_destroy(struct usb_dma_tag *udt) 416 { 417 bus_dma_tag_destroy(udt->tag); 418 } 419 420 /*------------------------------------------------------------------------* 421 * usb_pc_alloc_mem_cb - BUS-DMA callback function 422 *------------------------------------------------------------------------*/ 423 static void 424 usb_pc_alloc_mem_cb(void *arg, bus_dma_segment_t *segs, 425 int nseg, int error) 426 { 427 usb_pc_common_mem_cb(arg, segs, nseg, error, 0); 428 } 429 430 /*------------------------------------------------------------------------* 431 * usb_pc_load_mem_cb - BUS-DMA callback function 432 *------------------------------------------------------------------------*/ 433 static void 434 usb_pc_load_mem_cb(void *arg, bus_dma_segment_t *segs, 435 int nseg, int error) 436 { 437 usb_pc_common_mem_cb(arg, segs, nseg, error, 1); 438 } 439 440 /*------------------------------------------------------------------------* 441 * usb_pc_common_mem_cb - BUS-DMA callback function 442 *------------------------------------------------------------------------*/ 443 static void 444 usb_pc_common_mem_cb(void *arg, bus_dma_segment_t *segs, 445 int nseg, int error, uint8_t isload) 446 { 447 struct usb_dma_parent_tag *uptag; 448 struct usb_page_cache *pc; 449 struct usb_page *pg; 450 usb_size_t rem; 451 bus_size_t off; 452 uint8_t owned; 453 454 pc = arg; 455 uptag = pc->tag_parent; 456 457 /* 458 * XXX There is sometimes recursive locking here. 459 * XXX We should try to find a better solution. 460 * XXX Until further the "owned" variable does 461 * XXX the trick. 462 */ 463 464 if (error) { 465 goto done; 466 } 467 468 off = 0; 469 pg = pc->page_start; 470 pg->physaddr = rounddown2(segs->ds_addr, USB_PAGE_SIZE); 471 rem = segs->ds_addr & (USB_PAGE_SIZE - 1); 472 pc->page_offset_buf = rem; 473 pc->page_offset_end += rem; 474 #ifdef USB_DEBUG 475 if (nseg > 1) { 476 int x; 477 478 for (x = 0; x != nseg - 1; x++) { 479 if (((segs[x].ds_addr + segs[x].ds_len) & (USB_PAGE_SIZE - 1)) == 480 ((segs[x + 1].ds_addr & (USB_PAGE_SIZE - 1)))) 481 continue; 482 /* 483 * This check verifies there is no page offset 484 * hole between any of the segments. See the 485 * BUS_DMA_KEEP_PG_OFFSET flag. 486 */ 487 DPRINTFN(0, "Page offset was not preserved\n"); 488 error = 1; 489 goto done; 490 } 491 } 492 #endif 493 while (pc->ismultiseg) { 494 off += USB_PAGE_SIZE; 495 if (off >= (segs->ds_len + rem)) { 496 /* page crossing */ 497 nseg--; 498 segs++; 499 off = 0; 500 rem = 0; 501 if (nseg == 0) 502 break; 503 } 504 pg++; 505 pg->physaddr = rounddown2(segs->ds_addr + off, USB_PAGE_SIZE); 506 } 507 508 done: 509 owned = mtx_owned(uptag->mtx); 510 if (!owned) 511 USB_MTX_LOCK(uptag->mtx); 512 513 uptag->dma_error = (error ? 1 : 0); 514 if (isload) { 515 (uptag->func) (uptag); 516 } else { 517 cv_broadcast(uptag->cv); 518 } 519 if (!owned) 520 USB_MTX_UNLOCK(uptag->mtx); 521 } 522 523 /*------------------------------------------------------------------------* 524 * usb_pc_alloc_mem - allocate DMA'able memory 525 * 526 * Returns: 527 * 0: Success 528 * Else: Failure 529 *------------------------------------------------------------------------*/ 530 uint8_t 531 usb_pc_alloc_mem(struct usb_page_cache *pc, struct usb_page *pg, 532 usb_size_t size, usb_size_t align) 533 { 534 struct usb_dma_parent_tag *uptag; 535 struct usb_dma_tag *utag; 536 bus_dmamap_t map; 537 void *ptr; 538 int err; 539 540 uptag = pc->tag_parent; 541 542 if (align != 1) { 543 /* 544 * The alignment must be greater or equal to the 545 * "size" else the object can be split between two 546 * memory pages and we get a problem! 547 */ 548 while (align < size) { 549 align *= 2; 550 if (align == 0) { 551 goto error; 552 } 553 } 554 #if 1 555 /* 556 * XXX BUS-DMA workaround - FIXME later: 557 * 558 * We assume that that the aligment at this point of 559 * the code is greater than or equal to the size and 560 * less than two times the size, so that if we double 561 * the size, the size will be greater than the 562 * alignment. 563 * 564 * The bus-dma system has a check for "alignment" 565 * being less than "size". If that check fails we end 566 * up using contigmalloc which is page based even for 567 * small allocations. Try to avoid that to save 568 * memory, hence we sometimes to a large number of 569 * small allocations! 570 */ 571 if (size <= (USB_PAGE_SIZE / 2)) { 572 size *= 2; 573 } 574 #endif 575 } 576 /* get the correct DMA tag */ 577 utag = usb_dma_tag_find(uptag, size, align); 578 if (utag == NULL) { 579 goto error; 580 } 581 /* allocate memory */ 582 if (bus_dmamem_alloc( 583 utag->tag, &ptr, (BUS_DMA_WAITOK | BUS_DMA_COHERENT), &map)) { 584 goto error; 585 } 586 /* setup page cache */ 587 pc->buffer = ptr; 588 pc->page_start = pg; 589 pc->page_offset_buf = 0; 590 pc->page_offset_end = size; 591 pc->map = map; 592 pc->tag = utag->tag; 593 pc->ismultiseg = (align == 1); 594 595 USB_MTX_LOCK(uptag->mtx); 596 597 /* load memory into DMA */ 598 err = bus_dmamap_load( 599 utag->tag, map, ptr, size, &usb_pc_alloc_mem_cb, 600 pc, (BUS_DMA_WAITOK | BUS_DMA_COHERENT)); 601 602 if (err == EINPROGRESS) { 603 cv_wait(uptag->cv, uptag->mtx); 604 err = 0; 605 } 606 USB_MTX_UNLOCK(uptag->mtx); 607 608 if (err || uptag->dma_error) { 609 bus_dmamem_free(utag->tag, ptr, map); 610 goto error; 611 } 612 memset(ptr, 0, size); 613 614 usb_pc_cpu_flush(pc); 615 616 return (0); 617 618 error: 619 /* reset most of the page cache */ 620 pc->buffer = NULL; 621 pc->page_start = NULL; 622 pc->page_offset_buf = 0; 623 pc->page_offset_end = 0; 624 pc->map = NULL; 625 pc->tag = NULL; 626 return (1); 627 } 628 629 /*------------------------------------------------------------------------* 630 * usb_pc_free_mem - free DMA memory 631 * 632 * This function is NULL safe. 633 *------------------------------------------------------------------------*/ 634 void 635 usb_pc_free_mem(struct usb_page_cache *pc) 636 { 637 if (pc && pc->buffer) { 638 639 bus_dmamap_unload(pc->tag, pc->map); 640 641 bus_dmamem_free(pc->tag, pc->buffer, pc->map); 642 643 pc->buffer = NULL; 644 } 645 } 646 647 /*------------------------------------------------------------------------* 648 * usb_pc_load_mem - load virtual memory into DMA 649 * 650 * Return values: 651 * 0: Success 652 * Else: Error 653 *------------------------------------------------------------------------*/ 654 uint8_t 655 usb_pc_load_mem(struct usb_page_cache *pc, usb_size_t size, uint8_t sync) 656 { 657 /* setup page cache */ 658 pc->page_offset_buf = 0; 659 pc->page_offset_end = size; 660 pc->ismultiseg = 1; 661 662 USB_MTX_ASSERT(pc->tag_parent->mtx, MA_OWNED); 663 664 if (size > 0) { 665 if (sync) { 666 struct usb_dma_parent_tag *uptag; 667 int err; 668 669 uptag = pc->tag_parent; 670 671 /* 672 * We have to unload the previous loaded DMA 673 * pages before trying to load a new one! 674 */ 675 bus_dmamap_unload(pc->tag, pc->map); 676 677 /* 678 * Try to load memory into DMA. 679 */ 680 err = bus_dmamap_load( 681 pc->tag, pc->map, pc->buffer, size, 682 &usb_pc_alloc_mem_cb, pc, BUS_DMA_WAITOK); 683 if (err == EINPROGRESS) { 684 cv_wait(uptag->cv, uptag->mtx); 685 err = 0; 686 } 687 if (err || uptag->dma_error) { 688 return (1); 689 } 690 } else { 691 692 /* 693 * We have to unload the previous loaded DMA 694 * pages before trying to load a new one! 695 */ 696 bus_dmamap_unload(pc->tag, pc->map); 697 698 /* 699 * Try to load memory into DMA. The callback 700 * will be called in all cases: 701 */ 702 if (bus_dmamap_load( 703 pc->tag, pc->map, pc->buffer, size, 704 &usb_pc_load_mem_cb, pc, BUS_DMA_WAITOK)) { 705 } 706 } 707 } else { 708 if (!sync) { 709 /* 710 * Call callback so that refcount is decremented 711 * properly: 712 */ 713 pc->tag_parent->dma_error = 0; 714 (pc->tag_parent->func) (pc->tag_parent); 715 } 716 } 717 return (0); 718 } 719 720 /*------------------------------------------------------------------------* 721 * usb_pc_cpu_invalidate - invalidate CPU cache 722 *------------------------------------------------------------------------*/ 723 void 724 usb_pc_cpu_invalidate(struct usb_page_cache *pc) 725 { 726 if (pc->page_offset_end == pc->page_offset_buf) { 727 /* nothing has been loaded into this page cache! */ 728 return; 729 } 730 731 /* 732 * TODO: We currently do XXX_POSTREAD and XXX_PREREAD at the 733 * same time, but in the future we should try to isolate the 734 * different cases to optimise the code. --HPS 735 */ 736 bus_dmamap_sync(pc->tag, pc->map, BUS_DMASYNC_POSTREAD); 737 bus_dmamap_sync(pc->tag, pc->map, BUS_DMASYNC_PREREAD); 738 } 739 740 /*------------------------------------------------------------------------* 741 * usb_pc_cpu_flush - flush CPU cache 742 *------------------------------------------------------------------------*/ 743 void 744 usb_pc_cpu_flush(struct usb_page_cache *pc) 745 { 746 if (pc->page_offset_end == pc->page_offset_buf) { 747 /* nothing has been loaded into this page cache! */ 748 return; 749 } 750 bus_dmamap_sync(pc->tag, pc->map, BUS_DMASYNC_PREWRITE); 751 } 752 753 /*------------------------------------------------------------------------* 754 * usb_pc_dmamap_create - create a DMA map 755 * 756 * Returns: 757 * 0: Success 758 * Else: Failure 759 *------------------------------------------------------------------------*/ 760 uint8_t 761 usb_pc_dmamap_create(struct usb_page_cache *pc, usb_size_t size) 762 { 763 struct usb_xfer_root *info; 764 struct usb_dma_tag *utag; 765 766 /* get info */ 767 info = USB_DMATAG_TO_XROOT(pc->tag_parent); 768 769 /* sanity check */ 770 if (info == NULL) { 771 goto error; 772 } 773 utag = usb_dma_tag_find(pc->tag_parent, size, 1); 774 if (utag == NULL) { 775 goto error; 776 } 777 /* create DMA map */ 778 if (bus_dmamap_create(utag->tag, 0, &pc->map)) { 779 goto error; 780 } 781 pc->tag = utag->tag; 782 return 0; /* success */ 783 784 error: 785 pc->map = NULL; 786 pc->tag = NULL; 787 return 1; /* failure */ 788 } 789 790 /*------------------------------------------------------------------------* 791 * usb_pc_dmamap_destroy 792 * 793 * This function is NULL safe. 794 *------------------------------------------------------------------------*/ 795 void 796 usb_pc_dmamap_destroy(struct usb_page_cache *pc) 797 { 798 if (pc && pc->tag) { 799 bus_dmamap_destroy(pc->tag, pc->map); 800 pc->tag = NULL; 801 pc->map = NULL; 802 } 803 } 804 805 /*------------------------------------------------------------------------* 806 * usb_dma_tag_find - factored out code 807 *------------------------------------------------------------------------*/ 808 struct usb_dma_tag * 809 usb_dma_tag_find(struct usb_dma_parent_tag *udpt, 810 usb_size_t size, usb_size_t align) 811 { 812 struct usb_dma_tag *udt; 813 uint8_t nudt; 814 815 USB_ASSERT(align > 0, ("Invalid parameter align = 0\n")); 816 USB_ASSERT(size > 0, ("Invalid parameter size = 0\n")); 817 818 udt = udpt->utag_first; 819 nudt = udpt->utag_max; 820 821 while (nudt--) { 822 823 if (udt->align == 0) { 824 usb_dma_tag_create(udt, size, align); 825 if (udt->tag == NULL) { 826 return (NULL); 827 } 828 udt->align = align; 829 udt->size = size; 830 return (udt); 831 } 832 if ((udt->align == align) && (udt->size == size)) { 833 return (udt); 834 } 835 udt++; 836 } 837 return (NULL); 838 } 839 840 /*------------------------------------------------------------------------* 841 * usb_dma_tag_setup - initialise USB DMA tags 842 *------------------------------------------------------------------------*/ 843 void 844 usb_dma_tag_setup(struct usb_dma_parent_tag *udpt, 845 struct usb_dma_tag *udt, bus_dma_tag_t dmat, 846 struct mtx *mtx, usb_dma_callback_t *func, 847 uint8_t ndmabits, uint8_t nudt) 848 { 849 memset(udpt, 0, sizeof(*udpt)); 850 851 /* sanity checking */ 852 if ((nudt == 0) || 853 (ndmabits == 0) || 854 (mtx == NULL)) { 855 /* something is corrupt */ 856 return; 857 } 858 /* initialise condition variable */ 859 cv_init(udpt->cv, "USB DMA CV"); 860 861 /* store some information */ 862 udpt->mtx = mtx; 863 udpt->func = func; 864 udpt->tag = dmat; 865 udpt->utag_first = udt; 866 udpt->utag_max = nudt; 867 udpt->dma_bits = ndmabits; 868 869 while (nudt--) { 870 memset(udt, 0, sizeof(*udt)); 871 udt->tag_parent = udpt; 872 udt++; 873 } 874 } 875 876 /*------------------------------------------------------------------------* 877 * usb_bus_tag_unsetup - factored out code 878 *------------------------------------------------------------------------*/ 879 void 880 usb_dma_tag_unsetup(struct usb_dma_parent_tag *udpt) 881 { 882 struct usb_dma_tag *udt; 883 uint8_t nudt; 884 885 udt = udpt->utag_first; 886 nudt = udpt->utag_max; 887 888 while (nudt--) { 889 890 if (udt->align) { 891 /* destroy the USB DMA tag */ 892 usb_dma_tag_destroy(udt); 893 udt->align = 0; 894 } 895 udt++; 896 } 897 898 if (udpt->utag_max) { 899 /* destroy the condition variable */ 900 cv_destroy(udpt->cv); 901 } 902 } 903 904 /*------------------------------------------------------------------------* 905 * usb_bdma_work_loop 906 * 907 * This function handles loading of virtual buffers into DMA and is 908 * only called when "dma_refcount" is zero. 909 *------------------------------------------------------------------------*/ 910 void 911 usb_bdma_work_loop(struct usb_xfer_queue *pq) 912 { 913 struct usb_xfer_root *info; 914 struct usb_xfer *xfer; 915 usb_frcount_t nframes; 916 917 xfer = pq->curr; 918 info = xfer->xroot; 919 920 USB_MTX_ASSERT(info->xfer_mtx, MA_OWNED); 921 922 if (xfer->error) { 923 /* some error happened */ 924 USB_BUS_LOCK(info->bus); 925 usbd_transfer_done(xfer, 0); 926 USB_BUS_UNLOCK(info->bus); 927 return; 928 } 929 if (!xfer->flags_int.bdma_setup) { 930 struct usb_page *pg; 931 usb_frlength_t frlength_0; 932 uint8_t isread; 933 934 xfer->flags_int.bdma_setup = 1; 935 936 /* reset BUS-DMA load state */ 937 938 info->dma_error = 0; 939 940 if (xfer->flags_int.isochronous_xfr) { 941 /* only one frame buffer */ 942 nframes = 1; 943 frlength_0 = xfer->sumlen; 944 } else { 945 /* can be multiple frame buffers */ 946 nframes = xfer->nframes; 947 frlength_0 = xfer->frlengths[0]; 948 } 949 950 /* 951 * Set DMA direction first. This is needed to 952 * select the correct cache invalidate and cache 953 * flush operations. 954 */ 955 isread = USB_GET_DATA_ISREAD(xfer); 956 pg = xfer->dma_page_ptr; 957 958 if (xfer->flags_int.control_xfr && 959 xfer->flags_int.control_hdr) { 960 /* special case */ 961 if (xfer->flags_int.usb_mode == USB_MODE_DEVICE) { 962 /* The device controller writes to memory */ 963 xfer->frbuffers[0].isread = 1; 964 } else { 965 /* The host controller reads from memory */ 966 xfer->frbuffers[0].isread = 0; 967 } 968 } else { 969 /* default case */ 970 xfer->frbuffers[0].isread = isread; 971 } 972 973 /* 974 * Setup the "page_start" pointer which points to an array of 975 * USB pages where information about the physical address of a 976 * page will be stored. Also initialise the "isread" field of 977 * the USB page caches. 978 */ 979 xfer->frbuffers[0].page_start = pg; 980 981 info->dma_nframes = nframes; 982 info->dma_currframe = 0; 983 info->dma_frlength_0 = frlength_0; 984 985 pg += (frlength_0 / USB_PAGE_SIZE); 986 pg += 2; 987 988 while (--nframes > 0) { 989 xfer->frbuffers[nframes].isread = isread; 990 xfer->frbuffers[nframes].page_start = pg; 991 992 pg += (xfer->frlengths[nframes] / USB_PAGE_SIZE); 993 pg += 2; 994 } 995 996 } 997 if (info->dma_error) { 998 USB_BUS_LOCK(info->bus); 999 usbd_transfer_done(xfer, USB_ERR_DMA_LOAD_FAILED); 1000 USB_BUS_UNLOCK(info->bus); 1001 return; 1002 } 1003 if (info->dma_currframe != info->dma_nframes) { 1004 1005 if (info->dma_currframe == 0) { 1006 /* special case */ 1007 usb_pc_load_mem(xfer->frbuffers, 1008 info->dma_frlength_0, 0); 1009 } else { 1010 /* default case */ 1011 nframes = info->dma_currframe; 1012 usb_pc_load_mem(xfer->frbuffers + nframes, 1013 xfer->frlengths[nframes], 0); 1014 } 1015 1016 /* advance frame index */ 1017 info->dma_currframe++; 1018 1019 return; 1020 } 1021 /* go ahead */ 1022 usb_bdma_pre_sync(xfer); 1023 1024 /* start loading next USB transfer, if any */ 1025 usb_command_wrapper(pq, NULL); 1026 1027 /* finally start the hardware */ 1028 usbd_pipe_enter(xfer); 1029 } 1030 1031 /*------------------------------------------------------------------------* 1032 * usb_bdma_done_event 1033 * 1034 * This function is called when the BUS-DMA has loaded virtual memory 1035 * into DMA, if any. 1036 *------------------------------------------------------------------------*/ 1037 void 1038 usb_bdma_done_event(struct usb_dma_parent_tag *udpt) 1039 { 1040 struct usb_xfer_root *info; 1041 1042 info = USB_DMATAG_TO_XROOT(udpt); 1043 1044 USB_MTX_ASSERT(info->xfer_mtx, MA_OWNED); 1045 1046 /* copy error */ 1047 info->dma_error = udpt->dma_error; 1048 1049 /* enter workloop again */ 1050 usb_command_wrapper(&info->dma_q, 1051 info->dma_q.curr); 1052 } 1053 1054 /*------------------------------------------------------------------------* 1055 * usb_bdma_pre_sync 1056 * 1057 * This function handles DMA synchronisation that must be done before 1058 * an USB transfer is started. 1059 *------------------------------------------------------------------------*/ 1060 void 1061 usb_bdma_pre_sync(struct usb_xfer *xfer) 1062 { 1063 struct usb_page_cache *pc; 1064 usb_frcount_t nframes; 1065 1066 if (xfer->flags_int.isochronous_xfr) { 1067 /* only one frame buffer */ 1068 nframes = 1; 1069 } else { 1070 /* can be multiple frame buffers */ 1071 nframes = xfer->nframes; 1072 } 1073 1074 pc = xfer->frbuffers; 1075 1076 while (nframes--) { 1077 1078 if (pc->isread) { 1079 usb_pc_cpu_invalidate(pc); 1080 } else { 1081 usb_pc_cpu_flush(pc); 1082 } 1083 pc++; 1084 } 1085 } 1086 1087 /*------------------------------------------------------------------------* 1088 * usb_bdma_post_sync 1089 * 1090 * This function handles DMA synchronisation that must be done after 1091 * an USB transfer is complete. 1092 *------------------------------------------------------------------------*/ 1093 void 1094 usb_bdma_post_sync(struct usb_xfer *xfer) 1095 { 1096 struct usb_page_cache *pc; 1097 usb_frcount_t nframes; 1098 1099 if (xfer->flags_int.isochronous_xfr) { 1100 /* only one frame buffer */ 1101 nframes = 1; 1102 } else { 1103 /* can be multiple frame buffers */ 1104 nframes = xfer->nframes; 1105 } 1106 1107 pc = xfer->frbuffers; 1108 1109 while (nframes--) { 1110 if (pc->isread) { 1111 usb_pc_cpu_invalidate(pc); 1112 } 1113 pc++; 1114 } 1115 } 1116 1117 #endif 1118