1 /* $FreeBSD$ */ 2 /*- 3 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 4 * 5 * Copyright (c) 2008 Hans Petter Selasky. All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 */ 28 29 #ifdef USB_GLOBAL_INCLUDE_FILE 30 #include USB_GLOBAL_INCLUDE_FILE 31 #else 32 #include <sys/stdint.h> 33 #include <sys/stddef.h> 34 #include <sys/param.h> 35 #include <sys/queue.h> 36 #include <sys/types.h> 37 #include <sys/systm.h> 38 #include <sys/kernel.h> 39 #include <sys/bus.h> 40 #include <sys/module.h> 41 #include <sys/lock.h> 42 #include <sys/mutex.h> 43 #include <sys/condvar.h> 44 #include <sys/sysctl.h> 45 #include <sys/sx.h> 46 #include <sys/unistd.h> 47 #include <sys/callout.h> 48 #include <sys/malloc.h> 49 #include <sys/priv.h> 50 51 #include <dev/usb/usb.h> 52 #include <dev/usb/usbdi.h> 53 #include <dev/usb/usbdi_util.h> 54 55 #define USB_DEBUG_VAR usb_debug 56 57 #include <dev/usb/usb_core.h> 58 #include <dev/usb/usb_busdma.h> 59 #include <dev/usb/usb_process.h> 60 #include <dev/usb/usb_transfer.h> 61 #include <dev/usb/usb_device.h> 62 #include <dev/usb/usb_util.h> 63 #include <dev/usb/usb_debug.h> 64 65 #include <dev/usb/usb_controller.h> 66 #include <dev/usb/usb_bus.h> 67 #endif /* USB_GLOBAL_INCLUDE_FILE */ 68 69 #if USB_HAVE_BUSDMA 70 static void usb_dma_tag_create(struct usb_dma_tag *, usb_size_t, usb_size_t); 71 static void usb_dma_tag_destroy(struct usb_dma_tag *); 72 static void usb_dma_lock_cb(void *, bus_dma_lock_op_t); 73 static void usb_pc_alloc_mem_cb(void *, bus_dma_segment_t *, int, int); 74 static void usb_pc_load_mem_cb(void *, bus_dma_segment_t *, int, int); 75 static void usb_pc_common_mem_cb(void *, bus_dma_segment_t *, int, int, 76 uint8_t); 77 #endif 78 79 /*------------------------------------------------------------------------* 80 * usbd_get_page - lookup DMA-able memory for the given offset 81 * 82 * NOTE: Only call this function when the "page_cache" structure has 83 * been properly initialized ! 84 *------------------------------------------------------------------------*/ 85 void 86 usbd_get_page(struct usb_page_cache *pc, usb_frlength_t offset, 87 struct usb_page_search *res) 88 { 89 #if USB_HAVE_BUSDMA 90 struct usb_page *page; 91 92 if (pc->page_start) { 93 /* Case 1 - something has been loaded into DMA */ 94 95 if (pc->buffer) { 96 /* Case 1a - Kernel Virtual Address */ 97 98 res->buffer = USB_ADD_BYTES(pc->buffer, offset); 99 } 100 offset += pc->page_offset_buf; 101 102 /* compute destination page */ 103 104 page = pc->page_start; 105 106 if (pc->ismultiseg) { 107 page += (offset / USB_PAGE_SIZE); 108 109 offset %= USB_PAGE_SIZE; 110 111 res->length = USB_PAGE_SIZE - offset; 112 res->physaddr = page->physaddr + offset; 113 } else { 114 res->length = (usb_size_t)-1; 115 res->physaddr = page->physaddr + offset; 116 } 117 if (!pc->buffer) { 118 /* Case 1b - Non Kernel Virtual Address */ 119 120 res->buffer = USB_ADD_BYTES(page->buffer, offset); 121 } 122 return; 123 } 124 #endif 125 /* Case 2 - Plain PIO */ 126 127 res->buffer = USB_ADD_BYTES(pc->buffer, offset); 128 res->length = (usb_size_t)-1; 129 #if USB_HAVE_BUSDMA 130 res->physaddr = 0; 131 #endif 132 } 133 134 /*------------------------------------------------------------------------* 135 * usb_pc_buffer_is_aligned - verify alignment 136 * 137 * This function is used to check if a page cache buffer is properly 138 * aligned to reduce the use of bounce buffers in PIO mode. 139 *------------------------------------------------------------------------*/ 140 uint8_t 141 usb_pc_buffer_is_aligned(struct usb_page_cache *pc, usb_frlength_t offset, 142 usb_frlength_t len, usb_frlength_t mask) 143 { 144 struct usb_page_search buf_res; 145 146 while (len != 0) { 147 usbd_get_page(pc, offset, &buf_res); 148 149 if (buf_res.length > len) 150 buf_res.length = len; 151 if (USB_P2U(buf_res.buffer) & mask) 152 return (0); 153 if (buf_res.length & mask) 154 return (0); 155 156 offset += buf_res.length; 157 len -= buf_res.length; 158 } 159 return (1); 160 } 161 162 /*------------------------------------------------------------------------* 163 * usbd_copy_in - copy directly to DMA-able memory 164 *------------------------------------------------------------------------*/ 165 void 166 usbd_copy_in(struct usb_page_cache *cache, usb_frlength_t offset, 167 const void *ptr, usb_frlength_t len) 168 { 169 struct usb_page_search buf_res; 170 171 while (len != 0) { 172 usbd_get_page(cache, offset, &buf_res); 173 174 if (buf_res.length > len) { 175 buf_res.length = len; 176 } 177 memcpy(buf_res.buffer, ptr, buf_res.length); 178 179 offset += buf_res.length; 180 len -= buf_res.length; 181 ptr = USB_ADD_BYTES(ptr, buf_res.length); 182 } 183 } 184 185 /*------------------------------------------------------------------------* 186 * usbd_copy_in_user - copy directly to DMA-able memory from userland 187 * 188 * Return values: 189 * 0: Success 190 * Else: Failure 191 *------------------------------------------------------------------------*/ 192 #if USB_HAVE_USER_IO 193 int 194 usbd_copy_in_user(struct usb_page_cache *cache, usb_frlength_t offset, 195 const void *ptr, usb_frlength_t len) 196 { 197 struct usb_page_search buf_res; 198 int error; 199 200 while (len != 0) { 201 usbd_get_page(cache, offset, &buf_res); 202 203 if (buf_res.length > len) { 204 buf_res.length = len; 205 } 206 error = copyin(ptr, buf_res.buffer, buf_res.length); 207 if (error) 208 return (error); 209 210 offset += buf_res.length; 211 len -= buf_res.length; 212 ptr = USB_ADD_BYTES(ptr, buf_res.length); 213 } 214 return (0); /* success */ 215 } 216 #endif 217 218 /*------------------------------------------------------------------------* 219 * usbd_m_copy_in - copy a mbuf chain directly into DMA-able memory 220 *------------------------------------------------------------------------*/ 221 #if USB_HAVE_MBUF 222 struct usb_m_copy_in_arg { 223 struct usb_page_cache *cache; 224 usb_frlength_t dst_offset; 225 }; 226 227 static int 228 usbd_m_copy_in_cb(void *arg, void *src, uint32_t count) 229 { 230 struct usb_m_copy_in_arg *ua = arg; 231 232 usbd_copy_in(ua->cache, ua->dst_offset, src, count); 233 ua->dst_offset += count; 234 return (0); 235 } 236 237 void 238 usbd_m_copy_in(struct usb_page_cache *cache, usb_frlength_t dst_offset, 239 struct mbuf *m, usb_size_t src_offset, usb_frlength_t src_len) 240 { 241 struct usb_m_copy_in_arg arg = {cache, dst_offset}; 242 (void) m_apply(m, src_offset, src_len, &usbd_m_copy_in_cb, &arg); 243 } 244 #endif 245 246 /*------------------------------------------------------------------------* 247 * usb_uiomove - factored out code 248 *------------------------------------------------------------------------*/ 249 #if USB_HAVE_USER_IO 250 int 251 usb_uiomove(struct usb_page_cache *pc, struct uio *uio, 252 usb_frlength_t pc_offset, usb_frlength_t len) 253 { 254 struct usb_page_search res; 255 int error = 0; 256 257 while (len != 0) { 258 usbd_get_page(pc, pc_offset, &res); 259 260 if (res.length > len) { 261 res.length = len; 262 } 263 /* 264 * "uiomove()" can sleep so one needs to make a wrapper, 265 * exiting the mutex and checking things 266 */ 267 error = uiomove(res.buffer, res.length, uio); 268 269 if (error) { 270 break; 271 } 272 pc_offset += res.length; 273 len -= res.length; 274 } 275 return (error); 276 } 277 #endif 278 279 /*------------------------------------------------------------------------* 280 * usbd_copy_out - copy directly from DMA-able memory 281 *------------------------------------------------------------------------*/ 282 void 283 usbd_copy_out(struct usb_page_cache *cache, usb_frlength_t offset, 284 void *ptr, usb_frlength_t len) 285 { 286 struct usb_page_search res; 287 288 while (len != 0) { 289 usbd_get_page(cache, offset, &res); 290 291 if (res.length > len) { 292 res.length = len; 293 } 294 memcpy(ptr, res.buffer, res.length); 295 296 offset += res.length; 297 len -= res.length; 298 ptr = USB_ADD_BYTES(ptr, res.length); 299 } 300 } 301 302 /*------------------------------------------------------------------------* 303 * usbd_copy_out_user - copy directly from DMA-able memory to userland 304 * 305 * Return values: 306 * 0: Success 307 * Else: Failure 308 *------------------------------------------------------------------------*/ 309 #if USB_HAVE_USER_IO 310 int 311 usbd_copy_out_user(struct usb_page_cache *cache, usb_frlength_t offset, 312 void *ptr, usb_frlength_t len) 313 { 314 struct usb_page_search res; 315 int error; 316 317 while (len != 0) { 318 usbd_get_page(cache, offset, &res); 319 320 if (res.length > len) { 321 res.length = len; 322 } 323 error = copyout(res.buffer, ptr, res.length); 324 if (error) 325 return (error); 326 327 offset += res.length; 328 len -= res.length; 329 ptr = USB_ADD_BYTES(ptr, res.length); 330 } 331 return (0); /* success */ 332 } 333 #endif 334 335 /*------------------------------------------------------------------------* 336 * usbd_frame_zero - zero DMA-able memory 337 *------------------------------------------------------------------------*/ 338 void 339 usbd_frame_zero(struct usb_page_cache *cache, usb_frlength_t offset, 340 usb_frlength_t len) 341 { 342 struct usb_page_search res; 343 344 while (len != 0) { 345 usbd_get_page(cache, offset, &res); 346 347 if (res.length > len) { 348 res.length = len; 349 } 350 memset(res.buffer, 0, res.length); 351 352 offset += res.length; 353 len -= res.length; 354 } 355 } 356 357 #if USB_HAVE_BUSDMA 358 359 /*------------------------------------------------------------------------* 360 * usb_dma_lock_cb - dummy callback 361 *------------------------------------------------------------------------*/ 362 static void 363 usb_dma_lock_cb(void *arg, bus_dma_lock_op_t op) 364 { 365 /* we use "mtx_owned()" instead of this function */ 366 } 367 368 /*------------------------------------------------------------------------* 369 * usb_dma_tag_create - allocate a DMA tag 370 * 371 * NOTE: If the "align" parameter has a value of 1 the DMA-tag will 372 * allow multi-segment mappings. Else all mappings are single-segment. 373 *------------------------------------------------------------------------*/ 374 static void 375 usb_dma_tag_create(struct usb_dma_tag *udt, 376 usb_size_t size, usb_size_t align) 377 { 378 bus_dma_tag_t tag; 379 380 if (bus_dma_tag_create 381 ( /* parent */ udt->tag_parent->tag, 382 /* alignment */ align, 383 /* boundary */ 0, 384 /* lowaddr */ (2ULL << (udt->tag_parent->dma_bits - 1)) - 1, 385 /* highaddr */ BUS_SPACE_MAXADDR, 386 /* filter */ NULL, 387 /* filterarg */ NULL, 388 /* maxsize */ size, 389 /* nsegments */ (align == 1 && size > 1) ? 390 (2 + (size / USB_PAGE_SIZE)) : 1, 391 /* maxsegsz */ (align == 1 && size > USB_PAGE_SIZE) ? 392 USB_PAGE_SIZE : size, 393 /* flags */ BUS_DMA_KEEP_PG_OFFSET, 394 /* lockfn */ &usb_dma_lock_cb, 395 /* lockarg */ NULL, 396 &tag)) { 397 tag = NULL; 398 } 399 udt->tag = tag; 400 } 401 402 /*------------------------------------------------------------------------* 403 * usb_dma_tag_free - free a DMA tag 404 *------------------------------------------------------------------------*/ 405 static void 406 usb_dma_tag_destroy(struct usb_dma_tag *udt) 407 { 408 bus_dma_tag_destroy(udt->tag); 409 } 410 411 /*------------------------------------------------------------------------* 412 * usb_pc_alloc_mem_cb - BUS-DMA callback function 413 *------------------------------------------------------------------------*/ 414 static void 415 usb_pc_alloc_mem_cb(void *arg, bus_dma_segment_t *segs, 416 int nseg, int error) 417 { 418 usb_pc_common_mem_cb(arg, segs, nseg, error, 0); 419 } 420 421 /*------------------------------------------------------------------------* 422 * usb_pc_load_mem_cb - BUS-DMA callback function 423 *------------------------------------------------------------------------*/ 424 static void 425 usb_pc_load_mem_cb(void *arg, bus_dma_segment_t *segs, 426 int nseg, int error) 427 { 428 usb_pc_common_mem_cb(arg, segs, nseg, error, 1); 429 } 430 431 /*------------------------------------------------------------------------* 432 * usb_pc_common_mem_cb - BUS-DMA callback function 433 *------------------------------------------------------------------------*/ 434 static void 435 usb_pc_common_mem_cb(void *arg, bus_dma_segment_t *segs, 436 int nseg, int error, uint8_t isload) 437 { 438 struct usb_dma_parent_tag *uptag; 439 struct usb_page_cache *pc; 440 struct usb_page *pg; 441 usb_size_t rem; 442 bus_size_t off; 443 uint8_t owned; 444 445 pc = arg; 446 uptag = pc->tag_parent; 447 448 /* 449 * XXX There is sometimes recursive locking here. 450 * XXX We should try to find a better solution. 451 * XXX Until further the "owned" variable does 452 * XXX the trick. 453 */ 454 455 if (error) { 456 goto done; 457 } 458 459 off = 0; 460 pg = pc->page_start; 461 pg->physaddr = rounddown2(segs->ds_addr, USB_PAGE_SIZE); 462 rem = segs->ds_addr & (USB_PAGE_SIZE - 1); 463 pc->page_offset_buf = rem; 464 pc->page_offset_end += rem; 465 #ifdef USB_DEBUG 466 if (nseg > 1) { 467 int x; 468 469 for (x = 0; x != nseg - 1; x++) { 470 if (((segs[x].ds_addr + segs[x].ds_len) & (USB_PAGE_SIZE - 1)) == 471 ((segs[x + 1].ds_addr & (USB_PAGE_SIZE - 1)))) 472 continue; 473 /* 474 * This check verifies there is no page offset 475 * hole between any of the segments. See the 476 * BUS_DMA_KEEP_PG_OFFSET flag. 477 */ 478 DPRINTFN(0, "Page offset was not preserved\n"); 479 error = 1; 480 goto done; 481 } 482 } 483 #endif 484 while (pc->ismultiseg) { 485 off += USB_PAGE_SIZE; 486 if (off >= (segs->ds_len + rem)) { 487 /* page crossing */ 488 nseg--; 489 segs++; 490 off = 0; 491 rem = 0; 492 if (nseg == 0) 493 break; 494 } 495 pg++; 496 pg->physaddr = rounddown2(segs->ds_addr + off, USB_PAGE_SIZE); 497 } 498 499 done: 500 owned = mtx_owned(uptag->mtx); 501 if (!owned) 502 USB_MTX_LOCK(uptag->mtx); 503 504 uptag->dma_error = (error ? 1 : 0); 505 if (isload) { 506 (uptag->func) (uptag); 507 } else { 508 cv_broadcast(uptag->cv); 509 } 510 if (!owned) 511 USB_MTX_UNLOCK(uptag->mtx); 512 } 513 514 /*------------------------------------------------------------------------* 515 * usb_pc_alloc_mem - allocate DMA'able memory 516 * 517 * Returns: 518 * 0: Success 519 * Else: Failure 520 *------------------------------------------------------------------------*/ 521 uint8_t 522 usb_pc_alloc_mem(struct usb_page_cache *pc, struct usb_page *pg, 523 usb_size_t size, usb_size_t align) 524 { 525 struct usb_dma_parent_tag *uptag; 526 struct usb_dma_tag *utag; 527 bus_dmamap_t map; 528 void *ptr; 529 int err; 530 531 uptag = pc->tag_parent; 532 533 if (align != 1) { 534 /* 535 * The alignment must be greater or equal to the 536 * "size" else the object can be split between two 537 * memory pages and we get a problem! 538 */ 539 while (align < size) { 540 align *= 2; 541 if (align == 0) { 542 goto error; 543 } 544 } 545 #if 1 546 /* 547 * XXX BUS-DMA workaround - FIXME later: 548 * 549 * We assume that that the alignment at this point of 550 * the code is greater than or equal to the size and 551 * less than two times the size, so that if we double 552 * the size, the size will be greater than the 553 * alignment. 554 * 555 * The bus-dma system has a check for "alignment" 556 * being less than "size". If that check fails we end 557 * up using contigmalloc which is page based even for 558 * small allocations. Try to avoid that to save 559 * memory, hence we sometimes to a large number of 560 * small allocations! 561 */ 562 if (size <= (USB_PAGE_SIZE / 2)) { 563 size *= 2; 564 } 565 #endif 566 } 567 /* get the correct DMA tag */ 568 utag = usb_dma_tag_find(uptag, size, align); 569 if (utag == NULL) { 570 goto error; 571 } 572 /* allocate memory */ 573 if (bus_dmamem_alloc( 574 utag->tag, &ptr, (BUS_DMA_WAITOK | BUS_DMA_COHERENT), &map)) { 575 goto error; 576 } 577 /* setup page cache */ 578 pc->buffer = ptr; 579 pc->page_start = pg; 580 pc->page_offset_buf = 0; 581 pc->page_offset_end = size; 582 pc->map = map; 583 pc->tag = utag->tag; 584 pc->ismultiseg = (align == 1); 585 586 USB_MTX_LOCK(uptag->mtx); 587 588 /* load memory into DMA */ 589 err = bus_dmamap_load( 590 utag->tag, map, ptr, size, &usb_pc_alloc_mem_cb, 591 pc, (BUS_DMA_WAITOK | BUS_DMA_COHERENT)); 592 593 if (err == EINPROGRESS) { 594 cv_wait(uptag->cv, uptag->mtx); 595 err = 0; 596 } 597 USB_MTX_UNLOCK(uptag->mtx); 598 599 if (err || uptag->dma_error) { 600 bus_dmamem_free(utag->tag, ptr, map); 601 goto error; 602 } 603 pc->isloaded = 1; 604 memset(ptr, 0, size); 605 606 usb_pc_cpu_flush(pc); 607 608 return (0); 609 610 error: 611 /* reset most of the page cache */ 612 pc->buffer = NULL; 613 pc->page_start = NULL; 614 pc->page_offset_buf = 0; 615 pc->page_offset_end = 0; 616 pc->isloaded = 0; 617 pc->map = NULL; 618 pc->tag = NULL; 619 return (1); 620 } 621 622 /*------------------------------------------------------------------------* 623 * usb_pc_free_mem - free DMA memory 624 * 625 * This function is NULL safe. 626 *------------------------------------------------------------------------*/ 627 void 628 usb_pc_free_mem(struct usb_page_cache *pc) 629 { 630 if (pc && pc->buffer) { 631 if (pc->isloaded) 632 bus_dmamap_unload(pc->tag, pc->map); 633 634 bus_dmamem_free(pc->tag, pc->buffer, pc->map); 635 636 pc->buffer = NULL; 637 pc->isloaded = 0; 638 } 639 } 640 641 /*------------------------------------------------------------------------* 642 * usb_pc_load_mem - load virtual memory into DMA 643 * 644 * Return values: 645 * 0: Success 646 * Else: Error 647 *------------------------------------------------------------------------*/ 648 uint8_t 649 usb_pc_load_mem(struct usb_page_cache *pc, usb_size_t size, uint8_t sync) 650 { 651 /* setup page cache */ 652 pc->page_offset_buf = 0; 653 pc->page_offset_end = size; 654 pc->ismultiseg = 1; 655 656 USB_MTX_ASSERT(pc->tag_parent->mtx, MA_OWNED); 657 658 if (size > 0) { 659 if (sync) { 660 struct usb_dma_parent_tag *uptag; 661 int err; 662 663 uptag = pc->tag_parent; 664 665 /* 666 * We have to unload the previous loaded DMA 667 * pages before trying to load a new one! 668 */ 669 if (pc->isloaded) 670 bus_dmamap_unload(pc->tag, pc->map); 671 672 /* 673 * Try to load memory into DMA. 674 */ 675 err = bus_dmamap_load( 676 pc->tag, pc->map, pc->buffer, size, 677 &usb_pc_alloc_mem_cb, pc, BUS_DMA_WAITOK); 678 if (err == EINPROGRESS) { 679 cv_wait(uptag->cv, uptag->mtx); 680 err = 0; 681 } 682 if (err || uptag->dma_error) { 683 pc->isloaded = 0; 684 return (1); 685 } 686 } else { 687 /* 688 * We have to unload the previous loaded DMA 689 * pages before trying to load a new one! 690 */ 691 if (pc->isloaded) 692 bus_dmamap_unload(pc->tag, pc->map); 693 694 /* 695 * Try to load memory into DMA. The callback 696 * will be called in all cases: 697 */ 698 if (bus_dmamap_load( 699 pc->tag, pc->map, pc->buffer, size, 700 &usb_pc_load_mem_cb, pc, BUS_DMA_WAITOK)) { 701 } 702 } 703 pc->isloaded = 1; 704 } else { 705 if (!sync) { 706 /* 707 * Call callback so that refcount is decremented 708 * properly: 709 */ 710 pc->tag_parent->dma_error = 0; 711 (pc->tag_parent->func) (pc->tag_parent); 712 } 713 } 714 return (0); 715 } 716 717 /*------------------------------------------------------------------------* 718 * usb_pc_cpu_invalidate - invalidate CPU cache 719 *------------------------------------------------------------------------*/ 720 void 721 usb_pc_cpu_invalidate(struct usb_page_cache *pc) 722 { 723 if (pc->page_offset_end == pc->page_offset_buf) { 724 /* nothing has been loaded into this page cache! */ 725 return; 726 } 727 728 /* 729 * TODO: We currently do XXX_POSTREAD and XXX_PREREAD at the 730 * same time, but in the future we should try to isolate the 731 * different cases to optimise the code. --HPS 732 */ 733 bus_dmamap_sync(pc->tag, pc->map, BUS_DMASYNC_POSTREAD); 734 bus_dmamap_sync(pc->tag, pc->map, BUS_DMASYNC_PREREAD); 735 } 736 737 /*------------------------------------------------------------------------* 738 * usb_pc_cpu_flush - flush CPU cache 739 *------------------------------------------------------------------------*/ 740 void 741 usb_pc_cpu_flush(struct usb_page_cache *pc) 742 { 743 if (pc->page_offset_end == pc->page_offset_buf) { 744 /* nothing has been loaded into this page cache! */ 745 return; 746 } 747 bus_dmamap_sync(pc->tag, pc->map, BUS_DMASYNC_PREWRITE); 748 } 749 750 /*------------------------------------------------------------------------* 751 * usb_pc_dmamap_create - create a DMA map 752 * 753 * Returns: 754 * 0: Success 755 * Else: Failure 756 *------------------------------------------------------------------------*/ 757 uint8_t 758 usb_pc_dmamap_create(struct usb_page_cache *pc, usb_size_t size) 759 { 760 struct usb_xfer_root *info; 761 struct usb_dma_tag *utag; 762 763 /* get info */ 764 info = USB_DMATAG_TO_XROOT(pc->tag_parent); 765 766 /* sanity check */ 767 if (info == NULL) { 768 goto error; 769 } 770 utag = usb_dma_tag_find(pc->tag_parent, size, 1); 771 if (utag == NULL) { 772 goto error; 773 } 774 /* create DMA map */ 775 if (bus_dmamap_create(utag->tag, 0, &pc->map)) { 776 goto error; 777 } 778 pc->tag = utag->tag; 779 return 0; /* success */ 780 781 error: 782 pc->map = NULL; 783 pc->tag = NULL; 784 return 1; /* failure */ 785 } 786 787 /*------------------------------------------------------------------------* 788 * usb_pc_dmamap_destroy 789 * 790 * This function is NULL safe. 791 *------------------------------------------------------------------------*/ 792 void 793 usb_pc_dmamap_destroy(struct usb_page_cache *pc) 794 { 795 if (pc && pc->tag) { 796 if (pc->isloaded) 797 bus_dmamap_unload(pc->tag, pc->map); 798 bus_dmamap_destroy(pc->tag, pc->map); 799 pc->tag = NULL; 800 pc->map = NULL; 801 } 802 } 803 804 /*------------------------------------------------------------------------* 805 * usb_dma_tag_find - factored out code 806 *------------------------------------------------------------------------*/ 807 struct usb_dma_tag * 808 usb_dma_tag_find(struct usb_dma_parent_tag *udpt, 809 usb_size_t size, usb_size_t align) 810 { 811 struct usb_dma_tag *udt; 812 uint8_t nudt; 813 814 USB_ASSERT(align > 0, ("Invalid parameter align = 0\n")); 815 USB_ASSERT(size > 0, ("Invalid parameter size = 0\n")); 816 817 udt = udpt->utag_first; 818 nudt = udpt->utag_max; 819 820 while (nudt--) { 821 if (udt->align == 0) { 822 usb_dma_tag_create(udt, size, align); 823 if (udt->tag == NULL) { 824 return (NULL); 825 } 826 udt->align = align; 827 udt->size = size; 828 return (udt); 829 } 830 if ((udt->align == align) && (udt->size == size)) { 831 return (udt); 832 } 833 udt++; 834 } 835 return (NULL); 836 } 837 838 /*------------------------------------------------------------------------* 839 * usb_dma_tag_setup - initialise USB DMA tags 840 *------------------------------------------------------------------------*/ 841 void 842 usb_dma_tag_setup(struct usb_dma_parent_tag *udpt, 843 struct usb_dma_tag *udt, bus_dma_tag_t dmat, 844 struct mtx *mtx, usb_dma_callback_t *func, 845 uint8_t ndmabits, uint8_t nudt) 846 { 847 memset(udpt, 0, sizeof(*udpt)); 848 849 /* sanity checking */ 850 if ((nudt == 0) || 851 (ndmabits == 0) || 852 (mtx == NULL)) { 853 /* something is corrupt */ 854 return; 855 } 856 /* initialise condition variable */ 857 cv_init(udpt->cv, "USB DMA CV"); 858 859 /* store some information */ 860 udpt->mtx = mtx; 861 udpt->func = func; 862 udpt->tag = dmat; 863 udpt->utag_first = udt; 864 udpt->utag_max = nudt; 865 udpt->dma_bits = ndmabits; 866 867 while (nudt--) { 868 memset(udt, 0, sizeof(*udt)); 869 udt->tag_parent = udpt; 870 udt++; 871 } 872 } 873 874 /*------------------------------------------------------------------------* 875 * usb_bus_tag_unsetup - factored out code 876 *------------------------------------------------------------------------*/ 877 void 878 usb_dma_tag_unsetup(struct usb_dma_parent_tag *udpt) 879 { 880 struct usb_dma_tag *udt; 881 uint8_t nudt; 882 883 udt = udpt->utag_first; 884 nudt = udpt->utag_max; 885 886 while (nudt--) { 887 if (udt->align) { 888 /* destroy the USB DMA tag */ 889 usb_dma_tag_destroy(udt); 890 udt->align = 0; 891 } 892 udt++; 893 } 894 895 if (udpt->utag_max) { 896 /* destroy the condition variable */ 897 cv_destroy(udpt->cv); 898 } 899 } 900 901 /*------------------------------------------------------------------------* 902 * usb_bdma_work_loop 903 * 904 * This function handles loading of virtual buffers into DMA and is 905 * only called when "dma_refcount" is zero. 906 *------------------------------------------------------------------------*/ 907 void 908 usb_bdma_work_loop(struct usb_xfer_queue *pq) 909 { 910 struct usb_xfer_root *info; 911 struct usb_xfer *xfer; 912 usb_frcount_t nframes; 913 914 xfer = pq->curr; 915 info = xfer->xroot; 916 917 USB_MTX_ASSERT(info->xfer_mtx, MA_OWNED); 918 919 if (xfer->error) { 920 /* some error happened */ 921 USB_BUS_LOCK(info->bus); 922 usbd_transfer_done(xfer, 0); 923 USB_BUS_UNLOCK(info->bus); 924 return; 925 } 926 if (!xfer->flags_int.bdma_setup) { 927 struct usb_page *pg; 928 usb_frlength_t frlength_0; 929 uint8_t isread; 930 931 xfer->flags_int.bdma_setup = 1; 932 933 /* reset BUS-DMA load state */ 934 935 info->dma_error = 0; 936 937 if (xfer->flags_int.isochronous_xfr) { 938 /* only one frame buffer */ 939 nframes = 1; 940 frlength_0 = xfer->sumlen; 941 } else { 942 /* can be multiple frame buffers */ 943 nframes = xfer->nframes; 944 frlength_0 = xfer->frlengths[0]; 945 } 946 947 /* 948 * Set DMA direction first. This is needed to 949 * select the correct cache invalidate and cache 950 * flush operations. 951 */ 952 isread = USB_GET_DATA_ISREAD(xfer); 953 pg = xfer->dma_page_ptr; 954 955 if (xfer->flags_int.control_xfr && 956 xfer->flags_int.control_hdr) { 957 /* special case */ 958 if (xfer->flags_int.usb_mode == USB_MODE_DEVICE) { 959 /* The device controller writes to memory */ 960 xfer->frbuffers[0].isread = 1; 961 } else { 962 /* The host controller reads from memory */ 963 xfer->frbuffers[0].isread = 0; 964 } 965 } else { 966 /* default case */ 967 xfer->frbuffers[0].isread = isread; 968 } 969 970 /* 971 * Setup the "page_start" pointer which points to an array of 972 * USB pages where information about the physical address of a 973 * page will be stored. Also initialise the "isread" field of 974 * the USB page caches. 975 */ 976 xfer->frbuffers[0].page_start = pg; 977 978 info->dma_nframes = nframes; 979 info->dma_currframe = 0; 980 info->dma_frlength_0 = frlength_0; 981 982 pg += (frlength_0 / USB_PAGE_SIZE); 983 pg += 2; 984 985 while (--nframes > 0) { 986 xfer->frbuffers[nframes].isread = isread; 987 xfer->frbuffers[nframes].page_start = pg; 988 989 pg += (xfer->frlengths[nframes] / USB_PAGE_SIZE); 990 pg += 2; 991 } 992 } 993 if (info->dma_error) { 994 USB_BUS_LOCK(info->bus); 995 usbd_transfer_done(xfer, USB_ERR_DMA_LOAD_FAILED); 996 USB_BUS_UNLOCK(info->bus); 997 return; 998 } 999 if (info->dma_currframe != info->dma_nframes) { 1000 if (info->dma_currframe == 0) { 1001 /* special case */ 1002 usb_pc_load_mem(xfer->frbuffers, 1003 info->dma_frlength_0, 0); 1004 } else { 1005 /* default case */ 1006 nframes = info->dma_currframe; 1007 usb_pc_load_mem(xfer->frbuffers + nframes, 1008 xfer->frlengths[nframes], 0); 1009 } 1010 1011 /* advance frame index */ 1012 info->dma_currframe++; 1013 1014 return; 1015 } 1016 /* go ahead */ 1017 usb_bdma_pre_sync(xfer); 1018 1019 /* start loading next USB transfer, if any */ 1020 usb_command_wrapper(pq, NULL); 1021 1022 /* finally start the hardware */ 1023 usbd_pipe_enter(xfer); 1024 } 1025 1026 /*------------------------------------------------------------------------* 1027 * usb_bdma_done_event 1028 * 1029 * This function is called when the BUS-DMA has loaded virtual memory 1030 * into DMA, if any. 1031 *------------------------------------------------------------------------*/ 1032 void 1033 usb_bdma_done_event(struct usb_dma_parent_tag *udpt) 1034 { 1035 struct usb_xfer_root *info; 1036 1037 info = USB_DMATAG_TO_XROOT(udpt); 1038 1039 USB_MTX_ASSERT(info->xfer_mtx, MA_OWNED); 1040 1041 /* copy error */ 1042 info->dma_error = udpt->dma_error; 1043 1044 /* enter workloop again */ 1045 usb_command_wrapper(&info->dma_q, 1046 info->dma_q.curr); 1047 } 1048 1049 /*------------------------------------------------------------------------* 1050 * usb_bdma_pre_sync 1051 * 1052 * This function handles DMA synchronisation that must be done before 1053 * an USB transfer is started. 1054 *------------------------------------------------------------------------*/ 1055 void 1056 usb_bdma_pre_sync(struct usb_xfer *xfer) 1057 { 1058 struct usb_page_cache *pc; 1059 usb_frcount_t nframes; 1060 1061 if (xfer->flags_int.isochronous_xfr) { 1062 /* only one frame buffer */ 1063 nframes = 1; 1064 } else { 1065 /* can be multiple frame buffers */ 1066 nframes = xfer->nframes; 1067 } 1068 1069 pc = xfer->frbuffers; 1070 1071 while (nframes--) { 1072 if (pc->isread) { 1073 usb_pc_cpu_invalidate(pc); 1074 } else { 1075 usb_pc_cpu_flush(pc); 1076 } 1077 pc++; 1078 } 1079 } 1080 1081 /*------------------------------------------------------------------------* 1082 * usb_bdma_post_sync 1083 * 1084 * This function handles DMA synchronisation that must be done after 1085 * an USB transfer is complete. 1086 *------------------------------------------------------------------------*/ 1087 void 1088 usb_bdma_post_sync(struct usb_xfer *xfer) 1089 { 1090 struct usb_page_cache *pc; 1091 usb_frcount_t nframes; 1092 1093 if (xfer->flags_int.isochronous_xfr) { 1094 /* only one frame buffer */ 1095 nframes = 1; 1096 } else { 1097 /* can be multiple frame buffers */ 1098 nframes = xfer->nframes; 1099 } 1100 1101 pc = xfer->frbuffers; 1102 1103 while (nframes--) { 1104 if (pc->isread) { 1105 usb_pc_cpu_invalidate(pc); 1106 } 1107 pc++; 1108 } 1109 } 1110 1111 #endif 1112