1 /* $FreeBSD$ */ 2 /*- 3 * Copyright (c) 2008 Hans Petter Selasky. All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 */ 26 27 #include <dev/usb/usb_mfunc.h> 28 #include <dev/usb/usb_error.h> 29 #include <dev/usb/usb.h> 30 31 #define USB_DEBUG_VAR usb2_debug 32 33 #include <dev/usb/usb_core.h> 34 #include <dev/usb/usb_busdma.h> 35 #include <dev/usb/usb_process.h> 36 #include <dev/usb/usb_transfer.h> 37 #include <dev/usb/usb_device.h> 38 #include <dev/usb/usb_util.h> 39 #include <dev/usb/usb_debug.h> 40 41 #include <dev/usb/usb_controller.h> 42 #include <dev/usb/usb_bus.h> 43 44 #if USB_HAVE_BUSDMA 45 static void usb2_dma_tag_create(struct usb2_dma_tag *, usb2_size_t, usb2_size_t); 46 static void usb2_dma_tag_destroy(struct usb2_dma_tag *); 47 #endif 48 49 #if USB_HAVE_BUSDMA && defined(__FreeBSD__) 50 static void usb2_dma_lock_cb(void *, bus_dma_lock_op_t); 51 static void usb2_pc_alloc_mem_cb(void *, bus_dma_segment_t *, int, int); 52 static void usb2_pc_load_mem_cb(void *, bus_dma_segment_t *, int, int); 53 static void usb2_pc_common_mem_cb(void *, bus_dma_segment_t *, int, int, 54 uint8_t); 55 #endif 56 57 #if USB_HAVE_BUSDMA && defined(__NetBSD__) 58 static void usb2_pc_common_mem_cb(struct usb2_page_cache *, 59 bus_dma_segment_t *, int, int, uint8_t); 60 #endif 61 62 /*------------------------------------------------------------------------* 63 * usb2_get_page - lookup DMA-able memory for the given offset 64 * 65 * NOTE: Only call this function when the "page_cache" structure has 66 * been properly initialized ! 67 *------------------------------------------------------------------------*/ 68 void 69 usb2_get_page(struct usb2_page_cache *pc, usb2_frlength_t offset, 70 struct usb2_page_search *res) 71 { 72 struct usb2_page *page; 73 74 #if USB_HAVE_BUSDMA 75 if (pc->page_start) { 76 77 /* Case 1 - something has been loaded into DMA */ 78 79 if (pc->buffer) { 80 81 /* Case 1a - Kernel Virtual Address */ 82 83 res->buffer = USB_ADD_BYTES(pc->buffer, offset); 84 } 85 offset += pc->page_offset_buf; 86 87 /* compute destination page */ 88 89 page = pc->page_start; 90 91 if (pc->ismultiseg) { 92 93 page += (offset / USB_PAGE_SIZE); 94 95 offset %= USB_PAGE_SIZE; 96 97 res->length = USB_PAGE_SIZE - offset; 98 res->physaddr = page->physaddr + offset; 99 } else { 100 res->length = 0 - 1; 101 res->physaddr = page->physaddr + offset; 102 } 103 if (!pc->buffer) { 104 105 /* Case 1b - Non Kernel Virtual Address */ 106 107 res->buffer = USB_ADD_BYTES(page->buffer, offset); 108 } 109 return; 110 } 111 #endif 112 /* Case 2 - Plain PIO */ 113 114 res->buffer = USB_ADD_BYTES(pc->buffer, offset); 115 res->length = 0 - 1; 116 #if USB_HAVE_BUSDMA 117 res->physaddr = 0; 118 #endif 119 } 120 121 /*------------------------------------------------------------------------* 122 * usb2_copy_in - copy directly to DMA-able memory 123 *------------------------------------------------------------------------*/ 124 void 125 usb2_copy_in(struct usb2_page_cache *cache, usb2_frlength_t offset, 126 const void *ptr, usb2_frlength_t len) 127 { 128 struct usb2_page_search buf_res; 129 130 while (len != 0) { 131 132 usb2_get_page(cache, offset, &buf_res); 133 134 if (buf_res.length > len) { 135 buf_res.length = len; 136 } 137 bcopy(ptr, buf_res.buffer, buf_res.length); 138 139 offset += buf_res.length; 140 len -= buf_res.length; 141 ptr = USB_ADD_BYTES(ptr, buf_res.length); 142 } 143 } 144 145 /*------------------------------------------------------------------------* 146 * usb2_copy_in_user - copy directly to DMA-able memory from userland 147 * 148 * Return values: 149 * 0: Success 150 * Else: Failure 151 *------------------------------------------------------------------------*/ 152 #if USB_HAVE_USER_IO 153 int 154 usb2_copy_in_user(struct usb2_page_cache *cache, usb2_frlength_t offset, 155 const void *ptr, usb2_frlength_t len) 156 { 157 struct usb2_page_search buf_res; 158 int error; 159 160 while (len != 0) { 161 162 usb2_get_page(cache, offset, &buf_res); 163 164 if (buf_res.length > len) { 165 buf_res.length = len; 166 } 167 error = copyin(ptr, buf_res.buffer, buf_res.length); 168 if (error) 169 return (error); 170 171 offset += buf_res.length; 172 len -= buf_res.length; 173 ptr = USB_ADD_BYTES(ptr, buf_res.length); 174 } 175 return (0); /* success */ 176 } 177 #endif 178 179 /*------------------------------------------------------------------------* 180 * usb2_m_copy_in - copy a mbuf chain directly into DMA-able memory 181 *------------------------------------------------------------------------*/ 182 #if USB_HAVE_MBUF 183 struct usb2_m_copy_in_arg { 184 struct usb2_page_cache *cache; 185 usb2_frlength_t dst_offset; 186 }; 187 188 static int 189 #ifdef __FreeBSD__ 190 usb2_m_copy_in_cb(void *arg, void *src, uint32_t count) 191 #else 192 usb2_m_copy_in_cb(void *arg, caddr_t src, uint32_t count) 193 #endif 194 { 195 register struct usb2_m_copy_in_arg *ua = arg; 196 197 usb2_copy_in(ua->cache, ua->dst_offset, src, count); 198 ua->dst_offset += count; 199 return (0); 200 } 201 202 void 203 usb2_m_copy_in(struct usb2_page_cache *cache, usb2_frlength_t dst_offset, 204 struct mbuf *m, usb2_size_t src_offset, usb2_frlength_t src_len) 205 { 206 struct usb2_m_copy_in_arg arg = {cache, dst_offset}; 207 int error; 208 209 error = m_apply(m, src_offset, src_len, &usb2_m_copy_in_cb, &arg); 210 } 211 #endif 212 213 /*------------------------------------------------------------------------* 214 * usb2_uiomove - factored out code 215 *------------------------------------------------------------------------*/ 216 #if USB_HAVE_USER_IO 217 int 218 usb2_uiomove(struct usb2_page_cache *pc, struct uio *uio, 219 usb2_frlength_t pc_offset, usb2_frlength_t len) 220 { 221 struct usb2_page_search res; 222 int error = 0; 223 224 while (len != 0) { 225 226 usb2_get_page(pc, pc_offset, &res); 227 228 if (res.length > len) { 229 res.length = len; 230 } 231 /* 232 * "uiomove()" can sleep so one needs to make a wrapper, 233 * exiting the mutex and checking things 234 */ 235 error = uiomove(res.buffer, res.length, uio); 236 237 if (error) { 238 break; 239 } 240 pc_offset += res.length; 241 len -= res.length; 242 } 243 return (error); 244 } 245 #endif 246 247 /*------------------------------------------------------------------------* 248 * usb2_copy_out - copy directly from DMA-able memory 249 *------------------------------------------------------------------------*/ 250 void 251 usb2_copy_out(struct usb2_page_cache *cache, usb2_frlength_t offset, 252 void *ptr, usb2_frlength_t len) 253 { 254 struct usb2_page_search res; 255 256 while (len != 0) { 257 258 usb2_get_page(cache, offset, &res); 259 260 if (res.length > len) { 261 res.length = len; 262 } 263 bcopy(res.buffer, ptr, res.length); 264 265 offset += res.length; 266 len -= res.length; 267 ptr = USB_ADD_BYTES(ptr, res.length); 268 } 269 } 270 271 /*------------------------------------------------------------------------* 272 * usb2_copy_out_user - copy directly from DMA-able memory to userland 273 * 274 * Return values: 275 * 0: Success 276 * Else: Failure 277 *------------------------------------------------------------------------*/ 278 #if USB_HAVE_USER_IO 279 int 280 usb2_copy_out_user(struct usb2_page_cache *cache, usb2_frlength_t offset, 281 void *ptr, usb2_frlength_t len) 282 { 283 struct usb2_page_search res; 284 int error; 285 286 while (len != 0) { 287 288 usb2_get_page(cache, offset, &res); 289 290 if (res.length > len) { 291 res.length = len; 292 } 293 error = copyout(res.buffer, ptr, res.length); 294 if (error) 295 return (error); 296 297 offset += res.length; 298 len -= res.length; 299 ptr = USB_ADD_BYTES(ptr, res.length); 300 } 301 return (0); /* success */ 302 } 303 #endif 304 305 /*------------------------------------------------------------------------* 306 * usb2_bzero - zero DMA-able memory 307 *------------------------------------------------------------------------*/ 308 void 309 usb2_bzero(struct usb2_page_cache *cache, usb2_frlength_t offset, 310 usb2_frlength_t len) 311 { 312 struct usb2_page_search res; 313 314 while (len != 0) { 315 316 usb2_get_page(cache, offset, &res); 317 318 if (res.length > len) { 319 res.length = len; 320 } 321 bzero(res.buffer, res.length); 322 323 offset += res.length; 324 len -= res.length; 325 } 326 } 327 328 #if USB_HAVE_BUSDMA && defined(__FreeBSD__) 329 330 /*------------------------------------------------------------------------* 331 * usb2_dma_lock_cb - dummy callback 332 *------------------------------------------------------------------------*/ 333 static void 334 usb2_dma_lock_cb(void *arg, bus_dma_lock_op_t op) 335 { 336 /* we use "mtx_owned()" instead of this function */ 337 } 338 339 /*------------------------------------------------------------------------* 340 * usb2_dma_tag_create - allocate a DMA tag 341 * 342 * NOTE: If the "align" parameter has a value of 1 the DMA-tag will 343 * allow multi-segment mappings. Else all mappings are single-segment. 344 *------------------------------------------------------------------------*/ 345 static void 346 usb2_dma_tag_create(struct usb2_dma_tag *udt, 347 usb2_size_t size, usb2_size_t align) 348 { 349 bus_dma_tag_t tag; 350 351 if (bus_dma_tag_create 352 ( /* parent */ udt->tag_parent->tag, 353 /* alignment */ align, 354 /* boundary */ USB_PAGE_SIZE, 355 /* lowaddr */ (2ULL << (udt->tag_parent->dma_bits - 1)) - 1, 356 /* highaddr */ BUS_SPACE_MAXADDR, 357 /* filter */ NULL, 358 /* filterarg */ NULL, 359 /* maxsize */ size, 360 /* nsegments */ (align == 1) ? 361 (2 + (size / USB_PAGE_SIZE)) : 1, 362 /* maxsegsz */ (align == 1) ? 363 USB_PAGE_SIZE : size, 364 /* flags */ BUS_DMA_KEEP_PG_OFFSET, 365 /* lockfn */ &usb2_dma_lock_cb, 366 /* lockarg */ NULL, 367 &tag)) { 368 tag = NULL; 369 } 370 udt->tag = tag; 371 } 372 373 /*------------------------------------------------------------------------* 374 * usb2_dma_tag_free - free a DMA tag 375 *------------------------------------------------------------------------*/ 376 static void 377 usb2_dma_tag_destroy(struct usb2_dma_tag *udt) 378 { 379 bus_dma_tag_destroy(udt->tag); 380 } 381 382 /*------------------------------------------------------------------------* 383 * usb2_pc_alloc_mem_cb - BUS-DMA callback function 384 *------------------------------------------------------------------------*/ 385 static void 386 usb2_pc_alloc_mem_cb(void *arg, bus_dma_segment_t *segs, 387 int nseg, int error) 388 { 389 usb2_pc_common_mem_cb(arg, segs, nseg, error, 0); 390 } 391 392 /*------------------------------------------------------------------------* 393 * usb2_pc_load_mem_cb - BUS-DMA callback function 394 *------------------------------------------------------------------------*/ 395 static void 396 usb2_pc_load_mem_cb(void *arg, bus_dma_segment_t *segs, 397 int nseg, int error) 398 { 399 usb2_pc_common_mem_cb(arg, segs, nseg, error, 1); 400 } 401 402 /*------------------------------------------------------------------------* 403 * usb2_pc_common_mem_cb - BUS-DMA callback function 404 *------------------------------------------------------------------------*/ 405 static void 406 usb2_pc_common_mem_cb(void *arg, bus_dma_segment_t *segs, 407 int nseg, int error, uint8_t isload) 408 { 409 struct usb2_dma_parent_tag *uptag; 410 struct usb2_page_cache *pc; 411 struct usb2_page *pg; 412 usb2_size_t rem; 413 uint8_t owned; 414 415 pc = arg; 416 uptag = pc->tag_parent; 417 418 /* 419 * XXX There is sometimes recursive locking here. 420 * XXX We should try to find a better solution. 421 * XXX Until further the "owned" variable does 422 * XXX the trick. 423 */ 424 425 if (error) { 426 goto done; 427 } 428 pg = pc->page_start; 429 pg->physaddr = segs->ds_addr & ~(USB_PAGE_SIZE - 1); 430 rem = segs->ds_addr & (USB_PAGE_SIZE - 1); 431 pc->page_offset_buf = rem; 432 pc->page_offset_end += rem; 433 nseg--; 434 #if (USB_DEBUG != 0) 435 if (rem != (USB_P2U(pc->buffer) & (USB_PAGE_SIZE - 1))) { 436 /* 437 * This check verifies that the physical address is correct: 438 */ 439 DPRINTFN(0, "Page offset was not preserved!\n"); 440 error = 1; 441 goto done; 442 } 443 #endif 444 while (nseg > 0) { 445 nseg--; 446 segs++; 447 pg++; 448 pg->physaddr = segs->ds_addr & ~(USB_PAGE_SIZE - 1); 449 } 450 451 done: 452 owned = mtx_owned(uptag->mtx); 453 if (!owned) 454 mtx_lock(uptag->mtx); 455 456 uptag->dma_error = (error ? 1 : 0); 457 if (isload) { 458 (uptag->func) (uptag); 459 } else { 460 usb2_cv_broadcast(uptag->cv); 461 } 462 if (!owned) 463 mtx_unlock(uptag->mtx); 464 } 465 466 /*------------------------------------------------------------------------* 467 * usb2_pc_alloc_mem - allocate DMA'able memory 468 * 469 * Returns: 470 * 0: Success 471 * Else: Failure 472 *------------------------------------------------------------------------*/ 473 uint8_t 474 usb2_pc_alloc_mem(struct usb2_page_cache *pc, struct usb2_page *pg, 475 usb2_size_t size, usb2_size_t align) 476 { 477 struct usb2_dma_parent_tag *uptag; 478 struct usb2_dma_tag *utag; 479 bus_dmamap_t map; 480 void *ptr; 481 int err; 482 483 uptag = pc->tag_parent; 484 485 if (align != 1) { 486 /* 487 * The alignment must be greater or equal to the 488 * "size" else the object can be split between two 489 * memory pages and we get a problem! 490 */ 491 while (align < size) { 492 align *= 2; 493 if (align == 0) { 494 goto error; 495 } 496 } 497 #if 1 498 /* 499 * XXX BUS-DMA workaround - FIXME later: 500 * 501 * We assume that that the aligment at this point of 502 * the code is greater than or equal to the size and 503 * less than two times the size, so that if we double 504 * the size, the size will be greater than the 505 * alignment. 506 * 507 * The bus-dma system has a check for "alignment" 508 * being less than "size". If that check fails we end 509 * up using contigmalloc which is page based even for 510 * small allocations. Try to avoid that to save 511 * memory, hence we sometimes to a large number of 512 * small allocations! 513 */ 514 if (size <= (USB_PAGE_SIZE / 2)) { 515 size *= 2; 516 } 517 #endif 518 } 519 /* get the correct DMA tag */ 520 utag = usb2_dma_tag_find(uptag, size, align); 521 if (utag == NULL) { 522 goto error; 523 } 524 /* allocate memory */ 525 if (bus_dmamem_alloc( 526 utag->tag, &ptr, (BUS_DMA_WAITOK | BUS_DMA_COHERENT), &map)) { 527 goto error; 528 } 529 /* setup page cache */ 530 pc->buffer = ptr; 531 pc->page_start = pg; 532 pc->page_offset_buf = 0; 533 pc->page_offset_end = size; 534 pc->map = map; 535 pc->tag = utag->tag; 536 pc->ismultiseg = (align == 1); 537 538 mtx_lock(uptag->mtx); 539 540 /* load memory into DMA */ 541 err = bus_dmamap_load( 542 utag->tag, map, ptr, size, &usb2_pc_alloc_mem_cb, 543 pc, (BUS_DMA_WAITOK | BUS_DMA_COHERENT)); 544 545 if (err == EINPROGRESS) { 546 usb2_cv_wait(uptag->cv, uptag->mtx); 547 err = 0; 548 } 549 mtx_unlock(uptag->mtx); 550 551 if (err || uptag->dma_error) { 552 bus_dmamem_free(utag->tag, ptr, map); 553 goto error; 554 } 555 bzero(ptr, size); 556 557 usb2_pc_cpu_flush(pc); 558 559 return (0); 560 561 error: 562 /* reset most of the page cache */ 563 pc->buffer = NULL; 564 pc->page_start = NULL; 565 pc->page_offset_buf = 0; 566 pc->page_offset_end = 0; 567 pc->map = NULL; 568 pc->tag = NULL; 569 return (1); 570 } 571 572 /*------------------------------------------------------------------------* 573 * usb2_pc_free_mem - free DMA memory 574 * 575 * This function is NULL safe. 576 *------------------------------------------------------------------------*/ 577 void 578 usb2_pc_free_mem(struct usb2_page_cache *pc) 579 { 580 if (pc && pc->buffer) { 581 582 bus_dmamap_unload(pc->tag, pc->map); 583 584 bus_dmamem_free(pc->tag, pc->buffer, pc->map); 585 586 pc->buffer = NULL; 587 } 588 } 589 590 /*------------------------------------------------------------------------* 591 * usb2_pc_load_mem - load virtual memory into DMA 592 * 593 * Return values: 594 * 0: Success 595 * Else: Error 596 *------------------------------------------------------------------------*/ 597 uint8_t 598 usb2_pc_load_mem(struct usb2_page_cache *pc, usb2_size_t size, uint8_t sync) 599 { 600 /* setup page cache */ 601 pc->page_offset_buf = 0; 602 pc->page_offset_end = size; 603 pc->ismultiseg = 1; 604 605 mtx_assert(pc->tag_parent->mtx, MA_OWNED); 606 607 if (size > 0) { 608 if (sync) { 609 struct usb2_dma_parent_tag *uptag; 610 int err; 611 612 uptag = pc->tag_parent; 613 614 /* 615 * We have to unload the previous loaded DMA 616 * pages before trying to load a new one! 617 */ 618 bus_dmamap_unload(pc->tag, pc->map); 619 620 /* 621 * Try to load memory into DMA. 622 */ 623 err = bus_dmamap_load( 624 pc->tag, pc->map, pc->buffer, size, 625 &usb2_pc_alloc_mem_cb, pc, BUS_DMA_WAITOK); 626 if (err == EINPROGRESS) { 627 usb2_cv_wait(uptag->cv, uptag->mtx); 628 err = 0; 629 } 630 if (err || uptag->dma_error) { 631 return (1); 632 } 633 } else { 634 635 /* 636 * We have to unload the previous loaded DMA 637 * pages before trying to load a new one! 638 */ 639 bus_dmamap_unload(pc->tag, pc->map); 640 641 /* 642 * Try to load memory into DMA. The callback 643 * will be called in all cases: 644 */ 645 if (bus_dmamap_load( 646 pc->tag, pc->map, pc->buffer, size, 647 &usb2_pc_load_mem_cb, pc, BUS_DMA_WAITOK)) { 648 } 649 } 650 } else { 651 if (!sync) { 652 /* 653 * Call callback so that refcount is decremented 654 * properly: 655 */ 656 pc->tag_parent->dma_error = 0; 657 (pc->tag_parent->func) (pc->tag_parent); 658 } 659 } 660 return (0); 661 } 662 663 /*------------------------------------------------------------------------* 664 * usb2_pc_cpu_invalidate - invalidate CPU cache 665 *------------------------------------------------------------------------*/ 666 void 667 usb2_pc_cpu_invalidate(struct usb2_page_cache *pc) 668 { 669 if (pc->page_offset_end == pc->page_offset_buf) { 670 /* nothing has been loaded into this page cache! */ 671 return; 672 } 673 bus_dmamap_sync(pc->tag, pc->map, 674 BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD); 675 } 676 677 /*------------------------------------------------------------------------* 678 * usb2_pc_cpu_flush - flush CPU cache 679 *------------------------------------------------------------------------*/ 680 void 681 usb2_pc_cpu_flush(struct usb2_page_cache *pc) 682 { 683 if (pc->page_offset_end == pc->page_offset_buf) { 684 /* nothing has been loaded into this page cache! */ 685 return; 686 } 687 bus_dmamap_sync(pc->tag, pc->map, 688 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD); 689 } 690 691 /*------------------------------------------------------------------------* 692 * usb2_pc_dmamap_create - create a DMA map 693 * 694 * Returns: 695 * 0: Success 696 * Else: Failure 697 *------------------------------------------------------------------------*/ 698 uint8_t 699 usb2_pc_dmamap_create(struct usb2_page_cache *pc, usb2_size_t size) 700 { 701 struct usb2_xfer_root *info; 702 struct usb2_dma_tag *utag; 703 704 /* get info */ 705 info = USB_DMATAG_TO_XROOT(pc->tag_parent); 706 707 /* sanity check */ 708 if (info == NULL) { 709 goto error; 710 } 711 utag = usb2_dma_tag_find(pc->tag_parent, size, 1); 712 if (utag == NULL) { 713 goto error; 714 } 715 /* create DMA map */ 716 if (bus_dmamap_create(utag->tag, 0, &pc->map)) { 717 goto error; 718 } 719 pc->tag = utag->tag; 720 return 0; /* success */ 721 722 error: 723 pc->map = NULL; 724 pc->tag = NULL; 725 return 1; /* failure */ 726 } 727 728 /*------------------------------------------------------------------------* 729 * usb2_pc_dmamap_destroy 730 * 731 * This function is NULL safe. 732 *------------------------------------------------------------------------*/ 733 void 734 usb2_pc_dmamap_destroy(struct usb2_page_cache *pc) 735 { 736 if (pc && pc->tag) { 737 bus_dmamap_destroy(pc->tag, pc->map); 738 pc->tag = NULL; 739 pc->map = NULL; 740 } 741 } 742 743 #endif 744 745 #if USB_HAVE_BUSDMA && defined(__NetBSD__) 746 747 /*------------------------------------------------------------------------* 748 * usb2_dma_tag_create - allocate a DMA tag 749 * 750 * NOTE: If the "align" parameter has a value of 1 the DMA-tag will 751 * allow multi-segment mappings. Else all mappings are single-segment. 752 *------------------------------------------------------------------------*/ 753 static void 754 usb2_dma_tag_create(struct usb2_dma_tag *udt, 755 usb2_size_t size, usb2_size_t align) 756 { 757 usb2_size_t nseg; 758 759 if (align == 1) { 760 nseg = (2 + (size / USB_PAGE_SIZE)); 761 } else { 762 nseg = 1; 763 } 764 765 udt->p_seg = malloc(nseg * sizeof(*(udt->p_seg)), 766 M_USB, M_WAITOK | M_ZERO); 767 768 if (udt->p_seg == NULL) { 769 return; 770 } 771 udt->tag = udt->tag_parent->tag; 772 udt->n_seg = nseg; 773 } 774 775 /*------------------------------------------------------------------------* 776 * usb2_dma_tag_free - free a DMA tag 777 *------------------------------------------------------------------------*/ 778 static void 779 usb2_dma_tag_destroy(struct usb2_dma_tag *udt) 780 { 781 free(udt->p_seg, M_USB); 782 } 783 784 /*------------------------------------------------------------------------* 785 * usb2_pc_common_mem_cb - BUS-DMA callback function 786 *------------------------------------------------------------------------*/ 787 static void 788 usb2_pc_common_mem_cb(struct usb2_page_cache *pc, bus_dma_segment_t *segs, 789 int nseg, int error, uint8_t isload, uint8_t dolock) 790 { 791 struct usb2_dma_parent_tag *uptag; 792 struct usb2_page *pg; 793 usb2_size_t rem; 794 uint8_t ext_seg; /* extend last segment */ 795 796 uptag = pc->tag_parent; 797 798 if (error) { 799 goto done; 800 } 801 pg = pc->page_start; 802 pg->physaddr = segs->ds_addr & ~(USB_PAGE_SIZE - 1); 803 rem = segs->ds_addr & (USB_PAGE_SIZE - 1); 804 pc->page_offset_buf = rem; 805 pc->page_offset_end += rem; 806 if (nseg < ((pc->page_offset_end + 807 (USB_PAGE_SIZE - 1)) / USB_PAGE_SIZE)) { 808 ext_seg = 1; 809 } else { 810 ext_seg = 0; 811 } 812 nseg--; 813 #if (USB_DEBUG != 0) 814 if (rem != (USB_P2U(pc->buffer) & (USB_PAGE_SIZE - 1))) { 815 /* 816 * This check verifies that the physical address is correct: 817 */ 818 DPRINTFN(0, "Page offset was not preserved!\n"); 819 error = 1; 820 goto done; 821 } 822 #endif 823 while (nseg > 0) { 824 nseg--; 825 segs++; 826 pg++; 827 pg->physaddr = segs->ds_addr & ~(USB_PAGE_SIZE - 1); 828 } 829 830 /* 831 * XXX The segments we get from BUS-DMA are not aligned, 832 * XXX so we need to extend the last segment if we are 833 * XXX unaligned and cross the segment boundary! 834 */ 835 if (ext_seg && pc->ismultiseg) { 836 (pg + 1)->physaddr = pg->physaddr + USB_PAGE_SIZE; 837 } 838 done: 839 if (dolock) 840 mtx_lock(uptag->mtx); 841 842 uptag->dma_error = (error ? 1 : 0); 843 if (isload) { 844 (uptag->func) (uptag); 845 } 846 if (dolock) 847 mtx_unlock(uptag->mtx); 848 } 849 850 /*------------------------------------------------------------------------* 851 * usb2_pc_alloc_mem - allocate DMA'able memory 852 * 853 * Returns: 854 * 0: Success 855 * Else: Failure 856 *------------------------------------------------------------------------*/ 857 uint8_t 858 usb2_pc_alloc_mem(struct usb2_page_cache *pc, struct usb2_page *pg, 859 usb2_size_t size, usb2_size_t align) 860 { 861 struct usb2_dma_parent_tag *uptag; 862 struct usb2_dma_tag *utag; 863 caddr_t ptr = NULL; 864 bus_dmamap_t map; 865 int seg_count; 866 867 uptag = pc->tag_parent; 868 869 if (align != 1) { 870 /* 871 * The alignment must be greater or equal to the 872 * "size" else the object can be split between two 873 * memory pages and we get a problem! 874 */ 875 while (align < size) { 876 align *= 2; 877 if (align == 0) { 878 goto done_5; 879 } 880 } 881 } 882 /* get the correct DMA tag */ 883 utag = usb2_dma_tag_find(pc->tag_parent, size, align); 884 if (utag == NULL) { 885 goto done_5; 886 } 887 if (bus_dmamem_alloc(utag->tag, size, align, 0, utag->p_seg, 888 utag->n_seg, &seg_count, BUS_DMA_WAITOK)) { 889 goto done_4; 890 } 891 if (bus_dmamem_map(utag->tag, utag->p_seg, seg_count, size, 892 &ptr, BUS_DMA_WAITOK | BUS_DMA_COHERENT)) { 893 goto done_3; 894 } 895 if (bus_dmamap_create(utag->tag, size, utag->n_seg, (align == 1) ? 896 USB_PAGE_SIZE : size, 0, BUS_DMA_WAITOK, &map)) { 897 goto done_2; 898 } 899 if (bus_dmamap_load(utag->tag, map, ptr, size, NULL, 900 BUS_DMA_WAITOK)) { 901 goto done_1; 902 } 903 pc->p_seg = malloc(seg_count * sizeof(*(pc->p_seg)), 904 M_USB, M_WAITOK | M_ZERO); 905 if (pc->p_seg == NULL) { 906 goto done_0; 907 } 908 /* store number if actual segments used */ 909 pc->n_seg = seg_count; 910 911 /* make a copy of the segments */ 912 bcopy(utag->p_seg, pc->p_seg, 913 seg_count * sizeof(*(pc->p_seg))); 914 915 /* setup page cache */ 916 pc->buffer = ptr; 917 pc->page_start = pg; 918 pc->page_offset_buf = 0; 919 pc->page_offset_end = size; 920 pc->map = map; 921 pc->tag = utag->tag; 922 pc->ismultiseg = (align == 1); 923 924 usb2_pc_common_mem_cb(pc, utag->p_seg, seg_count, 0, 0, 1); 925 926 bzero(ptr, size); 927 928 usb2_pc_cpu_flush(pc); 929 930 return (0); 931 932 done_0: 933 bus_dmamap_unload(utag->tag, map); 934 done_1: 935 bus_dmamap_destroy(utag->tag, map); 936 done_2: 937 bus_dmamem_unmap(utag->tag, ptr, size); 938 done_3: 939 bus_dmamem_free(utag->tag, utag->p_seg, seg_count); 940 done_4: 941 /* utag is destroyed later */ 942 done_5: 943 /* reset most of the page cache */ 944 pc->buffer = NULL; 945 pc->page_start = NULL; 946 pc->page_offset_buf = 0; 947 pc->page_offset_end = 0; 948 pc->map = NULL; 949 pc->tag = NULL; 950 pc->n_seg = 0; 951 pc->p_seg = NULL; 952 return (1); 953 } 954 955 /*------------------------------------------------------------------------* 956 * usb2_pc_free_mem - free DMA memory 957 * 958 * This function is NULL safe. 959 *------------------------------------------------------------------------*/ 960 void 961 usb2_pc_free_mem(struct usb2_page_cache *pc) 962 { 963 if (pc && pc->buffer) { 964 bus_dmamap_unload(pc->tag, pc->map); 965 bus_dmamap_destroy(pc->tag, pc->map); 966 bus_dmamem_unmap(pc->tag, pc->buffer, 967 pc->page_offset_end - pc->page_offset_buf); 968 bus_dmamem_free(pc->tag, pc->p_seg, pc->n_seg); 969 free(pc->p_seg, M_USB); 970 pc->buffer = NULL; 971 } 972 } 973 974 /*------------------------------------------------------------------------* 975 * usb2_pc_load_mem - load virtual memory into DMA 976 * 977 * Return values: 978 * 0: Success 979 * Else: Error 980 *------------------------------------------------------------------------*/ 981 uint8_t 982 usb2_pc_load_mem(struct usb2_page_cache *pc, usb2_size_t size, uint8_t sync) 983 { 984 int error; 985 986 /* setup page cache */ 987 pc->page_offset_buf = 0; 988 pc->page_offset_end = size; 989 pc->ismultiseg = 1; 990 991 if (size > 0) { 992 993 /* 994 * We have to unload the previous loaded DMA 995 * pages before trying to load a new one! 996 */ 997 bus_dmamap_unload(pc->tag, pc->map); 998 999 /* try to load memory into DMA using using no wait option */ 1000 if (bus_dmamap_load(pc->tag, pc->map, pc->buffer, 1001 size, NULL, BUS_DMA_NOWAIT)) { 1002 error = ENOMEM; 1003 } else { 1004 error = 0; 1005 } 1006 1007 usb2_pc_common_mem_cb(pc, pc->map->dm_segs, 1008 pc->map->dm_nsegs, error, !sync); 1009 1010 if (error) { 1011 return (1); 1012 } 1013 } else { 1014 if (!sync) { 1015 /* 1016 * Call callback so that refcount is decremented 1017 * properly: 1018 */ 1019 pc->tag_parent->dma_error = 0; 1020 (pc->tag_parent->func) (pc->tag_parent); 1021 } 1022 } 1023 return (0); 1024 } 1025 1026 /*------------------------------------------------------------------------* 1027 * usb2_pc_cpu_invalidate - invalidate CPU cache 1028 *------------------------------------------------------------------------*/ 1029 void 1030 usb2_pc_cpu_invalidate(struct usb2_page_cache *pc) 1031 { 1032 usb2_size_t len; 1033 1034 len = pc->page_offset_end - pc->page_offset_buf; 1035 1036 if (len == 0) { 1037 /* nothing has been loaded into this page cache */ 1038 return; 1039 } 1040 bus_dmamap_sync(pc->tag, pc->map, 0, len, 1041 BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD); 1042 } 1043 1044 /*------------------------------------------------------------------------* 1045 * usb2_pc_cpu_flush - flush CPU cache 1046 *------------------------------------------------------------------------*/ 1047 void 1048 usb2_pc_cpu_flush(struct usb2_page_cache *pc) 1049 { 1050 usb2_size_t len; 1051 1052 len = pc->page_offset_end - pc->page_offset_buf; 1053 1054 if (len == 0) { 1055 /* nothing has been loaded into this page cache */ 1056 return; 1057 } 1058 bus_dmamap_sync(pc->tag, pc->map, 0, len, 1059 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD); 1060 } 1061 1062 /*------------------------------------------------------------------------* 1063 * usb2_pc_dmamap_create - create a DMA map 1064 * 1065 * Returns: 1066 * 0: Success 1067 * Else: Failure 1068 *------------------------------------------------------------------------*/ 1069 uint8_t 1070 usb2_pc_dmamap_create(struct usb2_page_cache *pc, usb2_size_t size) 1071 { 1072 struct usb2_xfer_root *info; 1073 struct usb2_dma_tag *utag; 1074 1075 /* get info */ 1076 info = USB_DMATAG_TO_XROOT(pc->tag_parent); 1077 1078 /* sanity check */ 1079 if (info == NULL) { 1080 goto error; 1081 } 1082 utag = usb2_dma_tag_find(pc->tag_parent, size, 1); 1083 if (utag == NULL) { 1084 goto error; 1085 } 1086 if (bus_dmamap_create(utag->tag, size, utag->n_seg, 1087 USB_PAGE_SIZE, 0, BUS_DMA_WAITOK, &pc->map)) { 1088 goto error; 1089 } 1090 pc->tag = utag->tag; 1091 pc->p_seg = utag->p_seg; 1092 pc->n_seg = utag->n_seg; 1093 return 0; /* success */ 1094 1095 error: 1096 pc->map = NULL; 1097 pc->tag = NULL; 1098 pc->p_seg = NULL; 1099 pc->n_seg = 0; 1100 return 1; /* failure */ 1101 } 1102 1103 /*------------------------------------------------------------------------* 1104 * usb2_pc_dmamap_destroy 1105 * 1106 * This function is NULL safe. 1107 *------------------------------------------------------------------------*/ 1108 void 1109 usb2_pc_dmamap_destroy(struct usb2_page_cache *pc) 1110 { 1111 if (pc && pc->tag) { 1112 bus_dmamap_destroy(pc->tag, pc->map); 1113 pc->tag = NULL; 1114 pc->map = NULL; 1115 } 1116 } 1117 1118 #endif 1119 1120 #if USB_HAVE_BUSDMA 1121 1122 /*------------------------------------------------------------------------* 1123 * usb2_dma_tag_find - factored out code 1124 *------------------------------------------------------------------------*/ 1125 struct usb2_dma_tag * 1126 usb2_dma_tag_find(struct usb2_dma_parent_tag *udpt, 1127 usb2_size_t size, usb2_size_t align) 1128 { 1129 struct usb2_dma_tag *udt; 1130 uint8_t nudt; 1131 1132 USB_ASSERT(align > 0, ("Invalid parameter align = 0!\n")); 1133 USB_ASSERT(size > 0, ("Invalid parameter size = 0!\n")); 1134 1135 udt = udpt->utag_first; 1136 nudt = udpt->utag_max; 1137 1138 while (nudt--) { 1139 1140 if (udt->align == 0) { 1141 usb2_dma_tag_create(udt, size, align); 1142 if (udt->tag == NULL) { 1143 return (NULL); 1144 } 1145 udt->align = align; 1146 udt->size = size; 1147 return (udt); 1148 } 1149 if ((udt->align == align) && (udt->size == size)) { 1150 return (udt); 1151 } 1152 udt++; 1153 } 1154 return (NULL); 1155 } 1156 1157 /*------------------------------------------------------------------------* 1158 * usb2_dma_tag_setup - initialise USB DMA tags 1159 *------------------------------------------------------------------------*/ 1160 void 1161 usb2_dma_tag_setup(struct usb2_dma_parent_tag *udpt, 1162 struct usb2_dma_tag *udt, bus_dma_tag_t dmat, 1163 struct mtx *mtx, usb2_dma_callback_t *func, 1164 uint8_t ndmabits, uint8_t nudt) 1165 { 1166 bzero(udpt, sizeof(*udpt)); 1167 1168 /* sanity checking */ 1169 if ((nudt == 0) || 1170 (ndmabits == 0) || 1171 (mtx == NULL)) { 1172 /* something is corrupt */ 1173 return; 1174 } 1175 #ifdef __FreeBSD__ 1176 /* initialise condition variable */ 1177 usb2_cv_init(udpt->cv, "USB DMA CV"); 1178 #endif 1179 1180 /* store some information */ 1181 udpt->mtx = mtx; 1182 udpt->func = func; 1183 udpt->tag = dmat; 1184 udpt->utag_first = udt; 1185 udpt->utag_max = nudt; 1186 udpt->dma_bits = ndmabits; 1187 1188 while (nudt--) { 1189 bzero(udt, sizeof(*udt)); 1190 udt->tag_parent = udpt; 1191 udt++; 1192 } 1193 } 1194 1195 /*------------------------------------------------------------------------* 1196 * usb2_bus_tag_unsetup - factored out code 1197 *------------------------------------------------------------------------*/ 1198 void 1199 usb2_dma_tag_unsetup(struct usb2_dma_parent_tag *udpt) 1200 { 1201 struct usb2_dma_tag *udt; 1202 uint8_t nudt; 1203 1204 udt = udpt->utag_first; 1205 nudt = udpt->utag_max; 1206 1207 while (nudt--) { 1208 1209 if (udt->align) { 1210 /* destroy the USB DMA tag */ 1211 usb2_dma_tag_destroy(udt); 1212 udt->align = 0; 1213 } 1214 udt++; 1215 } 1216 1217 if (udpt->utag_max) { 1218 #ifdef __FreeBSD__ 1219 /* destroy the condition variable */ 1220 usb2_cv_destroy(udpt->cv); 1221 #endif 1222 } 1223 } 1224 1225 /*------------------------------------------------------------------------* 1226 * usb2_bdma_work_loop 1227 * 1228 * This function handles loading of virtual buffers into DMA and is 1229 * only called when "dma_refcount" is zero. 1230 *------------------------------------------------------------------------*/ 1231 void 1232 usb2_bdma_work_loop(struct usb2_xfer_queue *pq) 1233 { 1234 struct usb2_xfer_root *info; 1235 struct usb2_xfer *xfer; 1236 usb2_frcount_t nframes; 1237 1238 xfer = pq->curr; 1239 info = xfer->xroot; 1240 1241 mtx_assert(info->xfer_mtx, MA_OWNED); 1242 1243 if (xfer->error) { 1244 /* some error happened */ 1245 USB_BUS_LOCK(info->bus); 1246 usb2_transfer_done(xfer, 0); 1247 USB_BUS_UNLOCK(info->bus); 1248 return; 1249 } 1250 if (!xfer->flags_int.bdma_setup) { 1251 struct usb2_page *pg; 1252 usb2_frlength_t frlength_0; 1253 uint8_t isread; 1254 1255 xfer->flags_int.bdma_setup = 1; 1256 1257 /* reset BUS-DMA load state */ 1258 1259 info->dma_error = 0; 1260 1261 if (xfer->flags_int.isochronous_xfr) { 1262 /* only one frame buffer */ 1263 nframes = 1; 1264 frlength_0 = xfer->sumlen; 1265 } else { 1266 /* can be multiple frame buffers */ 1267 nframes = xfer->nframes; 1268 frlength_0 = xfer->frlengths[0]; 1269 } 1270 1271 /* 1272 * Set DMA direction first. This is needed to 1273 * select the correct cache invalidate and cache 1274 * flush operations. 1275 */ 1276 isread = USB_GET_DATA_ISREAD(xfer); 1277 pg = xfer->dma_page_ptr; 1278 1279 if (xfer->flags_int.control_xfr && 1280 xfer->flags_int.control_hdr) { 1281 /* special case */ 1282 if (xfer->flags_int.usb2_mode == USB_MODE_DEVICE) { 1283 /* The device controller writes to memory */ 1284 xfer->frbuffers[0].isread = 1; 1285 } else { 1286 /* The host controller reads from memory */ 1287 xfer->frbuffers[0].isread = 0; 1288 } 1289 } else { 1290 /* default case */ 1291 xfer->frbuffers[0].isread = isread; 1292 } 1293 1294 /* 1295 * Setup the "page_start" pointer which points to an array of 1296 * USB pages where information about the physical address of a 1297 * page will be stored. Also initialise the "isread" field of 1298 * the USB page caches. 1299 */ 1300 xfer->frbuffers[0].page_start = pg; 1301 1302 info->dma_nframes = nframes; 1303 info->dma_currframe = 0; 1304 info->dma_frlength_0 = frlength_0; 1305 1306 pg += (frlength_0 / USB_PAGE_SIZE); 1307 pg += 2; 1308 1309 while (--nframes > 0) { 1310 xfer->frbuffers[nframes].isread = isread; 1311 xfer->frbuffers[nframes].page_start = pg; 1312 1313 pg += (xfer->frlengths[nframes] / USB_PAGE_SIZE); 1314 pg += 2; 1315 } 1316 1317 } 1318 if (info->dma_error) { 1319 USB_BUS_LOCK(info->bus); 1320 usb2_transfer_done(xfer, USB_ERR_DMA_LOAD_FAILED); 1321 USB_BUS_UNLOCK(info->bus); 1322 return; 1323 } 1324 if (info->dma_currframe != info->dma_nframes) { 1325 1326 if (info->dma_currframe == 0) { 1327 /* special case */ 1328 usb2_pc_load_mem(xfer->frbuffers, 1329 info->dma_frlength_0, 0); 1330 } else { 1331 /* default case */ 1332 nframes = info->dma_currframe; 1333 usb2_pc_load_mem(xfer->frbuffers + nframes, 1334 xfer->frlengths[nframes], 0); 1335 } 1336 1337 /* advance frame index */ 1338 info->dma_currframe++; 1339 1340 return; 1341 } 1342 /* go ahead */ 1343 usb2_bdma_pre_sync(xfer); 1344 1345 /* start loading next USB transfer, if any */ 1346 usb2_command_wrapper(pq, NULL); 1347 1348 /* finally start the hardware */ 1349 usb2_pipe_enter(xfer); 1350 } 1351 1352 /*------------------------------------------------------------------------* 1353 * usb2_bdma_done_event 1354 * 1355 * This function is called when the BUS-DMA has loaded virtual memory 1356 * into DMA, if any. 1357 *------------------------------------------------------------------------*/ 1358 void 1359 usb2_bdma_done_event(struct usb2_dma_parent_tag *udpt) 1360 { 1361 struct usb2_xfer_root *info; 1362 1363 info = USB_DMATAG_TO_XROOT(udpt); 1364 1365 mtx_assert(info->xfer_mtx, MA_OWNED); 1366 1367 /* copy error */ 1368 info->dma_error = udpt->dma_error; 1369 1370 /* enter workloop again */ 1371 usb2_command_wrapper(&info->dma_q, 1372 info->dma_q.curr); 1373 } 1374 1375 /*------------------------------------------------------------------------* 1376 * usb2_bdma_pre_sync 1377 * 1378 * This function handles DMA synchronisation that must be done before 1379 * an USB transfer is started. 1380 *------------------------------------------------------------------------*/ 1381 void 1382 usb2_bdma_pre_sync(struct usb2_xfer *xfer) 1383 { 1384 struct usb2_page_cache *pc; 1385 usb2_frcount_t nframes; 1386 1387 if (xfer->flags_int.isochronous_xfr) { 1388 /* only one frame buffer */ 1389 nframes = 1; 1390 } else { 1391 /* can be multiple frame buffers */ 1392 nframes = xfer->nframes; 1393 } 1394 1395 pc = xfer->frbuffers; 1396 1397 while (nframes--) { 1398 1399 if (pc->isread) { 1400 usb2_pc_cpu_invalidate(pc); 1401 } else { 1402 usb2_pc_cpu_flush(pc); 1403 } 1404 pc++; 1405 } 1406 } 1407 1408 /*------------------------------------------------------------------------* 1409 * usb2_bdma_post_sync 1410 * 1411 * This function handles DMA synchronisation that must be done after 1412 * an USB transfer is complete. 1413 *------------------------------------------------------------------------*/ 1414 void 1415 usb2_bdma_post_sync(struct usb2_xfer *xfer) 1416 { 1417 struct usb2_page_cache *pc; 1418 usb2_frcount_t nframes; 1419 1420 if (xfer->flags_int.isochronous_xfr) { 1421 /* only one frame buffer */ 1422 nframes = 1; 1423 } else { 1424 /* can be multiple frame buffers */ 1425 nframes = xfer->nframes; 1426 } 1427 1428 pc = xfer->frbuffers; 1429 1430 while (nframes--) { 1431 if (pc->isread) { 1432 usb2_pc_cpu_invalidate(pc); 1433 } 1434 pc++; 1435 } 1436 } 1437 1438 #endif 1439