1 /* $FreeBSD$ */ 2 /*- 3 * Copyright (c) 2013 Hans Petter Selasky. All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 */ 26 27 #include <bsd_global.h> 28 29 #if USB_HAVE_BUSDMA 30 static void usb_pc_common_mem_cb(struct usb_page_cache *pc, 31 void *vaddr, uint32_t length); 32 #endif 33 34 /*------------------------------------------------------------------------* 35 * usbd_get_page - lookup DMA-able memory for the given offset 36 * 37 * NOTE: Only call this function when the "page_cache" structure has 38 * been properly initialized ! 39 *------------------------------------------------------------------------*/ 40 void 41 usbd_get_page(struct usb_page_cache *pc, usb_frlength_t offset, 42 struct usb_page_search *res) 43 { 44 #if USB_HAVE_BUSDMA 45 struct usb_page *page; 46 47 if (pc->page_start) { 48 49 /* Case 1 - something has been loaded into DMA */ 50 51 if (pc->buffer) { 52 53 /* Case 1a - Kernel Virtual Address */ 54 55 res->buffer = USB_ADD_BYTES(pc->buffer, offset); 56 } 57 offset += pc->page_offset_buf; 58 59 /* compute destination page */ 60 61 page = pc->page_start; 62 63 if (pc->ismultiseg) { 64 65 page += (offset / USB_PAGE_SIZE); 66 67 offset %= USB_PAGE_SIZE; 68 69 res->length = USB_PAGE_SIZE - offset; 70 res->physaddr = page->physaddr + offset; 71 } else { 72 res->length = (usb_size_t)-1; 73 res->physaddr = page->physaddr + offset; 74 } 75 if (!pc->buffer) { 76 77 /* Case 1b - Non Kernel Virtual Address */ 78 79 res->buffer = USB_ADD_BYTES(page->buffer, offset); 80 } 81 return; 82 } 83 #endif 84 /* Case 2 - Plain PIO */ 85 86 res->buffer = USB_ADD_BYTES(pc->buffer, offset); 87 res->length = (usb_size_t)-1; 88 #if USB_HAVE_BUSDMA 89 res->physaddr = 0; 90 #endif 91 } 92 93 /*------------------------------------------------------------------------* 94 * usbd_copy_in - copy directly to DMA-able memory 95 *------------------------------------------------------------------------*/ 96 void 97 usbd_copy_in(struct usb_page_cache *cache, usb_frlength_t offset, 98 const void *ptr, usb_frlength_t len) 99 { 100 struct usb_page_search buf_res; 101 102 while (len != 0) { 103 104 usbd_get_page(cache, offset, &buf_res); 105 106 if (buf_res.length > len) { 107 buf_res.length = len; 108 } 109 memcpy(buf_res.buffer, ptr, buf_res.length); 110 111 offset += buf_res.length; 112 len -= buf_res.length; 113 ptr = USB_ADD_BYTES(ptr, buf_res.length); 114 } 115 } 116 117 /*------------------------------------------------------------------------* 118 * usbd_copy_out - copy directly from DMA-able memory 119 *------------------------------------------------------------------------*/ 120 void 121 usbd_copy_out(struct usb_page_cache *cache, usb_frlength_t offset, 122 void *ptr, usb_frlength_t len) 123 { 124 struct usb_page_search res; 125 126 while (len != 0) { 127 128 usbd_get_page(cache, offset, &res); 129 130 if (res.length > len) { 131 res.length = len; 132 } 133 memcpy(ptr, res.buffer, res.length); 134 135 offset += res.length; 136 len -= res.length; 137 ptr = USB_ADD_BYTES(ptr, res.length); 138 } 139 } 140 141 /*------------------------------------------------------------------------* 142 * usbd_frame_zero - zero DMA-able memory 143 *------------------------------------------------------------------------*/ 144 void 145 usbd_frame_zero(struct usb_page_cache *cache, usb_frlength_t offset, 146 usb_frlength_t len) 147 { 148 struct usb_page_search res; 149 150 while (len != 0) { 151 152 usbd_get_page(cache, offset, &res); 153 154 if (res.length > len) { 155 res.length = len; 156 } 157 memset(res.buffer, 0, res.length); 158 159 offset += res.length; 160 len -= res.length; 161 } 162 } 163 164 #if USB_HAVE_BUSDMA 165 166 /*------------------------------------------------------------------------* 167 * usb_pc_common_mem_cb - BUS-DMA callback function 168 *------------------------------------------------------------------------*/ 169 static void 170 usb_pc_common_mem_cb(struct usb_page_cache *pc, 171 void *vaddr, uint32_t length) 172 { 173 struct usb_page *pg; 174 usb_size_t rem; 175 bus_size_t off; 176 bus_addr_t phys = (uintptr_t)vaddr; /* XXX */ 177 uint32_t nseg; 178 179 if (length == 0) 180 nseg = 1; 181 else 182 nseg = ((length + USB_PAGE_SIZE - 1) / USB_PAGE_SIZE); 183 184 pg = pc->page_start; 185 pg->physaddr = phys & ~(USB_PAGE_SIZE - 1); 186 rem = phys & (USB_PAGE_SIZE - 1); 187 pc->page_offset_buf = rem; 188 pc->page_offset_end += rem; 189 length += rem; 190 191 for (off = USB_PAGE_SIZE; off < length; off += USB_PAGE_SIZE) { 192 pg++; 193 pg->physaddr = (phys + off) & ~(USB_PAGE_SIZE - 1); 194 } 195 } 196 197 /*------------------------------------------------------------------------* 198 * usb_pc_alloc_mem - allocate DMA'able memory 199 * 200 * Returns: 201 * 0: Success 202 * Else: Failure 203 *------------------------------------------------------------------------*/ 204 uint8_t 205 usb_pc_alloc_mem(struct usb_page_cache *pc, struct usb_page *pg, 206 usb_size_t size, usb_size_t align) 207 { 208 void *ptr; 209 uint32_t rem; 210 211 /* allocate zeroed memory */ 212 213 if (align != 1) { 214 ptr = malloc(size + align, XXX, XXX); 215 if (ptr == NULL) 216 goto error; 217 218 rem = (-((uintptr_t)ptr)) & (align - 1); 219 } else { 220 ptr = malloc(size, XXX, XXX); 221 if (ptr == NULL) 222 goto error; 223 rem = 0; 224 } 225 226 /* setup page cache */ 227 pc->buffer = ((uint8_t *)ptr) + rem; 228 pc->page_start = pg; 229 pc->page_offset_buf = 0; 230 pc->page_offset_end = size; 231 pc->map = NULL; 232 pc->tag = ptr; 233 pc->ismultiseg = (align == 1); 234 235 /* compute physical address */ 236 usb_pc_common_mem_cb(pc, pc->buffer, size); 237 238 usb_pc_cpu_flush(pc); 239 return (0); 240 241 error: 242 /* reset most of the page cache */ 243 pc->buffer = NULL; 244 pc->page_start = NULL; 245 pc->page_offset_buf = 0; 246 pc->page_offset_end = 0; 247 pc->map = NULL; 248 pc->tag = NULL; 249 return (1); 250 } 251 252 /*------------------------------------------------------------------------* 253 * usb_pc_free_mem - free DMA memory 254 * 255 * This function is NULL safe. 256 *------------------------------------------------------------------------*/ 257 void 258 usb_pc_free_mem(struct usb_page_cache *pc) 259 { 260 if (pc != NULL && pc->buffer != NULL) { 261 free(pc->tag, XXX); 262 pc->buffer = NULL; 263 } 264 } 265 266 /*------------------------------------------------------------------------* 267 * usb_pc_load_mem - load virtual memory into DMA 268 * 269 * Return values: 270 * 0: Success 271 * Else: Error 272 *------------------------------------------------------------------------*/ 273 uint8_t 274 usb_pc_load_mem(struct usb_page_cache *pc, usb_size_t size, uint8_t sync) 275 { 276 /* setup page cache */ 277 pc->page_offset_buf = 0; 278 pc->page_offset_end = size; 279 pc->ismultiseg = 1; 280 281 mtx_assert(pc->tag_parent->mtx, MA_OWNED); 282 283 if (size > 0) { 284 /* compute physical address */ 285 usb_pc_common_mem_cb(pc, pc->buffer, size); 286 } 287 if (sync == 0) { 288 /* 289 * Call callback so that refcount is decremented 290 * properly: 291 */ 292 pc->tag_parent->dma_error = 0; 293 (pc->tag_parent->func) (pc->tag_parent); 294 } 295 return (0); 296 } 297 298 /*------------------------------------------------------------------------* 299 * usb_pc_cpu_invalidate - invalidate CPU cache 300 *------------------------------------------------------------------------*/ 301 void 302 usb_pc_cpu_invalidate(struct usb_page_cache *pc) 303 { 304 if (pc->page_offset_end == pc->page_offset_buf) { 305 /* nothing has been loaded into this page cache! */ 306 return; 307 } 308 /* NOP */ 309 } 310 311 /*------------------------------------------------------------------------* 312 * usb_pc_cpu_flush - flush CPU cache 313 *------------------------------------------------------------------------*/ 314 void 315 usb_pc_cpu_flush(struct usb_page_cache *pc) 316 { 317 if (pc->page_offset_end == pc->page_offset_buf) { 318 /* nothing has been loaded into this page cache! */ 319 return; 320 } 321 /* NOP */ 322 } 323 324 /*------------------------------------------------------------------------* 325 * usb_pc_dmamap_create - create a DMA map 326 * 327 * Returns: 328 * 0: Success 329 * Else: Failure 330 *------------------------------------------------------------------------*/ 331 uint8_t 332 usb_pc_dmamap_create(struct usb_page_cache *pc, usb_size_t size) 333 { 334 return (0); /* NOP, success */ 335 } 336 337 /*------------------------------------------------------------------------* 338 * usb_pc_dmamap_destroy 339 * 340 * This function is NULL safe. 341 *------------------------------------------------------------------------*/ 342 void 343 usb_pc_dmamap_destroy(struct usb_page_cache *pc) 344 { 345 /* NOP */ 346 } 347 348 /*------------------------------------------------------------------------* 349 * usb_dma_tag_setup - initialise USB DMA tags 350 *------------------------------------------------------------------------*/ 351 void 352 usb_dma_tag_setup(struct usb_dma_parent_tag *udpt, 353 struct usb_dma_tag *udt, bus_dma_tag_t dmat, 354 struct mtx *mtx, usb_dma_callback_t *func, 355 uint8_t ndmabits, uint8_t nudt) 356 { 357 memset(udpt, 0, sizeof(*udpt)); 358 359 /* sanity checking */ 360 if ((nudt == 0) || 361 (ndmabits == 0) || 362 (mtx == NULL)) { 363 /* something is corrupt */ 364 return; 365 } 366 /* initialise condition variable */ 367 cv_init(udpt->cv, "USB DMA CV"); 368 369 /* store some information */ 370 udpt->mtx = mtx; 371 udpt->func = func; 372 udpt->tag = dmat; 373 udpt->utag_first = udt; 374 udpt->utag_max = nudt; 375 udpt->dma_bits = ndmabits; 376 377 while (nudt--) { 378 memset(udt, 0, sizeof(*udt)); 379 udt->tag_parent = udpt; 380 udt++; 381 } 382 } 383 384 /*------------------------------------------------------------------------* 385 * usb_bus_tag_unsetup - factored out code 386 *------------------------------------------------------------------------*/ 387 void 388 usb_dma_tag_unsetup(struct usb_dma_parent_tag *udpt) 389 { 390 struct usb_dma_tag *udt; 391 uint8_t nudt; 392 393 udt = udpt->utag_first; 394 nudt = udpt->utag_max; 395 396 while (nudt--) { 397 udt->align = 0; 398 udt++; 399 } 400 401 if (udpt->utag_max) { 402 /* destroy the condition variable */ 403 cv_destroy(udpt->cv); 404 } 405 } 406 407 /*------------------------------------------------------------------------* 408 * usb_bdma_work_loop 409 * 410 * This function handles loading of virtual buffers into DMA and is 411 * only called when "dma_refcount" is zero. 412 *------------------------------------------------------------------------*/ 413 void 414 usb_bdma_work_loop(struct usb_xfer_queue *pq) 415 { 416 struct usb_xfer_root *info; 417 struct usb_xfer *xfer; 418 usb_frcount_t nframes; 419 420 xfer = pq->curr; 421 info = xfer->xroot; 422 423 mtx_assert(info->xfer_mtx, MA_OWNED); 424 425 if (xfer->error) { 426 /* some error happened */ 427 USB_BUS_LOCK(info->bus); 428 usbd_transfer_done(xfer, 0); 429 USB_BUS_UNLOCK(info->bus); 430 return; 431 } 432 if (!xfer->flags_int.bdma_setup) { 433 struct usb_page *pg; 434 usb_frlength_t frlength_0; 435 uint8_t isread; 436 437 xfer->flags_int.bdma_setup = 1; 438 439 /* reset BUS-DMA load state */ 440 441 info->dma_error = 0; 442 443 if (xfer->flags_int.isochronous_xfr) { 444 /* only one frame buffer */ 445 nframes = 1; 446 frlength_0 = xfer->sumlen; 447 } else { 448 /* can be multiple frame buffers */ 449 nframes = xfer->nframes; 450 frlength_0 = xfer->frlengths[0]; 451 } 452 453 /* 454 * Set DMA direction first. This is needed to 455 * select the correct cache invalidate and cache 456 * flush operations. 457 */ 458 isread = USB_GET_DATA_ISREAD(xfer); 459 pg = xfer->dma_page_ptr; 460 461 if (xfer->flags_int.control_xfr && 462 xfer->flags_int.control_hdr) { 463 /* special case */ 464 if (xfer->flags_int.usb_mode == USB_MODE_DEVICE) { 465 /* The device controller writes to memory */ 466 xfer->frbuffers[0].isread = 1; 467 } else { 468 /* The host controller reads from memory */ 469 xfer->frbuffers[0].isread = 0; 470 } 471 } else { 472 /* default case */ 473 xfer->frbuffers[0].isread = isread; 474 } 475 476 /* 477 * Setup the "page_start" pointer which points to an array of 478 * USB pages where information about the physical address of a 479 * page will be stored. Also initialise the "isread" field of 480 * the USB page caches. 481 */ 482 xfer->frbuffers[0].page_start = pg; 483 484 info->dma_nframes = nframes; 485 info->dma_currframe = 0; 486 info->dma_frlength_0 = frlength_0; 487 488 pg += (frlength_0 / USB_PAGE_SIZE); 489 pg += 2; 490 491 while (--nframes > 0) { 492 xfer->frbuffers[nframes].isread = isread; 493 xfer->frbuffers[nframes].page_start = pg; 494 495 pg += (xfer->frlengths[nframes] / USB_PAGE_SIZE); 496 pg += 2; 497 } 498 499 } 500 if (info->dma_error) { 501 USB_BUS_LOCK(info->bus); 502 usbd_transfer_done(xfer, USB_ERR_DMA_LOAD_FAILED); 503 USB_BUS_UNLOCK(info->bus); 504 return; 505 } 506 if (info->dma_currframe != info->dma_nframes) { 507 508 if (info->dma_currframe == 0) { 509 /* special case */ 510 usb_pc_load_mem(xfer->frbuffers, 511 info->dma_frlength_0, 0); 512 } else { 513 /* default case */ 514 nframes = info->dma_currframe; 515 usb_pc_load_mem(xfer->frbuffers + nframes, 516 xfer->frlengths[nframes], 0); 517 } 518 519 /* advance frame index */ 520 info->dma_currframe++; 521 522 return; 523 } 524 /* go ahead */ 525 usb_bdma_pre_sync(xfer); 526 527 /* start loading next USB transfer, if any */ 528 usb_command_wrapper(pq, NULL); 529 530 /* finally start the hardware */ 531 usbd_pipe_enter(xfer); 532 } 533 534 /*------------------------------------------------------------------------* 535 * usb_bdma_done_event 536 * 537 * This function is called when the BUS-DMA has loaded virtual memory 538 * into DMA, if any. 539 *------------------------------------------------------------------------*/ 540 void 541 usb_bdma_done_event(struct usb_dma_parent_tag *udpt) 542 { 543 struct usb_xfer_root *info; 544 545 info = USB_DMATAG_TO_XROOT(udpt); 546 547 mtx_assert(info->xfer_mtx, MA_OWNED); 548 549 /* copy error */ 550 info->dma_error = udpt->dma_error; 551 552 /* enter workloop again */ 553 usb_command_wrapper(&info->dma_q, 554 info->dma_q.curr); 555 } 556 557 /*------------------------------------------------------------------------* 558 * usb_bdma_pre_sync 559 * 560 * This function handles DMA synchronisation that must be done before 561 * an USB transfer is started. 562 *------------------------------------------------------------------------*/ 563 void 564 usb_bdma_pre_sync(struct usb_xfer *xfer) 565 { 566 struct usb_page_cache *pc; 567 usb_frcount_t nframes; 568 569 if (xfer->flags_int.isochronous_xfr) { 570 /* only one frame buffer */ 571 nframes = 1; 572 } else { 573 /* can be multiple frame buffers */ 574 nframes = xfer->nframes; 575 } 576 577 pc = xfer->frbuffers; 578 579 while (nframes--) { 580 581 if (pc->isread) { 582 usb_pc_cpu_invalidate(pc); 583 } else { 584 usb_pc_cpu_flush(pc); 585 } 586 pc++; 587 } 588 } 589 590 /*------------------------------------------------------------------------* 591 * usb_bdma_post_sync 592 * 593 * This function handles DMA synchronisation that must be done after 594 * an USB transfer is complete. 595 *------------------------------------------------------------------------*/ 596 void 597 usb_bdma_post_sync(struct usb_xfer *xfer) 598 { 599 struct usb_page_cache *pc; 600 usb_frcount_t nframes; 601 602 if (xfer->flags_int.isochronous_xfr) { 603 /* only one frame buffer */ 604 nframes = 1; 605 } else { 606 /* can be multiple frame buffers */ 607 nframes = xfer->nframes; 608 } 609 610 pc = xfer->frbuffers; 611 612 while (nframes--) { 613 if (pc->isread) { 614 usb_pc_cpu_invalidate(pc); 615 } 616 pc++; 617 } 618 } 619 #endif 620