Lines Matching +full:dma +full:- +full:safe +full:- +full:map
1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
78 /*------------------------------------------------------------------------*
79 * usbd_get_page - lookup DMA-able memory for the given offset
83 *------------------------------------------------------------------------*/
91 if (pc->page_start) { in usbd_get_page()
92 /* Case 1 - something has been loaded into DMA */ in usbd_get_page()
94 if (pc->buffer) { in usbd_get_page()
95 /* Case 1a - Kernel Virtual Address */ in usbd_get_page()
97 res->buffer = USB_ADD_BYTES(pc->buffer, offset); in usbd_get_page()
99 offset += pc->page_offset_buf; in usbd_get_page()
103 page = pc->page_start; in usbd_get_page()
105 if (pc->ismultiseg) { in usbd_get_page()
110 res->length = USB_PAGE_SIZE - offset; in usbd_get_page()
111 res->physaddr = page->physaddr + offset; in usbd_get_page()
113 res->length = (usb_size_t)-1; in usbd_get_page()
114 res->physaddr = page->physaddr + offset; in usbd_get_page()
116 if (!pc->buffer) { in usbd_get_page()
117 /* Case 1b - Non Kernel Virtual Address */ in usbd_get_page()
119 res->buffer = USB_ADD_BYTES(page->buffer, offset); in usbd_get_page()
124 /* Case 2 - Plain PIO */ in usbd_get_page()
126 res->buffer = USB_ADD_BYTES(pc->buffer, offset); in usbd_get_page()
127 res->length = (usb_size_t)-1; in usbd_get_page()
129 res->physaddr = 0; in usbd_get_page()
133 /*------------------------------------------------------------------------*
134 * usb_pc_buffer_is_aligned - verify alignment
138 *------------------------------------------------------------------------*/
156 len -= buf_res.length; in usb_pc_buffer_is_aligned()
161 /*------------------------------------------------------------------------*
162 * usbd_copy_in - copy directly to DMA-able memory
163 *------------------------------------------------------------------------*/
179 len -= buf_res.length; in usbd_copy_in()
184 /*------------------------------------------------------------------------*
185 * usbd_copy_in_user - copy directly to DMA-able memory from userland
190 *------------------------------------------------------------------------*/
210 len -= buf_res.length; in usbd_copy_in_user()
217 /*------------------------------------------------------------------------*
218 * usbd_m_copy_in - copy a mbuf chain directly into DMA-able memory
219 *------------------------------------------------------------------------*/
231 usbd_copy_in(ua->cache, ua->dst_offset, src, count); in usbd_m_copy_in_cb()
232 ua->dst_offset += count; in usbd_m_copy_in_cb()
245 /*------------------------------------------------------------------------*
246 * usb_uiomove - factored out code
247 *------------------------------------------------------------------------*/
272 len -= res.length; in usb_uiomove()
278 /*------------------------------------------------------------------------*
279 * usbd_copy_out - copy directly from DMA-able memory
280 *------------------------------------------------------------------------*/
296 len -= res.length; in usbd_copy_out()
301 /*------------------------------------------------------------------------*
302 * usbd_copy_out_user - copy directly from DMA-able memory to userland
307 *------------------------------------------------------------------------*/
327 len -= res.length; in usbd_copy_out_user()
334 /*------------------------------------------------------------------------*
335 * usbd_frame_zero - zero DMA-able memory
336 *------------------------------------------------------------------------*/
352 len -= res.length; in usbd_frame_zero()
358 /*------------------------------------------------------------------------*
359 * usb_dma_lock_cb - dummy callback
360 *------------------------------------------------------------------------*/
367 /*------------------------------------------------------------------------*
368 * usb_dma_tag_create - allocate a DMA tag
370 * NOTE: If the "align" parameter has a value of 1 the DMA-tag will
371 * allow multi-segment mappings. Else all mappings are single-segment.
372 *------------------------------------------------------------------------*/
380 ( /* parent */ udt->tag_parent->tag, in usb_dma_tag_create()
383 /* lowaddr */ (2ULL << (udt->tag_parent->dma_bits - 1)) - 1, in usb_dma_tag_create()
398 udt->tag = tag; in usb_dma_tag_create()
401 /*------------------------------------------------------------------------*
402 * usb_dma_tag_free - free a DMA tag
403 *------------------------------------------------------------------------*/
407 bus_dma_tag_destroy(udt->tag); in usb_dma_tag_destroy()
410 /*------------------------------------------------------------------------*
411 * usb_pc_alloc_mem_cb - BUS-DMA callback function
412 *------------------------------------------------------------------------*/
420 /*------------------------------------------------------------------------*
421 * usb_pc_load_mem_cb - BUS-DMA callback function
422 *------------------------------------------------------------------------*/
430 /*------------------------------------------------------------------------*
431 * usb_pc_common_mem_cb - BUS-DMA callback function
432 *------------------------------------------------------------------------*/
445 uptag = pc->tag_parent; in usb_pc_common_mem_cb()
459 pg = pc->page_start; in usb_pc_common_mem_cb()
460 pg->physaddr = rounddown2(segs->ds_addr, USB_PAGE_SIZE); in usb_pc_common_mem_cb()
461 rem = segs->ds_addr & (USB_PAGE_SIZE - 1); in usb_pc_common_mem_cb()
462 pc->page_offset_buf = rem; in usb_pc_common_mem_cb()
463 pc->page_offset_end += rem; in usb_pc_common_mem_cb()
468 for (x = 0; x != nseg - 1; x++) { in usb_pc_common_mem_cb()
469 if (((segs[x].ds_addr + segs[x].ds_len) & (USB_PAGE_SIZE - 1)) == in usb_pc_common_mem_cb()
470 ((segs[x + 1].ds_addr & (USB_PAGE_SIZE - 1)))) in usb_pc_common_mem_cb()
483 while (pc->ismultiseg) { in usb_pc_common_mem_cb()
485 if (off >= (segs->ds_len + rem)) { in usb_pc_common_mem_cb()
487 nseg--; in usb_pc_common_mem_cb()
495 pg->physaddr = rounddown2(segs->ds_addr + off, USB_PAGE_SIZE); in usb_pc_common_mem_cb()
499 owned = mtx_owned(uptag->mtx); in usb_pc_common_mem_cb()
501 USB_MTX_LOCK(uptag->mtx); in usb_pc_common_mem_cb()
503 uptag->dma_error = (error ? 1 : 0); in usb_pc_common_mem_cb()
505 (uptag->func) (uptag); in usb_pc_common_mem_cb()
507 cv_broadcast(uptag->cv); in usb_pc_common_mem_cb()
510 USB_MTX_UNLOCK(uptag->mtx); in usb_pc_common_mem_cb()
513 /*------------------------------------------------------------------------*
514 * usb_pc_alloc_mem - allocate DMA'able memory
519 *------------------------------------------------------------------------*/
526 bus_dmamap_t map; in usb_pc_alloc_mem() local
530 uptag = pc->tag_parent; in usb_pc_alloc_mem()
546 * XXX BUS-DMA workaround - FIXME later: in usb_pc_alloc_mem()
554 * The bus-dma system has a check for "alignment" in usb_pc_alloc_mem()
566 /* get the correct DMA tag */ in usb_pc_alloc_mem()
573 utag->tag, &ptr, (BUS_DMA_WAITOK | BUS_DMA_COHERENT), &map)) { in usb_pc_alloc_mem()
577 pc->buffer = ptr; in usb_pc_alloc_mem()
578 pc->page_start = pg; in usb_pc_alloc_mem()
579 pc->page_offset_buf = 0; in usb_pc_alloc_mem()
580 pc->page_offset_end = size; in usb_pc_alloc_mem()
581 pc->map = map; in usb_pc_alloc_mem()
582 pc->tag = utag->tag; in usb_pc_alloc_mem()
583 pc->ismultiseg = (align == 1); in usb_pc_alloc_mem()
585 USB_MTX_LOCK(uptag->mtx); in usb_pc_alloc_mem()
587 /* load memory into DMA */ in usb_pc_alloc_mem()
589 utag->tag, map, ptr, size, &usb_pc_alloc_mem_cb, in usb_pc_alloc_mem()
593 cv_wait(uptag->cv, uptag->mtx); in usb_pc_alloc_mem()
596 USB_MTX_UNLOCK(uptag->mtx); in usb_pc_alloc_mem()
598 if (err || uptag->dma_error) { in usb_pc_alloc_mem()
599 bus_dmamem_free(utag->tag, ptr, map); in usb_pc_alloc_mem()
602 pc->isloaded = 1; in usb_pc_alloc_mem()
611 pc->buffer = NULL; in usb_pc_alloc_mem()
612 pc->page_start = NULL; in usb_pc_alloc_mem()
613 pc->page_offset_buf = 0; in usb_pc_alloc_mem()
614 pc->page_offset_end = 0; in usb_pc_alloc_mem()
615 pc->isloaded = 0; in usb_pc_alloc_mem()
616 pc->map = NULL; in usb_pc_alloc_mem()
617 pc->tag = NULL; in usb_pc_alloc_mem()
621 /*------------------------------------------------------------------------*
622 * usb_pc_free_mem - free DMA memory
624 * This function is NULL safe.
625 *------------------------------------------------------------------------*/
629 if (pc && pc->buffer) { in usb_pc_free_mem()
630 if (pc->isloaded) in usb_pc_free_mem()
631 bus_dmamap_unload(pc->tag, pc->map); in usb_pc_free_mem()
633 bus_dmamem_free(pc->tag, pc->buffer, pc->map); in usb_pc_free_mem()
635 pc->buffer = NULL; in usb_pc_free_mem()
636 pc->isloaded = 0; in usb_pc_free_mem()
640 /*------------------------------------------------------------------------*
641 * usb_pc_load_mem - load virtual memory into DMA
646 *------------------------------------------------------------------------*/
651 pc->page_offset_buf = 0; in usb_pc_load_mem()
652 pc->page_offset_end = size; in usb_pc_load_mem()
653 pc->ismultiseg = 1; in usb_pc_load_mem()
655 USB_MTX_ASSERT(pc->tag_parent->mtx, MA_OWNED); in usb_pc_load_mem()
662 uptag = pc->tag_parent; in usb_pc_load_mem()
665 * We have to unload the previous loaded DMA in usb_pc_load_mem()
668 if (pc->isloaded) in usb_pc_load_mem()
669 bus_dmamap_unload(pc->tag, pc->map); in usb_pc_load_mem()
672 * Try to load memory into DMA. in usb_pc_load_mem()
675 pc->tag, pc->map, pc->buffer, size, in usb_pc_load_mem()
678 cv_wait(uptag->cv, uptag->mtx); in usb_pc_load_mem()
681 if (err || uptag->dma_error) { in usb_pc_load_mem()
682 pc->isloaded = 0; in usb_pc_load_mem()
687 * We have to unload the previous loaded DMA in usb_pc_load_mem()
690 if (pc->isloaded) in usb_pc_load_mem()
691 bus_dmamap_unload(pc->tag, pc->map); in usb_pc_load_mem()
694 * Try to load memory into DMA. The callback in usb_pc_load_mem()
698 pc->tag, pc->map, pc->buffer, size, in usb_pc_load_mem()
702 pc->isloaded = 1; in usb_pc_load_mem()
709 pc->tag_parent->dma_error = 0; in usb_pc_load_mem()
710 (pc->tag_parent->func) (pc->tag_parent); in usb_pc_load_mem()
716 /*------------------------------------------------------------------------*
717 * usb_pc_cpu_invalidate - invalidate CPU cache
718 *------------------------------------------------------------------------*/
722 if (pc->page_offset_end == pc->page_offset_buf) { in usb_pc_cpu_invalidate()
730 * different cases to optimise the code. --HPS in usb_pc_cpu_invalidate()
732 bus_dmamap_sync(pc->tag, pc->map, BUS_DMASYNC_POSTREAD); in usb_pc_cpu_invalidate()
733 bus_dmamap_sync(pc->tag, pc->map, BUS_DMASYNC_PREREAD); in usb_pc_cpu_invalidate()
736 /*------------------------------------------------------------------------*
737 * usb_pc_cpu_flush - flush CPU cache
738 *------------------------------------------------------------------------*/
742 if (pc->page_offset_end == pc->page_offset_buf) { in usb_pc_cpu_flush()
746 bus_dmamap_sync(pc->tag, pc->map, BUS_DMASYNC_PREWRITE); in usb_pc_cpu_flush()
749 /*------------------------------------------------------------------------*
750 * usb_pc_dmamap_create - create a DMA map
755 *------------------------------------------------------------------------*/
763 info = USB_DMATAG_TO_XROOT(pc->tag_parent); in usb_pc_dmamap_create()
769 utag = usb_dma_tag_find(pc->tag_parent, size, 1); in usb_pc_dmamap_create()
773 /* create DMA map */ in usb_pc_dmamap_create()
774 if (bus_dmamap_create(utag->tag, 0, &pc->map)) { in usb_pc_dmamap_create()
777 pc->tag = utag->tag; in usb_pc_dmamap_create()
781 pc->map = NULL; in usb_pc_dmamap_create()
782 pc->tag = NULL; in usb_pc_dmamap_create()
786 /*------------------------------------------------------------------------*
789 * This function is NULL safe.
790 *------------------------------------------------------------------------*/
794 if (pc && pc->tag) { in usb_pc_dmamap_destroy()
795 if (pc->isloaded) in usb_pc_dmamap_destroy()
796 bus_dmamap_unload(pc->tag, pc->map); in usb_pc_dmamap_destroy()
797 bus_dmamap_destroy(pc->tag, pc->map); in usb_pc_dmamap_destroy()
798 pc->tag = NULL; in usb_pc_dmamap_destroy()
799 pc->map = NULL; in usb_pc_dmamap_destroy()
803 /*------------------------------------------------------------------------*
804 * usb_dma_tag_find - factored out code
805 *------------------------------------------------------------------------*/
816 udt = udpt->utag_first; in usb_dma_tag_find()
817 nudt = udpt->utag_max; in usb_dma_tag_find()
819 while (nudt--) { in usb_dma_tag_find()
820 if (udt->align == 0) { in usb_dma_tag_find()
822 if (udt->tag == NULL) { in usb_dma_tag_find()
825 udt->align = align; in usb_dma_tag_find()
826 udt->size = size; in usb_dma_tag_find()
829 if ((udt->align == align) && (udt->size == size)) { in usb_dma_tag_find()
837 /*------------------------------------------------------------------------*
838 * usb_dma_tag_setup - initialise USB DMA tags
839 *------------------------------------------------------------------------*/
856 cv_init(udpt->cv, "USB DMA CV"); in usb_dma_tag_setup()
859 udpt->mtx = mtx; in usb_dma_tag_setup()
860 udpt->func = func; in usb_dma_tag_setup()
861 udpt->tag = dmat; in usb_dma_tag_setup()
862 udpt->utag_first = udt; in usb_dma_tag_setup()
863 udpt->utag_max = nudt; in usb_dma_tag_setup()
864 udpt->dma_bits = ndmabits; in usb_dma_tag_setup()
866 while (nudt--) { in usb_dma_tag_setup()
868 udt->tag_parent = udpt; in usb_dma_tag_setup()
873 /*------------------------------------------------------------------------*
874 * usb_bus_tag_unsetup - factored out code
875 *------------------------------------------------------------------------*/
882 udt = udpt->utag_first; in usb_dma_tag_unsetup()
883 nudt = udpt->utag_max; in usb_dma_tag_unsetup()
885 while (nudt--) { in usb_dma_tag_unsetup()
886 if (udt->align) { in usb_dma_tag_unsetup()
887 /* destroy the USB DMA tag */ in usb_dma_tag_unsetup()
889 udt->align = 0; in usb_dma_tag_unsetup()
894 if (udpt->utag_max) { in usb_dma_tag_unsetup()
896 cv_destroy(udpt->cv); in usb_dma_tag_unsetup()
900 /*------------------------------------------------------------------------*
903 * This function handles loading of virtual buffers into DMA and is
905 *------------------------------------------------------------------------*/
913 xfer = pq->curr; in usb_bdma_work_loop()
914 info = xfer->xroot; in usb_bdma_work_loop()
916 USB_MTX_ASSERT(info->xfer_mtx, MA_OWNED); in usb_bdma_work_loop()
918 if (xfer->error) { in usb_bdma_work_loop()
920 USB_BUS_LOCK(info->bus); in usb_bdma_work_loop()
922 USB_BUS_UNLOCK(info->bus); in usb_bdma_work_loop()
925 if (!xfer->flags_int.bdma_setup) { in usb_bdma_work_loop()
930 xfer->flags_int.bdma_setup = 1; in usb_bdma_work_loop()
932 /* reset BUS-DMA load state */ in usb_bdma_work_loop()
934 info->dma_error = 0; in usb_bdma_work_loop()
936 if (xfer->flags_int.isochronous_xfr) { in usb_bdma_work_loop()
939 frlength_0 = xfer->sumlen; in usb_bdma_work_loop()
942 nframes = xfer->nframes; in usb_bdma_work_loop()
943 frlength_0 = xfer->frlengths[0]; in usb_bdma_work_loop()
947 * Set DMA direction first. This is needed to in usb_bdma_work_loop()
952 pg = xfer->dma_page_ptr; in usb_bdma_work_loop()
954 if (xfer->flags_int.control_xfr && in usb_bdma_work_loop()
955 xfer->flags_int.control_hdr) { in usb_bdma_work_loop()
957 if (xfer->flags_int.usb_mode == USB_MODE_DEVICE) { in usb_bdma_work_loop()
959 xfer->frbuffers[0].isread = 1; in usb_bdma_work_loop()
962 xfer->frbuffers[0].isread = 0; in usb_bdma_work_loop()
966 xfer->frbuffers[0].isread = isread; in usb_bdma_work_loop()
975 xfer->frbuffers[0].page_start = pg; in usb_bdma_work_loop()
977 info->dma_nframes = nframes; in usb_bdma_work_loop()
978 info->dma_currframe = 0; in usb_bdma_work_loop()
979 info->dma_frlength_0 = frlength_0; in usb_bdma_work_loop()
984 while (--nframes > 0) { in usb_bdma_work_loop()
985 xfer->frbuffers[nframes].isread = isread; in usb_bdma_work_loop()
986 xfer->frbuffers[nframes].page_start = pg; in usb_bdma_work_loop()
988 pg += (xfer->frlengths[nframes] / USB_PAGE_SIZE); in usb_bdma_work_loop()
992 if (info->dma_error) { in usb_bdma_work_loop()
993 USB_BUS_LOCK(info->bus); in usb_bdma_work_loop()
995 USB_BUS_UNLOCK(info->bus); in usb_bdma_work_loop()
998 if (info->dma_currframe != info->dma_nframes) { in usb_bdma_work_loop()
999 if (info->dma_currframe == 0) { in usb_bdma_work_loop()
1001 usb_pc_load_mem(xfer->frbuffers, in usb_bdma_work_loop()
1002 info->dma_frlength_0, 0); in usb_bdma_work_loop()
1005 nframes = info->dma_currframe; in usb_bdma_work_loop()
1006 usb_pc_load_mem(xfer->frbuffers + nframes, in usb_bdma_work_loop()
1007 xfer->frlengths[nframes], 0); in usb_bdma_work_loop()
1011 info->dma_currframe++; in usb_bdma_work_loop()
1025 /*------------------------------------------------------------------------*
1028 * This function is called when the BUS-DMA has loaded virtual memory
1029 * into DMA, if any.
1030 *------------------------------------------------------------------------*/
1038 USB_MTX_ASSERT(info->xfer_mtx, MA_OWNED); in usb_bdma_done_event()
1041 info->dma_error = udpt->dma_error; in usb_bdma_done_event()
1044 usb_command_wrapper(&info->dma_q, in usb_bdma_done_event()
1045 info->dma_q.curr); in usb_bdma_done_event()
1048 /*------------------------------------------------------------------------*
1051 * This function handles DMA synchronisation that must be done before
1053 *------------------------------------------------------------------------*/
1060 if (xfer->flags_int.isochronous_xfr) { in usb_bdma_pre_sync()
1065 nframes = xfer->nframes; in usb_bdma_pre_sync()
1068 pc = xfer->frbuffers; in usb_bdma_pre_sync()
1070 while (nframes--) { in usb_bdma_pre_sync()
1071 if (pc->isread) { in usb_bdma_pre_sync()
1080 /*------------------------------------------------------------------------*
1083 * This function handles DMA synchronisation that must be done after
1085 *------------------------------------------------------------------------*/
1092 if (xfer->flags_int.isochronous_xfr) { in usb_bdma_post_sync()
1097 nframes = xfer->nframes; in usb_bdma_post_sync()
1100 pc = xfer->frbuffers; in usb_bdma_post_sync()
1102 while (nframes--) { in usb_bdma_post_sync()
1103 if (pc->isread) { in usb_bdma_post_sync()