Lines Matching refs:dma
389 rootnex_dma_t *dma, ddi_dma_attr_t *attr, ddi_dma_obj_t *dmao, int kmflag);
391 rootnex_dma_t *dma, ddi_dma_attr_t *attr);
392 static void rootnex_teardown_copybuf(rootnex_dma_t *dma);
393 static int rootnex_setup_windows(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
395 static void rootnex_teardown_windows(rootnex_dma_t *dma);
396 static void rootnex_init_win(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
399 rootnex_dma_t *dma, ddi_dma_cookie_t *cookie, off_t cur_offset,
402 rootnex_dma_t *dma, rootnex_window_t **windowp, ddi_dma_cookie_t *cookie,
405 rootnex_dma_t *dma, rootnex_window_t **windowp,
408 rootnex_dma_t *dma, rootnex_window_t **windowp, ddi_dma_cookie_t *cookie);
411 static int rootnex_verify_buffer(rootnex_dma_t *dma);
1718 rootnex_dma_t *dma; in rootnex_coredma_allochdl() local
1746 dma = (rootnex_dma_t *)hp->dmai_private; in rootnex_coredma_allochdl()
1747 dma->dp_prealloc_buffer = (uchar_t *)(((uintptr_t)dma + in rootnex_coredma_allochdl()
1754 dma->dp_dip = rdip; in rootnex_coredma_allochdl()
1755 dma->dp_sglinfo.si_flags = attr->dma_attr_flags; in rootnex_coredma_allochdl()
1756 dma->dp_sglinfo.si_min_addr = attr->dma_attr_addr_lo; in rootnex_coredma_allochdl()
1766 dma->dp_sglinfo.si_max_addr = attr->dma_attr_seg; in rootnex_coredma_allochdl()
1767 dma->dp_sglinfo.si_flags &= ~_DDI_DMA_BOUNCE_ON_SEG; in rootnex_coredma_allochdl()
1769 dma->dp_sglinfo.si_max_addr = attr->dma_attr_addr_hi; in rootnex_coredma_allochdl()
1776 if (attr->dma_attr_seg >= dma->dp_sglinfo.si_max_addr) in rootnex_coredma_allochdl()
1777 dma->dp_sglinfo.si_cancross = B_FALSE; in rootnex_coredma_allochdl()
1779 dma->dp_sglinfo.si_cancross = B_TRUE; in rootnex_coredma_allochdl()
1782 mutex_init(&dma->dp_mutex, NULL, MUTEX_DRIVER, NULL); in rootnex_coredma_allochdl()
1810 dma->dp_granularity_power_2 = B_FALSE; in rootnex_coredma_allochdl()
1812 dma->dp_granularity_power_2 = B_TRUE; in rootnex_coredma_allochdl()
1822 if (dma->dp_granularity_power_2) { in rootnex_coredma_allochdl()
1823 dma->dp_maxxfer = attr->dma_attr_maxxfer - in rootnex_coredma_allochdl()
1827 dma->dp_maxxfer = attr->dma_attr_maxxfer - in rootnex_coredma_allochdl()
1831 dma->dp_maxxfer = attr->dma_attr_maxxfer; in rootnex_coredma_allochdl()
1834 maxsegmentsize_ll = MIN(seg, dma->dp_maxxfer); in rootnex_coredma_allochdl()
1841 dma->dp_sglinfo.si_max_cookie_size = maxsegmentsize; in rootnex_coredma_allochdl()
1842 dma->dp_sglinfo.si_segmask = attr->dma_attr_seg; in rootnex_coredma_allochdl()
1916 rootnex_dma_t *dma; in rootnex_coredma_freehdl() local
1920 dma = (rootnex_dma_t *)hp->dmai_private; in rootnex_coredma_freehdl()
1923 ASSERT(!dma->dp_inuse); in rootnex_coredma_freehdl()
1925 mutex_destroy(&dma->dp_mutex); in rootnex_coredma_freehdl()
1972 rootnex_dma_t *dma; in rootnex_coredma_bindhdl() local
1978 dma = (rootnex_dma_t *)hp->dmai_private; in rootnex_coredma_bindhdl()
1979 dmao = &dma->dp_dma; in rootnex_coredma_bindhdl()
1980 sinfo = &dma->dp_sglinfo; in rootnex_coredma_bindhdl()
1985 dma->dp_sleep_flags = kmflag = KM_SLEEP; in rootnex_coredma_bindhdl()
1987 dma->dp_sleep_flags = kmflag = KM_NOSLEEP; in rootnex_coredma_bindhdl()
2002 e = mutex_tryenter(&dma->dp_mutex); in rootnex_coredma_bindhdl()
2007 if (dma->dp_inuse) { in rootnex_coredma_bindhdl()
2008 mutex_exit(&dma->dp_mutex); in rootnex_coredma_bindhdl()
2012 dma->dp_inuse = B_TRUE; in rootnex_coredma_bindhdl()
2013 mutex_exit(&dma->dp_mutex); in rootnex_coredma_bindhdl()
2027 dma->dp_dma = dmareq->dmar_object; in rootnex_coredma_bindhdl()
2031 dmao = &dma->dp_dvma; in rootnex_coredma_bindhdl()
2038 dma->dp_dvma_used = B_TRUE; in rootnex_coredma_bindhdl()
2044 dma->dp_dvma_used = B_TRUE; in rootnex_coredma_bindhdl()
2045 dma->dp_need_to_free_cookie = B_FALSE; in rootnex_coredma_bindhdl()
2048 cookie = hp->dmai_cookie = dma->dp_cookies = in rootnex_coredma_bindhdl()
2049 (ddi_dma_cookie_t *)dma->dp_prealloc_buffer; in rootnex_coredma_bindhdl()
2071 sinfo->si_max_pages = mmu_btopr(dma->dp_dma.dmao_size) + 1; in rootnex_coredma_bindhdl()
2073 if (dma->dp_dvma_used) { in rootnex_coredma_bindhdl()
2086 ncookies = (dma->dp_dma.dmao_size / sinfo->si_max_cookie_size) in rootnex_coredma_bindhdl()
2090 (dma->dp_dma.dmao_size / attr->dma_attr_seg) + 1; in rootnex_coredma_bindhdl()
2102 dma->dp_cookies = (ddi_dma_cookie_t *)dma->dp_prealloc_buffer; in rootnex_coredma_bindhdl()
2103 dma->dp_need_to_free_cookie = B_FALSE; in rootnex_coredma_bindhdl()
2121 dma->dp_cookie_size = sinfo->si_max_pages * in rootnex_coredma_bindhdl()
2123 dma->dp_cookies = kmem_alloc(dma->dp_cookie_size, kmflag); in rootnex_coredma_bindhdl()
2124 if (dma->dp_cookies == NULL) { in rootnex_coredma_bindhdl()
2129 dma->dp_need_to_free_cookie = B_TRUE; in rootnex_coredma_bindhdl()
2133 hp->dmai_cookie = dma->dp_cookies; in rootnex_coredma_bindhdl()
2142 if (dma->dp_dvma_used) in rootnex_coredma_bindhdl()
2143 rootnex_dvma_get_sgl(dmao, dma->dp_cookies, &dma->dp_sglinfo); in rootnex_coredma_bindhdl()
2145 rootnex_get_sgl(dmao, dma->dp_cookies, &dma->dp_sglinfo); in rootnex_coredma_bindhdl()
2169 (dmao->dmao_size < dma->dp_maxxfer)) { in rootnex_coredma_bindhdl()
2184 *cookiep = dma->dp_cookies[0]; in rootnex_coredma_bindhdl()
2210 if (dma->dp_dvma_used) { in rootnex_coredma_bindhdl()
2212 &dma->dp_dvma); in rootnex_coredma_bindhdl()
2216 e = rootnex_bind_slowpath(hp, dmareq, dma, attr, &dma->dp_dma, in rootnex_coredma_bindhdl()
2222 if (dma->dp_need_to_free_cookie) { in rootnex_coredma_bindhdl()
2223 kmem_free(dma->dp_cookies, dma->dp_cookie_size); in rootnex_coredma_bindhdl()
2238 if ((dma->dp_window[dma->dp_current_win].wd_dosync) && in rootnex_coredma_bindhdl()
2256 *ccountp = dma->dp_window[dma->dp_current_win].wd_cookie_cnt; in rootnex_coredma_bindhdl()
2257 ASSERT(hp->dmai_nwin <= dma->dp_max_win); in rootnex_coredma_bindhdl()
2259 *cookiep = dma->dp_cookies[0]; in rootnex_coredma_bindhdl()
2304 rootnex_dma_t *dma; in rootnex_coredma_unbindhdl() local
2308 dma = (rootnex_dma_t *)hp->dmai_private; in rootnex_coredma_unbindhdl()
2312 e = rootnex_verify_buffer(dma); in rootnex_coredma_unbindhdl()
2320 if (dma->dp_window && dma->dp_window[dma->dp_current_win].wd_dosync && in rootnex_coredma_unbindhdl()
2330 rootnex_teardown_copybuf(dma); in rootnex_coredma_unbindhdl()
2331 rootnex_teardown_windows(dma); in rootnex_coredma_unbindhdl()
2336 &dma->dp_dvma); in rootnex_coredma_unbindhdl()
2343 if (dma->dp_need_to_free_cookie) { in rootnex_coredma_unbindhdl()
2344 kmem_free(dma->dp_cookies, dma->dp_cookie_size); in rootnex_coredma_unbindhdl()
2391 rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private; in rootnex_coredma_get_sleep_flags() local
2393 if (dma->dp_sleep_flags != KM_SLEEP && in rootnex_coredma_get_sleep_flags()
2394 dma->dp_sleep_flags != KM_NOSLEEP) in rootnex_coredma_get_sleep_flags()
2396 return (dma->dp_sleep_flags); in rootnex_coredma_get_sleep_flags()
2403 rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private; in rootnex_coredma_reset_cookies() local
2406 if (dma->dp_window) { in rootnex_coredma_reset_cookies()
2407 window = &dma->dp_window[dma->dp_current_win]; in rootnex_coredma_reset_cookies()
2410 hp->dmai_cookie = dma->dp_cookies; in rootnex_coredma_reset_cookies()
2423 rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private; in rootnex_coredma_get_cookies() local
2431 if (dma->dp_window) { in rootnex_coredma_get_cookies()
2432 window = &dma->dp_window[dma->dp_current_win]; in rootnex_coredma_get_cookies()
2436 cp = dma->dp_cookies; in rootnex_coredma_get_cookies()
2437 *ccountp = dma->dp_sglinfo.si_sgl_size; in rootnex_coredma_get_cookies()
2464 rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private; in rootnex_coredma_set_cookies() local
2470 ASSERT(dma->dp_need_to_switch_cookies == B_FALSE); in rootnex_coredma_set_cookies()
2472 if (dma->dp_window) { in rootnex_coredma_set_cookies()
2473 window = &dma->dp_window[dma->dp_current_win]; in rootnex_coredma_set_cookies()
2474 dma->dp_saved_cookies = window->wd_first_cookie; in rootnex_coredma_set_cookies()
2477 cur_cookiep = (hp->dmai_cookie - dma->dp_saved_cookies) in rootnex_coredma_set_cookies()
2480 dma->dp_saved_cookies = dma->dp_cookies; in rootnex_coredma_set_cookies()
2481 dma->dp_cookies = cookiep; in rootnex_coredma_set_cookies()
2482 ASSERT(ccount == dma->dp_sglinfo.si_sgl_size); in rootnex_coredma_set_cookies()
2483 cur_cookiep = (hp->dmai_cookie - dma->dp_saved_cookies) in rootnex_coredma_set_cookies()
2484 + dma->dp_cookies; in rootnex_coredma_set_cookies()
2487 dma->dp_need_to_switch_cookies = B_TRUE; in rootnex_coredma_set_cookies()
2498 rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private; in rootnex_coredma_clear_cookies() local
2505 if (dma->dp_need_to_switch_cookies == B_FALSE) in rootnex_coredma_clear_cookies()
2508 ASSERT(dma->dp_saved_cookies); in rootnex_coredma_clear_cookies()
2510 if (dma->dp_window) { in rootnex_coredma_clear_cookies()
2511 window = &dma->dp_window[dma->dp_current_win]; in rootnex_coredma_clear_cookies()
2513 window->wd_first_cookie = dma->dp_saved_cookies; in rootnex_coredma_clear_cookies()
2514 dma->dp_saved_cookies = NULL; in rootnex_coredma_clear_cookies()
2519 cookie_array = dma->dp_cookies; in rootnex_coredma_clear_cookies()
2520 dma->dp_cookies = dma->dp_saved_cookies; in rootnex_coredma_clear_cookies()
2521 dma->dp_saved_cookies = NULL; in rootnex_coredma_clear_cookies()
2522 ccount = dma->dp_sglinfo.si_sgl_size; in rootnex_coredma_clear_cookies()
2524 + dma->dp_cookies; in rootnex_coredma_clear_cookies()
2531 dma->dp_need_to_switch_cookies = B_FALSE; in rootnex_coredma_clear_cookies()
2562 rootnex_verify_buffer(rootnex_dma_t *dma) in rootnex_verify_buffer() argument
2573 if (dma->dp_dma.dmao_type == DMA_OTYP_PAGES) { in rootnex_verify_buffer()
2574 poff = dma->dp_dma.dmao_obj.pp_obj.pp_offset & MMU_PAGEOFFSET; in rootnex_verify_buffer()
2576 vaddr = dma->dp_dma.dmao_obj.virt_obj.v_addr; in rootnex_verify_buffer()
2579 pcnt = mmu_btopr(dma->dp_dma.dmao_size + poff); in rootnex_verify_buffer()
2581 switch (dma->dp_dma.dmao_type) { in rootnex_verify_buffer()
2587 pp = dma->dp_dma.dmao_obj.pp_obj.pp_pp; in rootnex_verify_buffer()
2598 pplist = dma->dp_dma.dmao_obj.virt_obj.v_priv; in rootnex_verify_buffer()
2612 if (rootnex_get_as(&dma->dp_dma) == &kas) { in rootnex_verify_buffer()
2640 rootnex_dma_t *dma; in rootnex_clean_dmahdl() local
2643 dma = (rootnex_dma_t *)hp->dmai_private; in rootnex_clean_dmahdl()
2646 dma->dp_current_cookie = 0; in rootnex_clean_dmahdl()
2647 dma->dp_copybuf_size = 0; in rootnex_clean_dmahdl()
2648 dma->dp_window = NULL; in rootnex_clean_dmahdl()
2649 dma->dp_cbaddr = NULL; in rootnex_clean_dmahdl()
2650 dma->dp_inuse = B_FALSE; in rootnex_clean_dmahdl()
2651 dma->dp_dvma_used = B_FALSE; in rootnex_clean_dmahdl()
2652 dma->dp_need_to_free_cookie = B_FALSE; in rootnex_clean_dmahdl()
2653 dma->dp_need_to_switch_cookies = B_FALSE; in rootnex_clean_dmahdl()
2654 dma->dp_saved_cookies = NULL; in rootnex_clean_dmahdl()
2655 dma->dp_sleep_flags = KM_PANIC; in rootnex_clean_dmahdl()
2656 dma->dp_need_to_free_window = B_FALSE; in rootnex_clean_dmahdl()
2657 dma->dp_partial_required = B_FALSE; in rootnex_clean_dmahdl()
2658 dma->dp_trim_required = B_FALSE; in rootnex_clean_dmahdl()
2659 dma->dp_sglinfo.si_copybuf_req = 0; in rootnex_clean_dmahdl()
2661 dma->dp_cb_remaping = B_FALSE; in rootnex_clean_dmahdl()
2662 dma->dp_kva = NULL; in rootnex_clean_dmahdl()
3251 rootnex_dma_t *dma, ddi_dma_attr_t *attr, ddi_dma_obj_t *dmao, int kmflag) in rootnex_bind_slowpath() argument
3266 sinfo = &dma->dp_sglinfo; in rootnex_bind_slowpath()
3275 e = rootnex_setup_copybuf(hp, dmareq, dma, attr); in rootnex_bind_slowpath()
3280 dma->dp_copybuf_size = 0; in rootnex_bind_slowpath()
3287 if ((dma->dp_copybuf_size < sinfo->si_copybuf_req) || in rootnex_bind_slowpath()
3288 (dmao->dmao_size > dma->dp_maxxfer) || in rootnex_bind_slowpath()
3290 dma->dp_partial_required = B_TRUE; in rootnex_bind_slowpath()
3292 dma->dp_trim_required = B_TRUE; in rootnex_bind_slowpath()
3295 dma->dp_partial_required = B_FALSE; in rootnex_bind_slowpath()
3296 dma->dp_trim_required = B_FALSE; in rootnex_bind_slowpath()
3300 if (dma->dp_partial_required && in rootnex_bind_slowpath()
3303 mnum = ddi_driver_major(dma->dp_dip); in rootnex_bind_slowpath()
3318 "/etc/system(4).", ddi_driver_name(dma->dp_dip)); in rootnex_bind_slowpath()
3327 e = rootnex_setup_windows(hp, dma, attr, dmao, kmflag); in rootnex_bind_slowpath()
3329 rootnex_teardown_copybuf(dma); in rootnex_bind_slowpath()
3333 window = &dma->dp_window[0]; in rootnex_bind_slowpath()
3334 cookie = &dma->dp_cookies[0]; in rootnex_bind_slowpath()
3336 rootnex_init_win(hp, dma, window, cookie, cur_offset); in rootnex_bind_slowpath()
3348 if (dma->dp_copybuf_size > 0) { in rootnex_bind_slowpath()
3349 rootnex_setup_cookie(dmao, dma, cookie, in rootnex_bind_slowpath()
3363 if (dma->dp_copybuf_size && in rootnex_bind_slowpath()
3364 (copybuf_used > dma->dp_copybuf_size)) { in rootnex_bind_slowpath()
3366 e = rootnex_copybuf_window_boundary(hp, dma, &window, in rootnex_bind_slowpath()
3369 rootnex_teardown_copybuf(dma); in rootnex_bind_slowpath()
3370 rootnex_teardown_windows(dma); in rootnex_bind_slowpath()
3382 dma->dp_dip); in rootnex_bind_slowpath()
3389 e = rootnex_sgllen_window_boundary(hp, dma, &window, in rootnex_bind_slowpath()
3392 rootnex_teardown_copybuf(dma); in rootnex_bind_slowpath()
3393 rootnex_teardown_windows(dma); in rootnex_bind_slowpath()
3405 dma->dp_dip); in rootnex_bind_slowpath()
3409 dma->dp_maxxfer) { in rootnex_bind_slowpath()
3411 e = rootnex_maxxfer_window_boundary(hp, dma, &window, in rootnex_bind_slowpath()
3414 rootnex_teardown_copybuf(dma); in rootnex_bind_slowpath()
3415 rootnex_teardown_windows(dma); in rootnex_bind_slowpath()
3427 dma->dp_dip); in rootnex_bind_slowpath()
3454 ASSERT(dma->dp_partial_required); in rootnex_bind_slowpath()
3465 rootnex_dma_t *dma, ddi_dma_attr_t *attr) in rootnex_setup_copybuf() argument
3476 ASSERT(!dma->dp_dvma_used); in rootnex_setup_copybuf()
3478 sinfo = &dma->dp_sglinfo; in rootnex_setup_copybuf()
3487 dma->dp_copybuf_size = MIN(sinfo->si_copybuf_req, max_copybuf); in rootnex_setup_copybuf()
3488 ASSERT((dma->dp_copybuf_size & MMU_PAGEOFFSET) == 0); in rootnex_setup_copybuf()
3507 dma->dp_kva = vmem_alloc(heap_arena, dma->dp_copybuf_size, in rootnex_setup_copybuf()
3509 if (dma->dp_kva == NULL) { in rootnex_setup_copybuf()
3538 e = i_ddi_mem_alloc(dma->dp_dip, &lattr, dma->dp_copybuf_size, cansleep, in rootnex_setup_copybuf()
3539 0, NULL, &dma->dp_cbaddr, &dma->dp_cbsize, NULL); in rootnex_setup_copybuf()
3542 if (dma->dp_kva != NULL) { in rootnex_setup_copybuf()
3543 vmem_free(heap_arena, dma->dp_kva, in rootnex_setup_copybuf()
3544 dma->dp_copybuf_size); in rootnex_setup_copybuf()
3550 ROOTNEX_DPROBE2(rootnex__alloc__copybuf, dev_info_t *, dma->dp_dip, in rootnex_setup_copybuf()
3551 size_t, dma->dp_copybuf_size); in rootnex_setup_copybuf()
3563 rootnex_setup_windows(ddi_dma_impl_t *hp, rootnex_dma_t *dma, in rootnex_setup_windows() argument
3578 sinfo = &dma->dp_sglinfo; in rootnex_setup_windows()
3580 dma->dp_current_win = 0; in rootnex_setup_windows()
3584 if (!dma->dp_partial_required) { in rootnex_setup_windows()
3585 dma->dp_max_win = 1; in rootnex_setup_windows()
3605 if (sinfo->si_copybuf_req > dma->dp_copybuf_size) { in rootnex_setup_windows()
3606 ASSERT(dma->dp_copybuf_size > 0); in rootnex_setup_windows()
3608 dma->dp_copybuf_size) + 1 + 2; in rootnex_setup_windows()
3633 if (dmao->dmao_size > dma->dp_maxxfer) { in rootnex_setup_windows()
3635 dma->dp_maxxfer) + 1 + 2; in rootnex_setup_windows()
3639 dma->dp_max_win = copybuf_win + sglwin + maxxfer_win; in rootnex_setup_windows()
3640 ASSERT(dma->dp_max_win > 0); in rootnex_setup_windows()
3642 win_state_size = dma->dp_max_win * sizeof (rootnex_window_t); in rootnex_setup_windows()
3653 if (dma->dp_need_to_free_cookie) { in rootnex_setup_windows()
3655 ASSERT(space_used <= dma->dp_cookie_size); in rootnex_setup_windows()
3656 if ((dma->dp_cookie_size - space_used) <= in rootnex_setup_windows()
3659 windowp = (rootnex_window_t *)dma->dp_prealloc_buffer; in rootnex_setup_windows()
3667 state_available = dma->dp_cookie_size - space_used; in rootnex_setup_windows()
3669 &dma->dp_cookies[sinfo->si_sgl_size]; in rootnex_setup_windows()
3677 &dma->dp_cookies[sinfo->si_sgl_size]; in rootnex_setup_windows()
3684 if (dma->dp_copybuf_size > 0) { in rootnex_setup_windows()
3695 dma->dp_window = windowp; in rootnex_setup_windows()
3696 dma->dp_need_to_free_window = B_FALSE; in rootnex_setup_windows()
3700 dma->dp_window = kmem_alloc(space_needed, kmflag); in rootnex_setup_windows()
3701 if (dma->dp_window == NULL) { in rootnex_setup_windows()
3704 dma->dp_need_to_free_window = B_TRUE; in rootnex_setup_windows()
3705 dma->dp_window_size = space_needed; in rootnex_setup_windows()
3707 dma->dp_dip, size_t, space_needed); in rootnex_setup_windows()
3714 if (dma->dp_copybuf_size > 0) { in rootnex_setup_windows()
3715 dma->dp_pgmap = (rootnex_pgmap_t *)(((uintptr_t) in rootnex_setup_windows()
3716 &dma->dp_window[dma->dp_max_win] + 0x7) & ~0x7); in rootnex_setup_windows()
3723 bzero(dma->dp_pgmap, copy_state_size); in rootnex_setup_windows()
3726 dma->dp_pgmap = NULL; in rootnex_setup_windows()
3738 rootnex_teardown_copybuf(rootnex_dma_t *dma) in rootnex_teardown_copybuf() argument
3748 if (dma->dp_kva != NULL) { in rootnex_teardown_copybuf()
3749 for (i = 0; i < dma->dp_sglinfo.si_max_pages; i++) { in rootnex_teardown_copybuf()
3750 if (dma->dp_pgmap[i].pm_mapped) { in rootnex_teardown_copybuf()
3751 hat_unload(kas.a_hat, dma->dp_pgmap[i].pm_kaddr, in rootnex_teardown_copybuf()
3753 dma->dp_pgmap[i].pm_mapped = B_FALSE; in rootnex_teardown_copybuf()
3757 vmem_free(heap_arena, dma->dp_kva, dma->dp_copybuf_size); in rootnex_teardown_copybuf()
3763 if (dma->dp_cbaddr != NULL) { in rootnex_teardown_copybuf()
3764 i_ddi_mem_free(dma->dp_cbaddr, NULL); in rootnex_teardown_copybuf()
3774 rootnex_teardown_windows(rootnex_dma_t *dma) in rootnex_teardown_windows() argument
3780 if (dma->dp_need_to_free_window) { in rootnex_teardown_windows()
3781 kmem_free(dma->dp_window, dma->dp_window_size); in rootnex_teardown_windows()
3793 rootnex_init_win(ddi_dma_impl_t *hp, rootnex_dma_t *dma, in rootnex_init_win() argument
3807 window->wd_remap_copybuf = dma->dp_cb_remaping; in rootnex_init_win()
3820 rootnex_setup_cookie(ddi_dma_obj_t *dmar_object, rootnex_dma_t *dma, in rootnex_setup_cookie() argument
3838 sinfo = &dma->dp_sglinfo; in rootnex_setup_cookie()
3867 if (!ISP2(dma->dp_copybuf_size)) { in rootnex_setup_cookie()
3874 dma->dp_pgmap[pidx].pm_uses_copybuf = B_TRUE; in rootnex_setup_cookie()
3881 if ((*copybuf_used + MMU_PAGESIZE) <= dma->dp_copybuf_size) { in rootnex_setup_cookie()
3882 dma->dp_pgmap[pidx].pm_cbaddr = dma->dp_cbaddr + in rootnex_setup_cookie()
3886 dma->dp_pgmap[pidx].pm_cbaddr = (caddr_t)( in rootnex_setup_cookie()
3887 (uintptr_t)dma->dp_cbaddr + in rootnex_setup_cookie()
3889 (dma->dp_copybuf_size - 1))); in rootnex_setup_cookie()
3891 dma->dp_pgmap[pidx].pm_cbaddr = (caddr_t)( in rootnex_setup_cookie()
3892 (uintptr_t)dma->dp_cbaddr + in rootnex_setup_cookie()
3893 (*copybuf_used % dma->dp_copybuf_size)); in rootnex_setup_cookie()
3903 dma->dp_pgmap[pidx].pm_cbaddr)) + poff; in rootnex_setup_cookie()
3914 dma->dp_pgmap[pidx].pm_kaddr = (caddr_t)(((uintptr_t) in rootnex_setup_cookie()
3922 dma->dp_pgmap[pidx].pm_mapped = B_FALSE; in rootnex_setup_cookie()
3932 dma->dp_pgmap[pidx].pm_kaddr = hat_kpm_pfn2va(pfn); in rootnex_setup_cookie()
3943 dma->dp_pgmap[pidx].pm_pp = *cur_pp; in rootnex_setup_cookie()
3944 dma->dp_pgmap[pidx].pm_vaddr = NULL; in rootnex_setup_cookie()
3946 dma->dp_pgmap[pidx].pm_pp = pplist[pidx]; in rootnex_setup_cookie()
3947 dma->dp_pgmap[pidx].pm_vaddr = NULL; in rootnex_setup_cookie()
3949 dma->dp_pgmap[pidx].pm_pp = NULL; in rootnex_setup_cookie()
3950 dma->dp_pgmap[pidx].pm_vaddr = (caddr_t) in rootnex_setup_cookie()
3965 dma->dp_copybuf_size) { in rootnex_setup_cookie()
3966 dma->dp_pgmap[pidx].pm_kaddr = (caddr_t) in rootnex_setup_cookie()
3967 (((uintptr_t)dma->dp_kva + *copybuf_used) & in rootnex_setup_cookie()
3971 dma->dp_pgmap[pidx].pm_kaddr = (caddr_t) in rootnex_setup_cookie()
3972 (((uintptr_t)dma->dp_kva + in rootnex_setup_cookie()
3974 (dma->dp_copybuf_size - 1))) & in rootnex_setup_cookie()
3977 dma->dp_pgmap[pidx].pm_kaddr = (caddr_t) in rootnex_setup_cookie()
3978 (((uintptr_t)dma->dp_kva + in rootnex_setup_cookie()
3980 dma->dp_copybuf_size)) & in rootnex_setup_cookie()
3989 if (!dma->dp_cb_remaping && ((*copybuf_used + in rootnex_setup_cookie()
3990 MMU_PAGESIZE) <= dma->dp_copybuf_size)) { in rootnex_setup_cookie()
3991 dma->dp_pgmap[pidx].pm_mapped = B_TRUE; in rootnex_setup_cookie()
3992 if (dma->dp_pgmap[pidx].pm_pp != NULL) { in rootnex_setup_cookie()
3993 i86_pp_map(dma->dp_pgmap[pidx].pm_pp, in rootnex_setup_cookie()
3994 dma->dp_pgmap[pidx].pm_kaddr); in rootnex_setup_cookie()
3996 i86_va_map(dma->dp_pgmap[pidx].pm_vaddr, in rootnex_setup_cookie()
3998 dma->dp_pgmap[pidx].pm_kaddr); in rootnex_setup_cookie()
4008 dma->dp_pgmap[pidx].pm_mapped = B_FALSE; in rootnex_setup_cookie()
4042 dma->dp_pgmap[pidx].pm_mapped = B_FALSE; in rootnex_setup_cookie()
4044 dma->dp_pgmap[pidx].pm_uses_copybuf = B_FALSE; in rootnex_setup_cookie()
4067 rootnex_sgllen_window_boundary(ddi_dma_impl_t *hp, rootnex_dma_t *dma, in rootnex_sgllen_window_boundary() argument
4080 if (!dma->dp_trim_required) { in rootnex_sgllen_window_boundary()
4082 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset); in rootnex_sgllen_window_boundary()
4090 if (dma->dp_granularity_power_2) { in rootnex_sgllen_window_boundary()
4099 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset); in rootnex_sgllen_window_boundary()
4147 rootnex_init_win(hp, dma, *windowp, cookie, new_offset); in rootnex_sgllen_window_boundary()
4169 if ((*windowp)->wd_size > dma->dp_maxxfer) { in rootnex_sgllen_window_boundary()
4175 trim_sz = (*windowp)->wd_size - dma->dp_maxxfer; in rootnex_sgllen_window_boundary()
4181 ASSERT((*windowp)->wd_size == dma->dp_maxxfer); in rootnex_sgllen_window_boundary()
4189 rootnex_init_win(hp, dma, *windowp, cookie, new_offset); in rootnex_sgllen_window_boundary()
4207 rootnex_copybuf_window_boundary(ddi_dma_impl_t *hp, rootnex_dma_t *dma, in rootnex_copybuf_window_boundary() argument
4220 sinfo = &dma->dp_sglinfo; in rootnex_copybuf_window_boundary()
4234 dma->dp_cb_remaping = B_TRUE; in rootnex_copybuf_window_boundary()
4245 if (!dma->dp_trim_required) { in rootnex_copybuf_window_boundary()
4247 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset); in rootnex_copybuf_window_boundary()
4261 if (dma->dp_granularity_power_2) { in rootnex_copybuf_window_boundary()
4275 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset); in rootnex_copybuf_window_boundary()
4331 dma->dp_pgmap[pidx].pm_cbaddr; in rootnex_copybuf_window_boundary()
4334 dma->dp_pgmap[pidx].pm_kaddr; in rootnex_copybuf_window_boundary()
4353 rootnex_init_win(hp, dma, *windowp, cookie, new_offset); in rootnex_copybuf_window_boundary()
4372 (*windowp)->wd_trim.tr_first_cbaddr = dma->dp_cbaddr; in rootnex_copybuf_window_boundary()
4375 paddr = pfn_to_pa(hat_getpfnum(kas.a_hat, dma->dp_cbaddr)) + in rootnex_copybuf_window_boundary()
4381 (*windowp)->wd_trim.tr_first_kaddr = dma->dp_kva; in rootnex_copybuf_window_boundary()
4408 dma->dp_pgmap[pidx + 1].pm_cbaddr += MMU_PAGESIZE; in rootnex_copybuf_window_boundary()
4412 dma->dp_pgmap[pidx + 1].pm_cbaddr)) + poff; in rootnex_copybuf_window_boundary()
4416 ASSERT(dma->dp_pgmap[pidx + 1].pm_mapped == B_FALSE); in rootnex_copybuf_window_boundary()
4417 dma->dp_pgmap[pidx + 1].pm_kaddr += MMU_PAGESIZE; in rootnex_copybuf_window_boundary()
4431 ASSERT((*windowp)->wd_size < dma->dp_maxxfer); in rootnex_copybuf_window_boundary()
4449 rootnex_maxxfer_window_boundary(ddi_dma_impl_t *hp, rootnex_dma_t *dma, in rootnex_maxxfer_window_boundary() argument
4463 trim_sz = ((*windowp)->wd_size + cookie->dmac_size) - dma->dp_maxxfer; in rootnex_maxxfer_window_boundary()
4465 ASSERT(trim_sz <= dma->dp_maxxfer); in rootnex_maxxfer_window_boundary()
4480 (*windowp)->wd_size = dma->dp_maxxfer; in rootnex_maxxfer_window_boundary()
4498 rootnex_init_win(hp, dma, *windowp, cookie, new_offset); in rootnex_maxxfer_window_boundary()
4521 rootnex_dma_t *dma; in rootnex_coredma_sync() local
4533 dma = (rootnex_dma_t *)hp->dmai_private; in rootnex_coredma_sync()
4534 sinfo = &dma->dp_sglinfo; in rootnex_coredma_sync()
4540 if (dma->dp_window == NULL) { in rootnex_coredma_sync()
4545 win = &dma->dp_window[dma->dp_current_win]; in rootnex_coredma_sync()
4596 cbpage = &dma->dp_pgmap[pidx]; in rootnex_coredma_sync()
4615 dev_info_t *, dma->dp_dip, size_t, psize); in rootnex_coredma_sync()
4626 dev_info_t *, dma->dp_dip, size_t, psize); in rootnex_coredma_sync()
4728 rootnex_dma_t *dma; in rootnex_coredma_win() local
4741 dma = (rootnex_dma_t *)hp->dmai_private; in rootnex_coredma_win()
4743 sinfo = &dma->dp_sglinfo; in rootnex_coredma_win()
4752 dmao = dma->dp_dvma_used ? &dma->dp_dvma : &dma->dp_dma; in rootnex_coredma_win()
4760 if (dma->dp_window == NULL) { in rootnex_coredma_win()
4766 hp->dmai_cookie = dma->dp_cookies; in rootnex_coredma_win()
4769 *ccountp = dma->dp_sglinfo.si_sgl_size; in rootnex_coredma_win()
4776 window = &dma->dp_window[dma->dp_current_win]; in rootnex_coredma_win()
4787 if (dma->dp_cb_remaping) { in rootnex_coredma_win()
4807 if (dma->dp_pgmap[pidx].pm_mapped) { in rootnex_coredma_win()
4809 dma->dp_pgmap[pidx].pm_kaddr, MMU_PAGESIZE, in rootnex_coredma_win()
4811 dma->dp_pgmap[pidx].pm_mapped = B_FALSE; in rootnex_coredma_win()
4822 dma->dp_current_win = win; in rootnex_coredma_win()
4823 window = &dma->dp_window[win]; in rootnex_coredma_win()
4836 dma->dp_pgmap[trim->tr_first_pidx].pm_cbaddr = in rootnex_coredma_win()
4839 dma->dp_pgmap[trim->tr_first_pidx].pm_kaddr = in rootnex_coredma_win()
4848 dma->dp_pgmap[trim->tr_last_pidx].pm_cbaddr = in rootnex_coredma_win()
4851 dma->dp_pgmap[trim->tr_last_pidx].pm_kaddr = in rootnex_coredma_win()
4871 if (dma->dp_cb_remaping) { in rootnex_coredma_win()
4885 pmap = &dma->dp_pgmap[pidx]; in rootnex_coredma_win()
4910 pmap = &dma->dp_pgmap[pidx]; in rootnex_coredma_win()
4967 rootnex_dma_t *dma; in rootnex_coredma_hdl_setprivate() local
4970 dma = (rootnex_dma_t *)hp->dmai_private; in rootnex_coredma_hdl_setprivate()
4971 dma->dp_iommu_private = v; in rootnex_coredma_hdl_setprivate()
4982 rootnex_dma_t *dma; in rootnex_coredma_hdl_getprivate() local
4985 dma = (rootnex_dma_t *)hp->dmai_private; in rootnex_coredma_hdl_getprivate()
4987 return (dma->dp_iommu_private); in rootnex_coredma_hdl_getprivate()
5050 rootnex_dma_t *dma; in rootnex_dma_check() local
5061 dma = (rootnex_dma_t *)hp->dmai_private; in rootnex_dma_check()
5070 if (dma->dp_window == NULL) { in rootnex_dma_check()
5072 for (i = 0; i < dma->dp_sglinfo.si_sgl_size; i++) { in rootnex_dma_check()
5077 if ((fault_addr >= dma->dp_cookies[i].dmac_laddress) && in rootnex_dma_check()
5078 (fault_addr <= (dma->dp_cookies[i].dmac_laddress + in rootnex_dma_check()
5079 dma->dp_cookies[i].dmac_size))) { in rootnex_dma_check()
5090 window = &dma->dp_window[i]; in rootnex_dma_check()