Lines Matching refs:dma
379 rootnex_dma_t *dma, ddi_dma_attr_t *attr, ddi_dma_obj_t *dmao, int kmflag);
381 rootnex_dma_t *dma, ddi_dma_attr_t *attr);
382 static void rootnex_teardown_copybuf(rootnex_dma_t *dma);
383 static int rootnex_setup_windows(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
385 static void rootnex_teardown_windows(rootnex_dma_t *dma);
386 static void rootnex_init_win(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
389 rootnex_dma_t *dma, ddi_dma_cookie_t *cookie, off_t cur_offset,
392 rootnex_dma_t *dma, rootnex_window_t **windowp, ddi_dma_cookie_t *cookie,
395 rootnex_dma_t *dma, rootnex_window_t **windowp,
398 rootnex_dma_t *dma, rootnex_window_t **windowp, ddi_dma_cookie_t *cookie);
401 static int rootnex_verify_buffer(rootnex_dma_t *dma);
1710 rootnex_dma_t *dma; in rootnex_coredma_allochdl() local
1738 dma = (rootnex_dma_t *)hp->dmai_private; in rootnex_coredma_allochdl()
1739 dma->dp_prealloc_buffer = (uchar_t *)(((uintptr_t)dma + in rootnex_coredma_allochdl()
1746 dma->dp_dip = rdip; in rootnex_coredma_allochdl()
1747 dma->dp_sglinfo.si_flags = attr->dma_attr_flags; in rootnex_coredma_allochdl()
1748 dma->dp_sglinfo.si_min_addr = attr->dma_attr_addr_lo; in rootnex_coredma_allochdl()
1758 dma->dp_sglinfo.si_max_addr = attr->dma_attr_seg; in rootnex_coredma_allochdl()
1759 dma->dp_sglinfo.si_flags &= ~_DDI_DMA_BOUNCE_ON_SEG; in rootnex_coredma_allochdl()
1761 dma->dp_sglinfo.si_max_addr = attr->dma_attr_addr_hi; in rootnex_coredma_allochdl()
1768 if (attr->dma_attr_seg >= dma->dp_sglinfo.si_max_addr) in rootnex_coredma_allochdl()
1769 dma->dp_sglinfo.si_cancross = B_FALSE; in rootnex_coredma_allochdl()
1771 dma->dp_sglinfo.si_cancross = B_TRUE; in rootnex_coredma_allochdl()
1774 mutex_init(&dma->dp_mutex, NULL, MUTEX_DRIVER, NULL); in rootnex_coredma_allochdl()
1802 dma->dp_granularity_power_2 = B_FALSE; in rootnex_coredma_allochdl()
1804 dma->dp_granularity_power_2 = B_TRUE; in rootnex_coredma_allochdl()
1814 if (dma->dp_granularity_power_2) { in rootnex_coredma_allochdl()
1815 dma->dp_maxxfer = attr->dma_attr_maxxfer - in rootnex_coredma_allochdl()
1819 dma->dp_maxxfer = attr->dma_attr_maxxfer - in rootnex_coredma_allochdl()
1823 dma->dp_maxxfer = attr->dma_attr_maxxfer; in rootnex_coredma_allochdl()
1826 maxsegmentsize_ll = MIN(seg, dma->dp_maxxfer); in rootnex_coredma_allochdl()
1833 dma->dp_sglinfo.si_max_cookie_size = maxsegmentsize; in rootnex_coredma_allochdl()
1834 dma->dp_sglinfo.si_segmask = attr->dma_attr_seg; in rootnex_coredma_allochdl()
1908 rootnex_dma_t *dma; in rootnex_coredma_freehdl() local
1912 dma = (rootnex_dma_t *)hp->dmai_private; in rootnex_coredma_freehdl()
1915 ASSERT(!dma->dp_inuse); in rootnex_coredma_freehdl()
1917 mutex_destroy(&dma->dp_mutex); in rootnex_coredma_freehdl()
1964 rootnex_dma_t *dma; in rootnex_coredma_bindhdl() local
1970 dma = (rootnex_dma_t *)hp->dmai_private; in rootnex_coredma_bindhdl()
1971 dmao = &dma->dp_dma; in rootnex_coredma_bindhdl()
1972 sinfo = &dma->dp_sglinfo; in rootnex_coredma_bindhdl()
1977 dma->dp_sleep_flags = kmflag = KM_SLEEP; in rootnex_coredma_bindhdl()
1979 dma->dp_sleep_flags = kmflag = KM_NOSLEEP; in rootnex_coredma_bindhdl()
1994 e = mutex_tryenter(&dma->dp_mutex); in rootnex_coredma_bindhdl()
1999 if (dma->dp_inuse) { in rootnex_coredma_bindhdl()
2000 mutex_exit(&dma->dp_mutex); in rootnex_coredma_bindhdl()
2004 dma->dp_inuse = B_TRUE; in rootnex_coredma_bindhdl()
2005 mutex_exit(&dma->dp_mutex); in rootnex_coredma_bindhdl()
2009 dma->dp_dma = dmareq->dmar_object; in rootnex_coredma_bindhdl()
2013 dmao = &dma->dp_dvma; in rootnex_coredma_bindhdl()
2020 dma->dp_dvma_used = B_TRUE; in rootnex_coredma_bindhdl()
2026 dma->dp_dvma_used = B_TRUE; in rootnex_coredma_bindhdl()
2027 dma->dp_need_to_free_cookie = B_FALSE; in rootnex_coredma_bindhdl()
2030 cookie = hp->dmai_cookie = dma->dp_cookies = in rootnex_coredma_bindhdl()
2031 (ddi_dma_cookie_t *)dma->dp_prealloc_buffer; in rootnex_coredma_bindhdl()
2053 sinfo->si_max_pages = mmu_btopr(dma->dp_dma.dmao_size) + 1; in rootnex_coredma_bindhdl()
2055 if (dma->dp_dvma_used) { in rootnex_coredma_bindhdl()
2068 ncookies = (dma->dp_dma.dmao_size / sinfo->si_max_cookie_size) in rootnex_coredma_bindhdl()
2072 (dma->dp_dma.dmao_size / attr->dma_attr_seg) + 1; in rootnex_coredma_bindhdl()
2084 dma->dp_cookies = (ddi_dma_cookie_t *)dma->dp_prealloc_buffer; in rootnex_coredma_bindhdl()
2085 dma->dp_need_to_free_cookie = B_FALSE; in rootnex_coredma_bindhdl()
2103 dma->dp_cookie_size = sinfo->si_max_pages * in rootnex_coredma_bindhdl()
2105 dma->dp_cookies = kmem_alloc(dma->dp_cookie_size, kmflag); in rootnex_coredma_bindhdl()
2106 if (dma->dp_cookies == NULL) { in rootnex_coredma_bindhdl()
2111 dma->dp_need_to_free_cookie = B_TRUE; in rootnex_coredma_bindhdl()
2115 hp->dmai_cookie = dma->dp_cookies; in rootnex_coredma_bindhdl()
2124 if (dma->dp_dvma_used) in rootnex_coredma_bindhdl()
2125 rootnex_dvma_get_sgl(dmao, dma->dp_cookies, &dma->dp_sglinfo); in rootnex_coredma_bindhdl()
2127 rootnex_get_sgl(dmao, dma->dp_cookies, &dma->dp_sglinfo); in rootnex_coredma_bindhdl()
2151 (dmao->dmao_size <= dma->dp_maxxfer)) { in rootnex_coredma_bindhdl()
2166 *cookiep = dma->dp_cookies[0]; in rootnex_coredma_bindhdl()
2194 if (dma->dp_dvma_used) { in rootnex_coredma_bindhdl()
2196 &dma->dp_dvma); in rootnex_coredma_bindhdl()
2200 e = rootnex_bind_slowpath(hp, dmareq, dma, attr, &dma->dp_dma, in rootnex_coredma_bindhdl()
2206 if (dma->dp_need_to_free_cookie) { in rootnex_coredma_bindhdl()
2207 kmem_free(dma->dp_cookies, dma->dp_cookie_size); in rootnex_coredma_bindhdl()
2222 if ((dma->dp_window[dma->dp_current_win].wd_dosync) && in rootnex_coredma_bindhdl()
2240 *ccountp = dma->dp_window[dma->dp_current_win].wd_cookie_cnt; in rootnex_coredma_bindhdl()
2241 ASSERT(hp->dmai_nwin <= dma->dp_max_win); in rootnex_coredma_bindhdl()
2243 *cookiep = dma->dp_cookies[0]; in rootnex_coredma_bindhdl()
2290 rootnex_dma_t *dma; in rootnex_coredma_unbindhdl() local
2294 dma = (rootnex_dma_t *)hp->dmai_private; in rootnex_coredma_unbindhdl()
2298 e = rootnex_verify_buffer(dma); in rootnex_coredma_unbindhdl()
2306 if (dma->dp_window && dma->dp_window[dma->dp_current_win].wd_dosync && in rootnex_coredma_unbindhdl()
2316 rootnex_teardown_copybuf(dma); in rootnex_coredma_unbindhdl()
2317 rootnex_teardown_windows(dma); in rootnex_coredma_unbindhdl()
2320 if (IOMMU_USED(rdip) && dma->dp_dvma_used) in rootnex_coredma_unbindhdl()
2322 &dma->dp_dvma); in rootnex_coredma_unbindhdl()
2329 if (dma->dp_need_to_free_cookie) { in rootnex_coredma_unbindhdl()
2330 kmem_free(dma->dp_cookies, dma->dp_cookie_size); in rootnex_coredma_unbindhdl()
2377 rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private; in rootnex_coredma_get_sleep_flags() local
2379 if (dma->dp_sleep_flags != KM_SLEEP && in rootnex_coredma_get_sleep_flags()
2380 dma->dp_sleep_flags != KM_NOSLEEP) in rootnex_coredma_get_sleep_flags()
2382 return (dma->dp_sleep_flags); in rootnex_coredma_get_sleep_flags()
2389 rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private; in rootnex_coredma_reset_cookies() local
2392 if (dma->dp_window) { in rootnex_coredma_reset_cookies()
2393 window = &dma->dp_window[dma->dp_current_win]; in rootnex_coredma_reset_cookies()
2396 hp->dmai_cookie = dma->dp_cookies; in rootnex_coredma_reset_cookies()
2410 rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private; in rootnex_coredma_get_cookies() local
2418 if (dma->dp_window) { in rootnex_coredma_get_cookies()
2419 window = &dma->dp_window[dma->dp_current_win]; in rootnex_coredma_get_cookies()
2423 cp = dma->dp_cookies; in rootnex_coredma_get_cookies()
2424 *ccountp = dma->dp_sglinfo.si_sgl_size; in rootnex_coredma_get_cookies()
2451 rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private; in rootnex_coredma_set_cookies() local
2457 ASSERT(dma->dp_need_to_switch_cookies == B_FALSE); in rootnex_coredma_set_cookies()
2459 if (dma->dp_window) { in rootnex_coredma_set_cookies()
2460 window = &dma->dp_window[dma->dp_current_win]; in rootnex_coredma_set_cookies()
2461 dma->dp_saved_cookies = window->wd_first_cookie; in rootnex_coredma_set_cookies()
2464 cur_cookiep = (hp->dmai_cookie - dma->dp_saved_cookies) in rootnex_coredma_set_cookies()
2467 dma->dp_saved_cookies = dma->dp_cookies; in rootnex_coredma_set_cookies()
2468 dma->dp_cookies = cookiep; in rootnex_coredma_set_cookies()
2469 ASSERT(ccount == dma->dp_sglinfo.si_sgl_size); in rootnex_coredma_set_cookies()
2470 cur_cookiep = (hp->dmai_cookie - dma->dp_saved_cookies) in rootnex_coredma_set_cookies()
2471 + dma->dp_cookies; in rootnex_coredma_set_cookies()
2474 dma->dp_need_to_switch_cookies = B_TRUE; in rootnex_coredma_set_cookies()
2485 rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private; in rootnex_coredma_clear_cookies() local
2492 if (dma->dp_need_to_switch_cookies == B_FALSE) in rootnex_coredma_clear_cookies()
2495 ASSERT(dma->dp_saved_cookies); in rootnex_coredma_clear_cookies()
2497 if (dma->dp_window) { in rootnex_coredma_clear_cookies()
2498 window = &dma->dp_window[dma->dp_current_win]; in rootnex_coredma_clear_cookies()
2500 window->wd_first_cookie = dma->dp_saved_cookies; in rootnex_coredma_clear_cookies()
2501 dma->dp_saved_cookies = NULL; in rootnex_coredma_clear_cookies()
2506 cookie_array = dma->dp_cookies; in rootnex_coredma_clear_cookies()
2507 dma->dp_cookies = dma->dp_saved_cookies; in rootnex_coredma_clear_cookies()
2508 dma->dp_saved_cookies = NULL; in rootnex_coredma_clear_cookies()
2509 ccount = dma->dp_sglinfo.si_sgl_size; in rootnex_coredma_clear_cookies()
2511 + dma->dp_cookies; in rootnex_coredma_clear_cookies()
2518 dma->dp_need_to_switch_cookies = B_FALSE; in rootnex_coredma_clear_cookies()
2549 rootnex_verify_buffer(rootnex_dma_t *dma) in rootnex_verify_buffer() argument
2560 if (dma->dp_dma.dmao_type == DMA_OTYP_PAGES) { in rootnex_verify_buffer()
2561 poff = dma->dp_dma.dmao_obj.pp_obj.pp_offset & MMU_PAGEOFFSET; in rootnex_verify_buffer()
2563 vaddr = dma->dp_dma.dmao_obj.virt_obj.v_addr; in rootnex_verify_buffer()
2566 pcnt = mmu_btopr(dma->dp_dma.dmao_size + poff); in rootnex_verify_buffer()
2568 switch (dma->dp_dma.dmao_type) { in rootnex_verify_buffer()
2574 pp = dma->dp_dma.dmao_obj.pp_obj.pp_pp; in rootnex_verify_buffer()
2585 pplist = dma->dp_dma.dmao_obj.virt_obj.v_priv; in rootnex_verify_buffer()
2599 if (rootnex_get_as(&dma->dp_dma) == &kas) { in rootnex_verify_buffer()
2627 rootnex_dma_t *dma; in rootnex_clean_dmahdl() local
2630 dma = (rootnex_dma_t *)hp->dmai_private; in rootnex_clean_dmahdl()
2633 dma->dp_current_cookie = 0; in rootnex_clean_dmahdl()
2634 dma->dp_copybuf_size = 0; in rootnex_clean_dmahdl()
2635 dma->dp_window = NULL; in rootnex_clean_dmahdl()
2636 dma->dp_cbaddr = NULL; in rootnex_clean_dmahdl()
2637 dma->dp_inuse = B_FALSE; in rootnex_clean_dmahdl()
2638 dma->dp_dvma_used = B_FALSE; in rootnex_clean_dmahdl()
2639 dma->dp_need_to_free_cookie = B_FALSE; in rootnex_clean_dmahdl()
2640 dma->dp_need_to_switch_cookies = B_FALSE; in rootnex_clean_dmahdl()
2641 dma->dp_saved_cookies = NULL; in rootnex_clean_dmahdl()
2642 dma->dp_sleep_flags = KM_PANIC; in rootnex_clean_dmahdl()
2643 dma->dp_need_to_free_window = B_FALSE; in rootnex_clean_dmahdl()
2644 dma->dp_partial_required = B_FALSE; in rootnex_clean_dmahdl()
2645 dma->dp_trim_required = B_FALSE; in rootnex_clean_dmahdl()
2646 dma->dp_sglinfo.si_copybuf_req = 0; in rootnex_clean_dmahdl()
3199 rootnex_dma_t *dma, ddi_dma_attr_t *attr, ddi_dma_obj_t *dmao, int kmflag) in rootnex_bind_slowpath() argument
3214 sinfo = &dma->dp_sglinfo; in rootnex_bind_slowpath()
3223 e = rootnex_setup_copybuf(hp, dmareq, dma, attr); in rootnex_bind_slowpath()
3228 dma->dp_copybuf_size = 0; in rootnex_bind_slowpath()
3235 if ((dma->dp_copybuf_size < sinfo->si_copybuf_req) || in rootnex_bind_slowpath()
3236 (dmao->dmao_size > dma->dp_maxxfer) || in rootnex_bind_slowpath()
3238 dma->dp_partial_required = B_TRUE; in rootnex_bind_slowpath()
3240 dma->dp_trim_required = B_TRUE; in rootnex_bind_slowpath()
3243 dma->dp_partial_required = B_FALSE; in rootnex_bind_slowpath()
3244 dma->dp_trim_required = B_FALSE; in rootnex_bind_slowpath()
3248 if (dma->dp_partial_required && in rootnex_bind_slowpath()
3251 mnum = ddi_driver_major(dma->dp_dip); in rootnex_bind_slowpath()
3266 "/etc/system(5).", ddi_driver_name(dma->dp_dip)); in rootnex_bind_slowpath()
3275 e = rootnex_setup_windows(hp, dma, attr, dmao, kmflag); in rootnex_bind_slowpath()
3277 rootnex_teardown_copybuf(dma); in rootnex_bind_slowpath()
3281 window = &dma->dp_window[0]; in rootnex_bind_slowpath()
3282 cookie = &dma->dp_cookies[0]; in rootnex_bind_slowpath()
3284 rootnex_init_win(hp, dma, window, cookie, cur_offset); in rootnex_bind_slowpath()
3296 if (dma->dp_copybuf_size > 0) { in rootnex_bind_slowpath()
3297 rootnex_setup_cookie(dmao, dma, cookie, in rootnex_bind_slowpath()
3311 if (dma->dp_copybuf_size && in rootnex_bind_slowpath()
3312 (copybuf_used > dma->dp_copybuf_size)) { in rootnex_bind_slowpath()
3314 e = rootnex_copybuf_window_boundary(hp, dma, &window, in rootnex_bind_slowpath()
3317 rootnex_teardown_copybuf(dma); in rootnex_bind_slowpath()
3318 rootnex_teardown_windows(dma); in rootnex_bind_slowpath()
3330 dma->dp_dip); in rootnex_bind_slowpath()
3337 e = rootnex_sgllen_window_boundary(hp, dma, &window, in rootnex_bind_slowpath()
3340 rootnex_teardown_copybuf(dma); in rootnex_bind_slowpath()
3341 rootnex_teardown_windows(dma); in rootnex_bind_slowpath()
3353 dma->dp_dip); in rootnex_bind_slowpath()
3357 dma->dp_maxxfer) { in rootnex_bind_slowpath()
3359 e = rootnex_maxxfer_window_boundary(hp, dma, &window, in rootnex_bind_slowpath()
3362 rootnex_teardown_copybuf(dma); in rootnex_bind_slowpath()
3363 rootnex_teardown_windows(dma); in rootnex_bind_slowpath()
3375 dma->dp_dip); in rootnex_bind_slowpath()
3402 ASSERT(dma->dp_partial_required); in rootnex_bind_slowpath()
3413 rootnex_dma_t *dma, ddi_dma_attr_t *attr) in rootnex_setup_copybuf() argument
3421 ASSERT(!dma->dp_dvma_used); in rootnex_setup_copybuf()
3423 sinfo = &dma->dp_sglinfo; in rootnex_setup_copybuf()
3432 dma->dp_copybuf_size = MIN(sinfo->si_copybuf_req, max_copybuf); in rootnex_setup_copybuf()
3433 ASSERT((dma->dp_copybuf_size & MMU_PAGEOFFSET) == 0); in rootnex_setup_copybuf()
3459 e = i_ddi_mem_alloc(dma->dp_dip, &lattr, dma->dp_copybuf_size, cansleep, in rootnex_setup_copybuf()
3460 0, NULL, &dma->dp_cbaddr, &dma->dp_cbsize, NULL); in rootnex_setup_copybuf()
3465 ROOTNEX_DPROBE2(rootnex__alloc__copybuf, dev_info_t *, dma->dp_dip, in rootnex_setup_copybuf()
3466 size_t, dma->dp_copybuf_size); in rootnex_setup_copybuf()
3478 rootnex_setup_windows(ddi_dma_impl_t *hp, rootnex_dma_t *dma, in rootnex_setup_windows() argument
3493 sinfo = &dma->dp_sglinfo; in rootnex_setup_windows()
3495 dma->dp_current_win = 0; in rootnex_setup_windows()
3499 if (!dma->dp_partial_required) { in rootnex_setup_windows()
3500 dma->dp_max_win = 1; in rootnex_setup_windows()
3520 if (sinfo->si_copybuf_req > dma->dp_copybuf_size) { in rootnex_setup_windows()
3521 ASSERT(dma->dp_copybuf_size > 0); in rootnex_setup_windows()
3523 dma->dp_copybuf_size) + 1 + 2; in rootnex_setup_windows()
3548 if (dmao->dmao_size > dma->dp_maxxfer) { in rootnex_setup_windows()
3550 dma->dp_maxxfer) + 1 + 2; in rootnex_setup_windows()
3554 dma->dp_max_win = copybuf_win + sglwin + maxxfer_win; in rootnex_setup_windows()
3555 ASSERT(dma->dp_max_win > 0); in rootnex_setup_windows()
3557 win_state_size = dma->dp_max_win * sizeof (rootnex_window_t); in rootnex_setup_windows()
3568 if (dma->dp_need_to_free_cookie) { in rootnex_setup_windows()
3570 ASSERT(space_used <= dma->dp_cookie_size); in rootnex_setup_windows()
3571 if ((dma->dp_cookie_size - space_used) <= in rootnex_setup_windows()
3574 windowp = (rootnex_window_t *)dma->dp_prealloc_buffer; in rootnex_setup_windows()
3582 state_available = dma->dp_cookie_size - space_used; in rootnex_setup_windows()
3584 &dma->dp_cookies[sinfo->si_sgl_size]; in rootnex_setup_windows()
3592 &dma->dp_cookies[sinfo->si_sgl_size]; in rootnex_setup_windows()
3599 if (dma->dp_copybuf_size > 0) { in rootnex_setup_windows()
3610 dma->dp_window = windowp; in rootnex_setup_windows()
3611 dma->dp_need_to_free_window = B_FALSE; in rootnex_setup_windows()
3615 dma->dp_window = kmem_alloc(space_needed, kmflag); in rootnex_setup_windows()
3616 if (dma->dp_window == NULL) { in rootnex_setup_windows()
3619 dma->dp_need_to_free_window = B_TRUE; in rootnex_setup_windows()
3620 dma->dp_window_size = space_needed; in rootnex_setup_windows()
3622 dma->dp_dip, size_t, space_needed); in rootnex_setup_windows()
3629 if (dma->dp_copybuf_size > 0) { in rootnex_setup_windows()
3630 dma->dp_pgmap = (rootnex_pgmap_t *)(((uintptr_t) in rootnex_setup_windows()
3631 &dma->dp_window[dma->dp_max_win] + 0x7) & ~0x7); in rootnex_setup_windows()
3634 dma->dp_pgmap = NULL; in rootnex_setup_windows()
3646 rootnex_teardown_copybuf(rootnex_dma_t *dma) in rootnex_teardown_copybuf() argument
3649 if (dma->dp_cbaddr != NULL) { in rootnex_teardown_copybuf()
3650 i_ddi_mem_free(dma->dp_cbaddr, NULL); in rootnex_teardown_copybuf()
3660 rootnex_teardown_windows(rootnex_dma_t *dma) in rootnex_teardown_windows() argument
3666 if (dma->dp_need_to_free_window) { in rootnex_teardown_windows()
3667 kmem_free(dma->dp_window, dma->dp_window_size); in rootnex_teardown_windows()
3679 rootnex_init_win(ddi_dma_impl_t *hp, rootnex_dma_t *dma, in rootnex_init_win() argument
3703 rootnex_setup_cookie(ddi_dma_obj_t *dmar_object, rootnex_dma_t *dma, in rootnex_setup_cookie() argument
3717 sinfo = &dma->dp_sglinfo; in rootnex_setup_cookie()
3744 if (!ISP2(dma->dp_copybuf_size)) { in rootnex_setup_cookie()
3751 dma->dp_pgmap[pidx].pm_uses_copybuf = B_TRUE; in rootnex_setup_cookie()
3758 if ((*copybuf_used + MMU_PAGESIZE) <= dma->dp_copybuf_size) { in rootnex_setup_cookie()
3759 dma->dp_pgmap[pidx].pm_cbaddr = dma->dp_cbaddr + in rootnex_setup_cookie()
3763 dma->dp_pgmap[pidx].pm_cbaddr = (caddr_t)( in rootnex_setup_cookie()
3764 (uintptr_t)dma->dp_cbaddr + in rootnex_setup_cookie()
3766 (dma->dp_copybuf_size - 1))); in rootnex_setup_cookie()
3768 dma->dp_pgmap[pidx].pm_cbaddr = (caddr_t)( in rootnex_setup_cookie()
3769 (uintptr_t)dma->dp_cbaddr + in rootnex_setup_cookie()
3770 (*copybuf_used % dma->dp_copybuf_size)); in rootnex_setup_cookie()
3780 dma->dp_pgmap[pidx].pm_cbaddr)) + poff; in rootnex_setup_cookie()
3791 dma->dp_pgmap[pidx].pm_kaddr = (caddr_t)(((uintptr_t) in rootnex_setup_cookie()
3801 dma->dp_pgmap[pidx].pm_kaddr = hat_kpm_pfn2va(pfn); in rootnex_setup_cookie()
3825 dma->dp_pgmap[pidx].pm_uses_copybuf = B_FALSE; in rootnex_setup_cookie()
3848 rootnex_sgllen_window_boundary(ddi_dma_impl_t *hp, rootnex_dma_t *dma, in rootnex_sgllen_window_boundary() argument
3861 if (!dma->dp_trim_required) { in rootnex_sgllen_window_boundary()
3863 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset); in rootnex_sgllen_window_boundary()
3871 if (dma->dp_granularity_power_2) { in rootnex_sgllen_window_boundary()
3880 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset); in rootnex_sgllen_window_boundary()
3928 rootnex_init_win(hp, dma, *windowp, cookie, new_offset); in rootnex_sgllen_window_boundary()
3950 if ((*windowp)->wd_size > dma->dp_maxxfer) { in rootnex_sgllen_window_boundary()
3956 trim_sz = (*windowp)->wd_size - dma->dp_maxxfer; in rootnex_sgllen_window_boundary()
3962 ASSERT((*windowp)->wd_size == dma->dp_maxxfer); in rootnex_sgllen_window_boundary()
3970 rootnex_init_win(hp, dma, *windowp, cookie, new_offset); in rootnex_sgllen_window_boundary()
3988 rootnex_copybuf_window_boundary(ddi_dma_impl_t *hp, rootnex_dma_t *dma, in rootnex_copybuf_window_boundary() argument
4001 sinfo = &dma->dp_sglinfo; in rootnex_copybuf_window_boundary()
4017 if (!dma->dp_trim_required) { in rootnex_copybuf_window_boundary()
4019 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset); in rootnex_copybuf_window_boundary()
4033 if (dma->dp_granularity_power_2) { in rootnex_copybuf_window_boundary()
4047 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset); in rootnex_copybuf_window_boundary()
4103 dma->dp_pgmap[pidx].pm_cbaddr; in rootnex_copybuf_window_boundary()
4121 rootnex_init_win(hp, dma, *windowp, cookie, new_offset); in rootnex_copybuf_window_boundary()
4140 (*windowp)->wd_trim.tr_first_cbaddr = dma->dp_cbaddr; in rootnex_copybuf_window_boundary()
4143 paddr = pfn_to_pa(hat_getpfnum(kas.a_hat, dma->dp_cbaddr)) + in rootnex_copybuf_window_boundary()
4173 dma->dp_pgmap[pidx + 1].pm_cbaddr += MMU_PAGESIZE; in rootnex_copybuf_window_boundary()
4177 dma->dp_pgmap[pidx + 1].pm_cbaddr)) + poff; in rootnex_copybuf_window_boundary()
4192 ASSERT((*windowp)->wd_size < dma->dp_maxxfer); in rootnex_copybuf_window_boundary()
4210 rootnex_maxxfer_window_boundary(ddi_dma_impl_t *hp, rootnex_dma_t *dma, in rootnex_maxxfer_window_boundary() argument
4224 trim_sz = ((*windowp)->wd_size + cookie->dmac_size) - dma->dp_maxxfer; in rootnex_maxxfer_window_boundary()
4226 ASSERT(trim_sz <= dma->dp_maxxfer); in rootnex_maxxfer_window_boundary()
4241 (*windowp)->wd_size = dma->dp_maxxfer; in rootnex_maxxfer_window_boundary()
4259 rootnex_init_win(hp, dma, *windowp, cookie, new_offset); in rootnex_maxxfer_window_boundary()
4282 rootnex_dma_t *dma; in rootnex_coredma_sync() local
4294 dma = (rootnex_dma_t *)hp->dmai_private; in rootnex_coredma_sync()
4295 sinfo = &dma->dp_sglinfo; in rootnex_coredma_sync()
4301 if (dma->dp_window == NULL) { in rootnex_coredma_sync()
4306 win = &dma->dp_window[dma->dp_current_win]; in rootnex_coredma_sync()
4357 cbpage = &dma->dp_pgmap[pidx]; in rootnex_coredma_sync()
4376 dev_info_t *, dma->dp_dip, size_t, psize); in rootnex_coredma_sync()
4387 dev_info_t *, dma->dp_dip, size_t, psize); in rootnex_coredma_sync()
4489 rootnex_dma_t *dma; in rootnex_coredma_win() local
4494 dma = (rootnex_dma_t *)hp->dmai_private; in rootnex_coredma_win()
4502 dmao = dma->dp_dvma_used ? &dma->dp_dvma : &dma->dp_dma; in rootnex_coredma_win()
4510 if (dma->dp_window == NULL) { in rootnex_coredma_win()
4516 hp->dmai_cookie = dma->dp_cookies; in rootnex_coredma_win()
4519 *ccountp = dma->dp_sglinfo.si_sgl_size; in rootnex_coredma_win()
4528 window = &dma->dp_window[dma->dp_current_win]; in rootnex_coredma_win()
4539 dma->dp_current_win = win; in rootnex_coredma_win()
4540 window = &dma->dp_window[win]; in rootnex_coredma_win()
4548 dma->dp_pgmap[trim->tr_first_pidx].pm_cbaddr = in rootnex_coredma_win()
4556 dma->dp_pgmap[trim->tr_last_pidx].pm_cbaddr = in rootnex_coredma_win()
4613 rootnex_dma_t *dma; in rootnex_coredma_hdl_setprivate() local
4616 dma = (rootnex_dma_t *)hp->dmai_private; in rootnex_coredma_hdl_setprivate()
4617 dma->dp_iommu_private = v; in rootnex_coredma_hdl_setprivate()
4628 rootnex_dma_t *dma; in rootnex_coredma_hdl_getprivate() local
4631 dma = (rootnex_dma_t *)hp->dmai_private; in rootnex_coredma_hdl_getprivate()
4633 return (dma->dp_iommu_private); in rootnex_coredma_hdl_getprivate()
4696 rootnex_dma_t *dma; in rootnex_dma_check() local
4707 dma = (rootnex_dma_t *)hp->dmai_private; in rootnex_dma_check()
4716 if (dma->dp_window == NULL) { in rootnex_dma_check()
4718 for (i = 0; i < dma->dp_sglinfo.si_sgl_size; i++) { in rootnex_dma_check()
4723 if ((fault_addr >= dma->dp_cookies[i].dmac_laddress) && in rootnex_dma_check()
4724 (fault_addr <= (dma->dp_cookies[i].dmac_laddress + in rootnex_dma_check()
4725 dma->dp_cookies[i].dmac_size))) { in rootnex_dma_check()
4736 window = &dma->dp_window[i]; in rootnex_dma_check()